[Python-modules-commits] [python-pex] 01/05: Import python-pex_1.0.1.orig.tar.gz
Barry Warsaw
barry at moszumanska.debian.org
Fri Jul 17 20:28:29 UTC 2015
This is an automated email from the git hooks/post-receive script.
barry pushed a commit to branch master
in repository python-pex.
commit 7ccc33838db1d5fcf9e16fb478a1be8f44a04fa3
Author: Barry Warsaw <barry at debian.org>
Date: Fri Jul 17 16:17:28 2015 -0400
Import python-pex_1.0.1.orig.tar.gz
---
CHANGES.rst | 14 ++++++++
PKG-INFO | 16 ++++++++-
docs/whatispex.rst | 2 +-
pex.egg-info/PKG-INFO | 16 ++++++++-
pex.egg-info/SOURCES.txt | 3 +-
pex/common.py | 3 +-
pex/compiler.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++
pex/environment.py | 5 ++-
pex/link.py | 3 ++
pex/marshaller.py | 86 --------------------------------------------
pex/pex_builder.py | 56 +++++++++++++++++++----------
pex/resolver.py | 29 ++++++++++++---
pex/testing.py | 15 ++++----
pex/version.py | 2 +-
tests/test_compiler.py | 72 +++++++++++++++++++++++++++++++++++++
tests/test_environment.py | 2 +-
tests/test_link.py | 5 +++
tests/test_pex_builder.py | 57 ++++++++++++++++++++++++++++++
tests/test_resolver.py | 34 ++++++++++++++++++
19 files changed, 388 insertions(+), 122 deletions(-)
diff --git a/CHANGES.rst b/CHANGES.rst
index cc25d1f..b6f9046 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -3,6 +3,20 @@ CHANGES
=======
-----
+1.0.1
+-----
+
+* Allow PEXBuilder to optionally copy files into the PEX environment instead of hard-linking them.
+
+* Allow PEXBuilder to optionally skip precompilation of .py files into .pyc files.
+
+* Bug fix: PEXBuilder did not respect the target interpreter when compiling source to bytecode.
+ Fixes `#127 <https://github.com/pantsbuild/pex/issues/127>`_.
+
+* Bug fix: Fix complex resolutions when using a cache.
+ Fixes: `#120 <https://github.com/pantsbuild/pex/issues/120>`_.
+
+-----
1.0.0
-----
diff --git a/PKG-INFO b/PKG-INFO
index 8dc49f7..b0fc2ca 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: pex
-Version: 1.0.0
+Version: 1.0.1
Summary: The PEX packaging toolchain.
Home-page: https://github.com/pantsbuild/pex
Author: UNKNOWN
@@ -11,6 +11,20 @@ Description: =======
=======
-----
+ 1.0.1
+ -----
+
+ * Allow PEXBuilder to optionally copy files into the PEX environment instead of hard-linking them.
+
+ * Allow PEXBuilder to optionally skip precompilation of .py files into .pyc files.
+
+ * Bug fix: PEXBuilder did not respect the target interpreter when compiling source to bytecode.
+ Fixes `#127 <https://github.com/pantsbuild/pex/issues/127>`_.
+
+ * Bug fix: Fix complex resolutions when using a cache.
+ Fixes: `#120 <https://github.com/pantsbuild/pex/issues/120>`_.
+
+ -----
1.0.0
-----
diff --git a/docs/whatispex.rst b/docs/whatispex.rst
index 7e0a4a0..d61037e 100644
--- a/docs/whatispex.rst
+++ b/docs/whatispex.rst
@@ -40,7 +40,7 @@ will execute ``my_module/__main__.py`` if it exists.
Because of the flexibility of the Python import subsystem, ``python -m
my_module`` works regardless if ``my_module`` is on disk or within a zip
file. Adding ``#!/usr/bin/env python`` to the top of a .zip file containing
-a ``__main__.py`` and and marking it executable will turn it into an
+a ``__main__.py`` and marking it executable will turn it into an
executable Python program. pex takes advantage of this feature in order to
build executable .pex files. This is described more thoroughly in
`PEP 441 <https://www.python.org/dev/peps/pep-0441/>`_.
diff --git a/pex.egg-info/PKG-INFO b/pex.egg-info/PKG-INFO
index 8dc49f7..b0fc2ca 100644
--- a/pex.egg-info/PKG-INFO
+++ b/pex.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: pex
-Version: 1.0.0
+Version: 1.0.1
Summary: The PEX packaging toolchain.
Home-page: https://github.com/pantsbuild/pex
Author: UNKNOWN
@@ -11,6 +11,20 @@ Description: =======
=======
-----
+ 1.0.1
+ -----
+
+ * Allow PEXBuilder to optionally copy files into the PEX environment instead of hard-linking them.
+
+ * Allow PEXBuilder to optionally skip precompilation of .py files into .pyc files.
+
+ * Bug fix: PEXBuilder did not respect the target interpreter when compiling source to bytecode.
+ Fixes `#127 <https://github.com/pantsbuild/pex/issues/127>`_.
+
+ * Bug fix: Fix complex resolutions when using a cache.
+ Fixes: `#120 <https://github.com/pantsbuild/pex/issues/120>`_.
+
+ -----
1.0.0
-----
diff --git a/pex.egg-info/SOURCES.txt b/pex.egg-info/SOURCES.txt
index 07727ef..bb267f2 100644
--- a/pex.egg-info/SOURCES.txt
+++ b/pex.egg-info/SOURCES.txt
@@ -17,6 +17,7 @@ pex/archiver.py
pex/base.py
pex/common.py
pex/compatibility.py
+pex/compiler.py
pex/crawler.py
pex/environment.py
pex/fetcher.py
@@ -26,7 +27,6 @@ pex/installer.py
pex/interpreter.py
pex/iterator.py
pex/link.py
-pex/marshaller.py
pex/orderedset.py
pex/package.py
pex/pep425.py
@@ -60,6 +60,7 @@ scripts/coverage.sh
scripts/do_nothing.py
tests/test_archiver.py
tests/test_compatibility.py
+tests/test_compiler.py
tests/test_crawler.py
tests/test_environment.py
tests/test_fetcher.py
diff --git a/pex/common.py b/pex/common.py
index b82f050..89f1941 100644
--- a/pex/common.py
+++ b/pex/common.py
@@ -293,7 +293,8 @@ class Chroot(object):
Has similar exceptional cases as Chroot.copy
"""
dst = self._normalize(dst)
- self.write('', dst, label, mode='a')
+ self._tag(dst, label)
+ touch(os.path.join(self.chroot, dst))
def get(self, label):
"""Get all files labeled with ``label``"""
diff --git a/pex/compiler.py b/pex/compiler.py
new file mode 100644
index 0000000..6346d0f
--- /dev/null
+++ b/pex/compiler.py
@@ -0,0 +1,90 @@
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import subprocess
+import tempfile
+
+from .compatibility import to_bytes
+
+
+_COMPILER_MAIN = """
+from __future__ import print_function
+
+import os
+import py_compile
+import sys
+
+
+def compile(root, relpaths):
+ compiled = []
+ errored = {}
+ for relpath in relpaths:
+ abspath = os.path.join(root, relpath)
+ # NB: We give the compiled bytecode file a `.pyc` extension, but if PYTHONOPTIMIZE is in play
+ # the generated bytecode will be optimized. Traditionally these optimized bytecode files would
+ # have a `.pyo` extension, but the extension only matters for location of the file to execute
+ # for a given module and not on the interpretation of its bytecode contents. As such we're
+ # safe to pick the `.pyc` extension for all bytecode file cases without a need to interpret the
+ # current optimization setting for the active python interpreter.
+ pyc_relpath = relpath + 'c'
+ pyc_abspath = os.path.join(root, pyc_relpath)
+ try:
+ py_compile.compile(abspath, cfile=pyc_abspath, dfile=relpath, doraise=True)
+ compiled.append(pyc_relpath)
+ except py_compile.PyCompileError as e:
+ errored[e.file] = e.msg
+ return compiled, errored
+
+
+def main(root, relpaths):
+ compiled, errored = compile(root, relpaths)
+ if not errored:
+ for path in compiled:
+ print(path)
+ sys.exit(0)
+
+ print('Encountered %%d errors compiling %%d files:' %% (len(errored), len(relpaths)),
+ file=sys.stderr)
+ for file, msg in errored.items():
+ print(' %%s: %%s' %% (file, msg), file=sys.stderr)
+ sys.exit(1)
+
+root = %(root)r
+relpaths = %(relpaths)r
+
+main(root, relpaths)
+"""
+
+
+class Compiler(object):
+ class Error(Exception):
+ """Indicates an error compiling one or more python source files."""
+
+ def __init__(self, interpreter):
+ """Creates a bytecode compiler for the given `interpreter`.
+
+ :param interpreter: The interpreter to use to compile sources with.
+ :type interpreter: :class:`pex.interpreter.PythonInterpreter`
+ """
+ self._interpreter = interpreter
+
+ def compile(self, root, relpaths):
+ """Compiles the given python source files using this compiler's interpreter.
+
+ :param string root: The root path all the source files are found under.
+ :param list relpaths: The realtive paths from the `root` of the source files to compile.
+ :returns: A list of relative paths of the compiled bytecode files.
+ :raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files.
+ """
+ with tempfile.NamedTemporaryFile() as fp:
+ fp.write(to_bytes(_COMPILER_MAIN % {'root': root, 'relpaths': relpaths}, encoding='utf-8'))
+ fp.flush()
+ process = subprocess.Popen([self._interpreter.binary, fp.name],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = process.communicate()
+ if process.returncode != 0:
+ raise self.Error(err)
+ return [pyc_relpath.decode('utf-8') for pyc_relpath in out.splitlines()]
diff --git a/pex/environment.py b/pex/environment.py
index efc92d9..162ab8e 100644
--- a/pex/environment.py
+++ b/pex/environment.py
@@ -148,7 +148,10 @@ class PEXEnvironment(Environment):
except DistributionNotFound as e:
TRACER.log('Failed to resolve a requirement: %s' % e)
unresolved_reqs.add(e.args[0].project_name)
- if e.args[1]:
+ # Older versions of pkg_resources just call `DistributionNotFound(req)` instead of the
+ # modern `DistributionNotFound(req, requirers)` and so we may not have the 2nd requirers
+ # slot at all.
+ if len(e.args) >= 2 and e.args[1]:
unresolved_reqs.update(e.args[1])
unresolved_reqs = set([req.lower() for req in unresolved_reqs])
diff --git a/pex/link.py b/pex/link.py
index be58927..2146a7f 100644
--- a/pex/link.py
+++ b/pex/link.py
@@ -76,6 +76,9 @@ class Link(object):
purl = urlparse.urlparse(self._normalize(url))
self._url = purl
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
def __eq__(self, link):
return self.__class__ == link.__class__ and self._url == link._url
diff --git a/pex/marshaller.py b/pex/marshaller.py
deleted file mode 100644
index 74b11a0..0000000
--- a/pex/marshaller.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
-# Licensed under the Apache License, Version 2.0 (see LICENSE).
-
-try:
- from imp import get_magic
- HAS_MAGIC = True
-except ImportError:
- HAS_MAGIC = False
-
-import struct
-import time
-
-import marshal
-
-from .compatibility import bytes as compatibility_bytes
-from .compatibility import BytesIO
-
-
-class CodeTimestamp(object):
- TIMESTAMP_RANGE = (4, 8)
-
- @classmethod
- def from_timestamp(cls, timestamp):
- return cls(timestamp)
-
- @classmethod
- def from_object(cls, pyc_object):
- stamp = time.localtime(
- struct.unpack('I', pyc_object[slice(*cls.TIMESTAMP_RANGE)])[0])
- return cls(stamp)
-
- def __init__(self, stamp=time.time()):
- self._stamp = stamp
-
- def to_object(self):
- return struct.pack('I', self._stamp)
-
-
-class CodeMarshaller(object):
- class Error(Exception): pass
- class InvalidCode(Error): pass
-
- if HAS_MAGIC:
- MAGIC = struct.unpack('I', get_magic())[0]
- MAGIC_RANGE = (0, 4)
- TIMESTAMP_RANGE = (4, 8)
-
- @classmethod
- def from_pyc(cls, pyc):
- if not HAS_MAGIC:
- raise cls.InvalidCode('Interpreter cannot unmarshal .pyc!')
- if not isinstance(pyc, compatibility_bytes) and not hasattr(pyc, 'read'):
- raise cls.InvalidCode(
- "CodeMarshaller.from_pyc expects a code or file-like object!")
- if not isinstance(pyc, compatibility_bytes):
- pyc = pyc.read()
- pyc_magic = struct.unpack('I', pyc[slice(*cls.MAGIC_RANGE)])[0]
- if pyc_magic != cls.MAGIC:
- raise cls.InvalidCode("Bad magic number! Got 0x%X" % pyc_magic)
- stamp = time.localtime(struct.unpack('I', pyc[slice(*cls.TIMESTAMP_RANGE)])[0])
- try:
- code = marshal.loads(pyc[8:])
- except ValueError as e:
- raise cls.InvalidCode("Unmarshaling error! %s" % e)
- return cls(code, stamp)
-
- @classmethod
- def from_py(cls, py, filename):
- stamp = int(time.time())
- code = compile(py.replace('\r\n', '\n').replace('\r', '\n'), filename, 'exec')
- return cls(code, stamp)
-
- def __init__(self, code, stamp):
- self._code = code
- self._stamp = stamp
-
- @property
- def code(self):
- return self._code
-
- def to_pyc(self):
- sio = BytesIO()
- sio.write(struct.pack('I', self.MAGIC))
- sio.write(struct.pack('I', self._stamp))
- sio.write(marshal.dumps(self._code))
- return sio.getvalue()
diff --git a/pex/pex_builder.py b/pex/pex_builder.py
index c2419ed..5410fc9 100644
--- a/pex/pex_builder.py
+++ b/pex/pex_builder.py
@@ -10,9 +10,9 @@ from pkg_resources import DefaultProvider, ZipProvider, get_provider
from .common import Chroot, chmod_plus_x, open_zip, safe_mkdir, safe_mkdtemp
from .compatibility import to_bytes
+from .compiler import Compiler
from .finders import get_entry_point_from_console_script, get_script_from_distributions
from .interpreter import PythonInterpreter
-from .marshaller import CodeMarshaller
from .pex_info import PexInfo
from .util import CacheHelper, DistributionHelper
@@ -55,7 +55,8 @@ class PEXBuilder(object):
BOOTSTRAP_DIR = ".bootstrap"
- def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, preamble=None):
+ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, preamble=None,
+ copy=False):
"""Initialize a pex builder.
:keyword path: The path to write the PEX as it is built. If ``None`` is specified,
@@ -67,6 +68,8 @@ class PEXBuilder(object):
:keyword preamble: If supplied, execute this code prior to bootstrapping this PEX
environment.
:type preamble: str
+ :keyword copy: If False, attempt to create the pex environment via hard-linking, falling
+ back to copying across devices. If True, always copy.
.. versionchanged:: 0.8
The temporary directory created when ``path`` is not specified is now garbage collected on
@@ -79,6 +82,7 @@ class PEXBuilder(object):
self._shebang = self._interpreter.identity.hashbang()
self._logger = logging.getLogger(__name__)
self._preamble = to_bytes(preamble or '')
+ self._copy = copy
self._distributions = set()
def _ensure_unfrozen(self, name='Operation'):
@@ -123,7 +127,6 @@ class PEXBuilder(object):
self._ensure_unfrozen('Changing PexInfo')
self._pex_info = value
- # TODO(wickman) Add option to not compile/marshal sources.
def add_source(self, filename, env_filename):
"""Add a source to the PEX environment.
@@ -132,12 +135,7 @@ class PEXBuilder(object):
must be a relative path.
"""
self._ensure_unfrozen('Adding source')
- self._chroot.link(filename, env_filename, "source")
- if filename.endswith('.py'):
- env_filename_pyc = os.path.splitext(env_filename)[0] + '.pyc'
- with open(filename) as fp:
- pyc_object = CodeMarshaller.from_py(fp.read(), env_filename)
- self._chroot.write(pyc_object.to_pyc(), env_filename_pyc, 'source')
+ self._copy_or_link(filename, env_filename, "source")
def add_resource(self, filename, env_filename):
"""Add a resource to the PEX environment.
@@ -147,7 +145,7 @@ class PEXBuilder(object):
must be a relative path.
"""
self._ensure_unfrozen('Adding a resource')
- self._chroot.link(filename, env_filename, "resource")
+ self._copy_or_link(filename, env_filename, "resource")
def add_requirement(self, req):
"""Add a requirement to the PEX environment.
@@ -178,7 +176,7 @@ class PEXBuilder(object):
if self._chroot.get("executable"):
raise self.InvalidExecutableSpecification(
"Setting executable on a PEXBuilder that already has one!")
- self._chroot.link(filename, env_filename, "executable")
+ self._copy_or_link(filename, env_filename, "executable")
entry_point = env_filename
entry_point.replace(os.path.sep, '.')
self._pex_info.entry_point = entry_point.rpartition('.')[0]
@@ -242,7 +240,7 @@ class PEXBuilder(object):
filename = os.path.join(root, f)
relpath = os.path.relpath(filename, path)
target = os.path.join(self._pex_info.internal_cache, dist_name, relpath)
- self._chroot.link(filename, target)
+ self._copy_or_link(filename, target)
return CacheHelper.dir_hash(path)
def _add_dist_zip(self, path, dist_name):
@@ -319,6 +317,15 @@ class PEXBuilder(object):
self._chroot.write(bytes(import_string, 'UTF-8'), sub_path)
init_digest.add(sub_path)
+ def _precompile_source(self):
+ source_relpaths = [path for label in ('source', 'executable', 'main', 'bootstrap')
+ for path in self._chroot.filesets.get(label, ()) if path.endswith('.py')]
+
+ compiler = Compiler(self.interpreter)
+ compiled_relpaths = compiler.compile(self._chroot.path(), source_relpaths)
+ for compiled in compiled_relpaths:
+ self._chroot.touch(compiled, label='bytecode')
+
def _prepare_manifest(self):
self._chroot.write(self._pex_info.dump().encode('utf-8'), PexInfo.PATH, label='manifest')
@@ -326,6 +333,12 @@ class PEXBuilder(object):
self._chroot.write(self._preamble + b'\n' + BOOTSTRAP_ENVIRONMENT,
'__main__.py', label='main')
+ def _copy_or_link(self, src, dst, label=None):
+ if self._copy:
+ self._chroot.copy(src, dst, label)
+ else:
+ self._chroot.link(src, dst, label)
+
# TODO(wickman) Ideally we unqualify our setuptools dependency and inherit whatever is
# bundled into the environment so long as it is compatible (and error out if not.)
#
@@ -346,8 +359,10 @@ class PEXBuilder(object):
for fn, content_stream in DistributionHelper.walk_data(setuptools):
if fn.startswith('pkg_resources') or fn.startswith('_markerlib'):
- self._chroot.write(content_stream.read(), os.path.join(self.BOOTSTRAP_DIR, fn), 'resource')
- wrote_setuptools = True
+ if not fn.endswith('.pyc'): # We'll compile our own .pyc's later.
+ dst = os.path.join(self.BOOTSTRAP_DIR, fn)
+ self._chroot.write(content_stream.read(), dst, 'bootstrap')
+ wrote_setuptools = True
if not wrote_setuptools:
raise RuntimeError(
@@ -366,11 +381,13 @@ class PEXBuilder(object):
for fn in provider.resource_listdir(''):
if fn.endswith('.py'):
self._chroot.write(provider.get_resource_string(source_name, fn),
- os.path.join(self.BOOTSTRAP_DIR, target_location, fn), 'resource')
+ os.path.join(self.BOOTSTRAP_DIR, target_location, fn), 'bootstrap')
- def freeze(self):
+ def freeze(self, bytecode_compile=True):
"""Freeze the PEX.
+ :param bytecode_compile: If True, precompile .py files into .pyc files when freezing code.
+
Freezing the PEX writes all the necessary metadata and environment bootstrapping code. It may
only be called once and renders the PEXBuilder immutable.
"""
@@ -380,18 +397,21 @@ class PEXBuilder(object):
self._prepare_manifest()
self._prepare_bootstrap()
self._prepare_main()
+ if bytecode_compile:
+ self._precompile_source()
self._frozen = True
- def build(self, filename):
+ def build(self, filename, bytecode_compile=True):
"""Package the PEX into a zipfile.
:param filename: The filename where the PEX should be stored.
+ :param bytecode_compile: If True, precompile .py files into .pyc files.
If the PEXBuilder is not yet frozen, it will be frozen by ``build``. This renders the
PEXBuilder immutable.
"""
if not self._frozen:
- self.freeze()
+ self.freeze(bytecode_compile=bytecode_compile)
try:
os.unlink(filename + '~')
self._logger.warn('Previous binary unexpectedly exists, cleaning: %s' % (filename + '~'))
diff --git a/pex/resolver.py b/pex/resolver.py
index 3a4e3b9..c8db1cf 100644
--- a/pex/resolver.py
+++ b/pex/resolver.py
@@ -18,6 +18,7 @@ from .platforms import Platform
from .resolvable import ResolvableRequirement, resolvables_from_iterable
from .resolver_options import ResolverOptionsBuilder
from .tracer import TRACER
+from .util import DistributionHelper
class Untranslateable(Exception):
@@ -55,9 +56,9 @@ class _ResolvedPackages(namedtuple('_ResolvedPackages', 'resolvable packages par
class _ResolvableSet(object):
- def __init__(self):
+ def __init__(self, tuples=None):
# A list of _ResolvedPackages
- self.__tuples = []
+ self.__tuples = tuples or []
def _collapse(self):
# Collapse all resolvables by name along with the intersection of all compatible packages.
@@ -112,6 +113,18 @@ class _ResolvableSet(object):
return set.union(
*[set(tup.resolvable.extras()) for tup in self.__tuples if tup.resolvable.name == name])
+ def replace_built(self, built_packages):
+ """Return a copy of this resolvable set but with built packages.
+
+ :param dict built_packages: A mapping from a resolved package to its locally built package.
+ :returns: A new resolvable set with built package replacements made.
+ """
+ def map_packages(resolved_packages):
+ packages = OrderedSet(built_packages.get(p, p) for p in resolved_packages.packages)
+ return _ResolvedPackages(resolved_packages.resolvable, packages, resolved_packages.parent)
+
+ return _ResolvableSet([map_packages(rp) for rp in self.__tuples])
+
class Resolver(object):
"""Interface for resolving resolvable entities into python packages."""
@@ -165,6 +178,7 @@ class Resolver(object):
resolvable_set.merge(resolvable, packages, parent)
processed_resolvables.add(resolvable)
+ built_packages = {}
for resolvable, packages, parent in resolvable_set.packages():
assert len(packages) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable
package = next(iter(packages))
@@ -174,13 +188,19 @@ class Resolver(object):
raise self.Error('Ambiguous resolvable: %s' % resolvable)
continue
if package not in distributions:
- distributions[package] = self.build(package, resolvable.options)
+ dist = self.build(package, resolvable.options)
+ built_package = Package.from_href(dist.location)
+ built_packages[package] = built_package
+ distributions[built_package] = dist
+ package = built_package
+
distribution = distributions[package]
processed_packages[resolvable.name] = package
new_parent = '%s->%s' % (parent, resolvable) if parent else str(resolvable)
resolvables.extend(
(ResolvableRequirement(req, resolvable.options), new_parent) for req in
distribution.requires(extras=resolvable_set.extras(resolvable.name)))
+ resolvable_set = resolvable_set.replace_built(built_packages)
return list(distributions.values())
@@ -233,7 +253,8 @@ class CachingResolver(Resolver):
shutil.copyfile(dist.location, target + '~')
os.rename(target + '~', target)
os.utime(target, None)
- return dist
+
+ return DistributionHelper.distribution_from_path(target)
def resolve(
diff --git a/pex/testing.py b/pex/testing.py
index 54c14c0..cded62c 100644
--- a/pex/testing.py
+++ b/pex/testing.py
@@ -77,6 +77,7 @@ PROJECT_CONTENT = {
'scripts/shell_script',
],
package_data={'my_package': ['package_data/*.dat']},
+ install_requires=%(install_requires)r,
)
'''),
'scripts/hello_world': '#!/usr/bin/env python\nprint("hello world!")\n',
@@ -89,21 +90,23 @@ PROJECT_CONTENT = {
@contextlib.contextmanager
-def make_installer(name='my_project', installer_impl=EggInstaller, zip_safe=True):
- interp = {'project_name': name, 'zip_safe': zip_safe}
+def make_installer(name='my_project', installer_impl=EggInstaller, zip_safe=True,
+ install_reqs=None):
+ interp = {'project_name': name, 'zip_safe': zip_safe, 'install_requires': install_reqs or []}
with temporary_content(PROJECT_CONTENT, interp=interp) as td:
yield installer_impl(td)
@contextlib.contextmanager
-def make_source_dir(name='my_project'):
- interp = {'project_name': name, 'zip_safe': True}
+def make_source_dir(name='my_project', install_reqs=None):
+ interp = {'project_name': name, 'zip_safe': True, 'install_requires': install_reqs or []}
with temporary_content(PROJECT_CONTENT, interp=interp) as td:
yield td
-def make_sdist(name='my_project', zip_safe=True):
- with make_installer(name=name, installer_impl=Packager, zip_safe=zip_safe) as packager:
+def make_sdist(name='my_project', zip_safe=True, install_reqs=None):
+ with make_installer(name=name, installer_impl=Packager, zip_safe=zip_safe,
+ install_reqs=install_reqs) as packager:
return packager.sdist()
diff --git a/pex/version.py b/pex/version.py
index 0d344a0..342aff8 100644
--- a/pex/version.py
+++ b/pex/version.py
@@ -1,7 +1,7 @@
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
-__version__ = '1.0.0'
+__version__ = '1.0.1'
SETUPTOOLS_REQUIREMENT = 'setuptools>=2.2,<16'
WHEEL_REQUIREMENT = 'wheel>=0.24.0,<0.25.0'
diff --git a/tests/test_compiler.py b/tests/test_compiler.py
new file mode 100644
index 0000000..6b55f03
--- /dev/null
+++ b/tests/test_compiler.py
@@ -0,0 +1,72 @@
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import contextlib
+import os
+
+import marshal
+import pytest
+from twitter.common.contextutil import temporary_dir
+
+from pex import compatibility
+from pex.common import safe_open
+from pex.compatibility import to_bytes
+from pex.compiler import Compiler
+from pex.interpreter import PythonInterpreter
+
+
+def write_source(path, valid=True):
+ with safe_open(path, 'wb') as fp:
+ fp.write(to_bytes('basename = %r\n' % os.path.basename(path)))
+ if not valid:
+ fp.write(to_bytes('invalid!\n'))
+
+
+ at contextlib.contextmanager
+def compilation(valid_paths=None, invalid_paths=None, compile_paths=None):
+ with temporary_dir() as root:
+ for path in valid_paths:
+ write_source(os.path.join(root, path))
+ for path in invalid_paths:
+ write_source(os.path.join(root, path), valid=False)
+ compiler = Compiler(PythonInterpreter.get())
+ yield root, compiler.compile(root, compile_paths)
+
+
+def test_compile_success():
+ with compilation(valid_paths=['a.py', 'c/c.py'],
+ invalid_paths=['b.py', 'd/d.py'],
+ compile_paths=['a.py', 'c/c.py']) as (root, compiled_relpaths):
+
+ assert 2 == len(compiled_relpaths)
+
+ results = {}
+ for compiled in compiled_relpaths:
+ compiled_abspath = os.path.join(root, compiled)
+ with open(compiled_abspath, 'rb') as fp:
+ fp.read(4) # Skip the magic header.
+ fp.read(4) # Skip the timestamp.
+ if compatibility.PY3:
+ fp.read(4) # Skip the size.
+ code = marshal.load(fp)
+ local_symbols = {}
+ exec(code, {}, local_symbols)
+ results[compiled] = local_symbols
+
+ assert {'basename': 'a.py'} == results.pop('a.pyc')
+ assert {'basename': 'c.py'} == results.pop('c/c.pyc')
+ assert 0 == len(results)
+
+
+def test_compile_failure():
+ with pytest.raises(Compiler.Error) as e:
+ with compilation(valid_paths=['a.py', 'c/c.py'],
+ invalid_paths=['b.py', 'd/d.py'],
+ compile_paths=['a.py', 'b.py', 'c/c.py', 'd/d.py']):
+ raise AssertionError('Should not reach here.')
+
+ message = str(e.value)
+ assert 'a.py' not in message
+ assert 'b.py' in message
+ assert 'c/c.py' not in message
+ assert 'd/d.py' in message
diff --git a/tests/test_environment.py b/tests/test_environment.py
index 77933ab..0c3baaa 100644
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -30,7 +30,7 @@ def test_force_local():
assert os.path.exists(pb.info.zip_unsafe_cache)
assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1
assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache)
- assert set(os.listdir(code_cache)) == set([PexInfo.PATH, '__main__.py'])
+ assert set(os.listdir(code_cache)) == set([PexInfo.PATH, '__main__.py', '__main__.pyc'])
# idempotence
assert PEXEnvironment.force_local(pex_file.name, pb.info) == code_cache
diff --git a/tests/test_link.py b/tests/test_link.py
index fb99894..586dada 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -52,3 +52,8 @@ def test_link_schemes():
assert link.scheme == 'file'
assert link.local
assert link.path == os.path.realpath('/foo/bar')
+
+
+def test_link_equality():
+ assert Link('http://www.google.com') == Link('http://www.google.com')
+ assert Link('http://www.google.com') != Link('http://www.twitter.com')
diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py
index 7eab1ef..5f2da3f 100644
--- a/tests/test_pex_builder.py
+++ b/tests/test_pex_builder.py
@@ -2,6 +2,7 @@
# Licensed under the Apache License, Version 2.0 (see LICENSE).
import os
+import stat
import zipfile
from contextlib import closing
@@ -65,3 +66,59 @@ def test_pex_builder_shebang():
expected_preamble = b'#!foobar\n'
with open(target, 'rb') as fp:
assert fp.read(len(expected_preamble)) == expected_preamble
+
+
+def test_pex_builder_compilation():
+ with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
+ src = os.path.join(td1, 'src.py')
+ with open(src, 'w') as fp:
+ fp.write(exe_main)
+
+ exe = os.path.join(td1, 'exe.py')
+ with open(exe, 'w') as fp:
+ fp.write(exe_main)
+
+ def build_and_check(path, precompile):
+ pb = PEXBuilder(path)
+ pb.add_source(src, 'lib/src.py')
+ pb.set_executable(exe, 'exe.py')
+ pb.freeze(bytecode_compile=precompile)
+ for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'):
+ pyc_exists = os.path.exists(os.path.join(path, pyc_file))
+ if precompile:
+ assert pyc_exists
+ else:
+ assert not pyc_exists
+ bootstrap_dir = os.path.join(path, PEXBuilder.BOOTSTRAP_DIR)
+ bootstrap_pycs = []
+ for _, _, files in os.walk(bootstrap_dir):
+ bootstrap_pycs.extend(f for f in files if f.endswith('.pyc'))
+ if precompile:
+ assert len(bootstrap_pycs) > 0
+ else:
+ assert 0 == len(bootstrap_pycs)
+
+ build_and_check(td2, False)
+ build_and_check(td3, True)
+
+
+def test_pex_builder_copy_or_link():
+ with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
+ src = os.path.join(td1, 'exe.py')
+ with open(src, 'w') as fp:
+ fp.write(exe_main)
+
+ def build_and_check(path, copy):
+ pb = PEXBuilder(path, copy=copy)
+ pb.add_source(src, 'exe.py')
+
+ s1 = os.stat(src)
+ s2 = os.stat(os.path.join(path, 'exe.py'))
+ is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
+ if copy:
+ assert not is_link
+ else:
+ assert is_link
+
+ build_and_check(td2, False)
+ build_and_check(td3, True)
diff --git a/tests/test_resolver.py b/tests/test_resolver.py
index 7d4fe06..5b88970 100644
--- a/tests/test_resolver.py
+++ b/tests/test_resolver.py
@@ -34,6 +34,20 @@ def test_simple_local_resolve():
assert len(dists) == 1
+def test_diamond_local_resolve_cached():
+ # This exercises the issue described here: https://github.com/pantsbuild/pex/issues/120
+ project1_sdist = make_sdist(name='project1', install_reqs=['project2<1.0.0'])
+ project2_sdist = make_sdist(name='project2')
+
+ with temporary_dir() as dd:
+ for sdist in (project1_sdist, project2_sdist):
+ safe_copy(sdist, os.path.join(dd, os.path.basename(sdist)))
+ fetchers = [Fetcher([dd])]
+ with temporary_dir() as cd:
+ dists = resolve(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000)
+ assert len(dists) == 2
+
+
def test_resolvable_set():
builder = ResolverOptionsBuilder()
rs = _ResolvableSet()
@@ -56,4 +70,24 @@ def test_resolvable_set():
rs.merge(rq, [binary_pkg])
+def test_resolvable_set_built():
+ builder = ResolverOptionsBuilder()
+ rs = _ResolvableSet()
+ rq = ResolvableRequirement.from_string('foo', builder)
+ source_pkg = SourcePackage.from_href('foo-2.3.4.tar.gz')
+ binary_pkg = EggPackage.from_href('foo-2.3.4-py3.4.egg')
+
+ rs.merge(rq, [source_pkg])
+ assert rs.get('foo') == set([source_pkg])
+ assert rs.packages() == [(rq, set([source_pkg]), None)]
+
+ with pytest.raises(Unsatisfiable):
+ rs.merge(rq, [binary_pkg])
+
+ updated_rs = rs.replace_built({source_pkg: binary_pkg})
+ updated_rs.merge(rq, [binary_pkg])
+ assert updated_rs.get('foo') == set([binary_pkg])
+ assert updated_rs.packages() == [(rq, set([binary_pkg]), None)]
+
+
# TODO(wickman) Write more than simple resolver test.
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-pex.git
More information about the Python-modules-commits
mailing list