[med-svn] [Git][med-team/python-xopen][master] 4 commits: routine-update: New upstream version
Nilesh Patra
gitlab at salsa.debian.org
Tue Nov 17 13:35:07 GMT 2020
Nilesh Patra pushed to branch master at Debian Med / python-xopen
Commits:
f41aa1d4 by Nilesh Patra at 2020-11-17T19:01:58+05:30
routine-update: New upstream version
- - - - -
bb602d16 by Nilesh Patra at 2020-11-17T19:01:59+05:30
New upstream version 1.0.1
- - - - -
ba979b19 by Nilesh Patra at 2020-11-17T19:02:00+05:30
Update upstream source from tag 'upstream/1.0.1'
Update to upstream version '1.0.1'
with Debian dir 2ba2eec8d99226257195cde260c581d9a46e5bb6
- - - - -
ed657d1d by Nilesh Patra at 2020-11-17T19:02:06+05:30
routine-update: Ready to upload to unstable
- - - - -
12 changed files:
- .travis.yml
- PKG-INFO
- debian/changelog
- setup.py
- src/xopen.egg-info/PKG-INFO
- src/xopen.egg-info/SOURCES.txt
- src/xopen/__init__.py
- src/xopen/_version.py
- + src/xopen/_version.pyi
- + src/xopen/py.typed
- tests/test_xopen.py
- tox.ini
Changes:
=====================================
.travis.yml
=====================================
@@ -52,6 +52,12 @@ jobs:
install: python3 -m pip install flake8
script: flake8 src/ tests/
+ - stage: test
+ name: mypy
+ python: "3.6"
+ install: python3 -m pip install mypy
+ script: mypy src/
+
- stage: test
name: igzip
python: "3.6"
=====================================
PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: xopen
-Version: 1.0.0
+Version: 1.0.1
Summary: Open compressed files transparently
Home-page: https://github.com/marcelm/xopen/
Author: Marcel Martin
=====================================
debian/changelog
=====================================
@@ -1,3 +1,9 @@
+python-xopen (1.0.1-1) unstable; urgency=medium
+
+ * New upstream version
+
+ -- Nilesh Patra <npatra974 at gmail.com> Tue, 17 Nov 2020 19:02:06 +0530
+
python-xopen (1.0.0-1) unstable; urgency=medium
* New upstream version
=====================================
setup.py
=====================================
@@ -16,6 +16,7 @@ setup(
license='MIT',
package_dir={'': 'src'},
packages=find_packages('src'),
+ package_data={"xopen": ["py.typed"]},
extras_require={
'dev': ['pytest'],
},
=====================================
src/xopen.egg-info/PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: xopen
-Version: 1.0.0
+Version: 1.0.1
Summary: Open compressed files transparently
Home-page: https://github.com/marcelm/xopen/
Author: Marcel Martin
=====================================
src/xopen.egg-info/SOURCES.txt
=====================================
@@ -10,6 +10,8 @@ setup.py
tox.ini
src/xopen/__init__.py
src/xopen/_version.py
+src/xopen/_version.pyi
+src/xopen/py.typed
src/xopen.egg-info/PKG-INFO
src/xopen.egg-info/SOURCES.txt
src/xopen.egg-info/dependency_links.txt
=====================================
src/xopen/__init__.py
=====================================
@@ -15,8 +15,9 @@ import signal
import pathlib
import subprocess
import tempfile
-from subprocess import Popen, PIPE
-from typing import Optional
+from abc import ABC, abstractmethod
+from subprocess import Popen, PIPE, DEVNULL
+from typing import Optional, TextIO, AnyStr, IO
from ._version import version as __version__
@@ -24,7 +25,7 @@ from ._version import version as __version__
try:
import lzma
except ImportError:
- lzma = None
+ lzma = None # type: ignore
try:
import fcntl
@@ -34,11 +35,11 @@ try:
if not hasattr(fcntl, "F_SETPIPE_SZ") and sys.platform == "linux":
setattr(fcntl, "F_SETPIPE_SZ", 1031)
except ImportError:
- fcntl = None
+ fcntl = None # type: ignore
_MAX_PIPE_SIZE_PATH = pathlib.Path("/proc/sys/fs/pipe-max-size")
if _MAX_PIPE_SIZE_PATH.exists():
- _MAX_PIPE_SIZE = int(_MAX_PIPE_SIZE_PATH.read_text())
+ _MAX_PIPE_SIZE = int(_MAX_PIPE_SIZE_PATH.read_text()) # type: Optional[int]
else:
_MAX_PIPE_SIZE = None
@@ -46,7 +47,7 @@ else:
try:
from os import fspath # Exists in Python 3.6+
except ImportError:
- def fspath(path):
+ def fspath(path): # type: ignore
if hasattr(path, "__fspath__"):
return path.__fspath__()
# Python 3.4 and 3.5 have pathlib, but do not support the file system
@@ -58,7 +59,7 @@ except ImportError:
return path
-def _available_cpu_count():
+def _available_cpu_count() -> int:
"""
Number of available virtual or physical CPUs on this system
Adapted from http://stackoverflow.com/a/1006301/715090
@@ -85,14 +86,17 @@ def _available_cpu_count():
return 1
-def _set_pipe_size_to_max(fd: int):
+def _set_pipe_size_to_max(fd: int) -> None:
"""
Set pipe size to maximum on platforms that support it.
:param fd: The file descriptor to increase the pipe size for.
"""
if not hasattr(fcntl, "F_SETPIPE_SZ") or not _MAX_PIPE_SIZE:
return
- fcntl.fcntl(fd, fcntl.F_SETPIPE_SZ, _MAX_PIPE_SIZE)
+ try:
+ fcntl.fcntl(fd, fcntl.F_SETPIPE_SZ, _MAX_PIPE_SIZE) # type: ignore
+ except OSError:
+ pass
def _can_read_concatenated_gz(program: str) -> bool:
@@ -118,7 +122,7 @@ def _can_read_concatenated_gz(program: str) -> bool:
os.remove(temp_path)
-class Closing:
+class Closing(ABC):
"""
Inherit from this class and implement a close() method to offer context
manager functionality.
@@ -136,12 +140,16 @@ class Closing:
except Exception:
pass
+ @abstractmethod
+ def close(self):
+ pass
+
class PipedCompressionWriter(Closing):
"""
Write Compressed files by running an external process and piping into it.
"""
- def __init__(self, path, program, mode='wt',
+ def __init__(self, path, program: str, mode='wt',
compresslevel: Optional[int] = None,
threads_flag: str = None,
threads: Optional[int] = None):
@@ -160,7 +168,6 @@ class PipedCompressionWriter(Closing):
# TODO use a context manager
self.outfile = open(path, mode)
- self.devnull = open(os.devnull, mode)
self.closed = False
self.name = path
self._mode = mode
@@ -169,22 +176,33 @@ class PipedCompressionWriter(Closing):
if threads is None:
threads = min(_available_cpu_count(), 4)
+ self._threads = threads
try:
self.process = self._open_process(
- mode, compresslevel, threads, self.outfile, self.devnull)
+ mode, compresslevel, threads, self.outfile)
except OSError:
self.outfile.close()
- self.devnull.close()
raise
-
+ assert self.process.stdin is not None
_set_pipe_size_to_max(self.process.stdin.fileno())
if 'b' not in mode:
- self._file = io.TextIOWrapper(self.process.stdin)
+ self._file = io.TextIOWrapper(self.process.stdin) # type: IO
else:
self._file = self.process.stdin
- def _open_process(self, mode, compresslevel, threads, outfile, devnull):
+ def __repr__(self):
+ return "{}('{}', mode='{}', program='{}', threads={})".format(
+ self.__class__.__name__,
+ self.name,
+ self._mode,
+ self._program,
+ self._threads,
+ )
+
+ def _open_process(
+ self, mode: str, compresslevel: Optional[int], threads: int, outfile: TextIO,
+ ) -> Popen:
program_args = [self._program]
if threads != 0 and self._threads_flag is not None:
program_args += [self._threads_flag, str(threads)]
@@ -192,7 +210,7 @@ class PipedCompressionWriter(Closing):
if 'w' in mode and compresslevel is not None:
extra_args += ['-' + str(compresslevel)]
- kwargs = dict(stdin=PIPE, stdout=outfile, stderr=devnull)
+ kwargs = dict(stdin=PIPE, stdout=outfile, stderr=DEVNULL)
# Setting close_fds to True in the Popen arguments is necessary due to
# <http://bugs.python.org/issue12786>.
@@ -201,26 +219,26 @@ class PipedCompressionWriter(Closing):
if sys.platform != 'win32':
kwargs['close_fds'] = True
- process = Popen(program_args + extra_args, **kwargs)
-
+ process = Popen(program_args + extra_args, **kwargs) # type: ignore
return process
- def write(self, arg):
+ def write(self, arg: AnyStr) -> None:
self._file.write(arg)
- def close(self):
+ def close(self) -> None:
if self.closed:
return
self.closed = True
self._file.close()
retcode = self.process.wait()
self.outfile.close()
- self.devnull.close()
if retcode != 0:
raise OSError(
"Output {} process terminated with exit code {}".format(self._program, retcode))
- def __iter__(self):
+ def __iter__(self): # type: ignore
+ # For compatibility with Pandas, which checks for an __iter__ method
+ # to determine whether an object is file-like.
return self
def __next__(self):
@@ -232,13 +250,20 @@ class PipedCompressionReader(Closing):
Open a pipe to a process for reading a compressed file.
"""
- def __init__(self, path, program, mode='r', threads_flag=None, threads=None):
+ def __init__(
+ self,
+ path,
+ program: str,
+ mode: str = "r",
+ threads_flag: Optional[str] = None,
+ threads: Optional[int] = None,
+ ):
"""
Raise an OSError when pigz could not be found.
"""
if mode not in ('r', 'rt', 'rb'):
raise ValueError("Mode is '{}', but it must be 'r', 'rt' or 'rb'".format(mode))
-
+ self._program = program
program_args = [program, '-cd', path]
if threads_flag is not None:
@@ -251,16 +276,19 @@ class PipedCompressionReader(Closing):
# clock time.
threads = 1
program_args += [threads_flag, str(threads)]
-
+ self._threads = threads
self.process = Popen(program_args, stdout=PIPE, stderr=PIPE)
self.name = path
+ assert self.process.stdout is not None
_set_pipe_size_to_max(self.process.stdout.fileno())
+ self._mode = mode
if 'b' not in mode:
- self._file = io.TextIOWrapper(self.process.stdout)
+ self._file = io.TextIOWrapper(self.process.stdout) # type: IO
else:
self._file = self.process.stdout
+ assert self.process.stderr is not None
self._stderr = io.TextIOWrapper(self.process.stderr)
self.closed = False
# Give the subprocess a little bit of time to report any errors (such as
@@ -268,7 +296,16 @@ class PipedCompressionReader(Closing):
time.sleep(0.01)
self._raise_if_error()
- def close(self):
+ def __repr__(self):
+ return "{}('{}', mode='{}', program='{}', threads={})".format(
+ self.__class__.__name__,
+ self.name,
+ self._mode,
+ self._program,
+ self._threads,
+ )
+
+ def close(self) -> None:
if self.closed:
return
self.closed = True
@@ -287,10 +324,10 @@ class PipedCompressionReader(Closing):
def __iter__(self):
return self
- def __next__(self):
+ def __next__(self) -> AnyStr:
return self._file.__next__()
- def _raise_if_error(self, allow_sigterm=False):
+ def _raise_if_error(self, allow_sigterm: bool = False) -> None:
"""
Raise IOError if process is not running anymore and the exit code is
nonzero. If allow_sigterm is set and a SIGTERM exit code is
@@ -306,28 +343,28 @@ class PipedCompressionReader(Closing):
self._stderr.close()
raise OSError("{} (exit code {})".format(message, retcode))
- def read(self, *args):
+ def read(self, *args) -> AnyStr:
return self._file.read(*args)
def readinto(self, *args):
return self._file.readinto(*args)
- def readline(self, *args):
+ def readline(self, *args) -> AnyStr:
return self._file.readline(*args)
- def seekable(self):
+ def seekable(self) -> bool:
return self._file.seekable()
- def peek(self, n=None):
- return self._file.peek(n)
+ def peek(self, n: int = None):
+ return self._file.peek(n) # type: ignore
- def readable(self):
+ def readable(self) -> bool:
return self._file.readable()
- def writable(self):
+ def writable(self) -> bool:
return self._file.writable()
- def flush(self):
+ def flush(self) -> None:
return None
@@ -338,7 +375,7 @@ class PipedGzipReader(PipedCompressionReader):
also faster when reading, even when forced to use a single thread
(ca. 2x speedup).
"""
- def __init__(self, path, mode='r', threads=None):
+ def __init__(self, path, mode: str = "r", threads: Optional[int] = None):
try:
super().__init__(path, "pigz", mode, "-p", threads)
except OSError:
@@ -354,7 +391,13 @@ class PipedGzipWriter(PipedCompressionWriter):
par with gzip itself, but running an external gzip can still reduce wall-clock
time because the compression happens in a separate process.
"""
- def __init__(self, path, mode='wt', compresslevel=None, threads=None):
+ def __init__(
+ self,
+ path,
+ mode: str = "wt",
+ compresslevel: Optional[int] = None,
+ threads: Optional[int] = None,
+ ):
"""
mode -- one of 'w', 'wt', 'wb', 'a', 'at', 'ab'
compresslevel -- compression level
@@ -377,7 +420,7 @@ class PipedIGzipReader(PipedCompressionReader):
can only run on x86 and ARM architectures, but is able to use more
architecture-specific optimizations as a result.
"""
- def __init__(self, path, mode="r"):
+ def __init__(self, path, mode: str = "r"):
if not _can_read_concatenated_gz("igzip"):
# Instead of elaborate version string checking once the problem is
# fixed, it is much easier to use this, "proof in the pudding" type
@@ -403,31 +446,31 @@ class PipedIGzipWriter(PipedCompressionWriter):
filesizes from their pigz/gzip counterparts.
See: https://gist.github.com/rhpvorderman/4f1201c3f39518ff28dde45409eb696b
"""
- def __init__(self, path, mode="wt", compresslevel=None):
+ def __init__(self, path, mode: str = "wt", compresslevel: Optional[int] = None):
if compresslevel is not None and compresslevel not in range(0, 4):
raise ValueError("compresslevel must be between 0 and 3")
super().__init__(path, "igzip", mode, compresslevel)
-def _open_stdin_or_out(mode):
+def _open_stdin_or_out(mode: str) -> IO:
# Do not return sys.stdin or sys.stdout directly as we want the returned object
# to be closable without closing sys.stdout.
std = dict(r=sys.stdin, w=sys.stdout)[mode[0]]
return open(std.fileno(), mode=mode, closefd=False)
-def _open_bz2(filename, mode):
+def _open_bz2(filename, mode: str) -> IO:
return bz2.open(filename, mode)
-def _open_xz(filename, mode):
+def _open_xz(filename, mode: str) -> IO:
if lzma is None:
raise ImportError(
"Cannot open xz files: The lzma module is not available (use Python 3.3 or newer)")
return lzma.open(filename, mode)
-def _open_gz(filename, mode, compresslevel, threads):
+def _open_gz(filename, mode: str, compresslevel, threads):
if threads != 0:
try:
if 'r' in mode:
@@ -454,7 +497,7 @@ def _open_gz(filename, mode, compresslevel, threads):
compresslevel=6 if compresslevel is None else compresslevel)
-def _detect_format_from_content(filename):
+def _detect_format_from_content(filename: str) -> Optional[str]:
"""
Attempts to detect file format from the content by reading the first
6 bytes. Returns None if no format could be detected.
@@ -473,10 +516,12 @@ def _detect_format_from_content(filename):
# https://tukaani.org/xz/xz-file-format.txt
return "xz"
except OSError:
- return None
+ pass
+
+ return None
-def _detect_format_from_extension(filename):
+def _detect_format_from_extension(filename: str) -> Optional[str]:
"""
Attempts to detect file format from the filename extension.
Returns None if no format could be detected.
@@ -491,7 +536,12 @@ def _detect_format_from_extension(filename):
return None
-def xopen(filename, mode='r', compresslevel=None, threads=None):
+def xopen(
+ filename,
+ mode: str = "r",
+ compresslevel: Optional[int] = None,
+ threads: Optional[int] = None,
+) -> IO:
"""
A replacement for the "open" function that can also read and write
compressed files transparently. The supported compression formats are gzip,
=====================================
src/xopen/_version.py
=====================================
@@ -1,4 +1,4 @@
# coding: utf-8
# file generated by setuptools_scm
# don't change, don't track in version control
-version = '1.0.0'
+version = '1.0.1'
=====================================
src/xopen/_version.pyi
=====================================
@@ -0,0 +1,4 @@
+# The _version.py file is generated on installation. By including this stub,
+# we can run mypy without having to install the package.
+
+version: str
=====================================
src/xopen/py.typed
=====================================
=====================================
tests/test_xopen.py
=====================================
@@ -194,8 +194,10 @@ def test_pipedgzipwriter_has_iter_method(tmpdir):
def test_iter_without_with(fname):
- it = iter(xopen(fname, "rt"))
+ f = xopen(fname, "rt")
+ it = iter(f)
assert CONTENT_LINES[0] == next(it)
+ f.close()
def test_pipedgzipreader_iter_without_with():
@@ -442,3 +444,14 @@ def test_pipesize_changed(tmpdir):
def test_xopen_falls_back_to_gzip_open(lacking_pigz_permissions):
with xopen("tests/file.txt.gz", "rb") as f:
assert f.readline() == CONTENT_LINES[0].encode("utf-8")
+
+
+def test_open_many_gzip_writers(tmp_path):
+ files = []
+ for i in range(1, 61):
+ path = tmp_path / "{:03d}.txt.gz".format(i)
+ f = xopen(path, "wb", threads=2)
+ f.write(b"hello")
+ files.append(f)
+ for f in files:
+ f.close()
=====================================
tox.ini
=====================================
@@ -1,5 +1,5 @@
[tox]
-envlist = flake8,py35,py36,py37,py38,py39,pypy3
+envlist = flake8,mypy,py35,py36,py37,py38,py39,pypy3
[testenv]
deps = pytest
@@ -11,7 +11,17 @@ basepython = python3.6
deps = flake8
commands = flake8 src/ tests/
+[testenv:mypy]
+basepython = python3.6
+deps = mypy
+commands = mypy src/
+
[flake8]
max-line-length = 99
max-complexity = 10
extend_ignore = E731
+
+[coverage:report]
+exclude_lines =
+ pragma: no cover
+ def __repr__
View it on GitLab: https://salsa.debian.org/med-team/python-xopen/-/compare/6db0f81505f5a02253d9a43952d33df6e94f0ff7...ed657d1d8ef6bad7fd8e162d5784438bbc014ccd
--
View it on GitLab: https://salsa.debian.org/med-team/python-xopen/-/compare/6db0f81505f5a02253d9a43952d33df6e94f0ff7...ed657d1d8ef6bad7fd8e162d5784438bbc014ccd
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20201117/71077acc/attachment-0001.html>
More information about the debian-med-commit
mailing list