[med-svn] [Git][med-team/python-xopen][upstream] New upstream version 1.0.1

Nilesh Patra gitlab at salsa.debian.org
Tue Nov 17 13:35:16 GMT 2020



Nilesh Patra pushed to branch upstream at Debian Med / python-xopen


Commits:
bb602d16 by Nilesh Patra at 2020-11-17T19:01:59+05:30
New upstream version 1.0.1
- - - - -


11 changed files:

- .travis.yml
- PKG-INFO
- setup.py
- src/xopen.egg-info/PKG-INFO
- src/xopen.egg-info/SOURCES.txt
- src/xopen/__init__.py
- src/xopen/_version.py
- + src/xopen/_version.pyi
- + src/xopen/py.typed
- tests/test_xopen.py
- tox.ini


Changes:

=====================================
.travis.yml
=====================================
@@ -52,6 +52,12 @@ jobs:
       install: python3 -m pip install flake8
       script: flake8 src/ tests/
 
+    - stage: test
+      name: mypy
+      python: "3.6"
+      install: python3 -m pip install mypy
+      script: mypy src/
+
     - stage: test
       name: igzip
       python: "3.6"


=====================================
PKG-INFO
=====================================
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xopen
-Version: 1.0.0
+Version: 1.0.1
 Summary: Open compressed files transparently
 Home-page: https://github.com/marcelm/xopen/
 Author: Marcel Martin


=====================================
setup.py
=====================================
@@ -16,6 +16,7 @@ setup(
     license='MIT',
     package_dir={'': 'src'},
     packages=find_packages('src'),
+    package_data={"xopen": ["py.typed"]},
     extras_require={
         'dev': ['pytest'],
     },


=====================================
src/xopen.egg-info/PKG-INFO
=====================================
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xopen
-Version: 1.0.0
+Version: 1.0.1
 Summary: Open compressed files transparently
 Home-page: https://github.com/marcelm/xopen/
 Author: Marcel Martin


=====================================
src/xopen.egg-info/SOURCES.txt
=====================================
@@ -10,6 +10,8 @@ setup.py
 tox.ini
 src/xopen/__init__.py
 src/xopen/_version.py
+src/xopen/_version.pyi
+src/xopen/py.typed
 src/xopen.egg-info/PKG-INFO
 src/xopen.egg-info/SOURCES.txt
 src/xopen.egg-info/dependency_links.txt


=====================================
src/xopen/__init__.py
=====================================
@@ -15,8 +15,9 @@ import signal
 import pathlib
 import subprocess
 import tempfile
-from subprocess import Popen, PIPE
-from typing import Optional
+from abc import ABC, abstractmethod
+from subprocess import Popen, PIPE, DEVNULL
+from typing import Optional, TextIO, AnyStr, IO
 
 from ._version import version as __version__
 
@@ -24,7 +25,7 @@ from ._version import version as __version__
 try:
     import lzma
 except ImportError:
-    lzma = None
+    lzma = None  # type: ignore
 
 try:
     import fcntl
@@ -34,11 +35,11 @@ try:
     if not hasattr(fcntl, "F_SETPIPE_SZ") and sys.platform == "linux":
         setattr(fcntl, "F_SETPIPE_SZ", 1031)
 except ImportError:
-    fcntl = None
+    fcntl = None  # type: ignore
 
 _MAX_PIPE_SIZE_PATH = pathlib.Path("/proc/sys/fs/pipe-max-size")
 if _MAX_PIPE_SIZE_PATH.exists():
-    _MAX_PIPE_SIZE = int(_MAX_PIPE_SIZE_PATH.read_text())
+    _MAX_PIPE_SIZE = int(_MAX_PIPE_SIZE_PATH.read_text())  # type: Optional[int]
 else:
     _MAX_PIPE_SIZE = None
 
@@ -46,7 +47,7 @@ else:
 try:
     from os import fspath  # Exists in Python 3.6+
 except ImportError:
-    def fspath(path):
+    def fspath(path):  # type: ignore
         if hasattr(path, "__fspath__"):
             return path.__fspath__()
         # Python 3.4 and 3.5 have pathlib, but do not support the file system
@@ -58,7 +59,7 @@ except ImportError:
         return path
 
 
-def _available_cpu_count():
+def _available_cpu_count() -> int:
     """
     Number of available virtual or physical CPUs on this system
     Adapted from http://stackoverflow.com/a/1006301/715090
@@ -85,14 +86,17 @@ def _available_cpu_count():
         return 1
 
 
-def _set_pipe_size_to_max(fd: int):
+def _set_pipe_size_to_max(fd: int) -> None:
     """
     Set pipe size to maximum on platforms that support it.
     :param fd: The file descriptor to increase the pipe size for.
     """
     if not hasattr(fcntl, "F_SETPIPE_SZ") or not _MAX_PIPE_SIZE:
         return
-    fcntl.fcntl(fd, fcntl.F_SETPIPE_SZ, _MAX_PIPE_SIZE)
+    try:
+        fcntl.fcntl(fd, fcntl.F_SETPIPE_SZ, _MAX_PIPE_SIZE)  # type: ignore
+    except OSError:
+        pass
 
 
 def _can_read_concatenated_gz(program: str) -> bool:
@@ -118,7 +122,7 @@ def _can_read_concatenated_gz(program: str) -> bool:
         os.remove(temp_path)
 
 
-class Closing:
+class Closing(ABC):
     """
     Inherit from this class and implement a close() method to offer context
     manager functionality.
@@ -136,12 +140,16 @@ class Closing:
         except Exception:
             pass
 
+    @abstractmethod
+    def close(self):
+        pass
+
 
 class PipedCompressionWriter(Closing):
     """
     Write Compressed files by running an external process and piping into it.
     """
-    def __init__(self, path, program, mode='wt',
+    def __init__(self, path, program: str, mode='wt',
                  compresslevel: Optional[int] = None,
                  threads_flag: str = None,
                  threads: Optional[int] = None):
@@ -160,7 +168,6 @@ class PipedCompressionWriter(Closing):
 
         # TODO use a context manager
         self.outfile = open(path, mode)
-        self.devnull = open(os.devnull, mode)
         self.closed = False
         self.name = path
         self._mode = mode
@@ -169,22 +176,33 @@ class PipedCompressionWriter(Closing):
 
         if threads is None:
             threads = min(_available_cpu_count(), 4)
+        self._threads = threads
         try:
             self.process = self._open_process(
-                mode, compresslevel, threads, self.outfile, self.devnull)
+                mode, compresslevel, threads, self.outfile)
         except OSError:
             self.outfile.close()
-            self.devnull.close()
             raise
-
+        assert self.process.stdin is not None
         _set_pipe_size_to_max(self.process.stdin.fileno())
 
         if 'b' not in mode:
-            self._file = io.TextIOWrapper(self.process.stdin)
+            self._file = io.TextIOWrapper(self.process.stdin)  # type: IO
         else:
             self._file = self.process.stdin
 
-    def _open_process(self, mode, compresslevel, threads, outfile, devnull):
+    def __repr__(self):
+        return "{}('{}', mode='{}', program='{}', threads={})".format(
+            self.__class__.__name__,
+            self.name,
+            self._mode,
+            self._program,
+            self._threads,
+        )
+
+    def _open_process(
+        self, mode: str, compresslevel: Optional[int], threads: int, outfile: TextIO,
+    ) -> Popen:
         program_args = [self._program]
         if threads != 0 and self._threads_flag is not None:
             program_args += [self._threads_flag, str(threads)]
@@ -192,7 +210,7 @@ class PipedCompressionWriter(Closing):
         if 'w' in mode and compresslevel is not None:
             extra_args += ['-' + str(compresslevel)]
 
-        kwargs = dict(stdin=PIPE, stdout=outfile, stderr=devnull)
+        kwargs = dict(stdin=PIPE, stdout=outfile, stderr=DEVNULL)
 
         # Setting close_fds to True in the Popen arguments is necessary due to
         # <http://bugs.python.org/issue12786>.
@@ -201,26 +219,26 @@ class PipedCompressionWriter(Closing):
         if sys.platform != 'win32':
             kwargs['close_fds'] = True
 
-        process = Popen(program_args + extra_args, **kwargs)
-
+        process = Popen(program_args + extra_args, **kwargs)  # type: ignore
         return process
 
-    def write(self, arg):
+    def write(self, arg: AnyStr) -> None:
         self._file.write(arg)
 
-    def close(self):
+    def close(self) -> None:
         if self.closed:
             return
         self.closed = True
         self._file.close()
         retcode = self.process.wait()
         self.outfile.close()
-        self.devnull.close()
         if retcode != 0:
             raise OSError(
                 "Output {} process terminated with exit code {}".format(self._program, retcode))
 
-    def __iter__(self):
+    def __iter__(self):  # type: ignore
+        # For compatibility with Pandas, which checks for an __iter__ method
+        # to determine whether an object is file-like.
         return self
 
     def __next__(self):
@@ -232,13 +250,20 @@ class PipedCompressionReader(Closing):
     Open a pipe to a process for reading a compressed file.
     """
 
-    def __init__(self, path, program, mode='r', threads_flag=None, threads=None):
+    def __init__(
+        self,
+        path,
+        program: str,
+        mode: str = "r",
+        threads_flag: Optional[str] = None,
+        threads: Optional[int] = None,
+    ):
         """
         Raise an OSError when pigz could not be found.
         """
         if mode not in ('r', 'rt', 'rb'):
             raise ValueError("Mode is '{}', but it must be 'r', 'rt' or 'rb'".format(mode))
-
+        self._program = program
         program_args = [program, '-cd', path]
 
         if threads_flag is not None:
@@ -251,16 +276,19 @@ class PipedCompressionReader(Closing):
                 #   clock time.
                 threads = 1
             program_args += [threads_flag, str(threads)]
-
+        self._threads = threads
         self.process = Popen(program_args, stdout=PIPE, stderr=PIPE)
         self.name = path
 
+        assert self.process.stdout is not None
         _set_pipe_size_to_max(self.process.stdout.fileno())
 
+        self._mode = mode
         if 'b' not in mode:
-            self._file = io.TextIOWrapper(self.process.stdout)
+            self._file = io.TextIOWrapper(self.process.stdout)  # type: IO
         else:
             self._file = self.process.stdout
+        assert self.process.stderr is not None
         self._stderr = io.TextIOWrapper(self.process.stderr)
         self.closed = False
         # Give the subprocess a little bit of time to report any errors (such as
@@ -268,7 +296,16 @@ class PipedCompressionReader(Closing):
         time.sleep(0.01)
         self._raise_if_error()
 
-    def close(self):
+    def __repr__(self):
+        return "{}('{}', mode='{}', program='{}', threads={})".format(
+            self.__class__.__name__,
+            self.name,
+            self._mode,
+            self._program,
+            self._threads,
+        )
+
+    def close(self) -> None:
         if self.closed:
             return
         self.closed = True
@@ -287,10 +324,10 @@ class PipedCompressionReader(Closing):
     def __iter__(self):
         return self
 
-    def __next__(self):
+    def __next__(self) -> AnyStr:
         return self._file.__next__()
 
-    def _raise_if_error(self, allow_sigterm=False):
+    def _raise_if_error(self, allow_sigterm: bool = False) -> None:
         """
         Raise IOError if process is not running anymore and the exit code is
         nonzero. If allow_sigterm is set and a SIGTERM exit code is
@@ -306,28 +343,28 @@ class PipedCompressionReader(Closing):
             self._stderr.close()
             raise OSError("{} (exit code {})".format(message, retcode))
 
-    def read(self, *args):
+    def read(self, *args) -> AnyStr:
         return self._file.read(*args)
 
     def readinto(self, *args):
         return self._file.readinto(*args)
 
-    def readline(self, *args):
+    def readline(self, *args) -> AnyStr:
         return self._file.readline(*args)
 
-    def seekable(self):
+    def seekable(self) -> bool:
         return self._file.seekable()
 
-    def peek(self, n=None):
-        return self._file.peek(n)
+    def peek(self, n: int = None):
+        return self._file.peek(n)  # type: ignore
 
-    def readable(self):
+    def readable(self) -> bool:
         return self._file.readable()
 
-    def writable(self):
+    def writable(self) -> bool:
         return self._file.writable()
 
-    def flush(self):
+    def flush(self) -> None:
         return None
 
 
@@ -338,7 +375,7 @@ class PipedGzipReader(PipedCompressionReader):
     also faster when reading, even when forced to use a single thread
     (ca. 2x speedup).
     """
-    def __init__(self, path, mode='r', threads=None):
+    def __init__(self, path, mode: str = "r", threads: Optional[int] = None):
         try:
             super().__init__(path, "pigz", mode, "-p", threads)
         except OSError:
@@ -354,7 +391,13 @@ class PipedGzipWriter(PipedCompressionWriter):
     par with gzip itself, but running an external gzip can still reduce wall-clock
     time because the compression happens in a separate process.
     """
-    def __init__(self, path, mode='wt', compresslevel=None, threads=None):
+    def __init__(
+        self,
+        path,
+        mode: str = "wt",
+        compresslevel: Optional[int] = None,
+        threads: Optional[int] = None,
+    ):
         """
         mode -- one of 'w', 'wt', 'wb', 'a', 'at', 'ab'
         compresslevel -- compression level
@@ -377,7 +420,7 @@ class PipedIGzipReader(PipedCompressionReader):
     can only run on x86 and ARM architectures, but is able to use more
     architecture-specific optimizations as a result.
     """
-    def __init__(self, path, mode="r"):
+    def __init__(self, path, mode: str = "r"):
         if not _can_read_concatenated_gz("igzip"):
             # Instead of elaborate version string checking once the problem is
             # fixed, it is much easier to use this, "proof in the pudding" type
@@ -403,31 +446,31 @@ class PipedIGzipWriter(PipedCompressionWriter):
     filesizes from their pigz/gzip counterparts.
     See: https://gist.github.com/rhpvorderman/4f1201c3f39518ff28dde45409eb696b
     """
-    def __init__(self, path, mode="wt", compresslevel=None):
+    def __init__(self, path, mode: str = "wt", compresslevel: Optional[int] = None):
         if compresslevel is not None and compresslevel not in range(0, 4):
             raise ValueError("compresslevel must be between 0 and 3")
         super().__init__(path, "igzip", mode, compresslevel)
 
 
-def _open_stdin_or_out(mode):
+def _open_stdin_or_out(mode: str) -> IO:
     # Do not return sys.stdin or sys.stdout directly as we want the returned object
     # to be closable without closing sys.stdout.
     std = dict(r=sys.stdin, w=sys.stdout)[mode[0]]
     return open(std.fileno(), mode=mode, closefd=False)
 
 
-def _open_bz2(filename, mode):
+def _open_bz2(filename, mode: str) -> IO:
     return bz2.open(filename, mode)
 
 
-def _open_xz(filename, mode):
+def _open_xz(filename, mode: str) -> IO:
     if lzma is None:
         raise ImportError(
             "Cannot open xz files: The lzma module is not available (use Python 3.3 or newer)")
     return lzma.open(filename, mode)
 
 
-def _open_gz(filename, mode, compresslevel, threads):
+def _open_gz(filename, mode: str, compresslevel, threads):
     if threads != 0:
         try:
             if 'r' in mode:
@@ -454,7 +497,7 @@ def _open_gz(filename, mode, compresslevel, threads):
                          compresslevel=6 if compresslevel is None else compresslevel)
 
 
-def _detect_format_from_content(filename):
+def _detect_format_from_content(filename: str) -> Optional[str]:
     """
     Attempts to detect file format from the content by reading the first
     6 bytes. Returns None if no format could be detected.
@@ -473,10 +516,12 @@ def _detect_format_from_content(filename):
                 # https://tukaani.org/xz/xz-file-format.txt
                 return "xz"
     except OSError:
-        return None
+        pass
+
+    return None
 
 
-def _detect_format_from_extension(filename):
+def _detect_format_from_extension(filename: str) -> Optional[str]:
     """
     Attempts to detect file format from the filename extension.
     Returns None if no format could be detected.
@@ -491,7 +536,12 @@ def _detect_format_from_extension(filename):
         return None
 
 
-def xopen(filename, mode='r', compresslevel=None, threads=None):
+def xopen(
+    filename,
+    mode: str = "r",
+    compresslevel: Optional[int] = None,
+    threads: Optional[int] = None,
+) -> IO:
     """
     A replacement for the "open" function that can also read and write
     compressed files transparently. The supported compression formats are gzip,


=====================================
src/xopen/_version.py
=====================================
@@ -1,4 +1,4 @@
 # coding: utf-8
 # file generated by setuptools_scm
 # don't change, don't track in version control
-version = '1.0.0'
+version = '1.0.1'


=====================================
src/xopen/_version.pyi
=====================================
@@ -0,0 +1,4 @@
+# The _version.py file is generated on installation. By including this stub,
+# we can run mypy without having to install the package.
+
+version: str


=====================================
src/xopen/py.typed
=====================================


=====================================
tests/test_xopen.py
=====================================
@@ -194,8 +194,10 @@ def test_pipedgzipwriter_has_iter_method(tmpdir):
 
 
 def test_iter_without_with(fname):
-    it = iter(xopen(fname, "rt"))
+    f = xopen(fname, "rt")
+    it = iter(f)
     assert CONTENT_LINES[0] == next(it)
+    f.close()
 
 
 def test_pipedgzipreader_iter_without_with():
@@ -442,3 +444,14 @@ def test_pipesize_changed(tmpdir):
 def test_xopen_falls_back_to_gzip_open(lacking_pigz_permissions):
     with xopen("tests/file.txt.gz", "rb") as f:
         assert f.readline() == CONTENT_LINES[0].encode("utf-8")
+
+
+def test_open_many_gzip_writers(tmp_path):
+    files = []
+    for i in range(1, 61):
+        path = tmp_path / "{:03d}.txt.gz".format(i)
+        f = xopen(path, "wb", threads=2)
+        f.write(b"hello")
+        files.append(f)
+    for f in files:
+        f.close()


=====================================
tox.ini
=====================================
@@ -1,5 +1,5 @@
 [tox]
-envlist = flake8,py35,py36,py37,py38,py39,pypy3
+envlist = flake8,mypy,py35,py36,py37,py38,py39,pypy3
 
 [testenv]
 deps = pytest
@@ -11,7 +11,17 @@ basepython = python3.6
 deps = flake8
 commands = flake8 src/ tests/
 
+[testenv:mypy]
+basepython = python3.6
+deps = mypy
+commands = mypy src/
+
 [flake8]
 max-line-length = 99
 max-complexity = 10
 extend_ignore = E731
+
+[coverage:report]
+exclude_lines =
+    pragma: no cover
+    def __repr__



View it on GitLab: https://salsa.debian.org/med-team/python-xopen/-/commit/bb602d1649b8598f05b6f7d964c34d83f01cf554

-- 
View it on GitLab: https://salsa.debian.org/med-team/python-xopen/-/commit/bb602d1649b8598f05b6f7d964c34d83f01cf554
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20201117/a574911b/attachment-0001.html>


More information about the debian-med-commit mailing list