[Piuparts-commits] [piuparts] 07/11: lib: use DecompressedStream to reduce memory usage

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch develop
in repository piuparts.

commit 8cecef0774dc592692974e195db34ed1ba5e0a39
Author: Andreas Beckmann <anbe at debian.org>
Date:   Mon Nov 25 13:22:16 2013 +0100

    lib: use DecompressedStream to reduce memory usage
    
    transparently decompress while downloading and reading
    
    switch from gzip to zlib for decompressing .gz
    
    Signed-off-by: Andreas Beckmann <anbe at debian.org>
---
 debian/changelog        |  4 ++++
 piupartslib/__init__.py | 23 ++++-------------------
 2 files changed, 8 insertions(+), 19 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 0971b31..0a324ef 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -15,6 +15,10 @@ piuparts (0.56) UNRELEASED; urgency=low
     - open_packages_url(): Try Packages.gz if Packages.bz2 is not available.
       (Closes: #711157)
     - Add new class DecompressedStream.
+    - open_packages_url(): Use DecompressedStream to transparently decompress
+      a Packages.{bz2,gz} file while downloading and reading the content
+      without requiring to store the complete file (compressed and/or
+      decompressed) in memory.
   * piupartslib/packagesdb.py:
     - PackagesFile: Allow restricting to a set of package names.
   * piuparts-slave.py:
diff --git a/piupartslib/__init__.py b/piupartslib/__init__.py
index b2278ba..b712eac 100644
--- a/piupartslib/__init__.py
+++ b/piupartslib/__init__.py
@@ -19,9 +19,8 @@
 
 
 import bz2
-import gzip
+import zlib
 import urllib2
-import cStringIO
 
 
 import conf
@@ -91,25 +90,11 @@ def open_packages_url(url):
         raise httperror
     url = socket.geturl()
     if ext == '.bz2':
-        decompressed = cStringIO.StringIO()
         decompressor = bz2.BZ2Decompressor()
-        while True:
-            data = socket.read(1024)
-            if not data:
-                socket.close()
-                break
-            decompressed.write(decompressor.decompress(data))
-        decompressed.seek(0)
+        decompressed = DecompressedStream(socket, decompressor)
     elif ext == '.gz':
-        compressed = cStringIO.StringIO()
-        while True:
-            data = socket.read(1024)
-            if not data:
-                socket.close()
-                break
-            compressed.write(data)
-        compressed.seek(0)
-        decompressed = gzip.GzipFile(fileobj=compressed)
+        decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
+        decompressed = DecompressedStream(socket, decompressor)
     else:
         raise ext
     return (url, decompressed)

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/piuparts/piuparts.git