[Git][security-tracker-team/security-tracker][master] 2 commits: debian_support: updateFile: support .xz files

[Emilio Pozuelo Monfort]

Emilio Pozuelo Monfort pushed to branch master at Debian Security Tracker / security-tracker


Commits:
b3e1e759 by Emilio Pozuelo Monfort at 2020-09-30T21:30:26+02:00
debian_support: updateFile: support .xz files

https://bugs.debian.org/931533

- - - - -
88ca724c by Emilio Pozuelo Monfort at 2020-09-30T21:30:26+02:00
Reenable the backport releases

- - - - -


2 changed files:

- lib/debian-releases.mk
- lib/python/debian_support.py


Changes:

=====================================
lib/debian-releases.mk
=====================================
@@ -5,12 +5,9 @@ define get_config =
 $(shell jq -r $(1) 'data/config.json')
 endef
 
-# backports suites only have Sources.xz and respective Packages.xz
-# available.
-# Cf. as well https://bugs.debian.org/664866
-#BACKPORT_RELEASES := $(OLDSTABLE) $(STABLE)
 MAIN_RELEASES = $(call get_config, '.distributions | to_entries[] | select(.value.release) | .key')
 SECURITY_RELEASES = $(filter-out sid, $(MAIN_RELEASES))
+BACKPORT_RELEASES = $(SECURITY_RELEASES)
 
 # Define the variables for the release on the main mirror
 define add_main_release =


=====================================
lib/python/debian_support.py
=====================================
@@ -18,7 +18,7 @@ from __future__ import print_function
 
 """This module implements facilities to deal with Debian-specific metadata."""
 
-import gzip
+import gzip, lzma
 import io
 import json
 import os.path
@@ -28,8 +28,10 @@ import tempfile
 
 try:
     from urllib.request import urlopen
+    from urllib.error import URLError
 except ImportError:
     from urllib2 import urlopen
+    from urllib2.error import URLError
 
 try:
     from cStringIO import StringIO as streamIO
@@ -316,9 +318,6 @@ def patchLines(lines, patches):
         lines[first:last] = args
 
 def replaceFile(lines, local):
-
-    import os.path
-
     local_new = local + '.new'
     new_file = open(local_new, 'w+')
 
@@ -331,32 +330,41 @@ def replaceFile(lines, local):
         if os.path.exists(local_new):
             os.unlink(local_new)
 
-def downloadGunzipLines(remote):
-    """Downloads a file from a remote location and gunzips it.
+def downloadCompressedLines(remote):
+    """Downloads a file from a remote location and uncompresses it.
 
     Returns the lines in the file."""
 
+    if remote.endswith('.gz'):
+        cls = gzip
+    elif remote.endswith('.xz'):
+        cls = lzma
+    else:
+        raise ValueError('file format not supported: %s' % remote)
+
     data = urlopen(remote, timeout=TIMEOUT)
     try:
-        gfile = gzip.GzipFile(fileobj=streamIO(data.read()))
-        try:
-            if sys.version_info.major == 3:
-                return io.TextIOWrapper(gfile).readlines()
-            else:
-                return gfile.readlines()
-        finally:
-            gfile.close()
+        b = io.BytesIO(cls.decompress(data.read()))
+        t = io.TextIOWrapper(b, 'utf-8')
+        return t.readlines()
     finally:
         data.close()
-        
+
+def downloadLines(remote):
+    try:
+        return downloadCompressedLines(remote + '.xz')
+    except URLError:
+        return downloadCompressedLines(remote + '.gz')
+
 def downloadFile(remote, local):
-    """Copies a gzipped remote file to the local system.
+    """Copies a compressed remote file to the local system.
 
-    remote - URL, without the .gz suffix
+    remote - URL, without compression suffix
     local - name of the local file
     """
-    
-    lines = downloadGunzipLines(remote + '.gz')
+
+    lines = downloadLines(remote)
+
     replaceFile(lines, local)
     return lines
 
@@ -440,8 +448,10 @@ def updateFile(remote, local, verbose=None):
         if verbose:
             print("updateFile: downloading patch " + repr(patch_name))
         try:
-            patch_contents = downloadGunzipLines(remote + '.diff/' + patch_name
-                                                 + '.gz')
+            # We could remove the extension here and call downloadLines
+            # when diff files come with another compression
+            patch_contents = downloadCompressedLines(remote + '.diff/'
+                                                     + patch_name + '.gz')
         except IOError:
             return downloadFile(remote, local)
         if readLinesSHA1(patch_contents ) != patch_hashes[patch_name]:



View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/-/compare/4613cf4f398570c7ba630b5648faf2fdadedfff3...88ca724c224790bbf96016aff16c11d4f025db5a

-- 
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/-/compare/4613cf4f398570c7ba630b5648faf2fdadedfff3...88ca724c224790bbf96016aff16c11d4f025db5a
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-security-tracker-commits/attachments/20200930/00e185bf/attachment-0001.html>