[Secure-testing-commits] r43770 - bin

[Chris Lamb]

Author: lamby
Date: 2016-08-04 17:49:13 +0000 (Thu, 04 Aug 2016)
New Revision: 43770

Modified:
   bin/lts-missing-uploads.py
Log:
lts-missing-uploads: parallelise downloads of announcements.

Modified: bin/lts-missing-uploads.py
===================================================================
--- bin/lts-missing-uploads.py	2016-08-04 17:49:11 UTC (rev 43769)
+++ bin/lts-missing-uploads.py	2016-08-04 17:49:13 UTC (rev 43770)
@@ -19,6 +19,7 @@
 import sys
 import gzip
 import datetime
+import eventlet
 import requests
 import dateutil.relativedelta
 
@@ -34,10 +35,15 @@
     re_version = re.compile(r'^Version.*: (?P<version>.*)')
 
     def __init__(self):
+        self.pool = eventlet.GreenPool(10)
         self.session = requests.session()
 
     def main(self, *args):
         dlas = {}
+        def download(x):
+            self.info("{source}: parsing announcement from {url} ...", **x)
+            x.update(self.get_dla(x['url'])[0])
+            dlas[x['source']] = x
 
         for idx in range(3):
             dt = datetime.datetime.utcnow().replace(day=1) - \
@@ -49,16 +55,13 @@
                 dt.month,
             )
 
+
             # Prefer later DLAs with reversed(..)
             for x in reversed(self.get_dlas(dt.year, dt.month)):
-                # Only comment on the latest upload
-                if x['source'] in dlas:
-                    continue
+                if x['source'] not in dlas:
+                    self.pool.spawn_n(download, x)
+            self.pool.waitall()
 
-                self.info("{source}: parsing announcement from {url} ...", **x)
-                x.update(self.get_dla(x['url'])[0])
-                dlas[x['source']] = x
-
         if not dlas:
             return 0
 
@@ -127,6 +130,8 @@
         print("I: " + msg.format(*args, **kwargs), file=sys.stderr)
 
 if __name__ == '__main__':
+    eventlet.monkey_patch(socket=True)
+
     try:
         sys.exit(LTSMissingUploads().main(*sys.argv[1:]))
     except KeyboardInterrupt: