[Piuparts-commits] [SCM] piuparts git repository branch, master, updated. 0.48-79-gdf94975

[Andreas Beckmann]

The following commit has been merged in the master branch:
commit 470d1b2dbce8c562666f6201285c4c3f53bfc8e8
Author: Andreas Beckmann <debian at abeckmann.de>
Date:   Mon Dec 17 01:41:13 2012 +0100

    lib/db: prefetch directory listings to speed up LogDB
    
    os.listdir() is much faster than n times os.path.exists()
    
    Signed-off-by: Andreas Beckmann <debian at abeckmann.de>

diff --git a/debian/changelog b/debian/changelog
index 5ffc541..4e9917c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,6 +19,8 @@ piuparts (0.49) UNRELEASED; urgency=low
   * piupartslib/conf.py:
     - Add get_final_distro() method.
     - Add optional distro argument to get_mirror() method.
+  * piupartslib/packagesdb.py:
+    - Speed up LogDB by fetching existing files with os.listdir().
   * piuparts-master.py:
   * piuparts-slave.py:
     - Disable X forwarding while connecting to master.
diff --git a/piupartslib/packagesdb.py b/piupartslib/packagesdb.py
index 6e4a0e8..9b4af0a 100644
--- a/piupartslib/packagesdb.py
+++ b/piupartslib/packagesdb.py
@@ -181,6 +181,16 @@ class LogDB:
         except AttributeError:
             pass
 
+    def bulk_load_dir(self, dirname):
+        try:
+            cache = self.exists_cache
+        except AttributeError:
+            self.exists_cache = {}
+            cache = self.exists_cache
+        for basename in os.listdir(dirname):
+            if basename.endswith(".log"):
+                cache[os.path.join(dirname, basename)] = True
+
     def open_file(self, pathname, mode):
         return file(pathname, mode)
 
@@ -501,6 +511,9 @@ class PackagesDB:
 
         self._stamp = time.time()
 
+        for subdir in self._all:
+            self._logdb.bulk_load_dir(subdir)
+
         self._find_all_packages()
 
         self._package_state = {}

-- 
piuparts git repository