[Piuparts-commits] [piuparts] 02/09: p-s: handle tarball-only sections

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch develop
in repository piuparts.

commit 8fa5da72aa6f811251f40daa05e8e9a191ff3ab1
Author: Andreas Beckmann <anbe at debian.org>
Date:   Mon May 4 20:05:49 2015 +0200

    p-s: handle tarball-only sections
    
    Signed-off-by: Andreas Beckmann <anbe at debian.org>
---
 README_server.txt | 10 ++++++++++
 debian/changelog  |  9 +++++++++
 piuparts-slave.py | 27 +++++++++++++++++++++++++--
 3 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/README_server.txt b/README_server.txt
index 2f614aa..230b582 100644
--- a/README_server.txt
+++ b/README_server.txt
@@ -284,6 +284,11 @@ used for all further sections.
  from first section a package is in is used for the source package
  html report.
 
+* "basetgz-sections" is an additional list of sections that are only
+ used to maintain the basetgz tarballs and will therefore be ignored
+ by all scripts except piuparts-slave.
+ This list is empty by default.
+
 * "master-host" is the host where the master exists. The slave will
  give this host to ssh. This option is mandatory.
 
@@ -405,6 +410,11 @@ section, too, and will serve as defaults for all other sections
  settings are the first entry (to test upgrades of "disappearing"
  packages) or the restricted set in a partial distribution (e.g.
  stable to backports to testing).
+ The special keyword "None" is used to denote that no packages
+ are to be tested, but only the basetgz tarball will be created
+ and refreshed regularily (for the distribution given in
+ 'upgrade-test-distros'). This reference basetgz can be shared
+ between several sections without being affected by their flags.
 
 * "area" is the archive area used to get the list of packages to
  be tested. The Packages file for this area will be loaded. The
diff --git a/debian/changelog b/debian/changelog
index f34c237..73bca68 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,17 @@
 piuparts (0.64) UNRELEASED; urgency=medium
  
+  [ Holger Levsen ]
   * Add FancyIndexing to piuparts-master.conf and enable apache syntax
     highlighting in vim. (Thanks DSA!)
 
+  [ Andreas Beckmann ]
+  * piuparts.conf:
+    - New global setting: basetgz-sections, used by piuparts-slave only.
+  * piuparts-slave.py:
+    - Add support for special sections that only create/refresh reference
+      basetgz tarballs without testing packages. The basetgz can be shared
+      between multiple sections without being affected by their flags.
+
  -- Holger Levsen <holger at debian.org>  Mon, 27 Apr 2015 17:01:28 +0200
 
 piuparts (0.63) unstable; urgency=medium
diff --git a/piuparts-slave.py b/piuparts-slave.py
index 32a4870..0312a05 100644
--- a/piuparts-slave.py
+++ b/piuparts-slave.py
@@ -76,6 +76,7 @@ class Config(piupartslib.conf.Config):
         piupartslib.conf.Config.__init__(self, section,
                                          {
                                          "sections": "slave",
+                                         "basetgz-sections": "",
                                          "idle-sleep": 300,
                                          "max-tgz-age": 2592000,
                                          "min-tgz-retry-delay": 21600,
@@ -349,6 +350,7 @@ class Section:
         self._error_wait_until = 0
         self._idle_wait_until = 0
         self._recycle_wait_until = 0
+        self._tarball_wait_until = 0
         self._slave_directory = os.path.abspath(section)
         if not os.path.exists(self._slave_directory):
             os.makedirs(self._slave_directory)
@@ -411,27 +413,35 @@ class Section:
         return os.path.join(self._config["basetgz-directory"], basetgz)
 
     def _check_tarball(self):
+        if int(self._config["max-tgz-age"]) < 0:
+            return
+
         oldcwd = os.getcwd()
         os.chdir(self._slave_directory)
 
         tgz = self._get_tarball()
         max_tgz_age = int(self._config["max-tgz-age"])
         min_tgz_retry_delay = int(self._config["min-tgz-retry-delay"])
+        ttl = 0
         needs_update = not os.path.exists(tgz)
         if not needs_update and max_tgz_age > 0:
             # tgz exists and age is limited, so check age
             now = time.time()
             age = now - os.path.getmtime(tgz)
+            ttl = max_tgz_age - age
             logging.info("Check-replace %s: age=%d vs. max=%d" % (tgz, age, max_tgz_age))
-            if age > max_tgz_age:
+            if ttl < 0:
                 if os.path.exists(tgz + ".log"):
                     age = now - os.path.getmtime(tgz + ".log")
+                ttl = min_tgz_retry_delay - age
                 logging.info("Limit-replace %s: last-retry=%d vs. min=%d" % (tgz, age, min_tgz_retry_delay))
-                if age > min_tgz_retry_delay:
+                if ttl < 0:
                     needs_update = True
                     logging.info("%s too old.  Forcing re-creation" % tgz)
         if needs_update:
             create_chroot(self._config, tgz, self._config.get_start_distro())
+            ttl = min_tgz_retry_delay
+        self._tarball_wait_until = time.time() + ttl
 
         os.chdir(oldcwd)
 
@@ -473,6 +483,18 @@ class Section:
         if interrupted or got_sighup:
             do_processing = False
 
+        if do_processing and time.time() > self._tarball_wait_until:
+            self._check_tarball()
+
+        if self._config.get_distro() == "None":
+            # section is for tarball creation only
+            self._idle_wait_until = self._tarball_wait_until + 60
+            self._recycle_wait_until = self._tarball_wait_until + 3600
+            return 0
+
+        if interrupted or got_sighup:
+            do_processing = False
+
         if not do_processing and self._count_submittable_logs() == 0:
             return 0
 
@@ -899,6 +921,7 @@ def main():
         section_names = sys.argv[1:]
     else:
         section_names = global_config["sections"].split()
+        section_names += global_config["basetgz-sections"].split()
 
     persistent_connection = Slave()
     sections = []

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/piuparts/piuparts.git