[Piuparts-commits] [SCM] piuparts git repository branch, develop, updated. 0.49-99-g9bcbbe2
Andreas Beckmann
anbe at debian.org
Mon Feb 25 10:56:48 UTC 2013
The following commit has been merged in the develop branch:
commit 9bcbbe21782162d30517cd019b8d7b855b40e07e
Author: Andreas Beckmann <anbe at debian.org>
Date: Tue Feb 5 16:14:13 2013 +0100
p-s: check load frequently and sleep if it exceeds threshold
tunable via [global] slave-load-max = 1.5
0 (default) disables
9.5 seems to be reasonable for 8 slaves running on 4 cores / 8 threads
Signed-off-by: Andreas Beckmann <anbe at debian.org>
diff --git a/README.txt b/README.txt
index d3cf224..1a3d640 100644
--- a/README.txt
+++ b/README.txt
@@ -489,6 +489,11 @@ used for all further sections.
* "doc-root" is the location where the webserver will serve the
piuparts report from. Default: "/".
+* "slave-load-max" specifies the system load limit when
+ piuparts-slave will enter sleep mode. Operation will be resumed
+ after load drops below 'slave-load-max - 1.0'. Floating point
+ value. Defaults to 0 (= disabled).
+
==== section specific configuration
The section specific settings will be reloaded each time a section
diff --git a/debian/changelog b/debian/changelog
index 7153fd8..49e4763 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,6 +19,7 @@ piuparts (0.50) UNRELEASED; urgency=low
+ known-problem-directory (for known_problem definitions)
+ backup-directory (for the master script prepare_backup)
+ PYTHONPATH (to find the python modules in non-default path setups)
+ + slave-load-max
- Setting tmpdir is now mandatory for master/slave operation.
* piupartslib/conf.py:
* piupartslib/packagesdb.py:
@@ -32,6 +33,10 @@ piuparts (0.50) UNRELEASED; urgency=low
- Fix SIGHUP handling while recycling.
- Flush and unreserve all sections on interrupt.
- Add support for running piuparts under /usr/bin/timeout.
+ - Check system load frequently and enter sleep mode if threshold
+ (piuparts.conf global setting slave-load-max) is exceeded. Operation
+ will be resumed after load drops below 'slave-load-max - 1.0'. Disabled
+ by default.
* piuparts-analyze.py:
- Add support for magic "$DISTRO/None" versions.
* piuparts-report.py:
diff --git a/piuparts-slave.py b/piuparts-slave.py
index 5445920..7d2d9d6 100644
--- a/piuparts-slave.py
+++ b/piuparts-slave.py
@@ -90,6 +90,7 @@ class Config(piupartslib.conf.Config):
"keep-sources-list": "no",
"arch": None,
"precedence": "1",
+ "slave-load-max": None,
},
defaults_section=defaults_section)
@@ -337,6 +338,35 @@ class Section:
self._slave = Slave()
+
+ def _throttle_if_overloaded(self):
+ global interrupted
+ if interrupted or got_sighup:
+ return
+ if self._config["slave-load-max"] is None:
+ return
+ load_max = float(self._config["slave-load-max"])
+ if load_max < 1.0:
+ return
+ if os.getloadavg()[0] <= load_max:
+ return
+ load_resume = max(load_max - 1.0, 0.9)
+ secs = random.randrange(30, 90)
+ while True:
+ load = os.getloadavg()[0]
+ if load <= load_resume:
+ break
+ logging.info("Sleeping due to high load (%.2f)" % load)
+ try:
+ time.sleep(secs)
+ except KeyboardInterrupt:
+ interrupted = True
+ if interrupted or got_sighup:
+ break
+ if secs < 300:
+ secs += random.randrange(30, 90)
+
+
def _connect_to_master(self, recycle=False):
self._slave.set_master_host(self._config["master-host"])
self._slave.set_master_user(self._config["master-user"])
@@ -386,6 +416,8 @@ class Section:
if time.time() < self.sleep_until(recycle=recycle):
return 0
+ self._throttle_if_overloaded()
+
if interrupted or got_sighup:
do_processing = False
@@ -534,13 +566,12 @@ class Section:
test_count = 0
self._check_tarball()
for package_name, version in self._slave.get_reserved():
- if got_sighup:
+ self._throttle_if_overloaded()
+ if interrupted or got_sighup:
break
test_count += 1
test_package(self._config, package_name, version, packages_files)
self._slave.forget_reserved(package_name, version)
- if interrupted:
- break
self._talk_to_master(unreserve=interrupted)
return test_count
--
piuparts git repository
More information about the Piuparts-commits
mailing list