[Piuparts-commits] [SCM] piuparts git repository branch, master, updated. 0.51
Andreas Beckmann
anbe at debian.org
Wed May 15 10:09:51 UTC 2013
The following commit has been merged in the master branch:
commit 4f6c3eda758d467c8cc63d42a7d1d81272f54281
Author: Andreas Beckmann <anbe at debian.org>
Date: Tue Apr 9 11:24:53 2013 +0200
p-s: use a persistent master connection
Keep the master connection open to reduce the number of ssh connections
opened. Close the connection after errors, before processing packages, or
before going to sleep.
Signed-off-by: Andreas Beckmann <anbe at debian.org>
diff --git a/debian/changelog b/debian/changelog
index 8b96a0d..d866206 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -31,6 +31,10 @@ piuparts (0.51) UNRELEASED; urgency=low
- Skip sections that don't exist in piuparts.conf and continue to run.
- Pass the section to piuparts-master via the new "section" command
instead of using a command line argument.
+ - Keep the connection to the master open while switching sections. This
+ reduces the number of ssh connections attempted while checking idle
+ sections. Close the connection before processing packages, before going
+ to sleep, and after communication errors.
* piuparts-analyze.py:
* piuparts-report.py:
- Skip sections that don't exist in piuparts.conf.
diff --git a/piuparts-slave.py b/piuparts-slave.py
index 5cbfba5..62f9f82 100644
--- a/piuparts-slave.py
+++ b/piuparts-slave.py
@@ -182,21 +182,35 @@ class Slave:
def set_master_host(self, host):
logging.debug("Setting master host to %s" % host)
- self._master_host = host
+ if self._master_host != host:
+ self.close()
+ self._master_host = host
def set_master_user(self, user):
logging.debug("Setting master user to %s" % user)
- self._master_user = user
+ if self._master_user != user:
+ self.close()
+ self._master_user = user
def set_master_command(self, cmd):
logging.debug("Setting master command to %s" % cmd)
- self._master_command = cmd
+ if self._master_command != cmd:
+ self.close()
+ self._master_command = cmd
def set_section(self, section):
logging.debug("Setting section to %s" % section)
self._section = section
def connect_to_master(self):
+ if not self._is_connected():
+ self._initial_connect()
+ self._select_section()
+
+ def _is_connected(self):
+ return self._to_master and self._from_master
+
+ def _initial_connect(self):
logging.info("Connecting to %s" % self._master_host)
ssh_command = ["ssh", "-x"]
if self._master_user:
@@ -209,6 +223,8 @@ class Slave:
line = self._readline()
if line != "hello\n":
raise MasterDidNotGreet()
+
+ def _select_section(self):
self._writeline("section", self._section)
line = self._readline()
if line == "busy\n":
@@ -218,6 +234,8 @@ class Slave:
logging.debug("Connected to master")
def close(self):
+ if self._from_master is None and self._to_master is None:
+ return
logging.debug("Closing connection to master")
if self._from_master is not None:
self._from_master.close()
@@ -315,7 +333,7 @@ class Slave:
class Section:
- def __init__(self, section):
+ def __init__(self, section, slave=None):
self._config = Config(section=section, defaults_section="global")
self._config.read(CONFIG_FILE)
self._distro_config = piupartslib.conf.DistroConfig(
@@ -339,7 +357,7 @@ class Section:
if not os.path.exists(rdir):
os.mkdir(rdir)
- self._slave = Slave()
+ self._slave = slave or Slave()
def _throttle_if_overloaded(self):
@@ -355,6 +373,7 @@ class Section:
return
load_resume = max(load_max - 1.0, 0.9)
secs = random.randrange(30, 90)
+ self._slave.close()
while True:
load = os.getloadavg()[0]
if load <= load_resume:
@@ -505,6 +524,7 @@ class Section:
except (MasterDidNotGreet, MasterIsCrazy, MasterCommunicationFailed):
logging.error("connection to master failed")
self._error_wait_until = time.time() + 900
+ self._slave.close()
else:
try:
for logdir in ["pass", "fail", "untestable"]:
@@ -539,17 +559,19 @@ class Section:
except MasterNotOK:
logging.error("master did not respond with 'ok'")
self._error_wait_until = time.time() + 900
+ self._slave.close()
except (MasterIsCrazy, MasterCommunicationFailed):
logging.error("communication with master failed")
self._error_wait_until = time.time() + 900
+ self._slave.close()
else:
return True
- finally:
- self._slave.close()
return False
def _process(self):
+ self._slave.close()
+
packages_files = {}
for distro in [self._config.get_distro()] + self._config.get_distros():
if distro not in packages_files:
@@ -846,10 +868,11 @@ def main():
else:
section_names = global_config["sections"].split()
+ persistent_connection = Slave()
sections = []
for section_name in section_names:
try:
- sections.append(Section(section_name))
+ sections.append(Section(section_name, persistent_connection))
except MissingSection:
# ignore unknown sections
pass
@@ -883,6 +906,7 @@ def main():
sleep_until = min([now + int(global_config["idle-sleep"])] + [section.sleep_until() for section in sections])
if (sleep_until > now):
to_sleep = max(60, sleep_until - now)
+ persistent_connection.close()
logging.info("Nothing to do, sleeping for %d seconds." % to_sleep)
time.sleep(to_sleep)
--
piuparts git repository
More information about the Piuparts-commits
mailing list