[Piuparts-devel] Bug#421801: piuparts autopkgtest (hence xenlvm)
support
Ian Jackson
iwj at ubuntu.com
Tue May 1 16:29:00 UTC 2007
Package: piuparts
Version: 0.20-3
Severity: wishlist
I have made a patch that allows piuparts to use any autopkgtest
virtualisation server instead of running `chroot' and accessing the
target filesystem directly.
The main benefit of this is that you can run piuparts using an
autopkgtest-xenlvm throwaway Xen guest rather than a chroot.
Ian.
-------------- next part --------------
diff --exclude='*~' -ruN orig/piuparts-0.20/debian/changelog piuparts-0.20/debian/changelog
--- orig/piuparts-0.20/debian/changelog 2007-04-13 17:21:40.000000000 +0100
+++ piuparts-0.20/debian/changelog 2007-04-25 16:20:11.000000000 +0100
@@ -1,3 +1,9 @@
+piuparts (0.20-4ubuntu1~iwj) unstable; urgency=low
+
+ * Can use autopkgtest Xen-based virtualisation (--adt-virt option).
+
+ -- Ian Jackson <ian at davenant.greenend.org.uk> Wed, 25 Apr 2007 16:20:11 +0100
+
piuparts (0.20-3) unstable; urgency=low
* New Maintainer(s): piuparts team. Closes: #390754.
diff --exclude='*~' -ruN orig/piuparts-0.20/piuparts.py piuparts-0.20/piuparts.py
--- orig/piuparts-0.20/piuparts.py 2006-09-22 11:19:15.000000000 +0100
+++ piuparts-0.20/piuparts.py 2007-04-27 18:56:07.000000000 +0100
@@ -48,6 +48,7 @@
import sets
import subprocess
import unittest
+import urllib
class Settings:
@@ -136,6 +137,7 @@
"/var/lib/logrotate/status",
"/var/lib/rbldns",
"/var/log/dpkg.log",
+ "/var/log/auth.log",
"/var/log/faillog",
"/var/log/lastlog",
"/var/spool/cron", # Temporary until at bug is fixed.
@@ -406,6 +408,12 @@
def __init__(self):
self.name = None
+ def create_file(self, path, contents): create_file(path, contents)
+ def chmod(self, path, mode): os.chmod(path, mode)
+ def remove_files(self, list): remove_files(list)
+ def copy_file(source, target): shutil.copy(source, target)
+ def create_temp_tgz_file(self): (fd, tgz) = create_temp_file(); return tgz
+
def create_temp_dir(self):
"""Create a temporary directory for the chroot."""
self.name = tempfile.mkdtemp(dir=settings.tmpdir)
@@ -465,20 +473,20 @@
for mirror, components in settings.debian_mirrors:
lines.append("deb %s %s %s\n" %
(mirror, distro, " ".join(components)))
- create_file(os.path.join(self.name, "etc/apt/sources.list"),
+ self.create_file(os.path.join(self.name, "etc/apt/sources.list"),
"".join(lines))
def create_apt_conf(self):
"""Create /etc/apt/apt.conf inside the chroot."""
- create_file(self.relative("etc/apt/apt.conf"),
+ self.create_file(self.relative("etc/apt/apt.conf"),
'APT::Get::AllowUnauthenticated "yes";\n' +
'APT::Get::Assume-Yes "yes";\n')
def create_policy_rc_d(self):
"""Create a policy-rc.d that prevents daemons from running."""
full_name = os.path.join(self.name, "usr/sbin/policy-rc.d")
- create_file(full_name, "#!/bin/sh\nexit 101\n")
- os.chmod(full_name, 0777)
+ self.create_file(full_name, "#!/bin/sh\nexit 101\n")
+ self.chmod(full_name, 0777)
logging.debug("Created policy-rc.d and chmodded it.")
def setup_minimal_chroot(self):
@@ -498,7 +506,7 @@
self.run(["apt-get", "install", "debfoster"])
self.run(["debfoster", "-o", "MaxPriority=required", "-o",
"UseRecommends=no", "-f", "-n", "apt", "debfoster"])
- remove_files([self.relative("var/lib/debfoster/keepers")])
+ self.remove_files([self.relative("var/lib/debfoster/keepers")])
self.run(["dpkg", "--purge", "debfoster"])
def configure_chroot(self):
@@ -544,7 +552,7 @@
(", ".join(source_names), target_name))
for source_name in source_names:
try:
- shutil.copy(source_name, target_name)
+ self.copy_file(source_name, target_name)
except IOError, detail:
logging.error("Error copying %s to %s: %s" %
(source_name, target_name, detail))
@@ -556,15 +564,16 @@
tmp_files = [os.path.basename(a) for a in filenames]
tmp_files = [os.path.join("tmp", name) for name in tmp_files]
tmp_files = [shellquote(x) for x in tmp_files]
- self.run(["dpkg", "-i"] + tmp_files, ignore_errors=True)
+ self.run(["dpkg", "-i"] + tmp_files)
self.run(["apt-get", "-yf", "--no-remove", "install"])
self.run(["apt-get", "clean"])
- remove_files([os.path.join(self.name, name)
+ print >>sys.stderr, "remove_files", `tmp_files`
+ self.remove_files([os.path.join(self.name, name)
for name in tmp_files])
def get_selections(self):
"""Get current package selections in a chroot."""
- (status, output) = self.run(["dpkg", "--get-selections", "*"])
+ (status, output) = self.run(["dpkg", "--get-selections", "\\*"])
list = [line.split() for line in output.split("\n") if line.strip()]
dict = {}
for name, status in list:
@@ -704,6 +713,264 @@
else:
logging.debug("No broken symlinks as far as we can find.")
+class VirtServ(Chroot):
+ # Provides a thing that looks to the rest of piuparts much like
+ # a chroot but is actually provided by an adt virtualisation server.
+ # See /usr/share/doc/autopkgtest/README.virtualisation-server.
+
+ def __init__(self, cmdline):
+ self._cmdline = cmdline
+ self.name = '/ADT-VIRT'
+ self._vs = None
+
+ def _awaitok(self, cmd):
+ r = self._vs.stdout.readline().rstrip('\n')
+ l = r.split(' ')
+ if l[0] != 'ok': self._fail('virtserver response to %s: %s' % (cmd,r))
+ logging.debug('adt-virt << %s', r)
+ return l[1:]
+
+ def _vs_send(self, cmd):
+ if type(cmd) == type([]):
+ def maybe_quote(a):
+ if type(a) != type(()): return a
+ (a,) = a
+ return urllib.quote(a)
+ cmd = ' '.join(map(maybe_quote,cmd))
+ logging.debug('adt-virt >> %s', cmd)
+ print >>self._vs.stdin, cmd
+ return cmd.split(' ')[0]
+
+ def _command(self, cmd):
+ # argument forms: complete-command-string
+ # [arg, ...] where arg may be (arg,) to quote it
+ cmdp = self._vs_send(cmd)
+ self._vs.stdin.flush()
+ return self._awaitok(cmdp)
+
+ def _getfilecontents(self, filename):
+ try:
+ (_,tf) = create_temp_file()
+ self._command(['copyup',(filename,),(tf,)])
+ f = file(tf)
+ d = f.read()
+ f.close()
+ finally:
+ os.remove(tf)
+ return d
+
+ def create_temp_dir(self):
+ if self._vs is None:
+ logging.debug('adt-virt || %s' % self._cmdline)
+ self._vs = subprocess.Popen(self._cmdline, shell=True,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
+ self._awaitok('banner')
+ self._caps = self._command('capabilities')
+
+ def shutdown(self):
+ if self._vs is None: return
+ self._vs_send('quit')
+ self._vs.stdin.close()
+ self._vs.stdout.close()
+ self._vs.wait()
+ self._vs = None
+
+ def remove(self):
+ self._command('close')
+
+ def _fail(self,m):
+ logging.error("adt-virt-* error: "+m)
+ panic()
+
+ def _open(self):
+ self._scratch = self._command('open')[0]
+
+ # this is a hack to make install_and_upgrade_between distros
+ # work; we pretend to save the chroot to a tarball but in
+ # fact we do nothing and then we can `restore' the `tarball' with
+ # adt-virt revert
+ def create_temp_tgz_file(self):
+ return self
+ def pack_into_tgz(self, tgz):
+ if tgz is not self: self._fail('packing into tgz not supported')
+ if not 'revert' in self._caps: self._fail('testbed cannot revert')
+ def unpack_from_tgz(self, tgz):
+ if tgz is not self: self._fail('unpacking from tgz not supported')
+ self._open()
+
+ def _execute(self, cmdl, tolerate_errors=False):
+ assert type(cmdl) == type([])
+ prefix = ['sh','-ec','''
+ LC_ALL=C
+ unset LANGUAGES
+ export LC_ALL
+ exec 2>&1
+ exec "$@"
+ ''','<command>']
+ ca = ','.join(map(urllib.quote, prefix + cmdl))
+ stdout = '%s/cmd-stdout' % self._scratch
+ stderr = '%s/cmd-stderr-base' % self._scratch
+ cmd = ['execute',ca,
+ '/dev/null',(stdout,),(stderr,),
+ '/root','timeout=600']
+ es = int(self._command(cmd)[0])
+ if es and not tolerate_errors:
+ stderr_data = self._getfilecontents(stderr)
+ logging.error("Execution failed (status=%d): %s\n%s" %
+ (es, `cmdl`, indent_string(stderr_data)))
+ panic()
+ return (es, stdout, stderr)
+
+ def _execute_getoutput(self, cmdl):
+ (es,stdout,stderr) = self._execute(cmdl)
+ stderr_data = self._getfilecontents(stderr)
+ if es or stderr_data:
+ logging.error('Internal command failed (status=%d): %s\n%s' %
+ (es, `cmdl`, indent_string(stderr_data)))
+ panic()
+ (_,tf) = create_temp_file()
+ try:
+ self._command(['copyup',(stdout,),(tf,)])
+ except:
+ os.remove(tf)
+ raise
+ return tf
+
+ def run(self, command, ignore_errors=False):
+ cmdl = ['sh','-ec','cd /\n' + ' '.join(command)]
+ (es,stdout,stderr) = self._execute(cmdl, tolerate_errors=True)
+ stdout_data = self._getfilecontents(stdout)
+ print >>sys.stderr, "VirtServ run", `command`,`cmdl`, '==>', `es`,`stdout`,`stderr`, '|', stdout_data
+ if es == 0 or ignore_errors: return (es, stdout_data)
+ stderr_data = self._getfilecontents(stderr)
+ logging.error('Command failed (status=%d): %s\n%s' %
+ (es, `command`, indent_string(stdout_data + stderr_data)))
+ panic()
+
+ def setup_minimal_chroot(self):
+ self._open()
+
+ def _tbpath(self, with_junk):
+ if not with_junk.startswith(self.name):
+ logging.error("Un-mangling testbed path `%s' but it does not"
+ "start with expected manglement `%s'" %
+ (with_junk, self.name))
+ panic()
+ return with_junk[len(self.name):]
+
+ def chmod(self, path, mode):
+ self._execute(['chmod', ('0%o' % mode), self._tbpath(path)])
+ def remove_files(self, paths):
+ self._execute(['rm','--'] + map(self._tbpath, paths))
+ def copy_file(self, our_src, tb_dest):
+ self._command(['copydown',(our_src,),
+ (self._tbpath(tb_dest)+'/'+os.path.basename(our_src),)])
+ def create_file(self, path, data):
+ path = self._tbpath(path)
+ try:
+ (_,tf) = create_temp_file()
+ f = file(tf,'w')
+ f.write(tf)
+ f.close()
+ self._command(['copydown',(tf,),(path,)])
+ finally:
+ os.remove(tf)
+
+ class DummyStat: pass
+
+ def save_meta_data(self):
+ mode_map = {
+ 's': stat.S_IFSOCK,
+ 'l': stat.S_IFLNK,
+ 'f': stat.S_IFREG,
+ 'b': stat.S_IFBLK,
+ 'd': stat.S_IFDIR,
+ 'c': stat.S_IFCHR,
+ 'p': stat.S_IFIFO,
+ }
+
+ dict = {}
+
+ tf = self._execute_getoutput(['find','/','-xdev','-printf',
+ "%y %m %U %G %s %p %l \\n".replace(' ','\\0')])
+ try:
+ f = file(tf)
+
+ while 1:
+ line = ''
+ while 1:
+ splut = line.split('\0')
+ if len(splut) == 8 and splut[7] == '\n': break
+ if len(splut) >= 8:
+ self._fail('aaargh wrong output from find: %s' %
+ urllib.quote(line), `splut`)
+ l = f.readline()
+ if not l:
+ if not line: break
+ self._fail('aargh missing final newline from find'
+ ': %s, %s' % (`l`[0:200], `splut`[0:200]))
+ line += l
+ if not line: break
+
+ st = VirtServ.DummyStat()
+ st.st_mode = mode_map[splut[0]] | int(splut[1],8)
+ (st.st_uid, st.st_gid, st.st_size) = map(int, splut[2:5])
+
+ dict[splut[5]] = (st, splut[6])
+
+ f.close()
+ finally:
+ os.remove(tf)
+
+ return dict
+
+ def get_files_owned_by_packages(self):
+ tf = self._execute_getoutput(['bash','-ec','''
+ cd /var/lib/dpkg/info
+ find . -name "*.list" -type f -print0 | \\
+ xargs -r0 egrep . /dev/null
+ test "${PIPESTATUS[*]}" = "0 0"
+ '''])
+ dict = {}
+ try:
+ f = file(tf)
+ for l in f:
+ (lf,pathname) = l.rstrip('\n').split(':',1)
+ assert lf.endswith('.list')
+ pkg = lf[:-5]
+ if pathname in dict:
+ dict[pathname].append(pkg)
+ else:
+ dict[pathname] = [pkg]
+
+ f.close()
+ finally:
+ os.remove(tf)
+ return dict
+
+ def check_for_broken_symlinks(self):
+ if not settings.check_broken_symlinks:
+ return
+ tf = self._execute_getoutput(['bash','-ec','''
+ find / -xdev -type l -print0 | \\
+ xargs -r0 -i'{}' \\
+ find '{}' -maxdepth 0 -follow -type l -ls
+ test "${PIPESTATUS[*]}" = "0 0"
+ '''])
+ try:
+ f = file(tf)
+ broken = False
+ for l in f:
+ logging.error("Broken symlink: " + l)
+ broken = True
+ if broken: panic()
+ logging.debug("No broken symlinks found.")
+ finally:
+ os.remove(tf)
+
+ def check_for_no_processes(self): pass # ?!
+ def mount_proc(self): pass
+ def unmount_proc(self): pass
def objects_are_different(pair1, pair2):
"""Are filesystem objects different based on their meta data?"""
@@ -894,16 +1161,16 @@
"""Install package and upgrade it between distributions, then remove.
Return True if successful, False if not."""
- chroot = Chroot()
+ chroot = get_chroot()
chroot.create()
id = do_on_panic(chroot.remove)
if settings.basetgz:
root_tgz = settings.basetgz
else:
- (fd, root_tgz) = create_temp_file()
+ root_tgz = chroot.create_temp_tgz()
chroot.pack_into_tgz(root_tgz)
-
+
if settings.endmeta:
root_info, selections = load_meta_data(settings.endmeta)
else:
@@ -917,13 +1184,13 @@
if settings.saveendmeta:
save_meta_data(settings.saveendmeta, root_info, selections)
-
- chroot.remove()
- dont_do_on_panic(id)
- chroot = Chroot()
- chroot.create_temp_dir()
- id = do_on_panic(chroot.remove)
- chroot.unpack_from_tgz(root_tgz)
+
+ chroot.remove()
+ dont_do_on_panic(id)
+ chroot = get_chroot()
+ chroot.create_temp_dir()
+ id = do_on_panic(chroot.remove)
+ chroot.unpack_from_tgz(root_tgz)
chroot.check_for_no_processes()
@@ -1062,6 +1329,11 @@
help="Use /var/cache/pbuilder/base.tgz as the base " +
"tarball.")
+ parser.add_option('', "--adt-virt",
+ metavar='CMDLINE', default=None,
+ help="Use CMDLINE via autopkgtest (adt-virt-*)"
+ " protocol instead of managing a chroot.")
+
parser.add_option("-s", "--save", metavar="FILENAME",
help="Save the chroot into FILENAME.")
@@ -1094,6 +1366,11 @@
settings.check_broken_symlinks = not opts.no_symlinks
settings.savetgz = opts.save
+ if opts.adt_virt is None:
+ settings.adt_virt = None
+ else:
+ settings.adt_virt = VirtServ(opts.adt_virt)
+
if opts.tmpdir is not None:
settings.tmpdir = opts.tmpdir
if not os.path.isdir(settings.tmpdir):
@@ -1137,6 +1414,10 @@
return args
+def get_chroot():
+ if settings.adt_virt is None: return Chroot()
+ return settings.adt_virt
+
def main():
"""Main program. But you knew that."""
@@ -1159,7 +1440,7 @@
args = []
if len(settings.debian_distros) == 1:
- chroot = Chroot()
+ chroot = get_chroot()
chroot.create()
id = do_on_panic(chroot.remove)
@@ -1192,6 +1473,8 @@
logging.error("FAIL: Upgrading between Debian distributions.")
panic()
+ if settings.adt_virt is not None: settings.adt_virt.shutdown()
+
logging.info("PASS: All tests.")
logging.info("piuparts run ends.")
More information about the Piuparts-devel
mailing list