[Piuparts-devel] [Git][debian/piuparts][helmutg/feature-nonroot] 10 commits: Run autopkgtest on all supported Python versions
Helmut Grohne (@helmutg)
gitlab at salsa.debian.org
Tue May 14 18:08:57 BST 2024
Helmut Grohne pushed to branch helmutg/feature-nonroot at Debian / piuparts
Commits:
160354cc by Nicolas Dandrimont at 2024-05-14T13:21:38+02:00
Run autopkgtest on all supported Python versions
- - - - -
f6216a2f by Nicolas Dandrimont at 2024-05-14T13:24:42+02:00
Add more tar ignore patterns to debian/source/options
- - - - -
958c35ab by Nicolas Dandrimont at 2024-05-14T13:24:58+02:00
Remove use of deprecated configparser.SafeConfigParser
This was deprecated back in 3.2 and finally got removed in 3.12.
Closes: #1067938
- - - - -
38f61053 by Nicolas Dandrimont at 2024-05-14T13:24:58+02:00
d/t/all-python-versions: turn on the CPython dev mode
This should enable all warnings
- - - - -
663e4c40 by Nicolas Dandrimont at 2024-05-14T13:24:58+02:00
run(): ensure that fds are properly closed under all circumstances
This removes a ResourceWarning in dev mode
- - - - -
36cca1b2 by Nicolas Dandrimont at 2024-05-14T12:16:41+00:00
Merge branch 'mr/python3.12' into 'develop'
python 3.12 support
See merge request debian/piuparts!59
- - - - -
63a02da6 by Helmut Grohne at 2024-05-14T18:14:34+02:00
piuparts: reexecute via unshare when run as non-root
When piuparts is run as root, it currently errors out and exits. Rather
than doing that, we now try reexecuting it inside an unshared namespace
since we recently made piuparts work when run in an unprivileged user
namespace. This enables running piuparts as non-root out of the box.
- - - - -
36957036 by Helmut Grohne at 2024-05-14T19:08:10+02:00
install piuparts to /usr/bin
Since we can now run piuparts as a regular user, install it in a place
where a user can run it.
- - - - -
cc8cf654 by Helmut Grohne at 2024-05-14T19:08:10+02:00
suggest installing uidmap
Unprivileged piuparts only works when uidmap is installed.
- - - - -
b9dd57ef by Helmut Grohne at 2024-05-14T19:08:10+02:00
piuparts.1: document what it takes run piuparts as non-root
Reported-by: Holger Levsen <holger at layer-acht.org>
- - - - -
17 changed files:
- Makefile
- conf/piuparts-slave.sudoers
- debian/control
- debian/piuparts.install
- debian/piuparts.lintian-overrides
- debian/source/options
- + debian/tests/all-python-versions
- + debian/tests/common.sh
- debian/tests/control
- debian/tests/smoke-test
- docs/README_pejacevic.txt
- docs/piuparts/piuparts.1.txt
- instances/piuparts.conf-template.pejacevic
- instances/piuparts.conf.anbe
- piuparts.py
- piupartslib/conf.py
- slave-bin/slave_stop.in
Changes:
=====================================
Makefile
=====================================
@@ -1,5 +1,5 @@
prefix = /usr/local
-sbindir = $(prefix)/sbin
+bindir = $(prefix)/bin
sharedir = $(prefix)/share
mandir = $(sharedir)/man
man1dir = $(mandir)/man1
@@ -30,7 +30,7 @@ define placeholder_substitution
-e 's/__PIUPARTS_VERSION__/$(version)/g' \
-e 's%@libdir@%$(libdir)%g' \
-e 's%@sharedir@%$(sharedir)%g' \
- -e 's%@sbindir@%$(sbindir)%g' \
+ -e 's%@bindir@%$(bindir)%g' \
$< > $@
endef
@@ -146,8 +146,8 @@ install-master: build-master-stamp install-common
#install -m 0644 known_problems/*.conf $(DESTDIR)$(etcdir)/piuparts/known_problems/
install-slave: install-common
- install -d $(DESTDIR)$(sbindir)
- install -m 0755 piuparts $(DESTDIR)$(sbindir)/
+ install -d $(DESTDIR)$(bindir)
+ install -m 0755 piuparts $(DESTDIR)$(bindir)/
install -d $(DESTDIR)$(sharedir)/piuparts
install -m 0755 piuparts-slave $(DESTDIR)$(sharedir)/piuparts/
=====================================
conf/piuparts-slave.sudoers
=====================================
@@ -3,7 +3,7 @@
#
# The piuparts slave needs to handle chroots.
-piupartss ALL = NOPASSWD: /usr/sbin/piuparts *, \
+piupartss ALL = NOPASSWD: /usr/bin/piuparts *, \
/bin/umount /srv/piuparts.debian.org/tmp/tmp*, \
/usr/bin/test -f /srv/piuparts.debian.org/tmp/tmp*, \
/usr/bin/rm -rf --one-file-system /srv/piuparts.debian.org/tmp/tmp*
=====================================
debian/control
=====================================
@@ -45,6 +45,7 @@ Recommends:
Suggests:
schroot,
docker.io,
+ uidmap,
Description: .deb package installation, upgrading, and removal testing tool
piuparts tests that .deb packages (as used by Debian) handle
installation, upgrading, and removal correctly. It does this by
=====================================
debian/piuparts.install
=====================================
@@ -1,2 +1,2 @@
-/usr/sbin/piuparts
+/usr/bin/piuparts
/etc/piuparts/scripts*
=====================================
debian/piuparts.lintian-overrides
=====================================
@@ -3,5 +3,5 @@ uses-dpkg-database-directly [etc/piuparts/scripts-debug-purge/post_remove_postrm
uses-dpkg-database-directly [etc/piuparts/scripts-debug-remove/pre_remove_prerm_postrm_set-x]
uses-dpkg-database-directly [etc/piuparts/scripts/post_remove_exceptions]
uses-dpkg-database-directly [etc/piuparts/scripts/pre_remove_40_find_missing_md5sums]
-uses-dpkg-database-directly [usr/sbin/piuparts]
+uses-dpkg-database-directly [usr/bin/piuparts]
debian-news-entry-has-unknown-version 0.45 [usr/share/doc/piuparts/NEWS.Debian.gz:1]
=====================================
debian/source/options
=====================================
@@ -1 +1 @@
-tar-ignore = .git
+tar-ignore = .git,.tox,.mypy_cache,__pycache__
=====================================
debian/tests/all-python-versions
=====================================
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -eu
+
+. "$(dirname "$0")/common.sh"
+
+echo running "$0"
+
+test_this piuparts --version
+
+cd "$AUTOPKGTEST_TMP"
+
+create_packages
+
+for pyvers in $(py3versions -vi); do
+ test_this "python$pyvers" -X dev /usr/bin/piuparts t.deb
+done
+
+exit 0
=====================================
debian/tests/common.sh
=====================================
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+set -e
+
+test_this() {
+ echo
+ echo "running $@"
+ $@
+}
+
+create_packages () {
+ # set up a very simple test package
+
+ mkdir -p t/DEBIAN t/usr t/etc
+ cat >t/DEBIAN/control <<EOF
+Package: t
+Version: 4
+Maintainer: Piu Parts <piuparts-devel at alioth-lists.debian.net>
+Priority: optional
+Architecture: all
+Installed-Size: 0
+Description: Auto Package Test Dummy
+ Extremely simple binary package for piuparts testing
+EOF
+
+ dpkg-deb -b t
+
+ rm -r t/
+
+ # another simple package, but set up for failure
+
+ mkdir -p f/DEBIAN f/usr f/etc
+ cat >f/DEBIAN/control <<EOF
+Package: f
+Version: 4
+Maintainer: Piu Parts <piuparts-devel at alioth-lists.debian.net>
+Priority: optional
+Architecture: all
+Installed-Size: 0
+Description: Auto Package Test Fail Dummy
+ Extremely simple binary package for piuparts testing - fail version
+EOF
+
+ cat >f/DEBIAN/postinst <<EOF
+#! /bin/sh
+mkdir -p /etc/f/
+touch /etc/f/ailure
+EOF
+
+ chmod +x f/DEBIAN/postinst
+
+ dpkg-deb -b f
+
+ rm -r f/
+}
=====================================
debian/tests/control
=====================================
@@ -1,3 +1,7 @@
Tests: smoke-test
-Depends: @
+Depends: piuparts
+Restrictions: needs-root
+
+Tests: all-python-versions
+Depends: piuparts, python3-all
Restrictions: needs-root
=====================================
debian/tests/smoke-test
=====================================
@@ -1,66 +1,20 @@
#!/bin/sh
-set -e
+set -eu
-echo running $0
+. "$(dirname "$0")/common.sh"
-test_this() {
- echo
- echo "running $@"
- $@
-}
+echo running "$0"
test_this piuparts --version
-WORKDIR=$(mktemp -d)
-trap "rm -rf $WORKDIR" 0 INT QUIT ABRT PIPE TERM
-cd $WORKDIR
+cd "$AUTOPKGTEST_TMP"
-
-# set up a very simple test package
-
-mkdir -p t/DEBIAN t/usr t/etc
-cat >t/DEBIAN/control <<EOF
-Package: t
-Version: 4
-Maintainer: Piu Parts <piuparts-devel at alioth-lists.debian.net>
-Priority: optional
-Architecture: all
-Installed-Size: 0
-Description: Auto Package Test Dummy
- Extremely simple binary package for piuparts testing
-EOF
-
-dpkg-deb -b t
+create_packages
# this should always succeed
test_this piuparts t.deb
-
-# another simple package, but set up for failure
-
-mkdir -p f/DEBIAN f/usr f/etc
-cat >f/DEBIAN/control <<EOF
-Package: f
-Version: 4
-Maintainer: Piu Parts <piuparts-devel at alioth-lists.debian.net>
-Priority: optional
-Architecture: all
-Installed-Size: 0
-Description: Auto Package Test Fail Dummy
- Extremely simple binary package for piuparts testing - fail version
-EOF
-
-cat >f/DEBIAN/postinst <<EOF
-#! /bin/sh
-mkdir -p /etc/f/
-touch /etc/f/ailure
-EOF
-
-chmod +x f/DEBIAN/postinst
-
-dpkg-deb -b f
-
# it is an error if this succeeds
test_this piuparts f.deb && false
=====================================
docs/README_pejacevic.txt
=====================================
@@ -103,7 +103,7 @@ This is actually done by DSA:
.. code-block:: text
# The piuparts slave needs to handle chroots.
- piupartss ALL = NOPASSWD: /usr/sbin/piuparts *, \
+ piupartss ALL = NOPASSWD: /usr/bin/piuparts *, \
/bin/umount /srv/piuparts.debian.org/tmp/tmp*, \
/usr/bin/test -f /srv/piuparts.debian.org/tmp/tmp*, \
/usr/bin/rm -rf --one-file-system /srv/piuparts.debian.org/tmp/tmp*
=====================================
docs/piuparts/piuparts.1.txt
=====================================
@@ -33,7 +33,7 @@ When processing changes files, by default, all packages in a changes file will b
:program:`piuparts` outputs to the standard output some log messages to show what is going on. If a log file is used, the messages go there as well.
-:program:`piuparts` needs to be run as root.
+:program:`piuparts` requires root rights to test packages. It does not have to be run in the initial namespace though. When running it as non-root, it'll create a new Linux namespace and rerun itself as root inside said namespace. For this to work, your user needs have an subuid range (which happens by default since a few years) and you need to install the `uidmap` package to provide setuid helpers :program:`newuidmap` and :program:`newgidmap`.
OPTIONS
-------
=====================================
instances/piuparts.conf-template.pejacevic
=====================================
@@ -274,7 +274,7 @@ piuparts-command =
sudo
env PYTHONPATH=%(PYTHONPATH)s
timeout -s INT -k 5m 80m
- /srv/piuparts.debian.org/sbin/piuparts
+ /srv/piuparts.debian.org/bin/piuparts
PYTHONPATH = /srv/piuparts.debian.org/lib/python3/dist-packages
master-directory = /srv/piuparts.debian.org/master
slave-directory = /srv/piuparts.debian.org/slave
=====================================
instances/piuparts.conf.anbe
=====================================
@@ -450,7 +450,7 @@ piuparts-command =
nice
env PYTHONPATH=%(PYTHONPATH)s
timeout -s INT -k 5m 110m
- /srv/piuparts/sbin/piuparts
+ /srv/piuparts/bin/piuparts
PYTHONPATH = /srv/piuparts/lib/python3/dist-packages
master-directory = /srv/piuparts/master
slave-directory = /srv/piuparts/slave
=====================================
piuparts.py
=====================================
@@ -49,6 +49,7 @@ import time
import traceback
import uuid
from collections import namedtuple
+from contextlib import ExitStack
from signal import SIGALRM, SIGKILL, SIGTERM, alarm, signal
from typing import Dict
@@ -549,59 +550,62 @@ def run(command, ignore_errors=False, timeout=0):
p.kill()
p.wait()
- assert isinstance(command, type([]))
+ assert isinstance(command, list)
logging.debug("Starting command: %s" % command)
env = get_clean_environment()
- devnull = open("/dev/null", "r")
- p = subprocess.Popen(
- command,
- env=env,
- stdin=devnull,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- universal_newlines=True,
- errors="backslashreplace",
- )
- output = ""
- excessive_output = False
- if timeout > 0:
- signal(SIGALRM, alarm_handler)
- alarm(timeout)
- try:
- while p.poll() is None:
- """Read 64 KB chunks, but depending on the output buffering behavior
- of the command we may get less even if more output is coming later.
- Abort after reading max_command_output_size bytes."""
- output += p.stdout.read(1 << 16)
- if len(output) > settings.max_command_output_size:
- excessive_output = True
- ignore_errors = False
- alarm(0)
- kill_subprocess(p, "excessive output")
- output += "\n\n***** Command was terminated after exceeding output limit (%.2f MB) *****\n" % (
- settings.max_command_output_size / 1024.0 / 1024.0
- )
- break
- if not excessive_output:
- output += p.stdout.read(settings.max_command_output_size)
- alarm(0)
- except Alarm:
- ignore_errors = False
- kill_subprocess(p, "excessive runtime")
- output += "\n\n***** Command was terminated after exceeding runtime limit (%s s) *****\n" % timeout
- devnull.close()
-
- if output:
- dump("\n" + indent_string(output.rstrip("\n")))
-
- if p.returncode == 0:
- logging.debug("Command ok: %s" % repr(command))
- elif ignore_errors:
- logging.debug("Command failed (status=%d), but ignoring error: %s" % (p.returncode, repr(command)))
- else:
- logging.error("Command failed (status=%d): %s\n%s" % (p.returncode, repr(command), indent_string(output)))
- panic()
- return p.returncode, output
+ with ExitStack() as s:
+ devnull = s.enter_context(open("/dev/null", "r"))
+
+ p = s.enter_context(
+ subprocess.Popen(
+ command,
+ env=env,
+ stdin=devnull,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ errors="backslashreplace",
+ )
+ )
+ output = ""
+ excessive_output = False
+ if timeout > 0:
+ signal(SIGALRM, alarm_handler)
+ alarm(timeout)
+ try:
+ while p.poll() is None:
+ """Read 64 KB chunks, but depending on the output buffering behavior
+ of the command we may get less even if more output is coming later.
+ Abort after reading max_command_output_size bytes."""
+ output += p.stdout.read(1 << 16)
+ if len(output) > settings.max_command_output_size:
+ excessive_output = True
+ ignore_errors = False
+ alarm(0)
+ kill_subprocess(p, "excessive output")
+ output += "\n\n***** Command was terminated after exceeding output limit (%.2f MB) *****\n" % (
+ settings.max_command_output_size / 1024.0 / 1024.0
+ )
+ break
+ if not excessive_output:
+ output += p.stdout.read(settings.max_command_output_size)
+ alarm(0)
+ except Alarm:
+ ignore_errors = False
+ kill_subprocess(p, "excessive runtime")
+ output += "\n\n***** Command was terminated after exceeding runtime limit (%s s) *****\n" % timeout
+
+ if output:
+ dump("\n" + indent_string(output.rstrip("\n")))
+
+ if p.returncode == 0:
+ logging.debug("Command ok: %s" % repr(command))
+ elif ignore_errors:
+ logging.debug("Command failed (status=%d), but ignoring error: %s" % (p.returncode, repr(command)))
+ else:
+ logging.error("Command failed (status=%d): %s\n%s" % (p.returncode, repr(command), indent_string(output)))
+ panic()
+ return p.returncode, output
def create_temp_file():
@@ -3611,7 +3615,30 @@ def main():
# check if user has root privileges
if os.getuid():
- print("You need to be root to use piuparts.")
+ path_env = os.environ["PATH"].split(":")
+ for d in ("/usr/sbin", "/sbin"):
+ if d not in path_env:
+ path_env.append(d)
+ os.environ["PATH"] = ":".join(path_env)
+ # Make debootstrap happy when it cannot do mknod
+ os.environ.setdefault("container", "lxc")
+ logging.info("Running non-root. Reexecuting in a usernamespace.")
+ os.execvp(
+ "unshare",
+ [
+ "unshare",
+ "--user",
+ "--map-auto",
+ "--setuid",
+ "0",
+ "--setgid",
+ "0",
+ "--mount",
+ "--pid",
+ "--fork",
+ *sys.argv,
+ ],
+ )
sys.exit(1)
logging.info("-" * 78)
=====================================
piupartslib/conf.py
=====================================
@@ -156,7 +156,7 @@ class DistroConfig(UserDict):
"depends": None,
"candidates": None,
}
- cp = configparser.SafeConfigParser()
+ cp = configparser.ConfigParser()
cp.read(filename)
for section in cp.sections():
self[section] = dict(self._defaults)
=====================================
slave-bin/slave_stop.in
=====================================
@@ -45,7 +45,7 @@ while pgrep --full '/usr/bin/python3 @sharedir@/piuparts/piuparts-slave' > /dev/
echo -n "$(date -u +%T) - "
pgrep --full '/usr/bin/python3 @sharedir@/piuparts/piuparts-slave' | xargs -r echo -n "slaves running: "
SLEEP=$(( $i * $i ))
- BUSY=$(ps fax | grep -v grep | grep '/usr/bin/python3 @sbindir@/piuparts' | awk '{print $NF}')
+ BUSY=$(ps fax | grep -v grep | grep '/usr/bin/python3 @bindir@/piuparts' | awk '{print $NF}')
if [ -n "$BUSY" ] ; then
# really/meaningful busy
echo
View it on GitLab: https://salsa.debian.org/debian/piuparts/-/compare/e6421d4a333fb4fdf13d6a31746e8e449d0b7d01...b9dd57ef9adb006f8a849f7d2b9d5c6f511f2cf0
--
View it on GitLab: https://salsa.debian.org/debian/piuparts/-/compare/e6421d4a333fb4fdf13d6a31746e8e449d0b7d01...b9dd57ef9adb006f8a849f7d2b9d5c6f511f2cf0
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/piuparts-devel/attachments/20240514/023ec64c/attachment-0001.htm>
More information about the Piuparts-devel
mailing list