[Git][qa/jenkins.debian.net][h01ger/wip-rebuilder-db-backup] WIP

Holger Levsen (@holger) gitlab at salsa.debian.org
Wed Apr 16 12:17:49 BST 2025 


Holger Levsen pushed to branch h01ger/wip-rebuilder-db-backup at Debian QA / jenkins.debian.net


Commits:
660108eb by Holger Levsen at 2025-04-16T13:17:32+02:00
WIP

- - - - -


9 changed files:

- TODO.r.d.n
- TODO.rebuilderd-systemd-units
- bin/rebuilderd_db_backup.sh
- + bin/rebuilderd_db_backup_compress.sh
- + hosts/osuosl5-amd64/etc/systemd/system/rebuilderd-db-backup-compress.service
- + hosts/osuosl5-amd64/etc/systemd/system/rebuilderd-db-backup-compress.timer
- + hosts/osuosl5-amd64/etc/systemd/system/rebuilderd-db-backup at .service
- + hosts/osuosl5-amd64/etc/systemd/system/rebuilderd-db-backup at .timer
- update_jdn.sh


Changes:

=====================================
TODO.r.d.n
=====================================
@@ -1,7 +1,4 @@
 RFH running r.d.n
-	- sync script to avoid pipe problems as discussed on irc
-	  --now to run now, else/by default sleep 1-30min at the beginning to be able to run them all 100min(?) after dinstall starts
-	- systemd units for everything, see TODO.rebuilderd-systemd-units
 	- index.html:
 	  - should show $arch+all combined stats (MR189)
 	  - the worker(s) should be links to thanks.html
@@ -15,6 +12,7 @@ RFH running r.d.n
 	- /status/$pkg - showing ones package status on all tested archs
 	- rebuilderd puts inputs and outputs in /tmp, should be in of ~/tmp
 	- shrink the db please, test with https://r.d.n/$arch/stats/db-backup/
+	- watch disk usage of /srv/rebuilderd/*/stats/db-backup/
 	- find a way to reschedule as (any) user
 
 file *important* bugs about arch:all issues just needing no source change uploads. justification: i plan to NMU :)
@@ -34,13 +32,8 @@ https://r.d.n/ improvements
 	link from workers to generic thanks page? :) and then tell sponsors about r.d.n and them being mentioned on the frontpage.
 	add rb & debian logos
 	add thanks to osuosl for o5 and ionos for jenkins.d.n too
-do TODO.rebuilderd-systemd-units
-	incl monthly db backup
-upload src:rebuilderd to unstable (and its depends need to go through NEW too)
 until debrebuild does it by itself:
-	purge 1000 files from cache in node health check if > $defined_size_4host
 	file wishlist bug for --max-cache-size option alongside with --cache
-test unstable & trixie
 drop i7 extra partition to save infomaniak credits
 check rebuilderd uid+gid everywhere
 update README and THANKS
@@ -54,5 +47,4 @@ the db is still 80gb after vacuuming and splitting into amd64 and all...
 Static content last modified: doesnt work
 wishlist: all bad src pkgs on all archs combined :) by maintainer+uploaders :)
 setup one rebuilder-snapshot instance and use that as cache for all workers?
-drop old db backups
-drop rebuilder-snapshot-metadata.service from o4
+drop rebuilder-snapshot-metadata.service from o4 and o5


=====================================
TODO.rebuilderd-systemd-units
=====================================
@@ -1,12 +1,5 @@
 rebuilderd worker
  user systemd.tmpfiles
-rebuilderd backup
- on script, which does it for all rebuilderd instances sequentially
- run on tuesdays and fridays
- service timer which which stops and disables the rebuilded service
-	check if backup from same day exist and exit:
-	[ -d /srv/rebuilderd/i386/stats/db-backup/db-back-2025-04-05 ] && exit
-
 debrebuild cache size limiter unit
 	run once a day
 	configurable


=====================================
bin/rebuilderd_db_backup.sh
=====================================
@@ -3,15 +3,14 @@
 # Copyright 2024-2025 Holger Levsen (holger at layer-acht.org)
 # released under the GPLv2
 
-set -e
-
 #
-# WIP. currently to be run manually as the db user...
-# should be called by a systemd timer on boot before rebuilderd
-# is started. (and then rebuilderd can be started once its compressing in background...)
-# currently its ment to be run manually.
-# rbuilderd should not be running when this is run to ensure db consistancy.
+# backup a rebuilderd instance - this is rather clumsy but best we can do until rebuilderd uses postgresql or some such
 #
+# rbuilderd should not be running when this is run to ensure db consistancy.
+# this is enforced by the service running this
+
+set -e
+
 if [ -z "$1" ] ; then
 	echo "need an architecture."
 	exit 1
@@ -24,18 +23,8 @@ BACKUP_DIR=$BASE_PATH/stats/db-backup
 mkdir -p $BACKUP_DIR
 cd $BACKUP_DIR
 
+[ -d /srv/rebuilderd/$ARCH/stats/db-backup/db-back-$TODAY ] && exit 0
 mkdir db-back-${TODAY}
 cp -av $BASE_PATH/rebuilderd.db* db-back-${TODAY}/
 sync
-echo 
-echo "it's safe to restart rebuilderd again."
-echo
-#cd db-back-${TODAY}
-#for i in rebuilderd.db* ; do
-#	zstdmt --rm -3 $i
-#done
-#echo
-#du -sch $BASE_PATH/rebuilderd.db*
-#echo
-#du -sch $BACKUP_DIR/*
 


=====================================
bin/rebuilderd_db_backup_compress.sh
=====================================
@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Copyright 2025 Holger Levsen (holger at layer-acht.org)
+# released under the GPLv2
+
+#
+# compress all rebuilderd backups from today
+#
+set -e
+
+# prevent concurrent runs
+COUNT=$(ps fax|grep $0|grep -v grep | wc -l)
+if [ $COUNT -gt 1 ] ; then
+	echo "$(date -u) $0 already running, exiting."
+	exit 1
+fi
+
+# actually compress those backups
+TODAY=$(date '+%Y-%m-%d' -u)
+FILES=$(find /srv/rebuilderd/*/stats/db-backup/db-back-$TODAY -type f ! -name '*.zst' ! -name '*.xz')
+for i in $FILES ; do
+	zstdmt --rm -4 $i
+done
+


=====================================
hosts/osuosl5-amd64/etc/systemd/system/rebuilderd-db-backup-compress.service
=====================================
@@ -0,0 +1,8 @@
+[Unit]
+Description=Periodically compress the backups of all rebuilderd databases
+
+[Service]
+ExecStart=/srv/jenkins/bin/rebuilderd_db_backup_compress.sh
+RemainAfterExit=no
+Type=oneshot
+


=====================================
hosts/osuosl5-amd64/etc/systemd/system/rebuilderd-db-backup-compress.timer
=====================================
@@ -0,0 +1,8 @@
+[Unit]
+Description=Periodically compress the backups of all rebuilderd databases
+
+[Timer]
+OnCalendar=Sun 12:00
+
+[Install]
+WantedBy=timers.target


=====================================
hosts/osuosl5-amd64/etc/systemd/system/rebuilderd-db-backup at .service
=====================================
@@ -0,0 +1,10 @@
+[Unit]
+Description=Periodically backup a rebuilderd database
+
+[Service]
+ExecStartPre=/usr/bin/systemctl stop rebuilderd@%i.service
+ExecStart=/srv/jenkins/bin/rebuilderd_db_backup.sh %i
+ExecStartPost=/usr/bin/systemctl start rebuilderd@%i.service
+RemainAfterExit=no
+Type=oneshot
+


=====================================
hosts/osuosl5-amd64/etc/systemd/system/rebuilderd-db-backup at .timer
=====================================
@@ -0,0 +1,11 @@
+[Unit]
+Description=Periodically backup a rebuilderd database
+
+[Timer]
+# dinstall runs 1|7|13|19:52 UTC - see https://salsa.debian.org/ftp-team/dak/blob/master/config/debian/crontab
+OnCalendar=Sun, 05:00
+# randomly distribute over an hour
+RandomizedDelaySec=3600
+
+[Install]
+WantedBy=timers.target


=====================================
update_jdn.sh
=====================================
@@ -959,6 +959,12 @@ deploy_rebuilderd_services() {
 						sudo ln -sf rebuilderd-pkgsync at .timer rebuilderd-pkgsync@${i}.timer
 						sudo systemctl enable --now rebuilderd-pkgsync@${i}.timer
 					done
+				elif [ "$1" = "db-backup" ] ; then
+					for i in $2 ; do
+						sudo ln -sf rebuilderd-db-backup at .service rebuilderd-db-backup@${i}.service
+						sudo ln -sf rebuilderd-db-backup at .timer rebuilderd-db-backup@${i}.timer
+						sudo systemctl enable --now rebuilderd-db-backup@${i}.timer
+					done
 				fi
 				cd -
 }
@@ -978,8 +984,10 @@ case $HOSTNAME in
 						;;
 	*ppc64el)				deploy_rebuilderd_services worker 3
 						;;
-	osuosl5*)				deploy_rebuilderd_services rebuilderd "all amd64 arm64 armhf i386 riscv64 ppc64el"
-						deploy_rebuilderd_services pkgsync    "all amd64 arm64 armhf i386 riscv64 ppc64el"
+	osuosl5*)				deploy_rebuilderd_services rebuilderd	"all amd64 arm64 armhf i386 riscv64 ppc64el"
+						deploy_rebuilderd_services pkgsync	"all amd64 arm64 armhf i386 riscv64 ppc64el"
+						deploy_rebuilderd_services db-backup	"all amd64 arm64 armhf i386 riscv64 ppc64el"
+						sudo systemctl enable --now rebuilderd-db-backup-compress.timer
 						;;
 	*) 					;;
 esac



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/660108eb5fac5b0544b1bccdd003a13b1295deb0

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/660108eb5fac5b0544b1bccdd003a13b1295deb0
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20250416/db6a0529/attachment-0001.htm>

More information about the Qa-jenkins-scm mailing list