[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: trbo: new script to bring nodes back online

[Holger Levsen]

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
72d50d01 by Holger Levsen at 2020-09-14T00:31:07+02:00
trbo: new script to bring nodes back online

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
6b2aca38 by Holger Levsen at 2020-09-14T15:14:58+02:00
reproducible Debian: stop deleting schroot base directories after 3 days (we need them for a month), instead monitor their amount

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


3 changed files:

- + bin/bring_back_node.sh
- bin/jenkins-shell-monitor.sh
- bin/reproducible_maintenance.sh


Changes:

=====================================
bin/bring_back_node.sh
=====================================
@@ -0,0 +1,29 @@
+#!/bin/bash
+# vim: set noexpandtab:
+
+# Copyright © 2020 Holger Levsen (holger at layer-acht.org)
+# released under the GPLv2
+#
+# small helper script to bring nodes back online
+# FIXME: this script should call the powercycling scripts too
+
+SHORTNAMES=$@
+for shortname in $SHORTNAMES ; do
+	case $shortname in
+		pb2|pb6|pb12|pb16)			i=$(echo $shortname | sed 's#pb##')
+							node=profitbricks-build$i-i386.debian.net
+							;;
+		pb1|pb3|pb5|pb7|pb9|pb10|pb11|pb15)	i=$(echo $shortname | sed 's#pb##')
+							node=profitbricks-build$i-amd64.debian.net
+							;;
+		ct*)					i=$(echo $shortname | sed 's#ct##')
+							node=codethink-sled$i-arm64.debian.net
+		*)					echo unsupported, please improve
+							;;
+	esac
+	echo "Marking $node as online for jenkins again."
+	ssh jenkins.debian.net "sudo -u jenkins -i sed -i 's#$node##' offline_nodes"
+done
+echo
+ssh jenkins.debian.net "sudo -u jenkins -i cat offline_nodes"
+


=====================================
bin/jenkins-shell-monitor.sh
=====================================
@@ -37,6 +37,7 @@ main_loop() {
 		SCHROOT_SESSIONS="\033[93m$SCHROOT_SESSIONS\033[0m"
 	fi
 	SCHROOT_MOUNTS=$(mount | grep /run/schroot| wc -l)
+	SCHROOT_BASE_DIRS=$(find /schroots/ -maxdepth 1 -type d | wc -l)
 	REPRO_JOBS=$(ls ~jenkins/jobs/reproducible_* -1d | wc -l)
 	JOBS_RUNNING_TOTAL=$(grep '_ /bin/bash /srv/jenkins/bin/' $PSFAX | egrep -v 'reproducible_worker.sh|reproducible_build.sh|jenkins-shell-monitor.sh' | wc -l)
 	if [ $JOBS_RUNNING_TOTAL -eq 0 ] ; then
@@ -98,7 +99,7 @@ main_loop() {
 		echo    "logged in user sessions:                      $(uptime | rev |cut -d ',' -f1-4 | rev | cut -d ',' -f1 | sed "s#users##" | xargs echo)"
 		echo    "logged in users:                              $(w -h | awk '{print $1}' | sort -u | xargs echo)"
 		echo    "number of open files:                         $LSOF"
-		echo -e "schroot: (sessions / mounts)                  $SCHROOT_SESSIONS / $SCHROOT_MOUNTS"
+		echo -e "schroot: (sessions / mounts / base dirs)      $SCHROOT_SESSIONS / $SCHROOT_MOUNTS / $SCHROOT_BASE_DIRS"
 		echo    "configured r-b jobs:                          $REPRO_JOBS"
 		echo -e "running jenkins jobs: (total/local/remote)    $JOBS_RUNNING_TOTAL / $JOBS_RUNNING_LOCAL / $JOBS_RUNNING_REMOTE"
 		echo    "running debian r-b workers:                   $REPRO_WORKERS"


=====================================
bin/reproducible_maintenance.sh
=====================================
@@ -448,21 +448,21 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
 fi
 
 # find old schroots
-echo "$(date -u) - Removing schroots older than 3 days."
+echo "$(date -u) - Removing schroots older than 31 days."
 regex="/schroots/(reproducible-.+-[0-9]{1,5}|schroot-install-.+)"
-OLDSTUFF=$(find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +2 -exec ls -lad {} \; || true)
+OLDSTUFF=$(find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +31 -exec ls -lad {} \; || true)
 if [ ! -z "$OLDSTUFF" ] ; then
 	# try to unmount mounts first
-	for MP in $(find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +2 ) ; do
+	for MP in $(find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +31 ) ; do
 		if [ -d $MP/proc ] ; then
 			sudo umount -l $MP/proc 2>/dev/null || echo "umount -l $MP/proc failed, continueing."
 		fi
 	done
 	echo
-	echo "schroots older than 3 days found in /schroots, which will be deleted:"
-	find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +2 -exec sudo rm -rf --one-file-system {} \; || true
+	echo "schroots older than 31 days found in /schroots, which will be deleted:"
+	find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +31 -exec sudo rm -rf --one-file-system {} \; || true
 	echo "$OLDSTUFF"
-	OLDSTUFF=$(find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +2 -exec ls -lad {} \; || true)
+	OLDSTUFF=$(find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +31 -exec ls -lad {} \; || true)
 	if [ ! -z "$OLDSTUFF" ] ; then
 		echo
 		echo "Warning: Tried, but failed to delete these schroots:"



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/e202f952e4524a8df409250f29cb40f3e774f2cc...6b2aca3826089981291a7e8c844ad2a2145ea121

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/e202f952e4524a8df409250f29cb40f3e774f2cc...6b2aca3826089981291a7e8c844ad2a2145ea121
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20200914/fad4982b/attachment-0001.html>