[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 5 commits: djm: add --no-future option

Holger Levsen (@holger) gitlab at salsa.debian.org
Tue May 16 23:39:51 BST 2023



Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
8e4f9bc4 by Holger Levsen at 2023-05-16T23:33:07+02:00
djm: add --no-future option

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
47a32d54 by Holger Levsen at 2023-05-16T23:53:37+02:00
djm: notice failure, even if moving to the future. (also run autoremove in the presence)

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
bee675e1 by Holger Levsen at 2023-05-17T00:01:28+02:00
djm: refactoring, introduce run_command_futureproof_on_host_in_xterm()

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
2254f1ae by Holger Levsen at 2023-05-17T00:03:55+02:00
djm: new reason: debug host problems

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
a96f60e3 by Holger Levsen at 2023-05-17T00:38:40+02:00
djm: new action: command - thus finally being able to drop deploy_jdn after 7y of service

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


3 changed files:

- TODO
- bin/djm
- − deploy_jdn


Changes:

=====================================
TODO
=====================================
@@ -32,10 +32,26 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
 ** maybe: rm /tmp/mmdebstrap.* older than 3 days
 * split TODO in TODO and TODO.legacy?
 * djm:
-** ACTION: c|command - run arbitrary command (with and without reset_clock)
-** then get rid of deploy_jdn
+** option: --today to be used with action shell
+** options: --press-enter --dont-wait-for-enter, default being wait for enter.
+** update documentation refering to deploy_jdn to djm
+*** ups, there's no doc on deploy_jdn anyway
+** show FIXMEs?
+show_fixmes() {
+	local BASEDIR="$(dirname "$(readlink -e $0)")"
+	# FIXME: if ! [ -f bin/djm...
+	local TMPFILE=$(mktemp)
+	rgrep FI[X]ME $BASEDIR/* | grep -v $BASEDIR/TODO | grep -v echo > $TMPFILE || true
+	if [ -s $TMPFILE ] ; then
+		echo
+		echo Existing FIXME statements:
+		echo
+		cat $TMPFILE | sed "s#$PWD#...#g"
+		echo
+	fi
+	rm -f $TMPFILE
+}
 ** also log an id, so that one command effecting several hosts is counted as on action
-** new option, to optionally not fix the time (on future hosts) after command, to see failing commands?
 *** make actions triggering 'all' be only one entry in the djm logfile -> more sensible stats
 ** automatically backup+cleanup logs on the first of the month
 ** new feature: --show-month 04


=====================================
bin/djm
=====================================
@@ -17,6 +17,7 @@ set -o pipefail		# see eg http://petereisentraut.blogspot.com/2010/11/pipefail.h
 DRY_RUN=false
 VERBOSE=false
 FETCH=false
+NO_FUTURE=false
 LOGFILE=~/.djm.log
 FAIL_REASON=""
 CONFIRM=false
@@ -24,6 +25,7 @@ BG=""
 TARGET=
 ACTION=
 REASON=
+COMMAND=
 LOCAL_LOGFILE=~/.djm.log
 UI_LOGFILE=~/.djm-jenkins-ui.log
 PARSER_LOGFILE=~/.djm-jenkins-parser.log
@@ -62,7 +64,7 @@ node_in_the_future () {
 #
 
 # thanks to /usr/share/doc/util-linux/examples/getopt-parse.bash
-TEMP=$(getopt -o 'dfhrv' --long 'dry-run,fetch,help,report,verbose' -n 'djm' -- "$@")
+TEMP=$(getopt -o 'dfhnrv' --long 'dry-run,fetch,help,no-future,report,verbose' -n 'djm' -- "$@")
 if [ $? -ne 0 ]; then
 	echo 'Terminating...' >&2
 	exit 1
@@ -87,6 +89,11 @@ while true; do
 			shift
 			continue
 			;;
+		'-n'|'--no-future')
+			NO_FUTURE=true
+			shift
+			continue
+			;;
 		'-r'|'--report')
 			MODE=report
 			shift
@@ -122,15 +129,19 @@ show_help(){
 	seperator "="
 	echo
 	echo Either of these are possible:
+	# FIXME: explain this better
 	echo "djm -d/--dry-run -f/--fetch -r/--report -v/--verbose -h/--help"
-	echo "djm -d/--dry-run \$TARGET \$ACTION \$REASON"
+	echo "djm -d/--dry-run -n/--no-future \$TARGET \$ACTION \$REASON"
+	# --no-future is useful, because setting the time to the future
+	# will succeed and thus hide earlier issues.
 	echo
 	echo "TARGET=	'all' or grepable (jenkins, amd64, ionos, osuosl3) from ./nodes/list_nodes"
 	echo "         	or special target '.' for jenkins-ui ACTION."
 	echo
 	echo "ACTION=   one of these:"
 	echo "         _a_utoremove             apt autoremove && apt clean"
-	echo "         _b_ring-back             mark host as online"
+	echo "         _b_ring-back             mark hosts as online"
+	echo "         _c_ommand                run \$4 on hosts"
 	echo "         _d_eploy-git             deploy jenkins.debian.net.git"
 	echo "         _f_fix-future            fix clock on hosts running in the future"
 	echo "         _j_enkins-ui             for when doing things manually in the jenkins-ui"
@@ -143,6 +154,7 @@ show_help(){
 	echo "REASON=   one of these:"
 	echo "          cs=clock skew"
 	echo "          dj=debug job problems"
+	echo "          dh=debug host problems"
 	echo "          hb=host is back"
 	echo "          ho=host is overloaded"
 	echo "          ku=kernel upgrade"
@@ -159,6 +171,7 @@ show_help(){
 		echo "\$1 = $1"
 		echo "\$2 = $2"
 		echo "\$3 = $3"
+		# FIXME: $4 is not explained here
 		echo
 		echo parsing problems:
 		echo
@@ -188,6 +201,7 @@ verify_target_action_reason(){
 	case $ACTION in
 		a|autoremove)	ACTION=autoremove				;;
 		b|bring-back)	ACTION=bring-back				;;
+		c|command)	ACTION=command		;	CONFIRM=true	;;
 		d|deploy-git)	ACTION=deploy-git				;;
 		f|fix-future)	ACTION=fix-future				;;
 		j|jenkins-ui)	ACTION=jenkins-ui				;;
@@ -201,6 +215,7 @@ verify_target_action_reason(){
 	case $REASON in
 		cs)	REASON="clock skew"				;;
 		dj)	REASON="debug job problems"			;;
+		dh)	REASON="debug host problems"			;;
 		hb)	REASON="host is back"				;;
 		ho)	REASON="host is overloaded"			;;
 		ku)	REASON="kernel upgrade"				;;
@@ -391,6 +406,17 @@ EOF
 	xterm -T "$SHORTNODE / $ACTION ($FILE)" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $FILE" &
 }
 
+run_command_futureproof_on_host_in_xterm() {
+	COMMAND="( $COMMAND ) || ( echo press enter ; read a )"
+	if node_in_the_future "$SHORTNODE" ; then
+		COMMAND="sudo ntpdate -b $NTP_SERVER ; $COMMAND"
+		if ! $NO_FUTURE ; then
+			COMMAND="$COMMAND ; sudo date --set='+398 days +6 hours + 23 minutes'"
+		fi
+	fi
+	xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -t $NODE \"$COMMAND\"" &
+}
+
 djm_do() {
 	#
 	# make sure we have target, action & reason
@@ -428,6 +454,9 @@ djm_do() {
 				SHORTNODE="jenkins web UI"
 			fi
 			echo "$(date -u '+%Y-%m-%d %H:%M UTC'), $SHORTNODE, $ACTION, $REASON"
+			if [ -n "$4" ] && [ "$ACTION" = "command" ] ; then
+				echo " - command to be run is \"$4\""
+			fi
 		done
 	fi
 	
@@ -490,8 +519,6 @@ djm_do() {
 					esac
 					run_xterm2wait4node_comeback $NODE
 					;;
-			autoremove)	xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE 'sudo apt-get autoremove || ( echo press enter ; read a ) sudo apt-get clean || ( echo press enter ; read a )'" &
-				;;
 			shell)	xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE" &
 				;;
 			jenkins-ui)	echo "Documenting one manual action in the jenkins-ui."
@@ -501,8 +528,12 @@ djm_do() {
 					echo "Trying to mark $SHORTNODE as online. This is the diff for $OFF_NODES on jenkins.debian.net:"
 					ssh jenkins.debian.net "grep -v $SHORTNODE $OFF_NODES > $TMPFILE ; diff $OFF_NODES $TMPFILE ; mv $TMPFILE $OFF_NODES ; chown jenkins:jenkins $OFF_NODES"
 					;;
-			fix-future)	COMMAND="date ; sudo ntpdate -b $NTP_SERVER"
+			fix-future)	if $NO_FUTURE ; then
+						echo "djm was called with --no-future and to fix hosts running in the future - this makes no sense, exiting."
+						exit 1
+					fi
 					if node_in_the_future "$SHORTNODE" ; then
+						COMMAND="date ; sudo ntpdate -b $NTP_SERVER"
 						COMMAND="$COMMAND && sudo date --set='+398 days +6 hours + 23 minutes'"
 						COMMAND="$COMMAND ; date -u ; echo press enter ; read a"
 						xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE \"$COMMAND\"" &
@@ -511,13 +542,12 @@ djm_do() {
 						continue
 					fi
 
+					;;
+			autoremove)	COMMAND="sudo apt-get autoremove"
+					run_command_futureproof_on_host_in_xterm
 					;;
 			apt-upgrade)	COMMAND="sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get clean"
-					if node_in_the_future "$SHORTNODE" ; then
-						COMMAND="sudo ntpdate -b $NTP_SERVER ; $COMMAND ; sudo date --set='+398 days +6 hours + 23 minutes'"
-					fi
-					COMMAND="( $COMMAND ) || ( echo press enter ; read a ) ; echo 'sleeping 10s now' ; sleep 10"
-					xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -t $NODE \"$COMMAND\"" &
+					run_command_futureproof_on_host_in_xterm
 					;;
 			deploy-git)	GIT_REPO="https://salsa.debian.org/qa/jenkins.debian.net.git"
 					unset GITOPTS
@@ -541,16 +571,15 @@ djm_do() {
 						fi
 						./update_jdn.sh 2>&1 | sudo tee -a /var/log/jenkins/update_jdn.log
 					EOF
-					echo -n "$COMMAND"
-					if node_in_the_future "$SHORTNODE" ; then
-						COMMAND="sudo ntpdate -b $NTP_SERVER ; $COMMAND ; sudo date --set='+398 days +6 hours + 23 minutes'"
-					fi
-					COMMAND="( $COMMAND ) || ( echo press enter ; read a ) ; echo 'sleeping 10s now' ; sleep 10"
-					xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -t $NODE \"$COMMAND\"" &
+					run_command_futureproof_on_host_in_xterm
 					;;
 			rmstamp)	COMMAND="sudo rm -f /var/log/jenkins/*stamp"
 					xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -t $NODE \"$COMMAND\"" &
 					;;
+			command)	COMMAND="$4"
+					COMMAND="$COMMAND ; ( echo press enter ; read a )"
+					run_command_futureproof_on_host_in_xterm
+					;;
 			*)	echo unsupported action, sorry, please improve.
 				exit 1
 				;;
@@ -567,7 +596,7 @@ djm_do() {
 
 djm_init() {
 	DUMMY=$(mktemp)
-	touch -d "$(date -u -d "18 hours ago" '+%Y-%m-%d %H:%M') UTC" $DUMMY
+	touch -d "$(date -u -d "12 hours ago" '+%Y-%m-%d %H:%M') UTC" $DUMMY
 	if $FETCH || [ ! -f $JOBS ] || [ $DUMMY -nt $JOBS ] ; then
 		if ! $DRY_RUN ; then
 			echo "Fetching remote logs."
@@ -583,7 +612,7 @@ djm_init() {
 # main
 #
 case $MODE in
-	report)		djm_init ; djm_report		;;
-	djm)		djm_init ; djm_do $1 $2 $3	;;
-	help)		show_help			;;
+	report)		djm_init ; djm_report			;;
+	djm)		djm_init ; djm_do "$1" "$2" "$3" "$4"	;;
+	help)		show_help				;;
 esac


=====================================
deploy_jdn deleted
=====================================
@@ -1,312 +0,0 @@
-#!/bin/bash
-# vim: set noexpandtab:
-#
-# Copyright 2014-2023 Holger Levsen <holger at layer-acht.org>
-#           ©    2018 Mattia Rizzolo <mattia at debian.org>
-#
-# released under the GPLv2
-#
-# deployment helper for jenkins.debian.net and build nodes
-# (misses freebsd node)
-
-# disclaimers:
-#   this script grew over the years…
-#   the code is horrible and was always meant to be a quick local hack and never to be published
-#   use at your own risk. it might eat your cats. you have been warned.
-#   the HOSTS lists below is yet another code duplication…
-
-help() {
-cat <<- EOF
-accepted params:
- ./deploy_jdn all		- deploy on all nodes (and handle 398 days hosts properly)
- ./deploy_jdn \$host		- deploy on \$host and jenkins (and handle 398 days hosts properly)
- ./deploy_jdn all \$foo		- run "\$foo" on all nodes (and handle 398 days hosts properly)
- ./deploy_jdn 			- deploy on jenkins only
- ./deploy_jdn jenkins		- deploy on jenkins only
- ./deploy_jdn jenkins ionos10	- deploy on jenkins and ionos10
- ./deploy_jdn jenkins o3	- deploy on jenkins and osuosl3
- ./deploy_jdn jenkins c9	- deploy on jenkins and codethink9
- ./deploy_jdn jenkins 10	- deploy on jenkins and ionos10
- ./deploy_jdn jenkins 5 6	- deploy on jenkins and ionos5 and ionos6
- ./deploy_jdn jenkins amd64	- deploy on jenkins and all amd64 nodes
- ./deploy_jdn only ionos10	- deploy on ionos10
- ./deploy_jdn only o3		- deploy on osuosl3
- ./deploy_jdn only c9		- deploy on codethink9
- ./deploy_jdn only i10		- deploy on ionos10
- ./deploy_jdn only 5 6		- deploy on ionos5 and ionos6
- ./deploy_jdn only amd64	- deploy on all amd64 only
- ./deploy_jdn upgrade|u		- run "apt-get update && upgrade && clean" everywhere
- ./deploy_jdn upgradey|uy	- run "apt-get upgrade -y" everywhere
- ./deploy_jdn rmstamp|rm	- delete stamp files everywhere
-
-Rebooting all nodes, except jenkins, is easily done like this:
- $ parallel -j 8 -i sh -c 'ssh {} sudo reboot' -- \$(./nodes/list_nodes | grep -v jenkins)
-
-EOF
-}
-
-
-LOGFILE=~/.djm.log
-LOGDATE="$(date -u '+%Y-%m-%d %H:%M UTC')"
-
-START=$(date +'%s')
-GIT_REPO="https://salsa.debian.org/qa/jenkins.debian.net.git"
-mapfile -t HOSTS < <(nodes/list_nodes | grep -v jenkins.debian.net)
-HOSTS+=(root at jenkins.debian.net)
-
-ALL_HOSTS=("${HOSTS[@]}")
-
-node_in_the_future () {
-	case "$1" in
-		ionos5-amd64*|ionos6-i386*|ionos15-amd64*|ionos16-i386*) true ;;
-		codethink9*|codethink11*|codethink13*|codethink15*) true ;;
-		osuosl2-amd64*) true ;;
-		*) false ;;
-	esac
-}
-
-show_fixmes() {
-	#
-	# There's always some work left...
-	#	echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
-	#
-	BASEDIR="$(dirname "$(readlink -e $0)")"
-	TMPFILE=$(mktemp)
-	rgrep FI[X]ME $BASEDIR/* | grep -v $BASEDIR/TODO | grep -v echo > $TMPFILE || true
-	if [ -s $TMPFILE ] ; then
-		echo
-		echo Existing FIXME statements:
-		echo
-		cat $TMPFILE | sed "s#$PWD#...#g"
-		echo
-	fi
-	rm -f $TMPFILE
-}
-
-echo
-echo -n "$(date) - "
-reset_clock=true
-if [ "$1" = "fixme" ] ; then
-	show_fixmes
-	exit 0
-elif [ "$1" = "all" ] ; then
-	ACTION="deploy jdn.git"
-	echo -n "Running j.d.n.git updates on ${HOSTS[@]} now"
-	# reset_clock can be false as update_jdn.sh sets the time
-	reset_clock=false
-	shift
-	if [ -n "$1" ] ; then
-		real_command="$@"
-		echo -n "Running '$real_command' on ${HOSTS[@]} now."
-		real_command="$@ && echo '__reallyreally=ok__'"
-	fi
-elif [ "$1" = "upgrade" ] || [ "$1" = "u" ] ; then
-	ACTION="apt upgrade"
-	real_command="export LANG=C && sudo apt-get update && sudo apt-get upgrade && sudo apt-get dist-upgrade && sudo apt-get clean"
-	shift
-elif [ "$1" = "upgradey" ] || [ "$1" = "uy" ] ; then
-	ACTION="apt upgrade"
-	real_command="export LANG=C && sudo apt-get update && sudo apt-get -y upgrade && sudo apt-get -y dist-upgrade && sudo apt-get clean"
-	shift
-elif [ "$1" = "rmstamp" ] || [ "$1" = "rm" ] ; then
-	ACTION="rm stamp"
-	real_command="sudo rm -f /var/log/jenkins/*stamp && echo '__reallyreally=ok__'"
-	reset_clock=false
-	shift
-elif [ "$1" = "" ] ; then
-	ACTION="deploy jdn.git"
-	HOSTS=(root at jenkins.debian.net)
-	echo -n "Running j.d.n.git updates on ${HOSTS[@]} now"
-elif [ "$1" = "jenkins" ] || [ "$1" = "only" ] ; then
-	ACTION="deploy jdn.git"
-	if [ "$1" = "jenkins" ] ; then
-		HOSTS=(root at jenkins.debian.net)
-	else
-		HOSTS=()
-	fi
-	shift
-	for i in "$@" ; do
-		case "$i" in
-			1|i1|ionos1)	HOSTS+=(ionos1-amd64.debian.net) ;;
-			2|i2|ionos2)	HOSTS+=(ionos2-i386.debian.net) ;;
-			3|i3|ionos3)	HOSTS+=(ionos3-i386.debian.net) ;;
-			5|i5|ionos5)	HOSTS+=(ionos5-amd64.debian.net) ;;
-			6|i6|ionos6)	HOSTS+=(ionos6-i386.debian.net) ;;
-			7|i7|ionos7)	HOSTS+=(ionos7-amd64.debian.net) ;;
-			9|i9|ionos9)	HOSTS+=(ionos9-amd64.debian.net) ;;
-			10|i10|ionos10)	HOSTS+=(ionos10-amd64.debian.net) ;;
-			11|i11|ionos11)	HOSTS+=(ionos11-amd64.debian.net) ;;
-			12|i12|ionos12)	HOSTS+=(ionos12-i386.debian.net) ;;
-			15|i15|ionos15)	HOSTS+=(ionos15-amd64.debian.net) ;;
-			16|i16|ionos16)	HOSTS+=(ionos16-i386.debian.net) ;;
-			o1)		HOSTS+=(osuosl1-amd64.debian.net) ;;
-			o2)		HOSTS+=(osuosl2-amd64.debian.net) ;;
-			o3)		HOSTS+=(osuosl3-amd64.debian.net) ;;
-			o4)		HOSTS+=(osuosl4-amd64.debian.net) ;;
-			o5)		HOSTS+=(osuosl5-amd64.debian.net) ;;
-			c9|cs9|ct9)	HOSTS+=(codethink9-arm64.debian.net) ;;
-			c10|cs10|ct10)	HOSTS+=(codethink10-arm64.debian.net) ;;
-			c11|cs11|ct11)	HOSTS+=(codethink11-arm64.debian.net) ;;
-			c12|cs12|ct12)	HOSTS+=(codethink12-arm64.debian.net) ;;
-			c13|cs13|ct13)	HOSTS+=(codethink13-arm64.debian.net) ;;
-			c14|cs14|ct14)	HOSTS+=(codethink14-arm64.debian.net) ;;
-			c15|cs15|ct15)	HOSTS+=(codethink15-arm64.debian.net) ;;
-			c16|cs16|ct16)	HOSTS+=(codethink16-arm64.debian.net) ;;
-			armhf|amd64|i386|arm64)	 HOSTS+=($(echo "${ALL_HOSTS[@]}" | sed -e 's# #\n#g' | grep "$i")) ;;
-			*) 	if ping -c 1 "$i" ; then HOSTS+=("$i") ; fi ;;
-		esac
-	done
-	echo -n "Running j.d.n.git updates on ${HOSTS[@]} now"
-else
-	echo "I dont understand what to do, please try again."
-	help
-	exit 1
-fi
-
-BG=""
-
-get_arch_color() {
-	case "$1" in
-		*amd64*)		BG=lightgreen ;;
-		*i386*)			BG=lightblue ;;
-		*arm64*)		BG=orange ;;
-		*armhf*)		BG=lightyellow ;;
-		*jenkins.debian.*)	BG=yellow ;;
-		*)				BG=white ;;
-	esac
-}
-
-LOG="$(mktemp -u)"
-# reverse the array
-STSOH=()
-for i in "${HOSTS[@]}" ; do
-	STSOH=("$i" "${STSOH[@]}")
-done
-HOSTS=("${STSOH[@]}")
-
-for i in "${HOSTS[@]}" ; do
-	if [ -z "$real_command" ]; then
-		if node_in_the_future "$i"; then
-			GITOPTS="-c http.sslVerify=false"
-		fi
-		# real command, for running manually: cd ~jenkins-adm/jenkins.debian.net/ ; sudo -H -u jenkins-adm git pull ; ./update_jdn.sh
-		read -r -d '' remote_command <<-EOF
-			set -e
-			set -o pipefail
-			export LANG=C
-			cd ~jenkins-adm
-			if [ ! -d jenkins.debian.net ]; then
-				[ -x /usr/bin/git ] || sudo apt-get install -y git
-				sudo -H -u jenkins-adm git ${GITOPTS:-} clone $GIT_REPO
-				cd jenkins.debian.net
-			else
-				cd jenkins.debian.net
-				sudo -H -u jenkins-adm git config pull.ff only
-				sudo -H -u jenkins-adm git ${GITOPTS:-} pull $GIT_REPO
-			fi
-			./update_jdn.sh 2>&1 | sudo tee -a /var/log/jenkins/update_jdn.log
-		EOF
-	else
-		remote_command="$real_command"
-	fi
-
-	echo -n "."
-	if $reset_clock ; then
-		if node_in_the_future "$i" ; then
-			#  set correct future date
-			case "$i" in
-				osuosl*)	NTP_SERVER=time.osuosl.org ;;
-				*)		NTP_SERVER=de.pool.ntp.org ;;
-			esac
-			remote_command="sudo ntpdate -b $NTP_SERVER && $remote_command && sudo date --set=\"+398 days +6 hours + 23 minutes\" && echo '__$(echo $i|cut -d '.' -f1)=ok__'"
-		fi
-	fi
-	get_arch_color $i
-	# djm logging
-	SHORTNODE=$(echo $i | sed 's#root@##g'|cut -d '.' -f1 )
-	echo "$LOGDATE, $SHORTNODE, $ACTION, regular maintenance" >> $LOGFILE
-	xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -o 'BatchMode = yes' -t $i '$remote_command' | tee $LOG.$i ; sleep 2 ; touch $LOG.$i.done" &
-	unset GITOPTS
-done
-sleep 3
-COUNTER=0
-for i in "${HOSTS[@]}" ; do
-	while ! test -f $LOG.$i.done ; do
-		let COUNTER+=1
-		sleep 1
-		echo -n "."
-		if [ $COUNTER -eq 42 ] ; then
-			echo 
-			echo -n "$LOG.$i.done still doesnt exist, how strange…"
-			COUNTER=0
-			continue
-		fi
-	done
-done
-echo
-
-JENKINS_OFFLINE_LIST="$(dirname $0)/jenkins-home/offline_nodes"
-echo
-echo JENKINS_OFFLINE_LIST=$JENKINS_OFFLINE_LIST
-echo
-OFFLINE=""
-PROBLEMS=""
-for i in "${HOSTS[@]}" ; do
-	HNAME1=$(echo $i | cut -d "@" -f2 | cut -d "." -f1|cut -d "-" -f1)	# IONOS nodes (h01ger)
-	HNAME2=$(echo $i | cut -d "@" -f2 | cut -d "." -f1)			# non -armhf ones (vagrant)
-	echo "Checking for problems on $i $HNAME1 $HNAME2..."
-	TAIL=$(tail -1 $LOG.$i 2>/dev/null)
-	if [ "$i" = "root at jenkins.debian.net" ] ; then
-		if ! ( [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] ) ; then
-			echo "Problems on $i:"
-		fi
-		cat $LOG.$i
-		rm $LOG.$i $LOG.$i.done > /dev/null
-	elif [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] || [ ! -f $LOG.$i ] ; then
-		rm $LOG.$i $LOG.$i.done > /dev/null
-	else
-		if [ -f "$JENKINS_OFFLINE_LIST" ]; then
-			if grep -q "$i" "$JENKINS_OFFLINE_LIST"; then
-				# node was known to be offline
-				if [ -z "$OFFLINE" ] ; then
-					OFFLINE=" $i ($LOG.$i)"
-				else
-					OFFLINE=" $i ($LOG.$i)\n$OFFLINE"
-				fi
-			else
-				# unexpected problem
-				if [ -z "$PROBLEMS" ] ; then
-					PROBLEMS=" $i"
-				else
-					PROBLEMS=" $i\n$PROBLEMS"
-				fi
-				get_arch_color $i
-				xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "less +G $LOG.$i ; rm $LOG.$i $LOG.$i.done" &
-			fi
-		fi
-	fi
-done
-echo
-echo
-
-echo "$(echo "${HOSTS[@]}" | sed -s "s# #\n#g" | wc -l) hosts updated."
-echo
-if [ -n "$PROBLEMS" ] ; then
-	echo "Problems on:"
-	echo -e "$PROBLEMS"
-	echo
-fi
-if [ -n "$OFFLINE" ] ; then
-	echo "Offline nodes with unsurprising problems encountered (not showing the failed deploy log):"
-	echo -e "$OFFLINE"
-	echo
-fi
-show_fixmes
-END=$(date +'%s')
-DURATION=$(( $END - $START ))
-HOUR=$(echo "$DURATION/3600"|bc)
-MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
-SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
-echo "$(date) - total duration: ${HOUR}h ${MIN}m ${SEC}s."
-echo
-



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/f44dd74b98bf4b700bdf80da8eea792773485381...a96f60e3799daf16bd6109ba7ace46a68a7efbc8

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/f44dd74b98bf4b700bdf80da8eea792773485381...a96f60e3799daf16bd6109ba7ace46a68a7efbc8
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230516/eaa845fc/attachment-0001.htm>


More information about the Qa-jenkins-scm mailing list