[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 5 commits: djm: add --no-future option
Holger Levsen (@holger)
gitlab at salsa.debian.org
Tue May 16 23:39:51 BST 2023
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
8e4f9bc4 by Holger Levsen at 2023-05-16T23:33:07+02:00
djm: add --no-future option
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
47a32d54 by Holger Levsen at 2023-05-16T23:53:37+02:00
djm: notice failure, even if moving to the future. (also run autoremove in the presence)
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
bee675e1 by Holger Levsen at 2023-05-17T00:01:28+02:00
djm: refactoring, introduce run_command_futureproof_on_host_in_xterm()
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
2254f1ae by Holger Levsen at 2023-05-17T00:03:55+02:00
djm: new reason: debug host problems
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
a96f60e3 by Holger Levsen at 2023-05-17T00:38:40+02:00
djm: new action: command - thus finally being able to drop deploy_jdn after 7y of service
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
3 changed files:
- TODO
- bin/djm
- − deploy_jdn
Changes:
=====================================
TODO
=====================================
@@ -32,10 +32,26 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
** maybe: rm /tmp/mmdebstrap.* older than 3 days
* split TODO in TODO and TODO.legacy?
* djm:
-** ACTION: c|command - run arbitrary command (with and without reset_clock)
-** then get rid of deploy_jdn
+** option: --today to be used with action shell
+** options: --press-enter --dont-wait-for-enter, default being wait for enter.
+** update documentation refering to deploy_jdn to djm
+*** ups, there's no doc on deploy_jdn anyway
+** show FIXMEs?
+show_fixmes() {
+ local BASEDIR="$(dirname "$(readlink -e $0)")"
+ # FIXME: if ! [ -f bin/djm...
+ local TMPFILE=$(mktemp)
+ rgrep FI[X]ME $BASEDIR/* | grep -v $BASEDIR/TODO | grep -v echo > $TMPFILE || true
+ if [ -s $TMPFILE ] ; then
+ echo
+ echo Existing FIXME statements:
+ echo
+ cat $TMPFILE | sed "s#$PWD#...#g"
+ echo
+ fi
+ rm -f $TMPFILE
+}
** also log an id, so that one command effecting several hosts is counted as on action
-** new option, to optionally not fix the time (on future hosts) after command, to see failing commands?
*** make actions triggering 'all' be only one entry in the djm logfile -> more sensible stats
** automatically backup+cleanup logs on the first of the month
** new feature: --show-month 04
=====================================
bin/djm
=====================================
@@ -17,6 +17,7 @@ set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.h
DRY_RUN=false
VERBOSE=false
FETCH=false
+NO_FUTURE=false
LOGFILE=~/.djm.log
FAIL_REASON=""
CONFIRM=false
@@ -24,6 +25,7 @@ BG=""
TARGET=
ACTION=
REASON=
+COMMAND=
LOCAL_LOGFILE=~/.djm.log
UI_LOGFILE=~/.djm-jenkins-ui.log
PARSER_LOGFILE=~/.djm-jenkins-parser.log
@@ -62,7 +64,7 @@ node_in_the_future () {
#
# thanks to /usr/share/doc/util-linux/examples/getopt-parse.bash
-TEMP=$(getopt -o 'dfhrv' --long 'dry-run,fetch,help,report,verbose' -n 'djm' -- "$@")
+TEMP=$(getopt -o 'dfhnrv' --long 'dry-run,fetch,help,no-future,report,verbose' -n 'djm' -- "$@")
if [ $? -ne 0 ]; then
echo 'Terminating...' >&2
exit 1
@@ -87,6 +89,11 @@ while true; do
shift
continue
;;
+ '-n'|'--no-future')
+ NO_FUTURE=true
+ shift
+ continue
+ ;;
'-r'|'--report')
MODE=report
shift
@@ -122,15 +129,19 @@ show_help(){
seperator "="
echo
echo Either of these are possible:
+ # FIXME: explain this better
echo "djm -d/--dry-run -f/--fetch -r/--report -v/--verbose -h/--help"
- echo "djm -d/--dry-run \$TARGET \$ACTION \$REASON"
+ echo "djm -d/--dry-run -n/--no-future \$TARGET \$ACTION \$REASON"
+ # --no-future is useful, because setting the time to the future
+ # will succeed and thus hide earlier issues.
echo
echo "TARGET= 'all' or grepable (jenkins, amd64, ionos, osuosl3) from ./nodes/list_nodes"
echo " or special target '.' for jenkins-ui ACTION."
echo
echo "ACTION= one of these:"
echo " _a_utoremove apt autoremove && apt clean"
- echo " _b_ring-back mark host as online"
+ echo " _b_ring-back mark hosts as online"
+ echo " _c_ommand run \$4 on hosts"
echo " _d_eploy-git deploy jenkins.debian.net.git"
echo " _f_fix-future fix clock on hosts running in the future"
echo " _j_enkins-ui for when doing things manually in the jenkins-ui"
@@ -143,6 +154,7 @@ show_help(){
echo "REASON= one of these:"
echo " cs=clock skew"
echo " dj=debug job problems"
+ echo " dh=debug host problems"
echo " hb=host is back"
echo " ho=host is overloaded"
echo " ku=kernel upgrade"
@@ -159,6 +171,7 @@ show_help(){
echo "\$1 = $1"
echo "\$2 = $2"
echo "\$3 = $3"
+ # FIXME: $4 is not explained here
echo
echo parsing problems:
echo
@@ -188,6 +201,7 @@ verify_target_action_reason(){
case $ACTION in
a|autoremove) ACTION=autoremove ;;
b|bring-back) ACTION=bring-back ;;
+ c|command) ACTION=command ; CONFIRM=true ;;
d|deploy-git) ACTION=deploy-git ;;
f|fix-future) ACTION=fix-future ;;
j|jenkins-ui) ACTION=jenkins-ui ;;
@@ -201,6 +215,7 @@ verify_target_action_reason(){
case $REASON in
cs) REASON="clock skew" ;;
dj) REASON="debug job problems" ;;
+ dh) REASON="debug host problems" ;;
hb) REASON="host is back" ;;
ho) REASON="host is overloaded" ;;
ku) REASON="kernel upgrade" ;;
@@ -391,6 +406,17 @@ EOF
xterm -T "$SHORTNODE / $ACTION ($FILE)" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $FILE" &
}
+run_command_futureproof_on_host_in_xterm() {
+ COMMAND="( $COMMAND ) || ( echo press enter ; read a )"
+ if node_in_the_future "$SHORTNODE" ; then
+ COMMAND="sudo ntpdate -b $NTP_SERVER ; $COMMAND"
+ if ! $NO_FUTURE ; then
+ COMMAND="$COMMAND ; sudo date --set='+398 days +6 hours + 23 minutes'"
+ fi
+ fi
+ xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -t $NODE \"$COMMAND\"" &
+}
+
djm_do() {
#
# make sure we have target, action & reason
@@ -428,6 +454,9 @@ djm_do() {
SHORTNODE="jenkins web UI"
fi
echo "$(date -u '+%Y-%m-%d %H:%M UTC'), $SHORTNODE, $ACTION, $REASON"
+ if [ -n "$4" ] && [ "$ACTION" = "command" ] ; then
+ echo " - command to be run is \"$4\""
+ fi
done
fi
@@ -490,8 +519,6 @@ djm_do() {
esac
run_xterm2wait4node_comeback $NODE
;;
- autoremove) xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE 'sudo apt-get autoremove || ( echo press enter ; read a ) sudo apt-get clean || ( echo press enter ; read a )'" &
- ;;
shell) xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE" &
;;
jenkins-ui) echo "Documenting one manual action in the jenkins-ui."
@@ -501,8 +528,12 @@ djm_do() {
echo "Trying to mark $SHORTNODE as online. This is the diff for $OFF_NODES on jenkins.debian.net:"
ssh jenkins.debian.net "grep -v $SHORTNODE $OFF_NODES > $TMPFILE ; diff $OFF_NODES $TMPFILE ; mv $TMPFILE $OFF_NODES ; chown jenkins:jenkins $OFF_NODES"
;;
- fix-future) COMMAND="date ; sudo ntpdate -b $NTP_SERVER"
+ fix-future) if $NO_FUTURE ; then
+ echo "djm was called with --no-future and to fix hosts running in the future - this makes no sense, exiting."
+ exit 1
+ fi
if node_in_the_future "$SHORTNODE" ; then
+ COMMAND="date ; sudo ntpdate -b $NTP_SERVER"
COMMAND="$COMMAND && sudo date --set='+398 days +6 hours + 23 minutes'"
COMMAND="$COMMAND ; date -u ; echo press enter ; read a"
xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE \"$COMMAND\"" &
@@ -511,13 +542,12 @@ djm_do() {
continue
fi
+ ;;
+ autoremove) COMMAND="sudo apt-get autoremove"
+ run_command_futureproof_on_host_in_xterm
;;
apt-upgrade) COMMAND="sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get clean"
- if node_in_the_future "$SHORTNODE" ; then
- COMMAND="sudo ntpdate -b $NTP_SERVER ; $COMMAND ; sudo date --set='+398 days +6 hours + 23 minutes'"
- fi
- COMMAND="( $COMMAND ) || ( echo press enter ; read a ) ; echo 'sleeping 10s now' ; sleep 10"
- xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -t $NODE \"$COMMAND\"" &
+ run_command_futureproof_on_host_in_xterm
;;
deploy-git) GIT_REPO="https://salsa.debian.org/qa/jenkins.debian.net.git"
unset GITOPTS
@@ -541,16 +571,15 @@ djm_do() {
fi
./update_jdn.sh 2>&1 | sudo tee -a /var/log/jenkins/update_jdn.log
EOF
- echo -n "$COMMAND"
- if node_in_the_future "$SHORTNODE" ; then
- COMMAND="sudo ntpdate -b $NTP_SERVER ; $COMMAND ; sudo date --set='+398 days +6 hours + 23 minutes'"
- fi
- COMMAND="( $COMMAND ) || ( echo press enter ; read a ) ; echo 'sleeping 10s now' ; sleep 10"
- xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -t $NODE \"$COMMAND\"" &
+ run_command_futureproof_on_host_in_xterm
;;
rmstamp) COMMAND="sudo rm -f /var/log/jenkins/*stamp"
xterm -T "$SHORTNODE / $ACTION" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -t $NODE \"$COMMAND\"" &
;;
+ command) COMMAND="$4"
+ COMMAND="$COMMAND ; ( echo press enter ; read a )"
+ run_command_futureproof_on_host_in_xterm
+ ;;
*) echo unsupported action, sorry, please improve.
exit 1
;;
@@ -567,7 +596,7 @@ djm_do() {
djm_init() {
DUMMY=$(mktemp)
- touch -d "$(date -u -d "18 hours ago" '+%Y-%m-%d %H:%M') UTC" $DUMMY
+ touch -d "$(date -u -d "12 hours ago" '+%Y-%m-%d %H:%M') UTC" $DUMMY
if $FETCH || [ ! -f $JOBS ] || [ $DUMMY -nt $JOBS ] ; then
if ! $DRY_RUN ; then
echo "Fetching remote logs."
@@ -583,7 +612,7 @@ djm_init() {
# main
#
case $MODE in
- report) djm_init ; djm_report ;;
- djm) djm_init ; djm_do $1 $2 $3 ;;
- help) show_help ;;
+ report) djm_init ; djm_report ;;
+ djm) djm_init ; djm_do "$1" "$2" "$3" "$4" ;;
+ help) show_help ;;
esac
=====================================
deploy_jdn deleted
=====================================
@@ -1,312 +0,0 @@
-#!/bin/bash
-# vim: set noexpandtab:
-#
-# Copyright 2014-2023 Holger Levsen <holger at layer-acht.org>
-# © 2018 Mattia Rizzolo <mattia at debian.org>
-#
-# released under the GPLv2
-#
-# deployment helper for jenkins.debian.net and build nodes
-# (misses freebsd node)
-
-# disclaimers:
-# this script grew over the years…
-# the code is horrible and was always meant to be a quick local hack and never to be published
-# use at your own risk. it might eat your cats. you have been warned.
-# the HOSTS lists below is yet another code duplication…
-
-help() {
-cat <<- EOF
-accepted params:
- ./deploy_jdn all - deploy on all nodes (and handle 398 days hosts properly)
- ./deploy_jdn \$host - deploy on \$host and jenkins (and handle 398 days hosts properly)
- ./deploy_jdn all \$foo - run "\$foo" on all nodes (and handle 398 days hosts properly)
- ./deploy_jdn - deploy on jenkins only
- ./deploy_jdn jenkins - deploy on jenkins only
- ./deploy_jdn jenkins ionos10 - deploy on jenkins and ionos10
- ./deploy_jdn jenkins o3 - deploy on jenkins and osuosl3
- ./deploy_jdn jenkins c9 - deploy on jenkins and codethink9
- ./deploy_jdn jenkins 10 - deploy on jenkins and ionos10
- ./deploy_jdn jenkins 5 6 - deploy on jenkins and ionos5 and ionos6
- ./deploy_jdn jenkins amd64 - deploy on jenkins and all amd64 nodes
- ./deploy_jdn only ionos10 - deploy on ionos10
- ./deploy_jdn only o3 - deploy on osuosl3
- ./deploy_jdn only c9 - deploy on codethink9
- ./deploy_jdn only i10 - deploy on ionos10
- ./deploy_jdn only 5 6 - deploy on ionos5 and ionos6
- ./deploy_jdn only amd64 - deploy on all amd64 only
- ./deploy_jdn upgrade|u - run "apt-get update && upgrade && clean" everywhere
- ./deploy_jdn upgradey|uy - run "apt-get upgrade -y" everywhere
- ./deploy_jdn rmstamp|rm - delete stamp files everywhere
-
-Rebooting all nodes, except jenkins, is easily done like this:
- $ parallel -j 8 -i sh -c 'ssh {} sudo reboot' -- \$(./nodes/list_nodes | grep -v jenkins)
-
-EOF
-}
-
-
-LOGFILE=~/.djm.log
-LOGDATE="$(date -u '+%Y-%m-%d %H:%M UTC')"
-
-START=$(date +'%s')
-GIT_REPO="https://salsa.debian.org/qa/jenkins.debian.net.git"
-mapfile -t HOSTS < <(nodes/list_nodes | grep -v jenkins.debian.net)
-HOSTS+=(root at jenkins.debian.net)
-
-ALL_HOSTS=("${HOSTS[@]}")
-
-node_in_the_future () {
- case "$1" in
- ionos5-amd64*|ionos6-i386*|ionos15-amd64*|ionos16-i386*) true ;;
- codethink9*|codethink11*|codethink13*|codethink15*) true ;;
- osuosl2-amd64*) true ;;
- *) false ;;
- esac
-}
-
-show_fixmes() {
- #
- # There's always some work left...
- # echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
- #
- BASEDIR="$(dirname "$(readlink -e $0)")"
- TMPFILE=$(mktemp)
- rgrep FI[X]ME $BASEDIR/* | grep -v $BASEDIR/TODO | grep -v echo > $TMPFILE || true
- if [ -s $TMPFILE ] ; then
- echo
- echo Existing FIXME statements:
- echo
- cat $TMPFILE | sed "s#$PWD#...#g"
- echo
- fi
- rm -f $TMPFILE
-}
-
-echo
-echo -n "$(date) - "
-reset_clock=true
-if [ "$1" = "fixme" ] ; then
- show_fixmes
- exit 0
-elif [ "$1" = "all" ] ; then
- ACTION="deploy jdn.git"
- echo -n "Running j.d.n.git updates on ${HOSTS[@]} now"
- # reset_clock can be false as update_jdn.sh sets the time
- reset_clock=false
- shift
- if [ -n "$1" ] ; then
- real_command="$@"
- echo -n "Running '$real_command' on ${HOSTS[@]} now."
- real_command="$@ && echo '__reallyreally=ok__'"
- fi
-elif [ "$1" = "upgrade" ] || [ "$1" = "u" ] ; then
- ACTION="apt upgrade"
- real_command="export LANG=C && sudo apt-get update && sudo apt-get upgrade && sudo apt-get dist-upgrade && sudo apt-get clean"
- shift
-elif [ "$1" = "upgradey" ] || [ "$1" = "uy" ] ; then
- ACTION="apt upgrade"
- real_command="export LANG=C && sudo apt-get update && sudo apt-get -y upgrade && sudo apt-get -y dist-upgrade && sudo apt-get clean"
- shift
-elif [ "$1" = "rmstamp" ] || [ "$1" = "rm" ] ; then
- ACTION="rm stamp"
- real_command="sudo rm -f /var/log/jenkins/*stamp && echo '__reallyreally=ok__'"
- reset_clock=false
- shift
-elif [ "$1" = "" ] ; then
- ACTION="deploy jdn.git"
- HOSTS=(root at jenkins.debian.net)
- echo -n "Running j.d.n.git updates on ${HOSTS[@]} now"
-elif [ "$1" = "jenkins" ] || [ "$1" = "only" ] ; then
- ACTION="deploy jdn.git"
- if [ "$1" = "jenkins" ] ; then
- HOSTS=(root at jenkins.debian.net)
- else
- HOSTS=()
- fi
- shift
- for i in "$@" ; do
- case "$i" in
- 1|i1|ionos1) HOSTS+=(ionos1-amd64.debian.net) ;;
- 2|i2|ionos2) HOSTS+=(ionos2-i386.debian.net) ;;
- 3|i3|ionos3) HOSTS+=(ionos3-i386.debian.net) ;;
- 5|i5|ionos5) HOSTS+=(ionos5-amd64.debian.net) ;;
- 6|i6|ionos6) HOSTS+=(ionos6-i386.debian.net) ;;
- 7|i7|ionos7) HOSTS+=(ionos7-amd64.debian.net) ;;
- 9|i9|ionos9) HOSTS+=(ionos9-amd64.debian.net) ;;
- 10|i10|ionos10) HOSTS+=(ionos10-amd64.debian.net) ;;
- 11|i11|ionos11) HOSTS+=(ionos11-amd64.debian.net) ;;
- 12|i12|ionos12) HOSTS+=(ionos12-i386.debian.net) ;;
- 15|i15|ionos15) HOSTS+=(ionos15-amd64.debian.net) ;;
- 16|i16|ionos16) HOSTS+=(ionos16-i386.debian.net) ;;
- o1) HOSTS+=(osuosl1-amd64.debian.net) ;;
- o2) HOSTS+=(osuosl2-amd64.debian.net) ;;
- o3) HOSTS+=(osuosl3-amd64.debian.net) ;;
- o4) HOSTS+=(osuosl4-amd64.debian.net) ;;
- o5) HOSTS+=(osuosl5-amd64.debian.net) ;;
- c9|cs9|ct9) HOSTS+=(codethink9-arm64.debian.net) ;;
- c10|cs10|ct10) HOSTS+=(codethink10-arm64.debian.net) ;;
- c11|cs11|ct11) HOSTS+=(codethink11-arm64.debian.net) ;;
- c12|cs12|ct12) HOSTS+=(codethink12-arm64.debian.net) ;;
- c13|cs13|ct13) HOSTS+=(codethink13-arm64.debian.net) ;;
- c14|cs14|ct14) HOSTS+=(codethink14-arm64.debian.net) ;;
- c15|cs15|ct15) HOSTS+=(codethink15-arm64.debian.net) ;;
- c16|cs16|ct16) HOSTS+=(codethink16-arm64.debian.net) ;;
- armhf|amd64|i386|arm64) HOSTS+=($(echo "${ALL_HOSTS[@]}" | sed -e 's# #\n#g' | grep "$i")) ;;
- *) if ping -c 1 "$i" ; then HOSTS+=("$i") ; fi ;;
- esac
- done
- echo -n "Running j.d.n.git updates on ${HOSTS[@]} now"
-else
- echo "I dont understand what to do, please try again."
- help
- exit 1
-fi
-
-BG=""
-
-get_arch_color() {
- case "$1" in
- *amd64*) BG=lightgreen ;;
- *i386*) BG=lightblue ;;
- *arm64*) BG=orange ;;
- *armhf*) BG=lightyellow ;;
- *jenkins.debian.*) BG=yellow ;;
- *) BG=white ;;
- esac
-}
-
-LOG="$(mktemp -u)"
-# reverse the array
-STSOH=()
-for i in "${HOSTS[@]}" ; do
- STSOH=("$i" "${STSOH[@]}")
-done
-HOSTS=("${STSOH[@]}")
-
-for i in "${HOSTS[@]}" ; do
- if [ -z "$real_command" ]; then
- if node_in_the_future "$i"; then
- GITOPTS="-c http.sslVerify=false"
- fi
- # real command, for running manually: cd ~jenkins-adm/jenkins.debian.net/ ; sudo -H -u jenkins-adm git pull ; ./update_jdn.sh
- read -r -d '' remote_command <<-EOF
- set -e
- set -o pipefail
- export LANG=C
- cd ~jenkins-adm
- if [ ! -d jenkins.debian.net ]; then
- [ -x /usr/bin/git ] || sudo apt-get install -y git
- sudo -H -u jenkins-adm git ${GITOPTS:-} clone $GIT_REPO
- cd jenkins.debian.net
- else
- cd jenkins.debian.net
- sudo -H -u jenkins-adm git config pull.ff only
- sudo -H -u jenkins-adm git ${GITOPTS:-} pull $GIT_REPO
- fi
- ./update_jdn.sh 2>&1 | sudo tee -a /var/log/jenkins/update_jdn.log
- EOF
- else
- remote_command="$real_command"
- fi
-
- echo -n "."
- if $reset_clock ; then
- if node_in_the_future "$i" ; then
- # set correct future date
- case "$i" in
- osuosl*) NTP_SERVER=time.osuosl.org ;;
- *) NTP_SERVER=de.pool.ntp.org ;;
- esac
- remote_command="sudo ntpdate -b $NTP_SERVER && $remote_command && sudo date --set=\"+398 days +6 hours + 23 minutes\" && echo '__$(echo $i|cut -d '.' -f1)=ok__'"
- fi
- fi
- get_arch_color $i
- # djm logging
- SHORTNODE=$(echo $i | sed 's#root@##g'|cut -d '.' -f1 )
- echo "$LOGDATE, $SHORTNODE, $ACTION, regular maintenance" >> $LOGFILE
- xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -o 'BatchMode = yes' -t $i '$remote_command' | tee $LOG.$i ; sleep 2 ; touch $LOG.$i.done" &
- unset GITOPTS
-done
-sleep 3
-COUNTER=0
-for i in "${HOSTS[@]}" ; do
- while ! test -f $LOG.$i.done ; do
- let COUNTER+=1
- sleep 1
- echo -n "."
- if [ $COUNTER -eq 42 ] ; then
- echo
- echo -n "$LOG.$i.done still doesnt exist, how strange…"
- COUNTER=0
- continue
- fi
- done
-done
-echo
-
-JENKINS_OFFLINE_LIST="$(dirname $0)/jenkins-home/offline_nodes"
-echo
-echo JENKINS_OFFLINE_LIST=$JENKINS_OFFLINE_LIST
-echo
-OFFLINE=""
-PROBLEMS=""
-for i in "${HOSTS[@]}" ; do
- HNAME1=$(echo $i | cut -d "@" -f2 | cut -d "." -f1|cut -d "-" -f1) # IONOS nodes (h01ger)
- HNAME2=$(echo $i | cut -d "@" -f2 | cut -d "." -f1) # non -armhf ones (vagrant)
- echo "Checking for problems on $i $HNAME1 $HNAME2..."
- TAIL=$(tail -1 $LOG.$i 2>/dev/null)
- if [ "$i" = "root at jenkins.debian.net" ] ; then
- if ! ( [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] ) ; then
- echo "Problems on $i:"
- fi
- cat $LOG.$i
- rm $LOG.$i $LOG.$i.done > /dev/null
- elif [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] || [ ! -f $LOG.$i ] ; then
- rm $LOG.$i $LOG.$i.done > /dev/null
- else
- if [ -f "$JENKINS_OFFLINE_LIST" ]; then
- if grep -q "$i" "$JENKINS_OFFLINE_LIST"; then
- # node was known to be offline
- if [ -z "$OFFLINE" ] ; then
- OFFLINE=" $i ($LOG.$i)"
- else
- OFFLINE=" $i ($LOG.$i)\n$OFFLINE"
- fi
- else
- # unexpected problem
- if [ -z "$PROBLEMS" ] ; then
- PROBLEMS=" $i"
- else
- PROBLEMS=" $i\n$PROBLEMS"
- fi
- get_arch_color $i
- xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "less +G $LOG.$i ; rm $LOG.$i $LOG.$i.done" &
- fi
- fi
- fi
-done
-echo
-echo
-
-echo "$(echo "${HOSTS[@]}" | sed -s "s# #\n#g" | wc -l) hosts updated."
-echo
-if [ -n "$PROBLEMS" ] ; then
- echo "Problems on:"
- echo -e "$PROBLEMS"
- echo
-fi
-if [ -n "$OFFLINE" ] ; then
- echo "Offline nodes with unsurprising problems encountered (not showing the failed deploy log):"
- echo -e "$OFFLINE"
- echo
-fi
-show_fixmes
-END=$(date +'%s')
-DURATION=$(( $END - $START ))
-HOUR=$(echo "$DURATION/3600"|bc)
-MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
-SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
-echo "$(date) - total duration: ${HOUR}h ${MIN}m ${SEC}s."
-echo
-
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/f44dd74b98bf4b700bdf80da8eea792773485381...a96f60e3799daf16bd6109ba7ace46a68a7efbc8
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/f44dd74b98bf4b700bdf80da8eea792773485381...a96f60e3799daf16bd6109ba7ace46a68a7efbc8
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230516/eaa845fc/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list