[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 4 commits: djm: refactoring
Holger Levsen (@holger)
gitlab at salsa.debian.org
Sun May 7 21:26:56 BST 2023
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
6208d4f4 by Holger Levsen at 2023-05-07T21:37:57+02:00
djm: refactoring
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
e8436074 by Holger Levsen at 2023-05-07T21:43:12+02:00
djm: refactoring
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
6e1b92aa by Holger Levsen at 2023-05-07T21:59:42+02:00
djm: refactoring
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
9491bcba by Holger Levsen at 2023-05-07T22:26:34+02:00
djm: refactoring, also start xterm to wait for comeback of powercycled nodes
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/djm
Changes:
=====================================
bin/djm
=====================================
@@ -12,12 +12,17 @@ set -e
set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html
#
-# define environment
+# define environment and global variables
#
+MODE=djm
DRY_MODE=false
LOGFILE=~/.djm.log
FAIL_REASON=""
CONFIRM=false
+BG=""
+TARGET=
+ACTION=
+REASON=
#
# parse parameters
@@ -52,64 +57,100 @@ while true; do
done
#
-# make sure we have target, action & reason
+# some helper functions
#
-TARGET=$1
-ACTION=$2
-REASON=$3
-
-case $TARGET in
- .) TARGET="."
- ;;
- all) TARGET="$(./nodes/list_nodes )"
- ;;
- *) TARGET="$(./nodes/list_nodes | egrep $TARGET || true)"
- ;;
-esac
+get_arch_color() {
+ case "$1" in
+ *amd64*) BG=lightgreen ;;
+ *i386*) BG=lightblue ;;
+ *arm64*) BG=orange ;;
+ *armhf*) BG=lightyellow ;;
+ *jenkins.debian.*) BG=yellow ;;
+ *) BG=white ;;
+ esac
+}
-if [ -z "$TARGET" ] ; then
- FAIL_REASON="${FAIL_REASON}Undefined target.\n"
-fi
+run_xterm2wait4node_comeback() {
+ #
+ # prepare tmpfile to be run in xterm
+ #
+ local FILE=$(mktemp --tmpdir=/tmp djm-XXXXXXXX)
+ local SLEEP=77
+ cat > $FILE << EOF
+#!/bin/bash
+echo -n "Initial ${SLEEP}s sleep..."
+for i in $(seq 1 $SLEEP | xargs echo) ; do
+ echo -n .
+ sleep 1
+done
+echo " now trying ssh to $NODE"
+while ! ssh $NODE 'uptime ; echo press enter, $SHORTNODE is back after $ACTION ; read a || true' ; do
+ for i in $(seq 1 30 | xargs echo) ; do
+ echo -n "."
+ sleep 1
+ done
+ echo "...and trying again."
+done
+rm $FILE
+EOF
+ #
+ # actually run ssh (via tmpfile) in xterm
+ #
+ echo xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $FILE"
+ xterm -T "$SHORTNODE / $ACTION ($FILE)" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $FILE" &
+}
-case $ACTION in
- r|reboot) ACTION=reboot ; CONFIRM=true ;;
- p|powercycle) ACTION=powercycle ; CONFIRM=true ;;
- a|autoremove) ACTION=autoremove ;;
- s|shell) ACTION=shell ;;
- j|jenkins-ui) ACTION=jenkins-ui ;;
- b|bring-back) ACTION=bring-back ;;
- *) FAIL_REASON="${FAIL_REASON}Undefined action.\n"
+verify_target_action_reason(){
+ #
+ # verify target, action & reason
+ #
+ TARGET=$1
+ ACTION=$2
+ REASON=$3
+ case $TARGET in
+ .) TARGET="."
;;
-esac
-
-case $REASON in
- ku) REASON="kernel upgrade" ;;
- np) REASON="no ping" ;;
- dj) REASON="debug job problems" ;;
- rm) REASON="regular maintenance" ;;
- cs) REASON="clock skew" ;;
- hb) REASON="host is back" ;;
- ho) REASON="host is overloaded" ;;
- nt) REASON="new things" ;;
- *) FAIL_REASON="${FAIL_REASON}Undefined reason.\n"
+ all) TARGET="$(./nodes/list_nodes )"
;;
-esac
-
-#
-# some targets only allow specific actions
-#
-
-if [ "$TARGET" = "." ] && [ "$ACTION" != "jenkins-ui" ] ; then
- FAIL_REASON="${FAIL_REASON}Target '.' can only be used with action 'jenkins-ui'.\n"
-elif [ "$TARGET" != "." ] && [ "$ACTION" = "jenkins-ui" ] ; then
- FAIL_REASON="${FAIL_REASON}Action 'jenkins-ui' can only be used with target '.'.\n"
-fi
-
-#
-# display help text in case of problems (or no params given)
-#
+ *) TARGET="$(./nodes/list_nodes | egrep $TARGET || true)"
+ ;;
+ esac
+ if [ -z "$TARGET" ] ; then
+ FAIL_REASON="${FAIL_REASON}Undefined target.\n"
+ fi
+ case $ACTION in
+ r|reboot) ACTION=reboot ; CONFIRM=true ;;
+ p|powercycle) ACTION=powercycle ; CONFIRM=true ;;
+ a|autoremove) ACTION=autoremove ;;
+ s|shell) ACTION=shell ;;
+ j|jenkins-ui) ACTION=jenkins-ui ;;
+ b|bring-back) ACTION=bring-back ;;
+ *) FAIL_REASON="${FAIL_REASON}Undefined action.\n"
+ ;;
+ esac
+ case $REASON in
+ ku) REASON="kernel upgrade" ;;
+ np) REASON="no ping" ;;
+ dj) REASON="debug job problems" ;;
+ rm) REASON="regular maintenance" ;;
+ cs) REASON="clock skew" ;;
+ hb) REASON="host is back" ;;
+ ho) REASON="host is overloaded" ;;
+ nt) REASON="new things" ;;
+ *) FAIL_REASON="${FAIL_REASON}Undefined reason.\n"
+ ;;
+ esac
+ #
+ # some targets only allow specific actions
+ #
+ if [ "$TARGET" = "." ] && [ "$ACTION" != "jenkins-ui" ] ; then
+ FAIL_REASON="${FAIL_REASON}Target '.' can only be used with action 'jenkins-ui'.\n"
+ elif [ "$TARGET" != "." ] && [ "$ACTION" = "jenkins-ui" ] ; then
+ FAIL_REASON="${FAIL_REASON}Action 'jenkins-ui' can only be used with target '.'.\n"
+ fi
+}
-if [ -n "$FAIL_REASON" ] ; then
+show_help(){
echo
echo problem parsing parameters, djm by default needs three:
echo "djm -d/--dry-run \$TARGET \$ACTION \$REASON"
@@ -129,28 +170,67 @@ if [ -n "$FAIL_REASON" ] ; then
echo
echo parsing problems:
echo
- echo -e "$FAIL_REASON"
+ echo -e "$4"
echo
- exit 1
-fi
+}
-#
-# main
-#
+if [ "$MODE" = "djm" ] ; then
+ #
+ # make sure we have target, action & reason
+ #
+ TARGET=$1
+ ACTION=$2
+ REASON=$3
+ verify_target_action_reason $TARGET $ACTION $REASON
+ # display help text in case of problems (or no params given)
+ if [ -n "$FAIL_REASON" ] ; then
+ show_help $1 $2 $3 "$FAIL_REASON"
+ exit 1
-if $DRY_MODE ; then
- echo "###"
- echo "###"
- echo "### running in dry-run mode, not doing anything for real."
- echo "###"
- echo "###"
-elif $CONFIRM ; then
- echo
- echo Please confirm you want to do this:
- echo
-fi
+ fi
-if $DRY_MODE || $CONFIRM ; then
+ #
+ # main
+ #
+ if $DRY_MODE ; then
+ echo "###"
+ echo "###"
+ echo "### running in dry-run mode, not doing anything for real."
+ echo "###"
+ echo "###"
+ elif $CONFIRM ; then
+ echo
+ echo Please confirm you want to do this:
+ echo
+ fi
+
+ if $DRY_MODE || $CONFIRM ; then
+ for NODE in $TARGET ; do
+ if [ "$NODE" != "." ] ; then
+ SHORTNODE=$(echo $NODE | cut -d '.' -f1)
+ else
+ SHORTNODE="jenkins web UI"
+ fi
+ echo "$(date -u '+%Y-%m-%d %H:%M UTC'), $SHORTNODE, $ACTION, $REASON"
+ done
+ fi
+
+ if $DRY_MODE ; then
+ echo
+ echo "# dry-run end."
+ exit 0
+ elif $CONFIRM ; then
+ echo
+ echo -n "confirm: y/N: "
+ read CONFIRM
+ echo
+ case $CONFIRM in
+ y|Y) : ;;
+ *) echo "Ok, aborting" ; exit 0 ;;
+ esac
+ fi
+
+ LOGDATE="$(date -u '+%Y-%m-%d %H:%M UTC')"
for NODE in $TARGET ; do
if [ "$NODE" != "." ] ; then
SHORTNODE=$(echo $NODE | cut -d '.' -f1)
@@ -158,112 +238,59 @@ if $DRY_MODE || $CONFIRM ; then
SHORTNODE="jenkins web UI"
fi
echo "$(date -u '+%Y-%m-%d %H:%M UTC'), $SHORTNODE, $ACTION, $REASON"
- done
-fi
-
-if $DRY_MODE ; then
- echo
- echo "# dry-run end."
- exit 0
-elif $CONFIRM ; then
- echo
- echo -n "confirm: y/N: "
- read CONFIRM
- echo
- case $CONFIRM in
- y|Y) : ;;
- *) echo "Ok, aborting" ; exit 0 ;;
- esac
-fi
-
-BG=""
-get_arch_color() {
- case "$1" in
- *amd64*) BG=lightgreen ;;
- *i386*) BG=lightblue ;;
- *arm64*) BG=orange ;;
- *armhf*) BG=lightyellow ;;
- *jenkins.debian.*) BG=yellow ;;
- *) BG=white ;;
- esac
-}
-
-LOGDATE="$(date -u '+%Y-%m-%d %H:%M UTC')"
-for NODE in $TARGET ; do
- if [ "$NODE" != "." ] ; then
- SHORTNODE=$(echo $NODE | cut -d '.' -f1)
- else
- SHORTNODE="jenkins web UI"
- fi
- echo "$(date -u '+%Y-%m-%d %H:%M UTC'), $SHORTNODE, $ACTION, $REASON"
- get_arch_color $NODE
- #
- # action
- #
- case $ACTION in
- reboot) ( ssh $NODE "sudo reboot || read enter" || true ) & sleep 2
- TMPFILE=$(mktemp --tmpdir=/tmp djm-XXXXXXXX)
- cat > $TMPFILE << EOF
-#!/bin/bash
-for i in $(seq 1 42 | xargs echo) ; do
- echo -n .
- sleep 1
-done
-while ! ssh $NODE 'uptime ; echo press enter ; read a' ; do
- for i in $(seq 1 30 | xargs echo) ; do
- echo -n "."
- sleep 1
- done
-done
-rm $TMPFILE
-EOF
- echo xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $TMPFILE"
- ( sleep 30 ; xterm -T "$SHORTNODE / reboot ($TMPFILE)" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $TMPFILE" ) &
- echo
- ;;
- powercycle) case $SHORTNODE in
- jenkins) ./bin/powercycle_x86_nodes.py jenkins
- ;;
- ionos*) node=$(echo $NODE | cut -b 6- | cut -d '-' -f1)
- ./bin/powercycle_x86_nodes.py $node
- ;;
- virt*-armhf-rb) echo "Sorry, I do not yet know how to handle $NODE, skipping."
- echo "But the steps are documented in README.infrastructure."
- echo "One can powercycle mst0X however...."
- exit 1
- ;;
- *-armhf-rb) ./bin/powercycle_armhf_nodes.sh $NODE
- ;;
- codethink*) ./bin/powercycle_arm64_nodes.sh $(echo ${SHORTNODE:9} | cut -d '-' -f1)
- ;;
- osuosl*) echo "We cannot powercycle nodes at OSUOSL on our own. Please contact #osuosl on libera.chat or send email to support at osuosl.org"
- exit 1
- ;;
- *) echo "This should not happen, please investigate and fixup."
- exit 1
- ;;
- esac
+ get_arch_color $NODE
+ #
+ # action
+ #
+ case $ACTION in
+ reboot) ( ssh $NODE "sudo reboot || read enter" || true ) & sleep 2
+ run_xterm2wait4node_comeback $NODE
;;
- autoremove) xterm -T "$SHORTNODE / autoremove" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE 'sudo apt-get autoremove || read enter; sudo apt-get clean || read enter'" &
- ;;
- shell) xterm -T "$SHORTNODE / shell" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE" &
- ;;
- jenkins-ui) echo "Documenting one manual action in the jenkins-ui."
+ powercycle) case $SHORTNODE in
+ jenkins) ./bin/powercycle_x86_nodes.py jenkins
+ ;;
+ ionos*) node=$(echo $NODE | cut -b 6- | cut -d '-' -f1)
+ ./bin/powercycle_x86_nodes.py $node
+ ;;
+ virt*-armhf-rb) echo "Sorry, I do not yet know how to handle $NODE, skipping."
+ echo "But the steps are documented in README.infrastructure."
+ echo "One can powercycle mst0X however...."
+ exit 1
+ ;;
+ *-armhf-rb) ./bin/powercycle_armhf_nodes.sh $NODE
+ ;;
+ codethink*) ./bin/powercycle_arm64_nodes.sh $(echo ${SHORTNODE:9} | cut -d '-' -f1)
+ ;;
+ osuosl*) echo "We cannot powercycle nodes at OSUOSL on our own. Please contact #osuosl on libera.chat or send email to support at osuosl.org"
+ exit 1
+ ;;
+ *) echo "This should not happen, please investigate and fixup."
+ exit 1
+ ;;
+ esac
+ run_xterm2wait4node_comeback $NODE
+ ;;
+ autoremove) xterm -T "$SHORTNODE / autoremove" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE 'sudo apt-get autoremove || read enter; sudo apt-get clean || read enter'" &
;;
- bring-back) TMPFILE=$(mktemp -u)
- OFF_NODES=~jenkins/offline_nodes
- echo "Trying to mark $SHORTNODE as online. This is the diff for $OFF_NODES on jenkins.debian.net:"
- ssh jenkins.debian.net "grep -v $SHORTNODE $OFF_NODES > $TMPFILE ; diff $OFF_NODES $TMPFILE ; mv $TMPFILE $OFF_NODES ; chown jenkins:jenkins $OFF_NODES"
+ shell) xterm -T "$SHORTNODE / shell" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE" &
;;
- *) echo unsupported action, sorry, please improve.
- exit 1
- ;;
- esac
-
- #
- # log
- #
- echo "$LOGDATE, $SHORTNODE, $ACTION, $REASON" >> $LOGFILE
-done
-
-echo Thank you for doing documented jenkins maintenance.
+ jenkins-ui) echo "Documenting one manual action in the jenkins-ui."
+ ;;
+ bring-back) TMPFILE=$(mktemp -u)
+ OFF_NODES=~jenkins/offline_nodes
+ echo "Trying to mark $SHORTNODE as online. This is the diff for $OFF_NODES on jenkins.debian.net:"
+ ssh jenkins.debian.net "grep -v $SHORTNODE $OFF_NODES > $TMPFILE ; diff $OFF_NODES $TMPFILE ; mv $TMPFILE $OFF_NODES ; chown jenkins:jenkins $OFF_NODES"
+ ;;
+ *) echo unsupported action, sorry, please improve.
+ exit 1
+ ;;
+ esac
+
+ #
+ # log
+ #
+ echo "$LOGDATE, $SHORTNODE, $ACTION, $REASON" >> $LOGFILE
+ done
+
+ echo Thank you for doing documented jenkins maintenance.
+fi
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/ef0235102795b7a481756eabddff5556ca0531c4...9491bcbaaadfed28eb47a2cd89cab8724a47c07c
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/ef0235102795b7a481756eabddff5556ca0531c4...9491bcbaaadfed28eb47a2cd89cab8724a47c07c
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230507/6dfabab9/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list