[Qa-jenkins-scm] [jenkins.debian.net] 01/02: node wrapper: check if a node is up, else sleep 15m and abort the job

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch master
in repository jenkins.debian.net.

commit 79d2ce7dc1dd3fa4bdf2c5cba388166dd2bb5592
Author: Holger Levsen <holger at layer-acht.org>
Date:   Wed Sep 16 09:42:28 2015 +0200

    node wrapper: check if a node is up, else sleep 15m and abort the job
---
 bin/jenkins_master_wrapper.sh | 31 ++++++++++++++++++++++++-------
 bin/jenkins_node_wrapper.sh   |  4 +++-
 2 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/bin/jenkins_master_wrapper.sh b/bin/jenkins_master_wrapper.sh
index 77c9002..12dcb14 100755
--- a/bin/jenkins_master_wrapper.sh
+++ b/bin/jenkins_master_wrapper.sh
@@ -2,29 +2,46 @@
 
 set -u
 set -e
+PARAMS="$JOB_NAME"
 
 # these nodes also need to be listed in bin/reproducible_common.sh where they define $BUILD_NODES
 case "$NODE_NAME" in
   bpi0-armhf-rb.debian.net)
-    exec ssh -p 2222 $NODE_NAME "$JOB_NAME"
+    PORT=2222
     ;;
   hb0-armhf-rb.debian.net)
-    exec ssh -p 2224 $NODE_NAME "$JOB_NAME"
+    PORT=2224
     ;;
   wbq0-armhf-rb.debian.net)
-    exec ssh -p 2225 $NODE_NAME "$JOB_NAME"
+    PORT=2225
     ;;
   cbxi4pro0-armhf-rb.debian.net)
-    exec ssh -p 2226 $NODE_NAME "$JOB_NAME"
+    PORT=2226
     ;;
   profitbricks-build?-amd64.debian.net)
+    PORT=22
     if [[ "$JOB_NAME" =~ rebootstrap_.* ]] ; then
-	    exec ssh $NODE_NAME "$JOB_NAME $@"
-    else
-	    exec ssh $NODE_NAME "$JOB_NAME"
+	   PARAMS="$PARAMS $@"
     fi
     ;;
   *)
     echo >&2 "Unknown node $NODE_NAME."
     exit 1
 esac
+
+#
+# main
+#
+set +e
+ssh -p $PORT $NODE_NAME /bin/true
+RESULT=$?
+# abort job if host is down
+if [ $RESULT -ne 0 ] then
+	echo "$(date -u) - $NODE_NAME seems to be down, sleeping 15min before aborting this job."
+	sleep 15m
+	/srv/jenkins/bin/abort.sh
+fi
+set -e
+# finally
+exec ssh -p $PORT $NODE_NAME "$PARAMS"
+
diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh
index b470669..b6ea9eb 100755
--- a/bin/jenkins_node_wrapper.sh
+++ b/bin/jenkins_node_wrapper.sh
@@ -58,7 +58,9 @@ shift
 
 allowed_cmds=()
 
-if [[ "$*" =~ /bin/nc\ localhost\ 4949 ]] ; then
+if [[ "$*" =~ /bin/true ]] ; then
+	exec /bin/true ; croak "Exec failed";
+elif [[ "$*" =~ /bin/nc\ localhost\ 4949 ]] ; then
 	exec /bin/nc localhost 4949 ; croak "Exec failed";
 elif [[ "$*" =~ rebootstrap_.* ]] ; then
 	shift

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git