[Qa-jenkins-scm] [jenkins.debian.net] 01/01: reproducible Debian: check for (known) zombie jobs and report them

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch master
in repository jenkins.debian.net.

commit 77c6891eb21c8fa5cbffe22f75d9e54fbe10b797
Author: Holger Levsen <holger at layer-acht.org>
Date:   Wed Jun 28 11:43:40 2017 +0200

    reproducible Debian: check for (known) zombie jobs and report them
    
    Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
 bin/reproducible_maintenance.sh       |  1 +
 bin/reproducible_node_health_check.sh | 19 +++++++++++++++++++
 logparse/reproducible.rules           |  1 +
 3 files changed, 21 insertions(+)

diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index 4543441..d93676c 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -336,6 +336,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
 		echo "Removed $OLDSTUFF jenkins html logs."
 		echo
 	fi
+
 fi
 
 # find+terminate processes which should not be there
diff --git a/bin/reproducible_node_health_check.sh b/bin/reproducible_node_health_check.sh
index 189bc98..1646682 100755
--- a/bin/reproducible_node_health_check.sh
+++ b/bin/reproducible_node_health_check.sh
@@ -137,6 +137,25 @@ if [ -z "$HAVEGED" ] ; then
 fi
 
 #
+# checks only for the main node
+#
+if [ "$HOSTNAME" = "$MAINNODE" ] ; then
+	#
+	# sometimes deleted jobs come back as zombies
+	# and we dont know why and when that happens,
+	# so just report those zombies here.
+	#
+	ZOMBIES=$(ls -1d var/lib/jenkins/jobs/ | egrep '(reproducible_builder_amd64|reproducible_builder_i386|reproducible_builder_armhf|reproducible_builder_arm64|chroot-installation_wheezy)')
+	if [ ! -z "$ZOMBIES" ] ; then
+		echo "Warning, rise of the jenkins job zombies has started again, these jobs should not exist:"
+		echo -e "$ZOMBIES"
+		DIRTY=TRUE
+		echo
+	fi
+fi
+
+
+#
 # finally
 #
 if ! $DIRTY ; then
diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules
index 39faf2e..ef19fef 100644
--- a/logparse/reproducible.rules
+++ b/logparse/reproducible.rules
@@ -24,4 +24,5 @@ warning /Warning: /.../shm.+please tell the jenkins admins to fix this./
 warning /Warning, today is the wrong future:.+/
 warning /Warning: curl http:..www.debian.org failed, probably the proxy is down for.+/
 warning /Warning, more than one kernel in.+/
+warning /Warning, rise of the jenkins job zombies.+/
 

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git