[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: improve notification about zombie jobs

[Holger Levsen]

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
42d7c71a by Holger Levsen at 2019-08-29T11:51:28Z
improve notification about zombie jobs

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
63f06e8c by Holger Levsen at 2019-08-29T11:53:22Z
add debug output

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


2 changed files:

- bin/reproducible_node_health_check.sh
- update_jdn.sh


Changes:

=====================================
bin/reproducible_node_health_check.sh
=====================================
@@ -164,41 +164,6 @@ if ! systemctl is-system-running > /dev/null; then
     DIRTY=true
 fi
 
-#
-# checks only for the main node
-#
-if [ "$HOSTNAME" = "$MAINNODE" ] ; then
-	#
-	# sometimes deleted jobs come back as zombies
-	# and we dont know why and when that happens,
-	# so just report those zombies here.
-	#
-	ZOMBIES=$(ls -1d /var/lib/jenkins/jobs/* | egrep 'strip-nondeterminism|reprotest|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|odc2a|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|piuparts_.*_jessie|udd_stretch' || true)
-	if [ ! -z "$ZOMBIES" ] ; then
-		echo "Warning, rise of the jenkins job zombies has started again, these jobs should not exist:"
-		for z in $ZOMBIES ; do
-			echo $(basename $z)
-		done
-		DIRTY=true
-		echo
-	fi
-	#
-	# /var/log/jenkins/jenkins.log sometimes grows very fast
-	# and we don't yet know why, so let's monitor this for now.
-	JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +42G)"
-	if [ -z "JENKINSLOG" ] ; then
-		echo "Warning, jenkins.log is larger than 42G, please fix, erroring out now."
-		exit 1
-	else
-		JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +23G)"
-		if [ -z "JENKINSLOG" ] ; then
-			echo "Warning, jenkins.log is larger than 23G, please do something…"
-			DIRTY=true
-		fi
-	fi
-fi
-
-
 #
 # finally
 #


=====================================
update_jdn.sh
=====================================
@@ -785,6 +785,43 @@ fi
 sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
 explain "$(date) - finished deployment."
 
+#
+# some final checks only for the jenkins
+#
+if [ "$HOSTNAME" = "jenkins" ] ; then
+	#
+	# sometimes deleted jobs come back as zombies
+	# and we dont know why and when that happens,
+	# so just report those zombies here.
+	#
+	ZOMBIES=$(ls -1d /var/lib/jenkins/jobs/* | egrep 'strip-nondeterminism|reprotest|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|odc2a|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|piuparts_.*_jessie|udd_stretch' || true)
+	if [ ! -z "$ZOMBIES" ] ; then
+		figlet 'zombies!!!'
+		echo "Warning, rise of the jenkins job zombies has started again, these jobs should not exist:"
+		for z in $ZOMBIES ; do
+			echo $(basename $z)
+		done
+		echo
+	fi
+	#
+	# /var/log/jenkins/jenkins.log sometimes grows very fast
+	# and we don't yet know why, so let's monitor this for now.
+	JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +42G)"
+	if [ -z "JENKINSLOG" ] ; then
+		echo "Warning, jenkins.log is larger than 42G, please fix, erroring out now."
+		exit 1
+	else
+		JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +23G)"
+		if [ -z "JENKINSLOG" ] ; then
+			figlet 'jenkins.log size'
+			echo "Warning, jenkins.log is larger than 23G, please do something…"
+		fi
+	fi
+	explain "$(date) - done checking for known jenkins bugs."
+fi
+
+
+
 # finally!
 case $HOSTNAME in
 	# set time back to the future



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/7fc9de6cb7245d50370fc9782882fa608f4911b9...63f06e8c1112c624d36c43c0a531addc4f4cc90b

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/7fc9de6cb7245d50370fc9782882fa608f4911b9...63f06e8c1112c624d36c43c0a531addc4f4cc90b
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190829/ce8cbc3f/attachment-0001.html>