[Qa-jenkins-scm] [jenkins.debian.net] 01/01: reproducible Debian: add node health check for too big jenkins.logs

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch master
in repository jenkins.debian.net.

commit 9b381aff743a88c9ebc2e400a345ee836a25bd7e
Author: Holger Levsen <holger at layer-acht.org>
Date:   Sat Jul 15 13:58:34 2017 +0200

    reproducible Debian: add node health check for too big jenkins.logs
    
    Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
 bin/reproducible_node_health_check.sh | 14 ++++++++++++++
 logparse/reproducible.rules           |  2 ++
 2 files changed, 16 insertions(+)

diff --git a/bin/reproducible_node_health_check.sh b/bin/reproducible_node_health_check.sh
index 1fc61fb..cc87584 100755
--- a/bin/reproducible_node_health_check.sh
+++ b/bin/reproducible_node_health_check.sh
@@ -152,6 +152,20 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
 		DIRTY=true
 		echo
 	fi
+	#
+	# /var/log/jenkins/jenkins.log sometimes grows very fast
+	# and we don't yet know why, so let's monitor this for now.
+	JENKINSLOG="find /var/log/jenkins -name jenkins.log -size +42G"
+	if [ -z "JENKINSLOG" ] ; then
+		echo "Warning, jenkins.log is larger than 42G, please fix, erroring out now."
+		exit 1
+	else
+		JENKINSLOG="find /var/log/jenkins -name jenkins.log -size +23G"
+		if [ -z "JENKINSLOG" ] ; then
+			echo "Warning, jenkins.log is larger than 23G, please do something…"
+			DIRTY=true
+		fi
+	fi
 fi
 
 
diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules
index ef19fef..eef8d5c 100644
--- a/logparse/reproducible.rules
+++ b/logparse/reproducible.rules
@@ -25,4 +25,6 @@ warning /Warning, today is the wrong future:.+/
 warning /Warning: curl http:..www.debian.org failed, probably the proxy is down for.+/
 warning /Warning, more than one kernel in.+/
 warning /Warning, rise of the jenkins job zombies.+/
+warning /Warning, jenkins.log is larger than.+/
+
 

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git