[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: reproducible trbo system health check: color highlight more conditions

[Holger Levsen]

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
ef5ef2fe by Holger Levsen at 2020-09-02T22:32:19+02:00
reproducible trbo system health check: color highlight more conditions

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
627f9c9a by Holger Levsen at 2020-09-02T22:33:55+02:00
reproducible trbo monitor: refactoring and whitespace changes

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


1 changed file:

- bin/jenkins-shell-monitor.sh


Changes:

=====================================
bin/jenkins-shell-monitor.sh
=====================================
@@ -38,6 +38,11 @@ main_loop() {
 	SCHROOT_MOUNTS=$(mount | grep /run/schroot| wc -l)
 	REPRO_JOBS=$(ls ~jenkins/jobs/reproducible_* -1d | wc -l)
 	JOBS_RUNNING_TOTAL=$(grep '_ /bin/bash /srv/jenkins/bin/' $PSFAX | egrep -v 'reproducible_worker.sh|reproducible_build.sh|jenkins-shell-monitor.sh' | wc -l)
+	if [ $JOBS_RUNNING_TOTAL -eq 0 ] ; then
+		JOBS_RUNNING_TOTAL="\033[91m$JOBS_RUNNING_TOTAL\033[0m"
+	elif [ $JOBS_RUNNING_TOTAL -lt 10 ] ; then
+		JOBS_RUNNING_TOTAL="\033[93m$JOBS_RUNNING_TOTAL\033[0m"
+	fi
 	JOBS_RUNNING_LOCAL=$(grep '_ /bin/bash /srv/jenkins/bin/' $PSFAX | egrep -v 'reproducible_worker.sh|reproducible_build.sh|jenkins-shell-monitor.sh|jenkins_master_wrapper.sh' | wc -l)
 	JOBS_RUNNING_REMOTE=$(grep jenkins_master_wrapper.sh $PSFAX | wc -l)
 	REPRO_WORKERS=$(grep reproducible_worker.sh $PSFAX | wc -l)
@@ -46,6 +51,19 @@ main_loop() {
 	SOME_AGE="$(date -u)"
 	for i in $SEQ1 ; do
 		clear
+		LOAD="$(uptime | rev | cut -d ',' -f1-3 | rev | cut -d ':' -f2-|xargs echo)"
+		INTEGER_LOAD=$(echo $LOAD | cut -d '.' -f1)
+		# based on 23 CPUs
+		if [ $INTEGER_LOAD -gt 69 ] ; then
+			LOAD="\033[91m$LOAD\033[0m"
+		elif [ $INTEGER_LOAD -gt 42 ] ; then
+			LOAD="\033[93m$LOAD\033[0m"
+		fi
+		NODES_OFFLINE="$(grep -v ^# ~jenkins/offline_nodes | grep -c debian)"
+		NODES_GIT_OFFLINE="$(grep -v ^# ~jenkins-adm/jenkins.debian.net/jenkins-home/offline_nodes | grep -c debian)"
+		if [ "$NODES_OFFLINE" != "$NODES_GIT_OFFLINE" ] ; then
+			NODES_OFFLINE="\033[93m$NODES_OFFLINE\033[0m"
+		fi
 		echo "=== jenkins/trbo shell monitor ==="
 		echo
 		echo "relevant lines from ~jenkins/offline_nodes:"
@@ -58,21 +76,16 @@ main_loop() {
 		figlet $(cat ~/userContent/reproducible/trbo.status) 
 		echo " ^^^ trbo status, see https://tests.reproducible-builds.org/trbo.status.html"
 		echo
-		echo "uptime:                                       $(uptime -p | cut -d ' ' -f2-)"
-		echo "load averages:                               $(uptime | rev | cut -d ',' -f1-3 | rev | cut -d ':' -f2-)"
-		echo "logged in user sessions:                      $(uptime | rev |cut -d ',' -f1-4 | rev | cut -d ',' -f1 | sed "s#users##" | xargs echo)"
-		echo "logged in users:                              $(w -h | awk '{print $1}' | sort -u | xargs echo)"
-		echo "number of open files:                         $LSOF"
+		echo    "uptime:                                       $(uptime -p | cut -d ' ' -f2-)"
+		echo -e "load averages:                                $LOAD"
+		echo    "logged in user sessions:                      $(uptime | rev |cut -d ',' -f1-4 | rev | cut -d ',' -f1 | sed "s#users##" | xargs echo)"
+		echo    "logged in users:                              $(w -h | awk '{print $1}' | sort -u | xargs echo)"
+		echo    "number of open files:                         $LSOF"
 		echo -e "schroot: (sessions / mounts)                  $SCHROOT_SESSIONS / $SCHROOT_MOUNTS"
-		echo "configured r-b jobs:                          $REPRO_JOBS"
-		echo "running jenkins jobs: (total/local/remote)    $JOBS_RUNNING_TOTAL / $JOBS_RUNNING_LOCAL / $JOBS_RUNNING_REMOTE"
-		echo "running debian r-b workers:                   $REPRO_WORKERS"
-		echo "running jenkings agents/nodes:                $JENKINS_AGENTS"
-		NODES_OFFLINE="$(grep -v ^# ~jenkins/offline_nodes | grep -c debian)"
-		NODES_GIT_OFFLINE="$(grep -v ^# ~jenkins-adm/jenkins.debian.net/jenkins-home/offline_nodes | grep -c debian)"
-		if [ "$NODES_OFFLINE" != "$NODES_GIT_OFFLINE" ] ; then
-			NODES_OFFLINE="\033[93m$NODES_OFFLINE\033[0m"
-		fi
+		echo    "configured r-b jobs:                          $REPRO_JOBS"
+		echo -e "running jenkins jobs: (total/local/remote)    $JOBS_RUNNING_TOTAL / $JOBS_RUNNING_LOCAL / $JOBS_RUNNING_REMOTE"
+		echo    "running debian r-b workers:                   $REPRO_WORKERS"
+		echo    "running jenkings agents/nodes:                $JENKINS_AGENTS"
 		echo -e "nodes: (total/auto-offline/offline in git)    $NODES_TOTAL / $NODES_OFFLINE / $NODES_GIT_OFFLINE"
 		echo
 		df -h  /dev/vda1 /var/lib/jenkins/userContent/reproducible /srv/workspace /tmp | awk '{printf "%-45s %-6s %-6s %s\n", $6, $2, $4, $5}'



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/9b0894295877994b52356085bbd8212997de4c14...627f9c9a1b303aa63e7e637c2907e53f1958aa0b

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/9b0894295877994b52356085bbd8212997de4c14...627f9c9a1b303aa63e7e637c2907e53f1958aa0b
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20200902/1dad1bef/attachment-0001.html>