[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 3 commits: reproducible trbo monitor: colorize critical diskspace situations
Holger Levsen
gitlab at salsa.debian.org
Mon Sep 28 10:59:40 BST 2020
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
07297a9a by Holger Levsen at 2020-09-28T11:52:30+02:00
reproducible trbo monitor: colorize critical diskspace situations
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
c4fe892d by Holger Levsen at 2020-09-28T11:55:56+02:00
reproducible trbo monitor: refactor color codes
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
73b53fcf by Holger Levsen at 2020-09-28T11:58:54+02:00
reproducible trbo monitor: refactor, only gather filesystem stats every 15min
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/jenkins-shell-monitor.sh
Changes:
=====================================
bin/jenkins-shell-monitor.sh
=====================================
@@ -15,10 +15,11 @@
# prepare a trap for cleaning up
# using a global tempfile
PSFAX=$(mktemp --tmpdir=$TMPDIR jenkins-shell-monitor-XXXXXXX)
+DISKSTATS=$(mktemp --tmpdir=$TMPDIR jenkins-shell-monitor-XXXXXXX)
cleanup_all() {
trap - INT TERM EXIT
echo
- rm -vf $PSFAX
+ rm -vf $PSFAX $DISKSTATS
echo "Terminated, good-bye."
exit 0
}
@@ -32,9 +33,9 @@ main_loop() {
LSOF=$(lsof -n | wc -l)
SCHROOT_SESSIONS=$(find /var/lib/schroot/session/ -mindepth 1 | wc -l)
if [ $SCHROOT_SESSIONS -gt 30000 ] ; then
- SCHROOT_SESSIONS="\033[91m$SCHROOT_SESSIONS\033[0m"
+ SCHROOT_SESSIONS="${RED}$SCHROOT_SESSIONS${FOREGROUND_COLOR}"
elif [ $SCHROOT_SESSIONS -gt 15000 ] ; then
- SCHROOT_SESSIONS="\033[93m$SCHROOT_SESSIONS\033[0m"
+ SCHROOT_SESSIONS="${YELLOW}$SCHROOT_SESSIONS${FOREGROUND_COLOR}"
fi
SCHROOT_MOUNTS=$(mount | grep /run/schroot| wc -l)
SCHROOT_BASE_DIRS=$(find /schroots/ -maxdepth 1 -mindepth 1 -type d | wc -l)
@@ -42,15 +43,37 @@ main_loop() {
REPRO_JOBS=$(ls ~jenkins/jobs/reproducible_* -1d | wc -l)
JOBS_RUNNING_TOTAL=$(grep '_ /bin/bash /srv/jenkins/bin/' $PSFAX | egrep -v 'reproducible_worker.sh|reproducible_build.sh|jenkins-shell-monitor.sh|reproducible_build_archlinux_pkg.sh' | wc -l)
if [ $JOBS_RUNNING_TOTAL -eq 0 ] ; then
- JOBS_RUNNING_TOTAL="\033[91m$JOBS_RUNNING_TOTAL\033[0m"
+ JOBS_RUNNING_TOTAL="${RED}$JOBS_RUNNING_TOTAL${FOREGROUND_COLOR}"
elif [ $JOBS_RUNNING_TOTAL -lt 10 ] ; then
- JOBS_RUNNING_TOTAL="\033[93m$JOBS_RUNNING_TOTAL\033[0m"
+ JOBS_RUNNING_TOTAL="${YELLOW}$JOBS_RUNNING_TOTAL${FOREGROUND_COLOR}"
fi
JOBS_RUNNING_LOCAL=$(grep '_ /bin/bash /srv/jenkins/bin/' $PSFAX | egrep -v 'reproducible_worker.sh|reproducible_build.sh|jenkins-shell-monitor.sh|reproducible_build_archlinux_pkg.sh|jenkins_master_wrapper.sh' | wc -l)
JOBS_RUNNING_REMOTE=$(grep -c jenkins_master_wrapper.sh $PSFAX || true)
REPRO_WORKERS=$(grep -c reproducible_worker.sh $PSFAX || true)
JENKINS_AGENTS=$(grep -c jenkins/agent.jar $PSFAX || true)
NODES_TOTAL=$(find ~/nodes -mindepth 1 -type d | wc -l)
+ ( echo $DF_HEADER
+ for FILESYSTEM in /dev/vda1 /var/lib/jenkins/userContent/reproducible /srv/workspace /tmp ; do
+ DF_FILESYSTEM=$(df -h $FILESYSTEM | tail -1)
+ FILESYSTEM_MOUNTPOINT=$(echo $DF_FILESYSTEM | awk '{ print $6 }')
+ FILESYSTEM_SIZE=$(echo $DF_FILESYSTEM | awk '{ print $2 }')
+ FILESYSTEM_AVAIL=$(echo $DF_FILESYSTEM | awk '{ print $4 }')
+ FILESYSTEM_USAGE=$(echo $DF_FILESYSTEM | awk '{ print $5 }' | cut -d '%' -f1)
+ if [ $FILESYSTEM_USAGE -gt 92 ] ; then
+ FILESYSTEM_USAGE="${RED}$FILESYSTEM_USAGE%${FOREGROUND_COLOR}"
+ FILESYSTEM_AVAIL="${RED}$FILESYSTEM_AVAIL${FOREGROUND_COLOR}"
+ WIDTH=16
+ elif [ $FILESYSTEM_USAGE -gt 77 ] ; then
+ FILESYSTEM_USAGE="${YELLOW}$FILESYSTEM_USAGE%${FOREGROUND_COLOR}"
+ FILESYSTEM_AVAIL="${YELLOW}$FILESYSTEM_AVAIL${FOREGROUND_COLOR}"
+ WIDTH=16
+ else
+ FILESYSTEM_USAGE="$FILESYSTEM_USAGE%"
+ WIDTH=5
+ fi
+ printf "%-45s %-5s %-${WIDTH}s %s\n" $FILESYSTEM_MOUNTPOINT $FILESYSTEM_SIZE $FILESYSTEM_AVAIL $FILESYSTEM_USAGE
+ done
+ ) > $DISKSTATS
SOME_AGE="$(date -u)"
for i in $SEQ1 ; do
clear
@@ -58,27 +81,27 @@ main_loop() {
INTEGER_LOAD=$(echo $LOAD | cut -d '.' -f1)
# based on 23 CPUs
if [ $INTEGER_LOAD -gt 69 ] ; then
- LOAD="\033[91m$LOAD\033[0m"
+ LOAD="${RED}$LOAD${FOREGROUND_COLOR}"
elif [ $INTEGER_LOAD -gt 42 ] ; then
- LOAD="\033[93m$LOAD\033[0m"
+ LOAD="${YELLOW}$LOAD${FOREGROUND_COLOR}"
fi
NODES_OFFLINE="$(grep -v ^# ~jenkins/offline_nodes | grep -c debian)"
NODES_GIT_OFFLINE="$(grep -v ^# ~jenkins-adm/jenkins.debian.net/jenkins-home/offline_nodes | grep -c debian)"
if [ "$NODES_OFFLINE" != "$NODES_GIT_OFFLINE" ] ; then
- NODES_OFFLINE="\033[93m$NODES_OFFLINE\033[0m"
+ NODES_OFFLINE="${YELLOW}$NODES_OFFLINE${FOREGROUND_COLOR}"
fi
SYSTEMCTLSTATUS=$(systemctl status|head -4|tail -3)
SYS_STATE=$(echo "$SYSTEMCTLSTATUS" | grep State | cut -d ':' -f2- | xargs echo)
if [ "$SYS_STATE" != "running" ] ; then
- SYS_STATE="\033[93m$SYS_STATE\033[0m"
+ SYS_STATE="${YELLOW}$SYS_STATE${FOREGROUND_COLOR}"
fi
SYS_JOBS=$(echo "$SYSTEMCTLSTATUS" | grep Jobs | cut -d ':' -f2- | xargs echo)
if [ "$SYS_JOBS" != "0 queued" ] ; then
- SYS_JOBS="\033[93m$SYS_JOBS\033[0m"
+ SYS_JOBS="${YELLOW}$SYS_JOBS${FOREGROUND_COLOR}"
fi
SYS_FAILED=$(echo "$SYSTEMCTLSTATUS" | grep Failed | cut -d ':' -f2- | xargs echo)
if [ "$SYS_FAILED" != "0 units" ] ; then
- SYS_FAILED="\033[93m$SYS_FAILED\033[0m"
+ SYS_FAILED="${YELLOW}$SYS_FAILED${FOREGROUND_COLOR}"
fi
echo "=== jenkins/trbo shell monitor ==="
echo
@@ -108,7 +131,7 @@ main_loop() {
echo "running jenkings agents/nodes: $JENKINS_AGENTS"
echo -e "nodes: (total/auto-offline/offline in git) $NODES_TOTAL / $NODES_OFFLINE / $NODES_GIT_OFFLINE"
echo
- df -h /dev/vda1 /var/lib/jenkins/userContent/reproducible /srv/workspace /tmp | awk '{printf "%-45s %-6s %-6s %s\n", $6, $2, $4, $5}'
+ cat $DISKSTATS
free -h | cut -b1-47
echo
echo "some information was gathered at $SOME_AGE."
@@ -120,12 +143,19 @@ main_loop() {
done
}
-# internal main
+# main init
export LANG=C
trap cleanup_all INT TERM EXIT
# initialize internal loop variables
SEQ1=$(seq 1 10) # outer loop
SEQ2=$(seq 1 60) # inner loop
+# static data
+DF_HEADER=$(df | head -1)
+# static colors
+FOREGROUND_COLOR=$(tput sgr0)
+RED=$(tput setaf 1)
+YELLOW=$(tput setaf 3)
+# main loop
while true ; do
main_loop
done
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/585ecee1bfd3003974ed6df01dbbeb582233b356...73b53fcf3cf2cc33ca8ae489ad51cb589f9a3d8c
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/585ecee1bfd3003974ed6df01dbbeb582233b356...73b53fcf3cf2cc33ca8ae489ad51cb589f9a3d8c
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20200928/2f90c052/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list