[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 3 commits: jenkins-shell-monitor: better explain loop runtimes
Holger Levsen (@holger)
gitlab at salsa.debian.org
Mon Feb 6 13:28:37 GMT 2023
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
8d882964 by Holger Levsen at 2023-02-06T14:27:51+01:00
jenkins-shell-monitor: better explain loop runtimes
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
7eafae2d by Holger Levsen at 2023-02-06T14:27:51+01:00
jenkins-shell-monitor: only query diskstats every 230min
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
5ed88c03 by Holger Levsen at 2023-02-06T14:27:51+01:00
jenkins-shell-monitor: explain disk usage update frequency
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/jenkins-shell-monitor.sh
Changes:
=====================================
bin/jenkins-shell-monitor.sh
=====================================
@@ -28,31 +28,11 @@ cleanup_all() {
# main
#
main_loop() {
- # most data is updated every 23 minutes in the outer loop
- ps fax > $PSFAX
- LSOF=$(lsof -n | wc -l)
- SCHROOT_SESSIONS=$(find /var/lib/schroot/session/ -mindepth 1 | wc -l)
- if [ $SCHROOT_SESSIONS -gt 30000 ] ; then
- SCHROOT_SESSIONS="${RED}$SCHROOT_SESSIONS${FOREGROUND_COLOR}"
- elif [ $SCHROOT_SESSIONS -gt 15000 ] ; then
- SCHROOT_SESSIONS="${YELLOW}$SCHROOT_SESSIONS${FOREGROUND_COLOR}"
- fi
- SCHROOT_MOUNTS=$(mount | grep /run/schroot| wc -l)
- SCHROOT_BASE_DIRS=$(find /schroots/ -maxdepth 1 -mindepth 1 -type d | wc -l)
- DIFFOSCOPE_PROCS=$(grep -c '/usr/bin/python3 /usr/bin/diffoscope' $PSFAX || true)
- REPRO_JOBS=$(ls ~jenkins/jobs/reproducible_* -1d | wc -l)
- JOBS_RUNNING_TOTAL=$(grep '_ /bin/bash /srv/jenkins/bin/' $PSFAX | egrep -v 'reproducible_worker.sh|reproducible_build.sh|jenkins-shell-monitor.sh|reproducible_build_archlinux_pkg.sh' | wc -l)
- if [ $JOBS_RUNNING_TOTAL -eq 0 ] ; then
- JOBS_RUNNING_TOTAL="${RED}$JOBS_RUNNING_TOTAL${FOREGROUND_COLOR}"
- elif [ $JOBS_RUNNING_TOTAL -lt 5 ] ; then
- JOBS_RUNNING_TOTAL="${YELLOW}$JOBS_RUNNING_TOTAL${FOREGROUND_COLOR}"
- fi
- JOBS_RUNNING_LOCAL=$(grep '_ /bin/bash /srv/jenkins/bin/' $PSFAX | egrep -v 'reproducible_worker.sh|reproducible_build.sh|jenkins-shell-monitor.sh|reproducible_build_archlinux_pkg.sh|jenkins_master_wrapper.sh' | wc -l)
- JOBS_RUNNING_REMOTE=$(grep -c jenkins_master_wrapper.sh $PSFAX || true)
- REPRO_WORKERS=$(grep -c reproducible_worker.sh $PSFAX || true)
- JENKINS_AGENTS=$(grep -c jenkins/agent.jar $PSFAX || true)
- NODES_TOTAL=$(find ~/nodes -mindepth 1 -type d | wc -l)
- ( printf "%-45s %-5s %-5s %s\n" "mountpoint" "size" "avail" "usage"
+ # most data is updated every 23 minutes in the outer loop,
+ # except some, which is only updated every 180 minutes...:
+ if [ $SEQ0 -eq 0 ] ; then
+ (
+ printf "%-45s %-5s %-5s %s\n" "mountpoint" "size" "avail" "usage"
for FILESYSTEM in /dev/vda1 /var/lib/jenkins/userContent/reproducible /srv/workspace /tmp ; do
DF_FILESYSTEM=$(df -h $FILESYSTEM | tail -1)
FILESYSTEM_MOUNTPOINT=$(echo $DF_FILESYSTEM | awk '{ print $6 }')
@@ -99,7 +79,32 @@ main_loop() {
;;
esac
done
- ) > $DISKSTATS
+ ) > $DISKSTATS
+ DISK_AGE="$(date -u)"
+ fi
+ ps fax > $PSFAX
+ LSOF=$(lsof -n | wc -l)
+ SCHROOT_SESSIONS=$(find /var/lib/schroot/session/ -mindepth 1 | wc -l)
+ if [ $SCHROOT_SESSIONS -gt 30000 ] ; then
+ SCHROOT_SESSIONS="${RED}$SCHROOT_SESSIONS${FOREGROUND_COLOR}"
+ elif [ $SCHROOT_SESSIONS -gt 15000 ] ; then
+ SCHROOT_SESSIONS="${YELLOW}$SCHROOT_SESSIONS${FOREGROUND_COLOR}"
+ fi
+ SCHROOT_MOUNTS=$(mount | grep /run/schroot| wc -l)
+ SCHROOT_BASE_DIRS=$(find /schroots/ -maxdepth 1 -mindepth 1 -type d | wc -l)
+ DIFFOSCOPE_PROCS=$(grep -c '/usr/bin/python3 /usr/bin/diffoscope' $PSFAX || true)
+ REPRO_JOBS=$(ls ~jenkins/jobs/reproducible_* -1d | wc -l)
+ JOBS_RUNNING_TOTAL=$(grep '_ /bin/bash /srv/jenkins/bin/' $PSFAX | egrep -v 'reproducible_worker.sh|reproducible_build.sh|jenkins-shell-monitor.sh|reproducible_build_archlinux_pkg.sh' | wc -l)
+ if [ $JOBS_RUNNING_TOTAL -eq 0 ] ; then
+ JOBS_RUNNING_TOTAL="${RED}$JOBS_RUNNING_TOTAL${FOREGROUND_COLOR}"
+ elif [ $JOBS_RUNNING_TOTAL -lt 5 ] ; then
+ JOBS_RUNNING_TOTAL="${YELLOW}$JOBS_RUNNING_TOTAL${FOREGROUND_COLOR}"
+ fi
+ JOBS_RUNNING_LOCAL=$(grep '_ /bin/bash /srv/jenkins/bin/' $PSFAX | egrep -v 'reproducible_worker.sh|reproducible_build.sh|jenkins-shell-monitor.sh|reproducible_build_archlinux_pkg.sh|jenkins_master_wrapper.sh' | wc -l)
+ JOBS_RUNNING_REMOTE=$(grep -c jenkins_master_wrapper.sh $PSFAX || true)
+ REPRO_WORKERS=$(grep -c reproducible_worker.sh $PSFAX || true)
+ JENKINS_AGENTS=$(grep -c jenkins/agent.jar $PSFAX || true)
+ NODES_TOTAL=$(find ~/nodes -mindepth 1 -type d | wc -l)
SOME_AGE="$(date -u)"
for i in $SEQ1 ; do
# the inner loop is run more often
@@ -163,11 +168,12 @@ main_loop() {
cat $DISKSTATS
free -h | cut -b1-47
echo
- echo "some information was gathered at $SOME_AGE. (updated every 23min)"
- echo "some other information at around $(date -u). (updated every 90sec)"
+ echo "directory stats were gathered at $DISK_AGE. (updated every 180min)"
+ echo "some information was gathered at $SOME_AGE. (updated every 23min)"
+ echo "some other information at around $(date -u). (updated every 90sec)"
for j in $SEQ2 ; do
echo -n "..."
- sleep 4.5
+ sleep 4.5 # -> this involves loop runtime... (see below)
done
done
}
@@ -176,8 +182,9 @@ main_loop() {
export LANG=C
trap cleanup_all INT TERM EXIT
# initialize internal loop variables
-SEQ1=$(seq 1 15) # outer loop
-SEQ2=$(seq 1 20) # inner loop
+SEQ2=$(seq 1 20) # -> inner loop takes 90sec because 20*4.5s=90s
+SEQ1=$(seq 1 15) # -> outer loop takes ~23min because 15*90s=23m
+SEQ0=0
# static colors
FOREGROUND_COLOR=$(tput sgr0)
RED=$(tput setaf 1)
@@ -186,4 +193,8 @@ YELLOW=$(tput setaf 3)
# main loop
while true ; do
main_loop
+ let SEQ0+=1
+ if [ $SEQ0 -eq 8 ] ; then # -> diskstats are only gathered every 180m
+ SEQ0=0
+ fi
done
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/65f523bb3ad82d37bbaf0ed34dcaaab3a2afee51...5ed88c03a0bbae2317c6fc83266781546c0e3dfb
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/65f523bb3ad82d37bbaf0ed34dcaaab3a2afee51...5ed88c03a0bbae2317c6fc83266781546c0e3dfb
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230206/d741482b/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list