[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 3 commits: djm-logparser: improve statistics about manually triggered jenkins jobs
Holger Levsen (@holger)
gitlab at salsa.debian.org
Fri Apr 14 16:56:09 BST 2023
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
ce036463 by Holger Levsen at 2023-04-14T17:53:47+02:00
djm-logparser: improve statistics about manually triggered jenkins jobs
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
194bd6a4 by Holger Levsen at 2023-04-14T17:54:52+02:00
TODO: update various tasks
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
afc378b1 by Holger Levsen at 2023-04-14T17:55:50+02:00
jenkins-shell-monitor: update various directory usage warning levels
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
4 changed files:
- TODO
- bin/djm-jenkins-parser
- bin/djm-logparser
- bin/jenkins-shell-monitor.sh
Changes:
=====================================
TODO
=====================================
@@ -24,21 +24,37 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
=== 2023 things
-* discouple t.r-b.o from jenkins.d.n (https hosting wise), maybe even www.r-b.o
+* decouple t.r-b.o from jenkins.d.n (https hosting wise), maybe even www.r-b.o
* rename jenkins to hudson? (the hostname)
-* maintenance job: rm /var/lib/schroot/unpack/d-i-manual* older than 3 days
-* /var/lib/schroot/unpack/jenkins-*/var/cache/apt/archives not empty, rather 96GB wasted atm
+* maintenance job:
+** maybe: rm /var/lib/schroot/unpack/d-i-manual* older than 5 days
+** maybe: rm /tmp/mmdebstrap.* older than 3 days
* split TODO in TODO and TODO.legacy?
+* djm:
+** -d/--dry-run so $4 is free...
+** get rif of deploy_jdn
+*** and while at it, make actions triggering 'all' be only one entry in the djm logfile -> more sensible stats
+** cleanup/backup logs on the first of the month
+** document running "mosh jenkins shell-monitor" processes :) (same with my_btop to be renamed and git added...)
+** create osuosl tickets?
+** add support for powercycling the others archs
+** document jenkins plugin updates -> manually / though my local script which calls the webpages can log this
+** document other jenkins ui actions. are there any?
+* djm-logparser:
+** include hours with manual jobs triggered
+** make no fetch mode the default, -n --no-fetch
+** include notmuch mails, "received and read". write djm-notmuch-parser.
+** include amount of job runs on jenkins
+* djm-jenknis-parser:
+** cleanup/backup logs on the first of the month
=== nodes at OSUOSL
-* five new nodes
-** o3: move /var/lib/qa on a dedicated 100GB partition and you move VM disks to /srv/workspace
-** mention o4+5 in THANKS and explain usage. mention facebook in past sponsors.
-** mv snapshot.r-b.o from osuosl4 to osuosl5
-*** setup xfs on o5, then copy snapshot over
-*** rebuilder on o4
-** setup netconsoles:
+* mention o4+5 in THANKS and explain usage. mention facebook in past sponsors.
+* mv snapshot.r-b.o from osuosl4 to osuosl5
+** setup xfs on o5, then copy snapshot over
+* rebuilder on o4
+* setup netconsoles:
----
<guerby> | Ramereth, h01ger netconsole is about the only way to diagnose this kind of issue in my experience
<guerby> | h01ger, setup rsyslogd on one of the gccserver then it's just one modprobe netconsole
=====================================
bin/djm-jenkins-parser
=====================================
@@ -37,8 +37,8 @@ if [ ! -f $MYLOG ] ; then
echo "Note: initial run detected, this will take a few minutes."
touch $MYLOG
else
- if [ $(cat $MYLOG | wc -l) -gt 15000 ] ; then
- echo "Note: $MYLOG has become very big, maybe time to delete $MYLOG and $MYRESULTS?"
+ if [ $(cat $MYLOG | wc -l) -gt 150000 ] ; then
+ echo "Note: $MYLOG has become very big, maybe time to backup $MYLOG and $MYRESULTS?"
fi
LOGS=$(find ./reproducible_*/builds/*/log -newer $MYLOG 2>/dev/null)
ZLOGS=$(find ./reproducible_*/builds/*/log.gz -newer $MYLOG 2>/dev/null)
=====================================
bin/djm-logparser
=====================================
@@ -20,6 +20,7 @@ if [ -z "$DJM_USER" ] ; then
fi
LOCAL_LOGFILE=~/.djm.log
UI_LOGFILE=~/.djm-jenkins-ui.log
+PARSER_LOGFILE=~/.djm-jenkins-parser.log
LOGMONTH="$(date -u '+%Y-%m')"
JOBS=~/.djm-jobs.txt
@@ -29,6 +30,7 @@ JOBS=~/.djm-jobs.txt
if [ -z "$1" ] ; then
ssh $DJM_USER at jenkins.debian.net "DJM_USER=$DJM_USER /srv/jenkins/bin/djm-jenkins-parser"
scp $DJM_USER at jenkins.debian.net:$(basename $UI_LOGFILE) $UI_LOGFILE
+ scp $DJM_USER at jenkins.debian.net:$(basename $PARSER_LOGFILE) $PARSER_LOGFILE
ssh $DJM_USER at jenkins.debian.net "cd ~jenkins/jobs ; ls -1d reproducible_* |wc -l" > $JOBS
fi
@@ -49,6 +51,17 @@ seperator() {
echo
}
+printf_if_not_zero() {
+ if [ $2 -ne 0 ] ; then
+ printf "$1" "$2" "$3"
+ fi
+}
+
+printf_heading() {
+ printf "$ONE_C" "$1"
+ printf "$ONE_C" "$(for i in $(seq 1 ${#1}) ; do echo -n '-' ; done)"
+}
+
#
# main
#
@@ -107,18 +120,32 @@ done
seperator "="
#
-# do statistics about the jobs triggered
+# statistics about jenkins jobs
#
-printf "$ONE_C" "jobs triggered:"
-printf "$ONE_C" "---------------"
+PATTERNS="setup_schroot setup_pbuilder debian_live maintenance node_health_check
+ html alpine archlinux builder_archlinux openwrt .*fedora builds opensuse .*strap
+ .*diffoscope reprotest disorderfs .*lfs" # .*(json|db|lfs)"
+PIPE_PATTERNS=$(echo $PATTERNS | sed 's# #|#g')
+HEADING="$(cat $LOGFILE | wc -l) jobs triggered manually:"
+printf_heading "$HEADING"
+(
+ for i in $PATTERNS ; do
+ printf_if_not_zero "$TWO_C" $(grep -c reproducible_$i $LOGFILE) "reproducible_${i}_.*"
+ done
+ for i in $(cut -d ',' -f5 $LOGFILE | grep -v -E $PIPE_PATTERNS|sort -u) ; do
+ printf_if_not_zero "$TWO_C" $(grep -c $i $LOGFILE) "$i"
+ done
+) | sort -n -r
+seperator
+
+HEADING="$(cat $PARSER_LOGFILE | wc -l) total jobs run:"
+printf_heading "$HEADING"
(
- PATTERNS="setup_schroot debian_live_build maintenance node_health_check html openwrt"
for i in $PATTERNS ; do
- printf "$TWO_C" "$(grep -c reproducible_$i $LOGFILE)" "reproducible_${i}_.*"
+ printf_if_not_zero "$TWO_C" $(grep -c reproducible_$i $PARSER_LOGFILE) "reproducible_${i}_.*"
done
- PATTERNS=$(echo $PATTERNS | sed 's# #|#g')
- for i in $(cut -d ',' -f5 $LOGFILE | grep -v -E $PATTERNS|sort -u) ; do
- printf "$TWO_C" "$(grep -c $i $LOGFILE)" "$i"
+ for i in $(cut -d '/' -f2 $PARSER_LOGFILE | grep -v -E $PIPE_PATTERNS|sort -u) ; do
+ printf_if_not_zero "$TWO_C" $(grep -c $i $PARSER_LOGFILE) "$i"
done
) | sort -n -r
seperator "="
=====================================
bin/jenkins-shell-monitor.sh
=====================================
@@ -14,16 +14,16 @@
# define some warning levels (in gigabytes) for some directories
declare -A WARNING_LEVEL
-WARNING_LEVEL['/var/lib/jenkins']=100
-WARNING_LEVEL['/var/lib/schroot']=100
+WARNING_LEVEL['/var/lib/jenkins']=80
+WARNING_LEVEL['/var/lib/schroot']=50
WARNING_LEVEL['/srv/schroots']=100
-WARNING_LEVEL['/srv/reproducible-results/rbuild-debian']=80
+WARNING_LEVEL['/srv/reproducible-results/rbuild-debian']=50
WARNING_LEVEL['r-b-userContent../dbd*/stretch']=3
WARNING_LEVEL['r-b-userContent../dbd*/buster']=5
WARNING_LEVEL['r-b-userContent../dbd*/bullseye']=40
WARNING_LEVEL['r-b-userContent../dbd*/bookworm']=60
-WARNING_LEVEL['r-b-userContent../dbd*/unstable']=140
-WARNING_LEVEL['r-b-userContent../dbd*/experimental']=40 # 30 outside freezes
+WARNING_LEVEL['r-b-userContent../dbd*/unstable']=145
+WARNING_LEVEL['r-b-userContent../dbd*/experimental']=50 # 30 outside freezes
# prepare a trap for cleaning up
# using a global tempfile
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/c9bd3a89b9acf7e4920474e271a84cc0c7b3a3e6...afc378b154038d4883f7168175bb769cca9c000d
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/c9bd3a89b9acf7e4920474e271a84cc0c7b3a3e6...afc378b154038d4883f7168175bb769cca9c000d
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230414/759cfbc1/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list