[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: djm: introduce djm-jenkins-parser.sh, to parse jenkins build logs for manually...

[Holger Levsen (@holger)]

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
610b2214 by Holger Levsen at 2023-04-10T18:15:39+02:00
djm: introduce djm-jenkins-parser.sh, to parse jenkins build logs for manually scheduled job runs and document them

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
b1e5a412 by Holger Levsen at 2023-04-10T19:09:48+02:00
djm-logparser: also include statistics about the jobs triggered manually

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


2 changed files:

- + bin/djm-jenkins-parser.sh
- bin/djm-logparser


Changes:

=====================================
bin/djm-jenkins-parser.sh
=====================================
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+cd ~/jobs
+
+HUMAN=holger
+
+export TZ="/usr/share/zoneinfo/UTC"
+MYLOG=~/.djm-jenkins-parser.log
+MYRESULTS=~/.djm-jenkins-ui.log
+
+if [ ! -f $MYLOG ] ; then
+	LOGS=$(find ./reproducible_*/builds/*/log 2>/dev/null)
+	ZLOGS=$(find ./reproducible_*/builds/*/log.gz 2>/dev/null)
+	echo "Note: initial run detected, this will take a few minutes."
+	touch $MYLOG
+else
+	if [ $(cat $MYLOG | wc -l) -gt 15000 ] ; then
+		echo "Note: $MYLOG has become very big, maybe time to delete $MYLOG and $MYRESULTS?"
+	fi
+	LOGS=$(find ./reproducible_*/builds/*/log -newer $MYLOG 2>/dev/null)
+	ZLOGS=$(find ./reproducible_*/builds/*/log.gz -newer $MYLOG 2>/dev/null)
+fi
+echo "Parsing $(echo $LOGS $ZLOGS | sed 's# #\n#g' | wc -l) logfiles now."
+
+(
+for i in $LOGS $ZLOGS ; do
+	DIRNAME="$(dirname $i)"
+	if [ ! -f $i ] || grep -q "$DIRNAME/log" $MYLOG ; then
+		: # echo $i already processed, continue.
+	else
+		echo "$DIRNAME/log" >> $MYLOG
+		if [ "$(basename $i)" == "log" ] ; then
+			RESULT=$(head -1 $i | grep -E -i "Started by user.*$HUMAN")
+		else
+			RESULT=$(zcat $i | head -1 | grep -E -i "Started by user.*$HUMAN")
+		fi
+		if [ -n "$RESULT" ] ; then
+			: # echo $i
+			echo "$(stat -c %w $i | cut -d ':' -f1-2) UTC, jenkins web UI, jenkins-ui, job triggered, $(echo $i | cut -d '/' -f2)"
+		else
+			: #echo autoscheduled: $i
+		fi
+	fi
+done
+ ) | sort | tee -a $MYRESULTS


=====================================
bin/djm-logparser
=====================================
@@ -15,15 +15,19 @@ set -o pipefail		# see eg http://petereisentraut.blogspot.com/2010/11/pipefail.h
 # define environment and parse parameters
 #
 
-LOGFILE=~/.djm.log
+LOCAL_LOGFILE=~/.djm.log
+UI_LOGFILE=~/.djm-jenkins-ui.log
+
+LOGFILE=$(mktemp)
 LOGMONTH="$(date -u '+%Y-%m')"
-THREE_C="%42s  %8s  %12s\n"
-TWO_C="%42s  %8s\n"
-ONE_C="%42s  %8s\n"
+cat $LOCAL_LOGFILE $UI_LOGFILE | grep ^$LOGMONTH | sort -u > $LOGFILE
+THREE_C="%42s  %12s  %12s\n"
+TWO_C="%42s  %12s\n"
+ONE_C="%42s  %12s\n"
 
 seperator() {
 	local CHAR="${1:--}"
-	for i in $(seq 1 66) ; do
+	for i in $(seq 1 79) ; do
 		echo -n "$CHAR"
 	done
 	echo
@@ -33,7 +37,7 @@ seperator() {
 # main
 #
 
-TOTAL=$(grep -c $LOGMONTH $LOGFILE)
+TOTAL=$(grep -c ^$LOGMONTH $LOGFILE)
 HOSTS=$(cut -d ',' -f2 $LOGFILE | sort -u | grep -v "web UI"| wc -l)
 HOURS=$(cut -b 1-13 $LOGFILE | sort -u | wc -l)
 TEN_MIN_SLICES=$(cut -b 1-15 $LOGFILE | sort -u | wc -l)
@@ -46,7 +50,9 @@ ACTIONS=$(cut -d ',' -f3 $LOGFILE | sort | sed 's#^#%#g'| uniq -c | sort -nr | c
 REASONS=$(cut -d ',' -f4 $LOGFILE | sort | sed 's#^#%#g'| uniq -c | sort -nr | cut -d '%' -f2| sed -s "s# #=#g")
 
 seperator "="
-echo "Still very simple statistics for djm logs at $LOGFILE"
+echo "Still very simple statistics for djm logs:"
+echo " - $LOCAL_LOGFILE"
+echo " - $UI_LOGFILE"
 echo
 echo "djm = documented jenkins maintenance"
 echo "      jenkins.debian.net exists since 2012, djm since 2023-04."
@@ -83,8 +89,26 @@ for reason in $REASONS ; do
 done
 seperator "="
 
-# todo: sort actions and reasons by highest amount
+#
+# do statistics about the jobs triggered
+#
+printf "$ONE_C" "jobs triggered:"
+printf "$ONE_C" "---------------"
+(
+	PATTERNS="setup_schroot debian_live_build maintenance node_health_check html openwrt"
+	for i in $PATTERNS  ; do
+		printf "$TWO_C" "$(grep -c reproducible_$i $LOGFILE)" "reproducible_${i}_.*"
+	done
+	PATTERNS=$(echo $PATTERNS | sed 's# #|#g')
+	for i in $(cut -d ',' -f5 $LOGFILE | grep -v -E $PATTERNS|sort -u) ; do
+		printf "$TWO_C" "$(grep -c $i $LOGFILE)" "$i"
+	done
+) | sort -n -r
+seperator "="
+
 # todo: include the number of reproducible related jenkins jobs
 # todo: include notmuch mails
 # - received but read
 # - written
+
+rm $LOGFILE



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/f8595599df4d4de0e20b29d0c642ceeec42b5c65...b1e5a412da304ff734aeb4cc6c9e00a68e6f93fa

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/f8595599df4d4de0e20b29d0c642ceeec42b5c65...b1e5a412da304ff734aeb4cc6c9e00a68e6f93fa
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230410/84a27c1e/attachment-0001.htm>