[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible: split html_nodes_info job into two jobs, one to gather data and…
Holger Levsen
gitlab at salsa.debian.org
Tue Sep 25 14:01:20 BST 2018
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
4f98a241 by Holger Levsen at 2018-09-25T13:00:50Z
reproducible: split html_nodes_info job into two jobs, one to gather data and one to generate the html
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
3 changed files:
- bin/reproducible_html_nodes_info.sh
- + bin/reproducible_nodes_info.sh
- job-cfg/reproducible.yaml
Changes:
=====================================
bin/reproducible_html_nodes_info.sh
=====================================
@@ -14,13 +14,6 @@ common_init "$@"
# common code defining db access
. /srv/jenkins/bin/reproducible_common.sh
-TARGET_DIR=/srv/reproducible-results/node-information/
-mkdir -p $TARGET_DIR
-TMPFILE_SRC=$(mktemp)
-TMPFILE_NODE=$(mktemp)
-# remove old node entries which are older than two weeks
-find $TARGET_DIR/ -type f -mtime +14 -exec rm -v {} \;
-
#
# build static webpages
#
@@ -204,65 +197,3 @@ for TYPE in daily weekly ; do
publish_page debian
done
-#
-# collect node information
-#
-echo "$(date -u) - Collecting information from nodes"
-for NODE in $BUILD_NODES jenkins.debian.net ; do
- if [ "$NODE" = "jenkins.debian.net" ] ; then
- echo "$(date -u) - Trying to update $TARGET_DIR/$NODE."
- /srv/jenkins/bin/reproducible_info.sh > $TARGET_DIR/$NODE
- echo "$(date -u) - $TARGET_DIR/$NODE updated:"
- cat $TARGET_DIR/$NODE
- continue
- fi
- # call jenkins_master_wrapper.sh so we only need to track different ssh ports in one place
- # jenkins_master_wrapper.sh needs NODE_NAME and JOB_NAME
- export NODE_NAME=$NODE
- export JOB_NAME=$JOB_NAME
- echo "$(date -u) - Trying to update $TARGET_DIR/$NODE."
- set +e
- /srv/jenkins/bin/jenkins_master_wrapper.sh /srv/jenkins/bin/reproducible_info.sh > $TMPFILE_SRC
- if [ $? -eq 1 ] ; then
- echo "$(date -u) - Warning: could not update $TARGET_DIR/$NODE."
- continue
- fi
- set -e
- for KEY in $BUILD_ENV_VARS ; do
- VALUE=$(egrep "^$KEY=" $TMPFILE_SRC | cut -d "=" -f2-)
- if [ ! -z "$VALUE" ] ; then
- echo "$KEY=$VALUE" >> $TMPFILE_NODE
- fi
- done
- if [ -s $TMPFILE_NODE ] ; then
- mv $TMPFILE_NODE $TARGET_DIR/$NODE
- echo "$(date -u) - $TARGET_DIR/$NODE updated:"
- cat $TARGET_DIR/$NODE
- fi
- rm -f $TMPFILE_SRC $TMPFILE_NODE
-done
-echo
-
-echo "$(date -u) - Showing node performance:"
-TMPFILE1=$(mktemp)
-TMPFILE2=$(mktemp)
-TMPFILE3=$(mktemp)
-NOW=$(date -u '+%Y-%m-%d %H:%m')
-for i in $BUILD_NODES ; do
- query_db "SELECT build_date FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' )" > $TMPFILE1 2>/dev/null
- j=$(wc -l $TMPFILE1|cut -d " " -f1)
- k=$(cat $TMPFILE1|cut -d " " -f1|sort -u|wc -l)
- l=$(echo "scale=1 ; ($j/$k)" | bc)
- echo "$l builds/day ($j/$k) on $i" >> $TMPFILE2
- DATE=$(date '+%Y-%m-%d %H:%M' -d "-1 days")
- m=$(query_db "SELECT count(build_date) FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' ) AND r.build_date > '$DATE' " 2>/dev/null)
- if [ "$m" = "" ] ; then m=0 ; fi
- echo "$m builds in the last 24h on $i" >> $TMPFILE3
-done
-rm $TMPFILE1 >/dev/null
-sort -g -r $TMPFILE2
-echo
-sort -g -r $TMPFILE3
-rm $TMPFILE2 $TMPFILE3 >/dev/null
-echo
-
=====================================
bin/reproducible_nodes_info.sh
=====================================
@@ -0,0 +1,85 @@
+#!/bin/bash
+# vim: set noexpandtab:
+
+# Copyright © 2015-2018 Holger Levsen <holger at layer-acht.org>
+# © 2018 Mattia Rizzolo <mattia at debian.org>
+# released under the GPLv=2
+
+set -e
+
+DEBUG=false
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+TARGET_DIR=/srv/reproducible-results/node-information/
+mkdir -p $TARGET_DIR
+TMPFILE_SRC=$(mktemp)
+TMPFILE_NODE=$(mktemp)
+# remove old node entries which are older than two weeks
+find $TARGET_DIR/ -type f -mtime +14 -exec rm -v {} \;
+
+#
+# collect node information
+#
+echo "$(date -u) - Collecting information from nodes"
+for NODE in $BUILD_NODES jenkins.debian.net ; do
+ if [ "$NODE" = "jenkins.debian.net" ] ; then
+ echo "$(date -u) - Trying to update $TARGET_DIR/$NODE."
+ /srv/jenkins/bin/reproducible_info.sh > $TARGET_DIR/$NODE
+ echo "$(date -u) - $TARGET_DIR/$NODE updated:"
+ cat $TARGET_DIR/$NODE
+ continue
+ fi
+ # call jenkins_master_wrapper.sh so we only need to track different ssh ports in one place
+ # jenkins_master_wrapper.sh needs NODE_NAME and JOB_NAME
+ export NODE_NAME=$NODE
+ export JOB_NAME=$JOB_NAME
+ echo "$(date -u) - Trying to update $TARGET_DIR/$NODE."
+ set +e
+ /srv/jenkins/bin/jenkins_master_wrapper.sh /srv/jenkins/bin/reproducible_info.sh > $TMPFILE_SRC
+ if [ $? -eq 1 ] ; then
+ echo "$(date -u) - Warning: could not update $TARGET_DIR/$NODE."
+ continue
+ fi
+ set -e
+ for KEY in $BUILD_ENV_VARS ; do
+ VALUE=$(egrep "^$KEY=" $TMPFILE_SRC | cut -d "=" -f2-)
+ if [ ! -z "$VALUE" ] ; then
+ echo "$KEY=$VALUE" >> $TMPFILE_NODE
+ fi
+ done
+ if [ -s $TMPFILE_NODE ] ; then
+ mv $TMPFILE_NODE $TARGET_DIR/$NODE
+ echo "$(date -u) - $TARGET_DIR/$NODE updated:"
+ cat $TARGET_DIR/$NODE
+ fi
+ rm -f $TMPFILE_SRC $TMPFILE_NODE
+done
+echo
+
+echo "$(date -u) - Showing node performance:"
+TMPFILE1=$(mktemp)
+TMPFILE2=$(mktemp)
+TMPFILE3=$(mktemp)
+NOW=$(date -u '+%Y-%m-%d %H:%m')
+for i in $BUILD_NODES ; do
+ query_db "SELECT build_date FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' )" > $TMPFILE1 2>/dev/null
+ j=$(wc -l $TMPFILE1|cut -d " " -f1)
+ k=$(cat $TMPFILE1|cut -d " " -f1|sort -u|wc -l)
+ l=$(echo "scale=1 ; ($j/$k)" | bc)
+ echo "$l builds/day ($j/$k) on $i" >> $TMPFILE2
+ DATE=$(date '+%Y-%m-%d %H:%M' -d "-1 days")
+ m=$(query_db "SELECT count(build_date) FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' ) AND r.build_date > '$DATE' " 2>/dev/null)
+ if [ "$m" = "" ] ; then m=0 ; fi
+ echo "$m builds in the last 24h on $i" >> $TMPFILE3
+done
+rm $TMPFILE1 >/dev/null
+sort -g -r $TMPFILE2
+echo
+sort -g -r $TMPFILE3
+rm $TMPFILE2 $TMPFILE3 >/dev/null
+echo
+
=====================================
job-cfg/reproducible.yaml
=====================================
@@ -518,8 +518,11 @@
my_timed: 'H/10 * * * *'
my_shellext: ".py"
- 'html_nodes_info':
- my_description: 'Collect various information from build nodes used for reproducible builds and generate static HTML pages with various information about nodes.'
+ my_description: 'Generate static HTML pages with various information about nodes and jenkins jobs.'
my_timed: '15 0,2,4,6,8,10,12,14,16,18,20,22 * * *'
+ - 'nodes_info':
+ my_description: 'Collect various information from build nodes used for reproducible builds.'
+ my_timed: '0 0 * * *'
- 'create_meta_pkg_sets':
my_description: 'Create meta package sets for later consumption when generating HTML stats.'
my_timed: '0 6 * * H/3'
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/4f98a241b5e748f6c6eee0ed6042b3ddbe0d88af
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/4f98a241b5e748f6c6eee0ed6042b3ddbe0d88af
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20180925/0479db44/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list