[Qa-jenkins-scm] [jenkins.debian.net] 01/02: reproducible: remove stale builds from at the beginning of the build and send a daily mail about these from the maintenance job
Holger Levsen
holger at moszumanska.debian.org
Mon Sep 21 15:06:39 UTC 2015
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit f521b26f9c308ca6c1baa4ee0589d78870717457
Author: Holger Levsen <holger at layer-acht.org>
Date: Mon Sep 21 14:17:10 2015 +0200
reproducible: remove stale builds from at the beginning of the build and send a daily mail about these from the maintenance job
---
TODO | 1 -
bin/reproducible_build.sh | 9 +++++++++
bin/reproducible_html_live_status.py | 2 +-
bin/reproducible_maintenance.sh | 10 ++++++++++
4 files changed, 20 insertions(+), 2 deletions(-)
diff --git a/TODO b/TODO
index 007f16a..efaf4fc 100644
--- a/TODO
+++ b/TODO
@@ -181,7 +181,6 @@ properties:
** new page: packages which are orphaned but have a reproducible usertagged patch
** cleanup old testing + diffscope schroots on armhf nodes, diffoscope schroots on pb-amd64 nodes and old pbuilder setups on jenkins
** reproducible_build.sh changes needed:
-*** should look for other builds "running" as this build job in the db on start and cleanup these from the db and prepare bits for a daily mail ...
*** fix: "DIFFOSCOPE='E: Failed to change to directory ‘/tmp’: Permission denied" - maybe by making sure the cause is gone…
*** race condition with itself? https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/console says Warning, package php5 in unstable on armhf is probably already building elsewhere, exiting. Please check https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/ which does not compute.
*** diffoscope needs to be run on the target arch...
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 6919b76..2d6bce0 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -415,6 +415,15 @@ init_package_build() {
echo "============================================================================="
echo "Initialising reproducibly build of ${SRCPACKAGE} in ${SUITE} on ${ARCH} on $(hostname -f) now. $ANNOUNCE"
echo "============================================================================="
+ # remove previous build attempts which didnt finish correctly
+ BUILDER_PREFIX="${JOB_NAME#reproducible_builder_}/"
+ BAD_BUILDS=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT package_id, date_build_started, builder FROM schedule WHERE builder LIKE '${BUILDER_PREFIX}%'")
+ if [ ! -z "$BAD_BUILDS" ] ; then
+ # stale_builds.txt is mailed once a day by reproducible_maintenance.sh
+ echo "$(date -u) - stale builds detected, cleaning up database from these entries:" | tee -a /var/lib/jenkins/stale_builds.txt
+ echo -n $BAD_BUILDS | tee -a /var/lib/jenkins/stale_builds.txt
+ sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='' WHERE builder LIKE '${BUILDER_PREFIX}%'"
+ fi
# mark build attempt
if [ -z "$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT date_build_started FROM schedule WHERE package_id = '$SRCPKGID'")" ] ; then
sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='$DATE', builder='$BUILDER' WHERE package_id = '$SRCPKGID'"
diff --git a/bin/reproducible_html_live_status.py b/bin/reproducible_html_live_status.py
index 334a31b..1d55bb3 100755
--- a/bin/reproducible_html_live_status.py
+++ b/bin/reproducible_html_live_status.py
@@ -78,7 +78,7 @@ def generate_live_status_table(arch):
counter += 1
# the numbers 17 and 9 should really be derived from /var/lib/jenkins/jobs/reproducible_builder_${arch}_* instead of being hard-coded here...
if ( arch == 'amd64' and counter == 17 ) or ( arch == 'armhf' and counter == 9 ):
- html += '<tr><td colspan="10">There are more builds marked as currently building in the database than there are ' + arch + ' build jobs. This does not compute. Please cleanup and please automate cleanup.</td></tr>'
+ html += '<tr><td colspan="10">There are more builds marked as currently building in the database than there are ' + arch + ' build jobs. This does not compute, please investigate and fix the cause.</td></tr>'
suite = row[1]
arch = row[2]
pkg = row[3]
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index c577a65..0fd3893 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -336,6 +336,16 @@ if [ ! -z "$BADPERMS" ] ; then
echo
fi
+# once a day, send mail about stale builds
+if [ "$HOSTNAME" = "jenkins" ] && [ $(date -u +%H) -eq 0 ] ; then
+ if [ -s /var/lib/jenkins/stale_builds.txt ] ; then
+ TMPFILE=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXXX)
+ mv /var/lib/jenkins/stale_builds.txt $TMPFILE
+ cat $TMPFILE | mail -s "stale builds found" qa-jenkins-scm at lists.alioth.debian.org
+ rm -f $TMPFILE
+ fi
+fi
+
if ! $DIRTY ; then
echo "$(date -u ) - Everything seems to be fine."
echo
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list