[Qa-jenkins-scm] [jenkins.debian.net] 03/03: reproducible debian: log _build.sh failures and notify us

[Mattia Rizzolo]

This is an automated email from the git hooks/post-receive script.

mattia pushed a commit to branch master
in repository jenkins.debian.net.

commit d6b5858df1845591c3c70729881b64091506c365
Author: Mattia Rizzolo <mattia at debian.org>
Date:   Thu Apr 5 21:51:21 2018 +0200

    reproducible debian: log _build.sh failures and notify us
    
    Signed-off-by: Mattia Rizzolo <mattia at debian.org>
---
 bin/reproducible_maintenance.sh |  3 ++-
 bin/reproducible_worker.sh      | 15 ++++++++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index 91dc6aa..f39a835 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -496,6 +496,7 @@ fi
 if [ "$HOSTNAME" = "$MAINNODE" ] && [ $(date -u +%H) -eq 0 ]  ; then
 	# once a day, send mail about builder problems
 	files_to_mail=(
+	/var/log/jenkins/reproducible-builder-errors.log
 	/var/log/jenkins/reproducible-stale-builds.log
 	/var/log/jenkins/reproducible-race-conditions.log
 	/var/log/jenkins/reproducible-diskspace-issues.log
@@ -510,7 +511,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] && [ $(date -u +%H) -eq 0 ]  ; then
 			if [ "$(dirname $PROBLEM)" = "/var/log/jenkins" ] ; then
 				if [ "$(basename $PROBLEM)" = "reproducible-diskspace-issues.log" ]; then
 					echo "diskspace issues should always be investigated." > $TMPFILE
-				elif [ "$(basename $PROBLEM)" != "postgresql-9.4-main.log " ]; then
+				elif [ "$(basename $PROBLEM)" != "postgresql-9.6-main.log " ]; then
 					echo "A few entries per day are normal, a few dozens or hundreds probably not." > $TMPFILE
 				fi
 				if grep -q https $PROBLEM ; then
diff --git a/bin/reproducible_worker.sh b/bin/reproducible_worker.sh
index 227ea5f..9d31b1f 100755
--- a/bin/reproducible_worker.sh
+++ b/bin/reproducible_worker.sh
@@ -21,6 +21,15 @@ common_init "$@"
 # common code defining db access
 . /srv/jenkins/bin/reproducible_common.sh
 
+notify_log_of_failure() {
+	tee -a /var/log/jenkins/reproducible-builder-errors.log <<-END
+
+		$WORKER_NAME/$BUILD_ID exited with an error.  Return code: $RETCODE
+		Check out the log at $BUILD_URL
+
+		END
+}
+
 # endless loop
 while true ; do
 	#
@@ -79,6 +88,10 @@ while true ; do
 	echo "                               see https://tests.reproducible-builds.org/cgi-bin/nph-logwatch?$WORKER_NAME/$BUILD_ID"
 	echo "================================================================================================"
 	echo
-	/srv/jenkins/bin/reproducible_build.sh $NODE1 $NODE2 >$BUILD_BASE/$BUILD_ID/console.log 2>&1
+	RETCODE=0
+	/srv/jenkins/bin/reproducible_build.sh $NODE1 $NODE2 >$BUILD_BASE/$BUILD_ID/console.log 2>&1 || RETCODE=$?
 	echo
+
+	[ "$RETCODE" -eq 0 ] |! notify_log_of_failure
+
 done

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git