[Qa-jenkins-scm] [jenkins.debian.net] 03/03: reproducible: detect enospace during builds, if so, send notifications, unregister this build, pause the builder and abort the job

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch master
in repository jenkins.debian.net.

commit 6026ac4931af6453c8f07717d6513a398325c1bb
Author: Holger Levsen <holger at layer-acht.org>
Date:   Sat Oct 3 11:43:46 2015 +0200

    reproducible: detect enospace during builds, if so, send notifications, unregister this build, pause the builder and abort the job
---
 TODO                      |  3 ---
 bin/reproducible_build.sh | 17 +++++++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/TODO b/TODO
index 375c32b..5e764d0 100644
--- a/TODO
+++ b/TODO
@@ -124,9 +124,6 @@ properties:
 === reproducible builds
 
 * make reproducible_build.sh rock solid again:
-** deal properly with out of diskspace problems, https://jenkins.debian.net/job/reproducible_builder_amd64_8/2575/ is an archived build which hit it
-*** '~jenkins/spacetest' has a list of affected logs in testing/amd64, though at least the syslinux one is a false positive
-*** this is important cause it causes packages to falsely ftbfs
 ** fix: "DIFFOSCOPE='E: Failed to change to directory /tmp: Permission denied' - maybe by making sure the cause is gone… https://jenkins.debian.net/job/reproducible_builder_amd64_14/909/ is an example for that
 ** fix disorderfs setup to *always* unmount+cleanup, this causes the full disks atm
 ** diffoscope needs to be run on the target arch..., this should probably be doable with qemu-static on the host
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 648cf36..722e9d3 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -203,6 +203,12 @@ handle_ftbfs() {
 			handle_depwait
 			return
 		fi
+		for NEEDLE in '^tar:.*Cannot write: No space left on device' 'fatal error: error writing to .* No space left on device' './configure: line .* printf: write error: No space left on device' 'cat: write error: No space left on device' ; do
+			if zgrep -e "$NEEDLE" "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then
+				handle_enospace
+				return
+			fi
+		done
 	done
 	calculate_build_duration
 	update_db_and_html "FTBFS"
@@ -266,6 +272,17 @@ handle_unhandled() {
 	exit 0
 }
 
+handle_enospace() {
+	unregister_build
+	MESSAGE="$BUILD_URL ran into diskspace problems, please investigate. Will send this builder to sleep for 3h now."
+	echo "$MESSAGE"
+	echo "$MESSAGE" | mail -s "$BUILD_URL ran into diskspace problems" qa-jenkins-scm at lists.alioth.debian.org
+	irc_msg "$MESSAGE"
+	sleep 3h
+	exec /srv/jenkins/bin/abort.sh
+	exit 0
+}
+
 dbd_timeout() {
 	local msg="$DIFFOSCOPE was killed after running into timeout after $1"
 	if [ ! -s ./${DBDREPORT} ] ; then

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git