[Qa-jenkins-scm] [jenkins.debian.net] 01/02: reproducibly: retry 'apt-get update' three times for each schroot before giving up, also fail gracefully

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch master
in repository jenkins.debian.net.

commit f2eb7acbbca71cf28dcfa8f689c3f8059e42670e
Author: Holger Levsen <holger at layer-acht.org>
Date:   Wed Sep 2 14:27:23 2015 +0200

    reproducibly: retry 'apt-get update' three times for each schroot before giving up, also fail gracefully
---
 TODO                            |  1 -
 bin/reproducible_maintenance.sh | 17 ++++++++++++++++-
 logparse/reproducible.rules     |  2 +-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/TODO b/TODO
index 9dd59a5..865b27a 100644
--- a/TODO
+++ b/TODO
@@ -238,7 +238,6 @@ properties:
 * reproducible_build.sh changes needed:
 ** diffoscope needs to be run on the target arch...
 *** can't this be run via qemu on the host?
-* maintenance job: deal with "hash sum mismatch" see https://jenkins.debian.net/job/reproducible_maintenance_amd64_profitbricks1/148/
 * exit 404 / * detect "bad gateway" as in https://jenkins.debian.net/job/reproducible_builder_amd64_4/222/ and deal with it properly
 * deal with rsync problems as in https://jenkins.debian.net/job/reproducible_builder_armhf_4/61/:
 ----
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index 492e6fe..890185f 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -267,5 +267,20 @@ for s in $SUITES ; do
 		continue
 	fi
 	echo "$(date -u) - updating the $s/$ARCH schroot now."
-	schroot --directory /root -u root -c source:jenkins-reproducible-$s -- apt-get update
+	set +e
+	for i in 1 2 3 ; do
+		schroot --directory /root -u root -c source:jenkins-reproducible-$s -- apt-get update
+		RESULT=$?
+		if [ $RESULT -eq 1 ] ; then
+			# sleep 31-100 secs
+			echo "Sleeping some time... (to workaround network problems like 'Hash Sum mismatch'...)"
+			/bin/sleep $(echo "scale=1 ; ($(shuf -i 1-700 -n 1)/10)+30" | bc )
+			echo "$(date -u) - Retrying to update the $s/$ARCH schroot."
+		elif [ $RESULT -eq 0 ] ; then
+			continue
+		fi
+	done
+	if [ $RESULT -eq 1 ] ; then
+		echo "Warning: failed to update the $s/$ARCH schroot."
+	fi
 done
diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules
index 37bedbd..1d34aad 100644
--- a/logparse/reproducible.rules
+++ b/logparse/reproducible.rules
@@ -16,4 +16,4 @@ warning /Warning: too much difference for .+, aborting. Please investigate and u
 warning /Warning: This query produces no results:.+/
 warning /Warning: couldn't delete old files from.+/
 warning /Warning: dose-deb-coinstall cannot.+/
-
+warning /Warning: failed to update the .+ schroot./

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git