[Qa-jenkins-scm] [jenkins.debian.net] 02/02: reproducible: update pbuilder in maintenance job too, run maintenance jobs more often, run pbuilder setup jobs less often
Holger Levsen
holger at moszumanska.debian.org
Wed Sep 2 13:35:40 UTC 2015
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit c150602dbdc879841b727fef2fe0618c23c5bfee
Author: Holger Levsen <holger at layer-acht.org>
Date: Wed Sep 2 15:35:13 2015 +0200
reproducible: update pbuilder in maintenance job too, run maintenance jobs more often, run pbuilder setup jobs less often
---
TODO | 2 +-
bin/reproducible_maintenance.sh | 34 +++++++++++-
bin/reproducible_setup_pbuilder.sh | 11 +---
job-cfg/reproducible.yaml | 104 ++++++++++++++++++-------------------
4 files changed, 86 insertions(+), 65 deletions(-)
diff --git a/TODO b/TODO
index 865b27a..dcfd2c0 100644
--- a/TODO
+++ b/TODO
@@ -182,7 +182,7 @@ properties:
* lesser prio
** get the content for freebsd/netbsd/openwrt/coreboot "<h2>status of $1</h2>" from notes.git/friends.yaml or such
** document (in README) the multihost setup
-** the setup pbuilder+schroot scripts should detect network problems such as "Bad gateway" and sleep 5m and retry once
+** the setup pbuilder+schroot setup scripts should detect network problems such as "Bad gateway" and sleep 5m and retry once
** more graphs:
*** graph average build duration by day
*** graph packages in testing+unstable which need to be fixed
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index 890185f..7b3ae8b 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -260,14 +260,17 @@ if ! $DIRTY ; then
echo
fi
-echo "$(date -u) - updating the schroots now..."
+echo "$(date -u) - updating the schroots and pbuilder now..."
+set +e
ARCH=$(dpkg --print-architecture)
for s in $SUITES ; do
if [ "$ARCH" = "armhf" ] && [ "$s" != "unstable" ] ; then
continue
fi
+ #
+ # schroot update
+ #
echo "$(date -u) - updating the $s/$ARCH schroot now."
- set +e
for i in 1 2 3 ; do
schroot --directory /root -u root -c source:jenkins-reproducible-$s -- apt-get update
RESULT=$?
@@ -283,4 +286,31 @@ for s in $SUITES ; do
if [ $RESULT -eq 1 ] ; then
echo "Warning: failed to update the $s/$ARCH schroot."
fi
+ #
+ # pbuilder update
+ #
+ echo "$(date -u) - updating pbuilder for $s/$ARCH now."
+ # use host apt proxy configuration for pbuilder
+ if [ ! -z "$http_proxy" ] ; then
+ pbuilder_http_proxy="--http-proxy $http_proxy"
+ fi
+ for i in 1 2 3 ; do
+ sudo pbuilder --update $pbuilder_http_proxy --basetgz /var/cache/pbuilder/$s-reproducible-base.tgz
+ RESULT=$?
+ if [ $RESULT -eq 1 ] ; then
+ # sleep 31-100 secs
+ echo "Sleeping some time... (to workaround network problems like 'Hash Sum mismatch'...)"
+ /bin/sleep $(echo "scale=1 ; ($(shuf -i 1-700 -n 1)/10)+30" | bc )
+ echo "$(date -u) - Retrying to update pbuilder for $s/$ARCH."
+ elif [ $RESULT -eq 0 ] ; then
+ continue
+ fi
+ done
+ if [ $RESULT -eq 1 ] ; then
+ echo "Warning: failed to update pbuilder for $s/$ARCH."
+ fi
done
+set -e
+echo "$(date -u) - the end."
+
+
diff --git a/bin/reproducible_setup_pbuilder.sh b/bin/reproducible_setup_pbuilder.sh
index 837f3b4..f30d686 100755
--- a/bin/reproducible_setup_pbuilder.sh
+++ b/bin/reproducible_setup_pbuilder.sh
@@ -114,14 +114,6 @@ setup_pbuilder() {
}
#
-# update pbuilder for reproducible builds
-#
-update_pbuilder() {
- NAME=$1
- sudo pbuilder --update --http-proxy $http_proxy --basetgz /var/cache/pbuilder/${NAME}.tgz
-}
-
-#
# main
#
BASETGZ=/var/cache/pbuilder/$SUITE-reproducible-base.tgz
@@ -135,7 +127,6 @@ if [ -n "$OLDSTAMP" ] || [ ! -f $BASETGZ ] || [ ! -f $STAMP ] ; then
fi
setup_pbuilder $SUITE $SUITE-reproducible-base dpkg dpkg-dev debhelper
else
- echo "Updating $BASETGZ..."
- update_pbuilder $SUITE-reproducible-base
+ echo "Echo $BASETGZ not old enough, doing nothing..."
fi
echo
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index a779194..623a664 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -440,128 +440,128 @@
name: reproducible
jobs:
- '{name}_maintenance':
- my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
- my_timed: '5 0,4,8,12,16,20 * * *'
+ my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+ my_timed: '5 0,3,6,9,12,15,18,21 * * *'
my_shell: '/srv/jenkins/bin/reproducible_maintenance.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: ''
- '{name}_maintenance_armhf_bpi0':
- my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
- my_timed: '5 0,4,8,12,16,20 * * *'
+ my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+ my_timed: '5 0,3,6,9,12,15,18,21 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'bpi0'
- '{name}_maintenance_armhf_cbxi4pro0':
- my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
- my_timed: '5 0,4,8,12,16,20 * * *'
+ my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+ my_timed: '5 0,3,6,9,12,15,18,21 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'cbxi4pro0'
- '{name}_maintenance_armhf_hb0':
- my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
- my_timed: '5 0,4,8,12,16,20 * * *'
+ my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+ my_timed: '5 0,3,6,9,12,15,18,21 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'hb0'
- '{name}_maintenance_armhf_wbq0':
- my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
- my_timed: '5 0,4,8,12,16,20 * * *'
+ my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+ my_timed: '5 0,3,6,9,12,15,18,21 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'wbq0'
- '{name}_maintenance_amd64_profitbricks1':
- my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
- my_timed: '5 0,4,8,12,16,20 * * *'
+ my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+ my_timed: '5 0,3,6,9,12,15,18,21 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks1'
- '{name}_maintenance_amd64_profitbricks2':
- my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
- my_timed: '5 0,4,8,12,16,20 * * *'
+ my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+ my_timed: '5 0,3,6,9,12,15,18,21 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks2'
- '{name}_setup_pbuilder_testing':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '23 7 * * *'
my_shell: '/srv/jenkins/bin/reproducible_setup_pbuilder.sh testing'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: ''
- '{name}_setup_pbuilder_unstable_armhf_bpi0':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '42 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'bpi0'
- '{name}_setup_pbuilder_unstable_armhf_cbxi4pro0':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '42 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'cbxi4pro0'
- '{name}_setup_pbuilder_unstable_armhf_hb0':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '42 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'hb0'
- '{name}_setup_pbuilder_unstable_armhf_wbq0':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '42 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'wbq0'
- '{name}_setup_pbuilder_testing_amd64_profitbricks1':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '23 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks1'
- '{name}_setup_pbuilder_testing_amd64_profitbricks2':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '23 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks2'
- '{name}_setup_pbuilder_unstable':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '42 7 * * *'
my_shell: '/srv/jenkins/bin/reproducible_setup_pbuilder.sh unstable'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: ''
- '{name}_setup_pbuilder_unstable_amd64_profitbricks1':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '42 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks1'
- '{name}_setup_pbuilder_unstable_amd64_profitbricks2':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '42 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks2'
- '{name}_setup_pbuilder_experimental':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '33 7 * * *'
my_shell: '/srv/jenkins/bin/reproducible_setup_pbuilder.sh experimental'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: ''
- '{name}_setup_pbuilder_experimental_amd64_profitbricks1':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '33 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks1'
- '{name}_setup_pbuilder_experimental_amd64_profitbricks2':
- my_description: 'Setup and update pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
- my_timed: '23 0,4,8,12,16,20 * * *'
+ my_description: 'Setup pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+ my_timed: '33 7 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks2'
- '{name}_setup_schroot_unstable_diffoscope_amd64_jenkins':
my_description: 'Setup schroot for running diffoscope in a unstable environment (this is needed to be able to correctly investigate haskell binaries...)'
- my_timed: '23 0 * * *'
+ my_timed: '42 0 * * *'
my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable-diffoscope unstable diffoscope locales-all'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: ''
@@ -573,19 +573,19 @@
my_node: ''
- '{name}_setup_schroot_unstable':
my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '42 1 * * *'
my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable unstable botch'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: ''
- '{name}_setup_schroot_unstable_amd64_profitbricks1':
my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '42 1 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks1'
- '{name}_setup_schroot_unstable_amd64_profitbricks2':
my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '42 1 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks2'
@@ -597,31 +597,31 @@
my_node: ''
- '{name}_setup_schroot_unstable_armhf_bpi0':
my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '42 1 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'bpi0'
- '{name}_setup_schroot_unstable_armhf_cbxi4pro0':
my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '42 1 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'cbxi4pro0'
- '{name}_setup_schroot_unstable_armhf_hb0':
my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '42 1 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'hb0'
- '{name}_setup_schroot_unstable_armhf_wbq0':
my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '42 1 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'wbq0'
- '{name}_setup_schroot_testing_amd64_profitbricks1':
my_description: 'Setup testing schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '42 1 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks1'
@@ -633,19 +633,19 @@
my_node: 'profitbricks2'
- '{name}_setup_schroot_experimental':
my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '33 1 * * *'
my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-experimental experimental'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: ''
- '{name}_setup_schroot_experimental_amd64_profitbricks1':
my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '33 1 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks1'
- '{name}_setup_schroot_experimental_amd64_profitbricks2':
my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.'
- my_timed: '23 1 * * *'
+ my_timed: '33 1 * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: 'profitbricks2'
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list