[Qa-jenkins-scm] [jenkins.debian.net] 01/01: reproducible armhf: enable testing of testing
Holger Levsen
holger at moszumanska.debian.org
Fri Jan 1 17:02:14 UTC 2016
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 3b5b249d024c1ce454f3f8ae43f3a8f820b405b2
Author: Holger Levsen <holger at layer-acht.org>
Date: Fri Jan 1 17:59:56 2016 +0100
reproducible armhf: enable testing of testing
---
README | 2 +-
bin/reproducible_blacklist.sh | 4 +---
bin/reproducible_common.py | 2 --
bin/reproducible_common.sh | 10 +++-------
bin/reproducible_html_dashboard.sh | 16 +---------------
bin/reproducible_html_indexes.py | 4 ----
bin/reproducible_html_notes.py | 2 --
bin/reproducible_html_packages.py | 6 ------
bin/reproducible_maintenance.sh | 6 ------
bin/reproducible_scheduler.py | 26 +++++++-------------------
job-cfg/reproducible.yaml | 15 +++------------
11 files changed, 16 insertions(+), 77 deletions(-)
diff --git a/README b/README
index 39b6cc4..32c2dde 100644
--- a/README
+++ b/README
@@ -137,7 +137,7 @@ Installation tests inside chroot environments.
* The (current) purpose of https://reproducible.debian.net is to show the prospects of reproducible builds for Debian. IOW: this is research, showing what could (and should) be done... check https://wiki.debian.org/ReproducibleBuilds for the real status of the project!
-* Currently, three suites are tested on amd64: 'testing', 'unstable' and 'experimental'. The tests are done using 'pbuilder' using link:https://wiki.debian.org/ReproducibleBuilds/ExperimentalToolchain[our toolchain] through concurrent builder jobs, 32 for 'amd64' and 21 for 'armhf', which are each constantly testing packages and saving the results of these tests.
+* Currently, three suites are tested on 'amd64' and 'armhf' architectures: 'testing', 'unstable' and 'experimental'. The tests are done using 'pbuilder' using link:https://wiki.debian.org/ReproducibleBuilds/ExperimentalToolchain[our toolchain] through concurrent builder jobs, 32 for 'amd64' and 21 for 'armhf', which are each constantly testing packages and saving the results of these tests.
** These builds on remote nodes run on very different hardware: for 'amd64' we are now using four virtual machines, profitbricks-build(1+2+5+6)-amd64, which have 16 or 17 cores and 48gb ram each and are sponsored by link:https://jenkins.debian.net/userContent/thanks.html[Profitbricks].
** To test 'armhf' we are using ten small boards donated by vagrant at d.o: six quad cores (wbq0, cbxi4pro0, ff2a, odxu4, odxu4b and odxu4c) with 2gb ram, another quad-core (rpi2b) with 1gb ram and three dual cores (bpi0, hb0 and wbd0) with 1gb ram, each. We would love to have more or more powerful ARM hardware in the future, if you can help, please talk to us!
diff --git a/bin/reproducible_blacklist.sh b/bin/reproducible_blacklist.sh
index a22b48f..0efa56d 100755
--- a/bin/reproducible_blacklist.sh
+++ b/bin/reproducible_blacklist.sh
@@ -69,9 +69,7 @@ case $SUITE in
sid) echo "WARNING: sid has been renamed to unstable."
SUITE=unstable
;;
- unstable|experimental) ;;
- testing) if [ "$ARCH" = "armhf" ] ; then echo "Testing is not yet tested on $ARCH, exiting." ; exit 0 ; fi
- ;;
+ testing|unstable|experimental) ;;
*) echo "$SUITE is not a valid suite".
explain_syntax
exit 1
diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py
index 9c60f58..2676e81 100755
--- a/bin/reproducible_common.py
+++ b/bin/reproducible_common.py
@@ -295,8 +295,6 @@ def _gen_links(suite, arch):
continue
html += link[1].format(suite=suite, arch=arch) + '\n'
for i in SUITES: # suite links
- if arch == 'armhf' and i == 'testing':
- continue
html += '<li><a href="/' + i + '/index_suite_' + arch + '_stats.html">suite: ' + i + '</a></li>'
if arch == 'amd64':
html += '<li><a href="/unstable/index_suite_armhf_stats.html\">arch: armhf</a></li>'
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 5fdac33..4cdc28f 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -235,7 +235,7 @@ write_page_header() {
continue
fi
if [ "$TARGET" = "pkg_sets" ] && [ "$ARCH" = "armhf" ] && [ "$SUITE" = "testing" ] ; then
- # testing/armhf is not being tested yet _yet_ (so I think this is never met…)
+ # no pkg_sets for testing/amd64 yet…
continue
fi
SPOKEN_TARGET=${SPOKENTARGET[$TARGET]}
@@ -253,10 +253,6 @@ write_page_header() {
done
if [ "$TARGET" = "suite_stats" ] ; then
for i in $SUITES ; do
- if [ "$i" = "testing" ] && [ "$ARCH" = "armhf" ] ; then
- # only unstable and experimental are tested on armhf atm
- continue
- fi
write_page "<li><a href=\"/$i/index_suite_${ARCH}_stats.html\">suite: $i</a></li>"
done
elif [ "$TARGET" = "scheduled" ] ; then
@@ -618,10 +614,10 @@ create_png_from_table() {
WHERE_EXTRA="$WHERE_EXTRA AND architecture = \"$ARCH\""
if [ "$ARCH" = "armhf" ] ; then
if [ $1 -eq 2 ] ; then
- # unstable/armhf was only build since 2015-08-30 (and experimental/armhf since 2015-12-19)
+ # unstable/armhf was only build since 2015-08-30 (and experimental/armhf since 2015-12-19 and testing/armhf since 2016-01-01)
WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2015-08-30'"
elif [ $1 -eq 6 ] ; then
- # armhf only has pkg sets since its complete, aka 2015-12-22
+ # armhf only has pkg sets since its complete (and only for unstable), aka 2015-12-22
WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2015-12-22'"
fi
fi
diff --git a/bin/reproducible_html_dashboard.sh b/bin/reproducible_html_dashboard.sh
index 1140af7..8c2d895 100755
--- a/bin/reproducible_html_dashboard.sh
+++ b/bin/reproducible_html_dashboard.sh
@@ -295,11 +295,7 @@ write_build_performance_stats() {
PERF_STATS[$ARCH]=$(mktemp -t reproducible-dashboard-perf-XXXXXXXX)
AGE_UNSTABLE=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='unstable' AND architecture='$ARCH' AND datum='$DATE'")
AGE_EXPERIMENTAL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='experimental' AND architecture='$ARCH' AND datum='$DATE'")
- if [ "$ARCH" != "armhf" ] ; then
- AGE_TESTING=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='testing' AND architecture='$ARCH' AND datum='$DATE'")
- else
- AGE_TESTING="-"
- fi
+ AGE_TESTING=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='testing' AND architecture='$ARCH' AND datum='$DATE'")
write_page "<td>$AGE_TESTING / $AGE_UNSTABLE / $AGE_EXPERIMENTAL days</td>"
done
write_page "</tr><tr><td>average test duration (on $DATE)</td>"
@@ -339,9 +335,6 @@ write_suite_table() {
write_page "<p>"
write_page "<table class=\"main\"><tr><th>suite</th><th>all sources packages</th><th>reproducible packages</th><th>unreproducible packages</th><th>packages failing to build</th><th>other packages</th></tr>"
for SUITE in $SUITES ; do
- if [ "$ARCH" = "armhf" ] && [ "$SUITE" = "testing" ] ; then
- continue
- fi
gather_suite_arch_stats
write_page "<tr><td>$SUITE/$ARCH</td><td>$AMOUNT"
if [ $(echo $PERCENT_TOTAL/1|bc) -lt 99 ] ; then
@@ -510,9 +503,6 @@ create_dashboard_page() {
write_page "</p><p style=\"clear:both;\">"
for ARCH in ${ARCHS} ; do
for SUITE in $SUITES ; do
- if [ "$ARCH" = "armhf" ] && [ "$SUITE" = testing ] ; then
- continue
- fi
write_page " <a href=\"/$SUITE/$ARCH/${TABLE[2]}.png\"><img src=\"/$SUITE/$ARCH/${TABLE[2]}.png\" class=\"overview\" alt=\"age of oldest reproducible build result in $SUITE/$ARCH\"></a>"
done
write_page "</p><p style=\"clear:both;\">"
@@ -533,10 +523,6 @@ update_bug_stats
update_notes_stats
for ARCH in ${ARCHS} ; do
for SUITE in $SUITES ; do
- if [ "$SUITE" = "testing" ] && [ "$ARCH" = "armhf" ] ; then
- # we only test unstable and experimental on armhf atm
- continue
- fi
update_suite_arch_stats
gather_suite_arch_stats
create_suite_arch_stats_page
diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py
index 0578676..a6bd442 100755
--- a/bin/reproducible_html_indexes.py
+++ b/bin/reproducible_html_indexes.py
@@ -512,8 +512,6 @@ def build_page(page, suite=None, arch=None):
else:
for suite in SUITES:
for arch in ARCHS:
- if arch == 'armhf' and suite == 'testing':
- continue
log.debug('global page §' + section['db_status'] +
' in ' + page + ' for ' + suite + '/' + arch)
html += build_page_section(page, section, suite, arch)[0]
@@ -539,8 +537,6 @@ bugs = get_bugs() # this variable should not be global, else merely importing _h
if __name__ == '__main__':
for arch in ARCHS:
for suite in SUITES:
- if arch == 'armhf' and suite == 'testing':
- continue
for page in pages.keys():
if 'global' not in pages[page] or not pages[page]['global']:
build_page(page, suite, arch)
diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py
index 320f75a..f7d045f 100755
--- a/bin/reproducible_html_notes.py
+++ b/bin/reproducible_html_notes.py
@@ -439,8 +439,6 @@ if __name__ == '__main__':
gen_packages_html([Package(x) for x in notes])
for suite in SUITES:
for arch in ARCHS:
- if arch == 'armhf' and suite == 'testing':
- continue
build_page('notes', suite, arch)
build_page('no_notes', suite, arch)
build_page('FTBFS', suite, arch)
diff --git a/bin/reproducible_html_packages.py b/bin/reproducible_html_packages.py
index 7d6db71..95d7e49 100755
--- a/bin/reproducible_html_packages.py
+++ b/bin/reproducible_html_packages.py
@@ -172,8 +172,6 @@ def gen_suites_links(package, current_suite, current_arch):
html += tab + '<li>{}\n'.format(a)
html += tab + '<ul class="children">\n'
for s in SUITES:
- if a == 'armhf' and s == 'testing':
- continue
status = package.get_status(s, a)
if not status: # The package is not available in that suite/arch
continue
@@ -249,8 +247,6 @@ def gen_packages_html(packages, no_clean=False):
pkg = package.name
for suite in SUITES:
for arch in ARCHS:
- if arch == 'armhf' and suite == 'testing':
- continue
status = package.get_status(suite, arch)
version = package.get_tested_version(suite, arch)
build_date = package.get_build_date(suite, arch)
@@ -306,8 +302,6 @@ def gen_all_rb_pkg_pages(no_clean=False):
def purge_old_pages():
for suite in SUITES:
for arch in ARCHS:
- if arch == 'armhf' and suite == 'testing':
- continue
log.info('Removing old pages from ' + suite + '/' + arch + '.')
try:
presents = sorted(os.listdir(RB_PKG_PATH + '/' + suite + '/' +
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index ba777dd..2a60a70 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -51,9 +51,6 @@ if [ ! -z "$http_proxy" ] ; then
pbuilder_http_proxy="--http-proxy $http_proxy"
fi
for s in $SUITES ; do
- if [ "$ARCH" = "armhf" ] && [ "$s" = "testing" ] ; then
- continue
- fi
#
# schroot update
#
@@ -177,9 +174,6 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
REASON="maintenance reschedule: reschedule builds which failed due to network errors"
for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f8 | sort -u) ; do
for ARCH in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f9 | sort -u) ; do
- if [ "$ARCH" = "armhf" ] && [ "$SUITE" = "testing" ] ; then
- continue
- fi
CANDIDATES=$(for PKG in $(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/$ARCH/" | cut -d "/" -f10 | cut -d "_" -f1) ; do echo "$PKG" ; done)
# double check those builds actually failed
TO_SCHEDULE=""
diff --git a/bin/reproducible_scheduler.py b/bin/reproducible_scheduler.py
index 6e18f5c..f6c447e 100755
--- a/bin/reproducible_scheduler.py
+++ b/bin/reproducible_scheduler.py
@@ -71,7 +71,7 @@ LIMITS = {
'experimental': {'*': 440},
},
'armhf': {
- 'testing': {'*': 0},
+ 'testing': {'*': 42},
'unstable': {'*': 150},
'experimental': {'*': 150},
},
@@ -83,7 +83,7 @@ LIMITS = {
'experimental': {1: (100, 250), 2: (200, 200), '*': 150},
},
'armhf': {
- 'testing': {1: (100, 0), 2: (200, 0), '*': 0},
+ 'testing': {1: (100, 42), 2: (200, 42), '*': 0},
'unstable': {1: (100, 75), 2: (200, 60), '*': 45},
'experimental': {1: (100, 75), 2: (200, 60), '*': 45},
},
@@ -95,7 +95,7 @@ LIMITS = {
'experimental': {1: (250, 40), 2: (350, 20), '*': 0},
},
'armhf': {
- 'testing': {1: (250, 0), 2: (350, 0), '*': 0},
+ 'testing': {1: (250, 12), 2: (350, 6), '*': 0},
'unstable': {1: (250, 12), 2: (350, 6), '*': 0},
'experimental': {1: (250, 12), 2: (350, 6), '*': 0},
}
@@ -172,11 +172,8 @@ def update_sources(suite):
sources = lzma.decompress(urlopen(remotefile).read()).decode('utf8')
log.debug('\tdownloaded')
for arch in ARCHS:
- if arch == 'armhf' and suite == 'testing':
- continue
- else:
- log.info('Updating sources db for %s/%s...', suite, arch)
- update_sources_db(suite, arch, sources)
+ log.info('Updating sources db for %s/%s...', suite, arch)
+ update_sources_db(suite, arch, sources)
def update_sources_db(suite, arch, sources):
@@ -312,8 +309,6 @@ def add_up_numbers(packages, arch):
packages_sum = '+'.join([str(len(packages[x])) for x in SUITES])
if packages_sum == '0+0+0':
packages_sum = '0'
- elif arch == 'armhf':
- packages_sum = str(len(packages['unstable']))+'+'+str(len(packages['experimental']))
return packages_sum
@@ -497,9 +492,6 @@ def scheduler(arch):
if suite not in priotized_suite_order:
priotized_suite_order.append(suite)
for suite in priotized_suite_order:
- if arch == 'armhf' and suite == 'testing':
- now_queued_here[suite] = 0
- continue
query = 'SELECT count(*) ' \
'FROM schedule AS p JOIN sources AS s ON p.package_id=s.id ' \
'WHERE s.suite="{suite}" AND s.architecture="{arch}"'
@@ -516,10 +508,7 @@ def scheduler(arch):
# update the scheduled page
generate_schedule(arch) # from reproducible_html_indexes
# build the kgb message text
- if arch != 'armhf':
- message = 'Scheduled in ' + '+'.join(SUITES) + ' (' + arch + '): '
- else:
- message = 'Scheduled in unstable+experimental (' + arch + '): '
+ message = 'Scheduled in ' + '+'.join(SUITES) + ' (' + arch + '): '
if msg_untested:
message += msg_untested
message += ' and ' if msg_new and not msg_old_ftbfs_and_depwait and not msg_old else ''
@@ -535,8 +524,7 @@ def scheduler(arch):
message += msg_old
total = [now_queued_here[x] for x in SUITES]
message += ', for ' + str(sum(total))
- if arch != 'armhf':
- message += ' or ' + '+'.join([str(now_queued_here[x]) for x in SUITES])
+ message += ' or ' + '+'.join([str(now_queued_here[x]) for x in SUITES])
message += ' packages in total.'
# only notifiy irc if there were packages scheduled in any suite
for x in SUITES:
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index e433228..396b044 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -280,18 +280,6 @@
- 'setup_pbuilder_testing':
my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
my_timed: '23 7 * * *'
- - 'setup_schroot_testing':
- my_description: 'Setup testing schroot for fetching source packages for the builder jobs.'
- my_timed: '0 1 * * *'
- my_hname:
- - 'profitbricks1': { my_arch: 'amd64' }
- - 'profitbricks2': { my_arch: 'amd64' }
- - 'profitbricks5': { my_arch: 'amd64' }
- - 'profitbricks6': { my_arch: 'amd64' }
- my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
-
- - '{name}_{my_task}_{my_arch}_{my_hname}':
- my_task:
- 'setup_pbuilder_unstable':
my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
my_timed: '42 7 * * *'
@@ -301,6 +289,9 @@
- 'setup_schroot_unstable':
my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
my_timed: '20 1 * * *'
+ - 'setup_schroot_testing':
+ my_description: 'Setup testing schroot for fetching source packages for the builder jobs.'
+ my_timed: '0 1 * * *'
- 'setup_schroot_experimental':
my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.'
my_timed: '40 1 * * *'
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list