[Qa-jenkins-scm] [jenkins.debian.net] 01/01: reproducible OpenWRT: initial test
Holger Levsen
holger at moszumanska.debian.org
Fri Jun 12 13:15:14 UTC 2015
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 922f670b42ac4fa6812bca17687cb7fa22cd1c38
Author: Holger Levsen <holger at layer-acht.org>
Date: Fri Jun 12 15:14:41 2015 +0200
reproducible OpenWRT: initial test
---
TODO | 2 -
bin/reproducible_coreboot.sh | 2 +-
bin/reproducible_openwrt.sh | 149 ++++++++++++++++++++++++++-----------------
job-cfg/reproducible.yaml | 9 +++
4 files changed, 99 insertions(+), 63 deletions(-)
diff --git a/TODO b/TODO
index 3fc0042..4b7ab56 100644
--- a/TODO
+++ b/TODO
@@ -251,8 +251,6 @@ egrep -R -l '(debbindiff had trouble comparing|maybe there is still )' /var/lib/
----
<nbd> wenn man einmal make defconfig ausgeführt hat, gibt's paket und target metadaten in tmp/.packageinfo und tmp/.targetinfo
<nbd> man kann ne .config mit der target auswahl und CONFIG_ALL=y erstellen und mit make defconfig auffüllen
- <nbd> macht wahrscheinlich sinn vom prozess her das bauen von paketen und das bauen der toolchain zusammen laufen zu lassen
- <nbd> weil openwrt das ja alles in einem durchgang macht
<nbd> reicht ja erstmal, mit dem basis paketsatz anzufangen
<nbd> das wird für ein target deutlich weniger als 12h brauchen
<nbd> default config ohne CONFIG_ALL dauert bei meinem laptop in der regel <30 min
diff --git a/bin/reproducible_coreboot.sh b/bin/reproducible_coreboot.sh
index f612c13..972bb8a 100755
--- a/bin/reproducible_coreboot.sh
+++ b/bin/reproducible_coreboot.sh
@@ -276,7 +276,7 @@ rm -f $ROMS_HTML $TOOLCHAIN_HTML
# the end
calculate_build_duration
print_out_duration
-irc_message "$REPRODUCIBLE_URL/coreboot/ has been updated."
+irc_message "$REPRODUCIBLE_URL/coreboot/ has been updated. ($GOOD_PERCENT% reproducible)"
echo "============================================================================="
# remove everything, we don't need it anymore...
diff --git a/bin/reproducible_openwrt.sh b/bin/reproducible_openwrt.sh
index 322c5e3..82e8bc3 100755
--- a/bin/reproducible_openwrt.sh
+++ b/bin/reproducible_openwrt.sh
@@ -22,6 +22,7 @@ create_results_dirs() {
}
call_debbindiff() {
+ mkdir -p $TMPDIR/$1
local TMPLOG=(mktemp --tmpdir=$TMPDIR)
local msg=""
set +e
@@ -29,9 +30,9 @@ call_debbindiff() {
--directory $TMPDIR \
-c source:jenkins-reproducible-${DBDSUITE}-debbindiff \
debbindiff -- \
- --html $TMPDIR/$1.html \
- $TMPDIR/b1/$1/openwrt.rom \
- $TMPDIR/b2/$1/openwrt.rom 2>&1 \
+ --html $TMPDIR/$1/$2.html \
+ $TMPDIR/b1/$1/$2 \
+ $TMPDIR/b2/$1/$2 2>&1 \
) 2>&1 >> $TMPLOG
RESULT=$?
if ! "$DEBUG" ; then set +x ; fi
@@ -39,27 +40,27 @@ call_debbindiff() {
cat $TMPLOG # print dbd output
rm -f $TMPLOG
case $RESULT in
- 0) echo "$(date -u) - $1/openwrt.rom is reproducible, yay!"
+ 0) echo "$(date -u) - $1/$2 is reproducible, yay!"
;;
1)
- echo "$(date -u) - $DBDVERSION found issues, please investigate $1/openwrt.rom"
+ echo "$(date -u) - $DBDVERSION found issues, please investigate $1/$2"
;;
2)
- msg="$(date -u) - $DBDVERSION had trouble comparing the two builds. Please investigate $1/openwrt.rom"
+ msg="$(date -u) - $DBDVERSION had trouble comparing the two builds. Please investigate $1/$2"
;;
124)
if [ ! -s $TMPDIR/$1.html ] ; then
- msg="$(date -u) - $DBDVERSION produced no output for $1/openwrt.rom and was killed after running into timeout after ${TIMEOUT}..."
+ msg="$(date -u) - $DBDVERSION produced no output for $1/$2 and was killed after running into timeout after ${TIMEOUT}..."
else
- msg="$DBDVERSION was killed after running into timeout after $TIMEOUT, but there is still $TMPDIR/$1.html"
+ msg="$DBDVERSION was killed after running into timeout after $TIMEOUT, but there is still $TMPDIR/$1/$2.html"
fi
;;
*)
- msg="$(date -u) - Something weird happened when running $DBDVERSION on $1/openwrt.rom (which exited with $RESULT) and I don't know how to handle it."
+ msg="$(date -u) - Something weird happened when running $DBDVERSION on $1/$2 (which exited with $RESULT) and I don't know how to handle it."
;;
esac
if [ ! -z $msg ] ; then
- echo $msg | tee -a $TMPDIR/$1.html
+ echo $msg | tee -a $TMPDIR/$1/$2.html
fi
}
@@ -96,14 +97,22 @@ make -j $NUM_CPU tools/install
# create html about toolchain used
#
TOOLCHAIN_HTML=$(mktemp)
-echo "<table><tr><th>Debian $(cat /etc/debian_version) package on $(dpkg --print-architecture)</th><th>installed version</th></tr>" > $TOOLCHAIN_HTML
+TARGET=$(ls -1d staging_dir/toolchain*|cut -d "-" -f2-)
+echo "<table><tr><th>Contents of <pre>build_dir/host/</pre></th></tr>" > $TOOLCHAIN_HTML
+for i in $(ls -1 build_dir/host/) ; do
+ echo " <tr><td>$i</td></tr>" >> $TOOLCHAIN_HTML
+echo "</table>" >> $TOOLCHAIN_HTML
+echo "<table><tr><th>Downloaded software built for <pre>$TARGET</pre></th></tr>" >> $TOOLCHAIN_HTML
+for i in $(ls -1 dl/) ; do
+ echo " <tr><td>$i</td></tr>" >> $TOOLCHAIN_HTML
+echo "</table>" >> $TOOLCHAIN_HTML
+echo "<table><tr><th>Debian $(cat /etc/debian_version) package on $(dpkg --print-architecture)</th><th>installed version</th></tr>" >> $TOOLCHAIN_HTML
for i in gcc binutils bzip2 flex python perl make findutils grep diff unzip gawk util-linux zlib1g-dev libc6-dev git subversion ; do
echo " <tr><td>$i</td><td>" >> $TOOLCHAIN_HTML
dpkg -s $i|grep '^Version'|cut -d " " -f2 >> $TOOLCHAIN_HTML
echo " </td></tr>" >> $TOOLCHAIN_HTML
done
echo "</table>" >> $TOOLCHAIN_HTML
-cd ../../..
echo "============================================================================="
echo "$(date -u) - Building openwrt ${OPENWRT_VERSION} images now - first build run."
@@ -116,19 +125,30 @@ nice ionice -c 3 \
make -j $NUM_CPU package/cleanup
nice ionice -c 3 \
make -j $NUM_CPU package/compile
+nice ionice -c 3 \
+ make -j $NUM_CPU package/install
+nice ionice -c 3 \
+ make -j $NUM_CPU target/install
+nice ionice -c 3 \
+ make -j $NUM_CPU package/index
-cd openwrt-builds
+cd bin
for i in * ; do
- # abuild and sharedutils are build results but not the results we are looking for...
- if [ "$i" != "abuild" ] && [ "$i" != "sharedutils" ] ; then
- mkdir $TMPDIR/b1/$i
- if [ -f $i/openwrt.rom ] ; then
- cp -p $i/openwrt.rom $TMPDIR/b1/$i/
- fi
- fi
+ cd $i
+ mkdir $TMPDIR/b1/$i
+ for j in $(find . -name "*.bin") ; do
+ cp -p $j $TMPDIR/b1/$i/
+ done
+ cd ..
done
cd ..
-rm openwrt-builds -rf
+rm bin -r
+
+#
+# clean up between builds
+#
+rm staging_dir -r
+rm target -r
echo "============================================================================="
echo "$(date -u) - Building openwrt images now - second build run."
@@ -150,6 +170,15 @@ nice ionice -c 3 \
nice ionice -c 3 \
linux64 --uname-2.6 \
make -j $NEW_NUM_CPU package/compile
+nice ionice -c 3 \
+ linux64 --uname-2.6 \
+ make -j $NEW_NUM_CPU package/install
+nice ionice -c 3 \
+ linux64 --uname-2.6 \
+ make -j $NEW_NUM_CPU target/install
+nice ionice -c 3 \
+ linux64 --uname-2.6 \
+ make -j $NEW_NUM_CPU package/index
# reset environment to default values again
export LANG="en_GB.UTF-8"
@@ -158,15 +187,17 @@ export TZ="/usr/share/zoneinfo/UTC"
export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:"
umask 0022
-cd openwrt-builds
+cd bin
for i in * ; do
- if [ -f $i/openwrt.rom ] ; then
- mkdir $TMPDIR/b2/$i
- cp -p $i/openwrt.rom $TMPDIR/b2/$i/
- fi
+ cd $i
+ mkdir $TMPDIR/b2/$i
+ for j in $(find . -name "*.bin") ; do
+ cp -p $j $TMPDIR/b2/$i/
+ done
+ cd ..
done
cd ..
-rm openwrt-builds -r
+rm bin -r
# run debbindiff on the results
TIMEOUT="30m"
@@ -175,36 +206,34 @@ DBDVERSION="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE
echo "============================================================================="
echo "$(date -u) - Running $DBDVERSION on openwrt images now"
echo "============================================================================="
-ROMS_HTML=$(mktemp)
-echo " <ul>" > $ROMS_HTML
-BAD_ROMS=0
-GOOD_ROMS=0
-ALL_ROMS=0
+IMAGES_HTML=$(mktemp)
+echo " <ul>" > $IMAGES_HTML
+GOOD_IMAGES=0
+ALL_IMAGES=0
create_results_dirs
cd $TMPDIR/b1
for i in * ; do
- let ALL_ROMS+=1
- if [ -f $i/openwrt.rom ] ; then
- call_debbindiff $i
- SIZE="$(du -h -b $i/openwrt.rom | cut -f1)"
+ cd $i
+ for j in $(find . -name "*.bin") ; do
+ let ALL_IMAGES+=1
+ call_debbindiff $i $j
+ SIZE="$(du -h -b $j | cut -f1)"
SIZE="$(echo $SIZE/1024|bc)"
- if [ -f $TMPDIR/$i.html ] ; then
- mv $TMPDIR/$i.html $BASE/openwrt/dbd/$i.html
- echo " <li><a href=\"dbd/$i.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $i</a> (${SIZE}K) is unreproducible.</li>" >> $ROMS_HTML
+ if [ -f $TMPDIR/$i/$j.html ] ; then
+ mkdir -p $BASE/openwrt/dbd/$i
+ mv $TMPDIR/$i/$j.html $BASE/openwrt/dbd/$i/$j.html
+ echo " <li><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> (${SIZE}K) is unreproducible.</li>" >> $IMAGES_HTML
else
- SHASUM=$(sha256sum $i/openwrt.rom|cut -d " " -f1)
- echo " <li><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $i ($SHASUM, ${SIZE}K) is reproducible.</li>" >> $ROMS_HTML
- let GOOD_ROMS+=1
- rm -f $BASE/openwrt/dbd/$i.html # cleanup from previous (unreproducible) tests - if needed
+ SHASUM=$(sha256sum $j|cut -d " " -f1)
+ echo " <li><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, ${SIZE}K) is reproducible.</li>" >> $IMAGES_HTML
+ let GOOD_IMAGES+=1
+ rm -f $BASE/openwrt/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed
fi
- else
- echo " <li><img src=\"/userContent/static/weather-storm.png\" alt=\"FTBFS icon\" /> $i <a href=\"${BUILD_URL}console\">failed to build</a> from source.</li>" >> $ROMS_HTML
- let BAD_ROMS+=1
- fi
+ done
+ cd ..
done
-echo " </ul>" >> $ROMS_HTML
-GOOD_PERCENT=$(echo "scale=1 ; ($GOOD_ROMS*100/$ALL_ROMS)" | bc)
-BAD_PERCENT=$(echo "scale=1 ; ($BAD_ROMS*100/$ALL_ROMS)" | bc)
+echo " </ul>" >> $IMAGES_HTML
+GOOD_PERCENT=$(echo "scale=1 ; ($GOOD_IMAGES*100/$ALL_IMAGES)" | bc)
#
# finally create the webpage
@@ -218,40 +247,40 @@ cat > $PAGE <<- EOF
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width">
<title>openwrt</title>
- <link rel='stylesheet' id='twentyfourteen-style-css' href='landing_style.css?ver=4.0' type='text/css' media='all' />
+ <!-- link rel='stylesheet' id='twentyfourteen-style-css' href='landing_style.css?ver=4.0' type='text/css' media='all' /-->
</head>
<body>
<div class="content">
<div class="page-content">
<p> </p>
- <p><center><img src="openwrt.png" width="300" class="alignnone size-medium wp-image-6" alt="openwrt" height="231" /><br />
+ <p><center><img src="logo.png" alt="openwrt logo"" /><br />
<blockquote>
<br />
- <strong>openwrt™</strong>: fast, flexible <em>and reproducible</em> Open Source firmware?
+ <strong>OpenWRT</strong>: <em>reproducible</em> wireless freedom?
</blockquote>
</center></p>
EOF
write_page " <h1>Reproducible OpenWRT</h1>"
write_page " <p><em>Reproducible builds</em> enable anyone to reproduce bit by bit identical binary packages from a given source, so that anyone can verify that a given binary derived from the source it was said to be derived. There is a lot more information about <a href=\"https://wiki.debian.org/ReproducibleBuilds\">reproducible builds on the Debian wiki</a> and on <a href=\"https://reproducible.debian.net\">https://reproducible.debian.net</a>. The wiki has a lot more information [...]
-write_page " <em>Reproducible OpenWRT</em> is an effort to apply this to openwrt. Thus each openwrt.rom is build twice (without payloads), with a few varitations added and then those two ROMs are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
-write_page " <p>There is a monthly run <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_openwrt/\">jenkins job</a> to test the <code>master</code> branch of <a href=\"https://review.openwrt.org/p/openwrt.git\">openwrt.git</a>. Currently this job is triggered more often though, because this is still under development and brand new. The jenkins job is simply running <a href=\"http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/tree/bin/reproducible_openwrt [...]
-write_page " <p>$GOOD_ROMS ($GOOD_PERCENT%) out of $ALL_ROMS built openwrt images were reproducible in our test setup, while $BAD_ROMS ($BAD_PERCENT%) failed to build from source."
+write_page " <em>Reproducible OpenWRT</em> is an effort to apply this to OpenWRT. Thus each OpenWR target is build twice, with a few varitations added and then the resulting images from the two builds are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
+write_page " <p>There is a monthly run <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_openwrt/\">jenkins job</a> to test the <code>master</code> branch of <a href=\"git://git.openwrt.org/openwrt.git\">openwrt.git</a>. Currently this job is triggered more often though, because this is still under development and brand new. The jenkins job is simply running <a href=\"http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/tree/bin/reproducible_openwrt.sh\">r [...]
+write_page " <p>$GOOD_IMAGES ($GOOD_PERCENT%) out of $ALL_IMAGES built openwrt images were reproducible in our test setup."
write_page " These tests were last run on $DATE for version ${OPENWRT_VERSION}.</p>"
write_explaination_table openwrt
-cat $ROMS_HTML >> $PAGE
+cat $IMAGES_HTML >> $PAGE
write_page " <p><pre>"
echo -n "$OPENWRT" >> $PAGE
write_page " </pre></p>"
cat $TOOLCHAIN_HTML >> $PAGE
write_page " </div></div>"
-write_page_footer openwrt
+write_page_footer OpenWRT
publish_page
-rm -f $ROMS_HTML $TOOLCHAIN_HTML
+rm -f $IMAGES_HTML $TOOLCHAIN_HTML
# the end
calculate_build_duration
print_out_duration
-irc_message "$REPRODUCIBLE_URL/openwrt/ has been updated."
+irc_message "$REPRODUCIBLE_URL/openwrt/ has been updated. ($GOOD_PERCENT% reproducible)"
echo "============================================================================="
# remove everything, we don't need it anymore...
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 1187724..5b52fbd 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -266,6 +266,10 @@
defaults: reproducible_other_projects
name: '{name}_coreboot'
+- job-template:
+ defaults: reproducible_other_projects
+ name: '{name}_openwrt'
+
- project:
name: reproducible
jobs:
@@ -410,4 +414,9 @@
my_timed: '23 42 23 * *'
my_shell: '/srv/jenkins/bin/reproducible_coreboot.sh'
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
+ - '{name}_coreboot':
+ my_description: 'Try to rebuild OpenWRT (some targets, some packages) twice (with variations) and investigate them regarding reproducibility.'
+ my_timed: '23 42 23 * *'
+ my_shell: '/srv/jenkins/bin/reproducible_openwrt.sh'
+ my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list