[Qa-jenkins-scm] [jenkins.debian.net] 01/01: reproducible: s#debbindiff#diffoscope#g in most of the output
Holger Levsen
holger at moszumanska.debian.org
Sat Aug 15 13:20:36 UTC 2015
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 571b063077bc37e7b85478cbfc9fa3a4c41821db
Author: Holger Levsen <holger at layer-acht.org>
Date: Sat Aug 15 15:18:58 2015 +0200
reproducible: s#debbindiff#diffoscope#g in most of the output
---
README | 2 +-
TODO | 2 +-
bin/reproducible_breakages.py | 20 ++++++++++----------
bin/reproducible_build.sh | 16 ++++++++--------
bin/reproducible_common.sh | 8 ++++----
bin/reproducible_coreboot.sh | 4 ++--
bin/reproducible_freebsd.sh | 4 ++--
bin/reproducible_html_packages.py | 2 +-
bin/reproducible_html_repository_comparison.sh | 6 +++---
bin/reproducible_netbsd.sh | 4 ++--
bin/reproducible_openwrt.sh | 10 +++++-----
bin/schroot-create.sh | 8 ++++----
12 files changed, 43 insertions(+), 43 deletions(-)
diff --git a/README b/README
index 5084a57..6b3781d 100644
--- a/README
+++ b/README
@@ -152,7 +152,7 @@ Installation tests inside chroot environments.
* There are suite specific jobs to create the pbuilder base.tgz's per suite, which have the reproducible apt repo added. Similarly there's another job per suite to create the schroots used by the builder jobs to download the packages sources to build.
-* Then there are two more jobs to create sid and testing schroots to run debbindiff on the the two results. This is necessary since to investigate haskell binaries, debbindiff needs access to the same haskell compiler version as the investigated packages have been built with.
+* Then there are two more jobs to create sid and testing schroots to run diffoscope on the the two results. This is necessary since to investigate haskell binaries, diffoscope needs access to the same haskell compiler version as the investigated packages have been built with.
* For making sure things are considerably under control at any time, there is a maintenance job running every 4h, mostly doing cleanups.
diff --git a/TODO b/TODO
index b2cc5d5..81d8038 100644
--- a/TODO
+++ b/TODO
@@ -173,7 +173,7 @@ properties:
* higher prio:
** fix https://jenkins.debian.net/munin/debian.net/jenkins.debian.net/jenkins_builds.html which is broken since jessie upgrade
** repo-comparison: check for binaries without source
-** s#debbindiff#diffoscope#g and s#dbd#ds#g everywhere and manually rename the jobs accordingly...
+** s#debbindiff#diffoscope#g and s#dbd#ds#g everywhere and manually rename the jobs accordingly... also the .debbindiff.html+txt files need to be renamed, as well as the dbd directories...
* lesser prio
** document (in README) the multihost setup
diff --git a/bin/reproducible_breakages.py b/bin/reproducible_breakages.py
index 0551c9e..4cbfa70 100755
--- a/bin/reproducible_breakages.py
+++ b/bin/reproducible_breakages.py
@@ -27,14 +27,14 @@ def unrep_with_dbd_issues():
if not os.access(dbd, os.R_OK):
without_dbd.append((pkg, version, suite, arch))
log.warning(pkg + '/' + suite + ' (' + version + ') is '
- 'unreproducible without debbindiff file.')
+ 'unreproducible without diffoscope file.')
else:
log.debug(dbd + ' found.')
data = open(dbd, 'br').read(3)
if b'<' not in data:
bad_dbd.append((pkg, version, suite, arch))
log.warning(pkg + '/' + suite + ' (' + version + ') has '
- 'debbindiff output, but it does not seem to '
+ 'diffoscope output, but it does not seem to '
'be an html page.')
return without_dbd, bad_dbd
@@ -54,7 +54,7 @@ def not_unrep_with_dbd_file():
if os.access(dbd, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(pkg + '/' + suite + ' (' + version + ') has a '
- 'debbindiff file but it\'s not unreproducible.')
+ 'diffoscope file but it\'s not unreproducible.')
return bad_pkgs
@@ -262,28 +262,28 @@ def gen_html():
# files that should not be there (e.g. removed package without cleanup)
html += _gen_section('log files that should not be there', None,
entries=alien_log())
- html += _gen_section('debbindiff files that should not be there:', None,
+ html += _gen_section('diffoscope files that should not be there:', None,
entries=alien_dbd())
html += _gen_section('rb-pkg pages that should not be there:', None,
entries=alien_rbpkg())
html += _gen_section('buildinfo files that should not be there:', None,
entries=alien_buildinfo())
- # debbindiff report where it shouldn't be
+ # diffoscope report where it shouldn't be
html += _gen_section('are not marked as unreproducible, but they ' +
- 'have a debbindiff file:', not_unrep_with_dbd_file())
+ 'have a diffoscope file:', not_unrep_with_dbd_file())
# missing files
html += _gen_section('are built but don\'t have a buildlog:',
lack_rbuild())
html += _gen_section('are built but don\'t have a .buildinfo file:',
lack_buildinfo())
- # debbindiff troubles
+ # diffoscope troubles
without_dbd, bad_dbd = unrep_with_dbd_issues()
html += _gen_section('are marked as unreproducible, but without ' +
- 'debbindiff output - so probably debbindiff ' +
+ 'diffoscope output - so probably diffoscope ' +
'crashed:', without_dbd)
html += _gen_section('are marked as unreproducible, but their ' +
- 'debbindiff output does not seem to be an html ' +
- 'file - so probably debbindiff ran into a ' +
+ 'diffoscope output does not seem to be an html ' +
+ 'file - so probably diffoscope ran into a ' +
'timeout:', bad_dbd)
# pbuilder-satisfydepends failed
html += _gen_section('failed to match their build-dependencies:',
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 8b142ab..dc4690c 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -81,7 +81,7 @@ save_artifacts() {
# irc message
if [ ! -z "$NOTIFY" ] ; then
local MESSAGE="$URL published"
- if [ "$NOTIFY" = "debbindiff" ] ; then
+ if [ "$NOTIFY" = "diffoscope" ] ; then
MESSAGE="$MESSAGE, $DBDVERSION had troubles with these..."
fi
irc_message "$MESSAGE"
@@ -118,8 +118,8 @@ update_db_and_html() {
MESSAGE="${REPRODUCIBLE_URL}/${SUITE}/${ARCH}/${SRCPACKAGE} : reproducible ➤ ${STATUS}"
echo "\n$MESSAGE" | tee -a ${RBUILDLOG}
irc_message "$MESSAGE"
- # disable ("regular") irc notification unless it's due to debbindiff problems
- if [ ! -z "$NOTIFY" ] && [ "$NOTIFY" != "debbindiff" ] ; then
+ # disable ("regular") irc notification unless it's due to diffoscope problems
+ if [ ! -z "$NOTIFY" ] && [ "$NOTIFY" != "diffoscope" ] ; then
NOTIFY=""
fi
fi
@@ -255,7 +255,7 @@ handle_reproducible() {
calculate_build_duration
update_db_and_html "reproducible"
elif [ -f ./$DBDREPORT ] ; then
- echo "Debbindiff says the build is reproducible, but there is a debbindiff file. Please investigate" | tee -a $RBUILDLOG
+ echo "Debbindiff says the build is reproducible, but there is a diffoscope file. Please investigate" | tee -a $RBUILDLOG
handle_ftbr
elif [ ! -f b1/$BUILDINFO ] ; then
echo "Debbindiff says the build is reproducible, but there is no .buildinfo file. Please investigate" | tee -a $RBUILDLOG
@@ -271,11 +271,11 @@ dbd_timeout() {
msg="$msg, but there is still $REPRODUCIBLE_URL/dbd/$SUITE/$ARCH/$DDBREPORT"
fi
SAVE_ARTIFACTS=1
- NOTIFY="debbindiff"
+ NOTIFY="diffoscope"
handle_ftbr "$msg"
}
-call_debbindiff_on_changes_files() {
+call_diffoscope_on_changes_files() {
local TMPLOG=(mktemp --tmpdir=$TMPDIR)
echo | tee -a ${RBUILDLOG}
local TIMEOUT="30m"
@@ -314,7 +314,7 @@ call_debbindiff_on_changes_files() {
;;
2)
SAVE_ARTIFACTS=1
- NOTIFY="debbindiff"
+ NOTIFY="diffoscope"
handle_ftbr "$DBDVERSION had trouble comparing the two builds. Please investigate $REPRODUCIBLE_URL/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log"
;;
124)
@@ -561,7 +561,7 @@ update_rbuildlog
if [ $FTBFS -eq 1 ] ; then
handle_ftbfs
elif [ $FTBFS -eq 0 ] ; then
- call_debbindiff_on_changes_files # defines DBDVERSION, update_db_and_html defines STATUS
+ call_diffoscope_on_changes_files # defines DBDVERSION, update_db_and_html defines STATUS
fi
check_for_race_conditions
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 1ce6f63..a87bc81 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -237,15 +237,15 @@ write_page_header() {
write_page_intro() {
write_page " <p><em>Reproducible builds</em> enable anyone to reproduce bit by bit identical binary packages from a given source, so that anyone can verify that a given binary derived from the source it was said to be derived. There is a lot more information about <a href=\"https://wiki.debian.org/ReproducibleBuilds\">reproducible builds on the Debian wiki</a> and on <a href=\"https://reproducible.debian.net\">https://reproducible.debian.net</a>. The wiki explains in more depth wh [...]
if [ "$1" = "coreboot" ] ; then
- write_page " <em>Reproducible Coreboot</em> is an effort to apply this to coreboot. Thus each coreboot.rom is build twice (without payloads), with a few varitations added and then those two ROMs are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
+ write_page " <em>Reproducible Coreboot</em> is an effort to apply this to coreboot. Thus each coreboot.rom is build twice (without payloads), with a few varitations added and then those two ROMs are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
local PROJECTNAME="$1"
local PROJECTURL="https://review.coreboot.org/p/coreboot.git"
elif [ "$1" = "OpenWrt" ] ; then
- write_page " <em>Reproducible OpenWrt</em> is an effort to apply this to OpenWrt. Thus each OpenWrt target is build twice, with a few varitations added and then the resulting images and packages from the two builds are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>, <em>which currently cannot detect <code>.bin</code> files as squashfs filesystems.</em> Thus the resulting debbindiff output is not nearly as clear as it could be - hopefully this lim [...]
+ write_page " <em>Reproducible OpenWrt</em> is an effort to apply this to OpenWrt. Thus each OpenWrt target is build twice, with a few varitations added and then the resulting images and packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>, <em>which currently cannot detect <code>.bin</code> files as squashfs filesystems.</em> Thus the resulting diffoscope output is not nearly as clear as it could be - hopefully this lim [...]
local PROJECTNAME="openwrt"
local PROJECTURL="git://git.openwrt.org/openwrt.git"
elif [ "$1" = "NetBSD" ] ; then
- write_page " <em>Reproducible NetBSD</em> is an effort to apply this to NetBSD. Thus each NetBSD target is build twice, with a few varitations added and then the resulting files from the two builds are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
+ write_page " <em>Reproducible NetBSD</em> is an effort to apply this to NetBSD. Thus each NetBSD target is build twice, with a few varitations added and then the resulting files from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
local PROJECTNAME="netbsd"
local PROJECTURL="https://github.com/jsonn/src"
fi
@@ -371,7 +371,7 @@ irc_message() {
kgb-client --conf /srv/jenkins/kgb/debian-reproducible.conf --relay-msg "$MESSAGE" || true # don't fail the whole job
}
-call_debbindiff() {
+call_diffoscope() {
mkdir -p $TMPDIR/$1/$(dirname $2)
local TMPLOG=(mktemp --tmpdir=$TMPDIR)
local msg=""
diff --git a/bin/reproducible_coreboot.sh b/bin/reproducible_coreboot.sh
index cb36880..fb7e011 100755
--- a/bin/reproducible_coreboot.sh
+++ b/bin/reproducible_coreboot.sh
@@ -148,7 +148,7 @@ save_coreboot_results b2
# clean up builddir to save space on tmpfs
rm -r $TMPBUILDDIR/coreboot
-# run debbindiff on the results
+# run diffoscope on the results
TIMEOUT="30m"
DBDSUITE="unstable"
DBDVERSION="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-debbindiff debbindiff -- --version 2>&1)"
@@ -166,7 +166,7 @@ cd $TMPDIR/b1
for i in $(ls -1d *| sort -u) ; do
let ALL_ROMS+=1
if [ -f $i/coreboot.rom ] ; then
- call_debbindiff $i coreboot.rom
+ call_diffoscope $i coreboot.rom
get_filesize $i/coreboot.rom
if [ -f $TMPDIR/$i.html ] ; then
mv $TMPDIR/$i.html $BASE/coreboot/dbd/$i.html
diff --git a/bin/reproducible_freebsd.sh b/bin/reproducible_freebsd.sh
index 5f08ce5..b4d90df 100755
--- a/bin/reproducible_freebsd.sh
+++ b/bin/reproducible_freebsd.sh
@@ -111,7 +111,7 @@ umask 0022
# cleanup on the node
$RSSH 'sudo rm -r /usr/src /usr/obj'
-# run debbindiff on the results
+# run diffoscope on the results
TIMEOUT="30m"
DBDSUITE="unstable"
DBDVERSION="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-debbindiff debbindiff -- --version 2>&1)"
@@ -131,7 +131,7 @@ for i in * ; do
echo " <table><tr><th>Release files for <code>$i</code></th></tr>" >> $FILES_HTML
for j in $(find * -type f |sort -u ) ; do
let ALL_FILES+=1
- call_debbindiff $i $j
+ call_diffoscope $i $j
get_filesize $j
if [ -f $TMPDIR/$i/$j.html ] ; then
mkdir -p $BASE/freebsd/dbd/$i/$(dirname $j)
diff --git a/bin/reproducible_html_packages.py b/bin/reproducible_html_packages.py
index 92b9a43..0d46cce 100755
--- a/bin/reproducible_html_packages.py
+++ b/bin/reproducible_html_packages.py
@@ -116,7 +116,7 @@ def gen_extra_links(package, version, suite, arch, status):
log.debug('debbindiff not detetected at ' + dbd)
if status == 'unreproducible' and not args.ignore_missing_files:
log.critical(REPRODUCIBLE_URL + '/' + suite + '/' + arch + '/' + package +
- ' is unreproducible, but without debbindiff output.')
+ ' is unreproducible, but without diffoscope output.')
if os.access(dbdtxt, os.R_OK):
url = DBDTXT_URI + '/' + suite + '/' + arch + '/' + package + '_' + \
eversion + '.debbindiff.txt'
diff --git a/bin/reproducible_html_repository_comparison.sh b/bin/reproducible_html_repository_comparison.sh
index 8be5bb4..af3a2ca 100755
--- a/bin/reproducible_html_repository_comparison.sh
+++ b/bin/reproducible_html_repository_comparison.sh
@@ -147,7 +147,7 @@ for PKG in $SOURCEPKGS ; do
write_row "<tr><td><pre>src:$PKG</pre></td>"
write_row " <td>"
case $PKG in
- strip-nondeterminism|debbindiff)
+ strip-nondeterminism|debbindiff|diffoscope)
URL="http://anonscm.debian.org/cgit/reproducible/$PKG.git" ;;
*)
URL="http://anonscm.debian.org/cgit/reproducible/$PKG.git/?h=pu/reproducible_builds" ;;
@@ -177,7 +177,7 @@ for PKG in $SOURCEPKGS ; do
fi
else
write_row "<a href=\"$URL\">$PKG.git</a>"
- if [ "$PKG" != "strip-nondeterminism" ] && [ "$PKG" != "debbindiff" ] ; then
+ if [ "$PKG" != "strip-nondeterminism" ] && [ "$PKG" != "diffoscope" ] ; then
if $OBSOLETE_IN_TESTING && $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then
write_row "<br />(unused?"
write_row "<br /><span class=\"purple\">Then the branch should probably renamed.</span>"
@@ -187,7 +187,7 @@ for PKG in $SOURCEPKGS ; do
elif $OBSOLETE_IN_EXP ; then
write_row "<br />(only used in testing and unstable, fixed in experimental)"
fi
- elif ( [ "$PKG" = "strip-nondeterminism" ] || [ "$PKG" = "debbindiff" ] ) && $OBSOLETE_IN_SID ; then
+ elif ( [ "$PKG" = "strip-nondeterminism" ] || [ "$PKG" = "diffoscope" ] ) && $OBSOLETE_IN_SID ; then
write_row "<br />(this repo is always used)"
fi
fi
diff --git a/bin/reproducible_netbsd.sh b/bin/reproducible_netbsd.sh
index 2ad6d6a..151c4ac 100755
--- a/bin/reproducible_netbsd.sh
+++ b/bin/reproducible_netbsd.sh
@@ -117,7 +117,7 @@ umask 0022
# clean up builddir to save space on tmpfs
rm -r $TMPBUILDDIR/netbsd
-# run debbindiff on the results
+# run diffoscope on the results
TIMEOUT="30m"
DBDSUITE="unstable"
DBDVERSION="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-debbindiff debbindiff -- --version 2>&1)"
@@ -137,7 +137,7 @@ for i in * ; do
echo " <table><tr><th>Release files for <code>$i</code></th></tr>" >> $FILES_HTML
for j in $(find * -type f |sort -u ) ; do
let ALL_FILES+=1
- call_debbindiff $i $j
+ call_diffoscope $i $j
get_filesize $j
if [ -f $TMPDIR/$i/$j.html ] ; then
mkdir -p $BASE/netbsd/dbd/$i/$(dirname $j)
diff --git a/bin/reproducible_openwrt.sh b/bin/reproducible_openwrt.sh
index cf73f03..d9daca6 100755
--- a/bin/reproducible_openwrt.sh
+++ b/bin/reproducible_openwrt.sh
@@ -201,7 +201,7 @@ cat $(find build_dir/ -name banner | grep etc/banner|head -1) >> $BANNER_HTML
# clean up builddir to save space on tmpfs
rm -r $TMPBUILDDIR/openwrt
-# run debbindiff on the results
+# run diffoscope on the results
# (this needs refactoring rather badly)
TIMEOUT="30m"
DBDSUITE="unstable"
@@ -210,7 +210,7 @@ echo "==========================================================================
echo "$(date -u) - Running $DBDVERSION on OpenWrt images and packages."
echo "============================================================================="
DBD_HTML=$(mktemp --tmpdir=$TMPDIR)
-# run debbindiff on the images
+# run diffoscope on the images
GOOD_IMAGES=0
ALL_IMAGES=0
SIZE=""
@@ -234,7 +234,7 @@ for i in * ; do
rm -f $BASE/openwrt/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed
continue
fi
- call_debbindiff $i $j
+ call_diffoscope $i $j
get_filesize $j
if [ -f $TMPDIR/$i/$j.html ] ; then
mkdir -p $BASE/openwrt/dbd/$i
@@ -251,7 +251,7 @@ for i in * ; do
echo " </table>" >> $DBD_HTML
done
GOOD_PERCENT_IMAGES=$(echo "scale=1 ; ($GOOD_IMAGES*100/$ALL_IMAGES)" | bc)
-# run debbindiff on the packages
+# run diffoscope on the packages
GOOD_PACKAGES=0
ALL_PACKAGES=0
create_results_dirs
@@ -273,7 +273,7 @@ for i in * ; do
rm -f $BASE/openwrt/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed
continue
fi
- call_debbindiff $i $j
+ call_diffoscope $i $j
get_filesize $j
if [ -f $TMPDIR/$i/$j.html ] ; then
mkdir -p $BASE/openwrt/dbd/$i/$(dirname $j)
diff --git a/bin/schroot-create.sh b/bin/schroot-create.sh
index 25e7fa4..adecb72 100755
--- a/bin/schroot-create.sh
+++ b/bin/schroot-create.sh
@@ -127,17 +127,17 @@ bootstrap() {
done
set -x
sudo chroot $CHROOT_TARGET apt-get update
- # install debbindiff with all recommends...
+ # install diffoscope with all recommends...
if [ "$1" = "debbindiff" ] ; then
sudo chroot $CHROOT_TARGET apt-get install -y --install-recommends debbindiff
fi
sudo chroot $CHROOT_TARGET apt-get install -y --no-install-recommends "$@" sudo
- # always use debbindiff from unstable
+ # always use diffoscope from unstable
if [ "$SUITE" = "testing" ] && [ "$1" = "debbindiff" ] ; then
echo "deb $MIRROR unstable main" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list > /dev/null
sudo chroot $CHROOT_TARGET apt-get update
- # install debbindiff from unstable without re-adding all recommends...
- sudo chroot $CHROOT_TARGET apt-get install -y -t unstable --no-install-recommends debbindiff || echo "Warning: debbindiff from unstable is uninstallable at the moment."
+ # install diffoscope from unstable without re-adding all recommends...
+ sudo chroot $CHROOT_TARGET apt-get install -y -t unstable --no-install-recommends debbindiff || echo "Warning: diffoscope from unstable is uninstallable at the moment."
fi
if ! $DEBUG ; then set +x ; fi
if [ "$1" = "debbindiff" ] ; then
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list