[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible NetBSD: various updates by Jan-Benedict Glaw <jbglaw at lug-owl.de>
Holger Levsen (@holger)
gitlab at salsa.debian.org
Fri Jun 16 23:25:44 BST 2023
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
52df83b2 by Holger Levsen at 2023-06-17T00:25:31+02:00
reproducible NetBSD: various updates by Jan-Benedict Glaw <jbglaw at lug-owl.de>
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/reproducible_netbsd.sh
Changes:
=====================================
bin/reproducible_netbsd.sh
=====================================
@@ -1,8 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
# Copyright 2014-2022 Holger Levsen <holger at layer-acht.org>
# released under the GPLv2
+
+PAGE=netbsd/netbsd.html
DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"
@@ -12,25 +14,29 @@ common_init "$@"
set -e
-# build for these architectures
-MACHINES="sparc64 amd64"
+# Build for these architectures/ports.
+declare -a machines # Contains <arch>-<machine> pairs.
+machines=( sparc64-sparc64 x86_64-amd64 )
+
cleanup_tmpdirs() {
cd
- rm -r $TMPDIR
- rm -r $TMPBUILDDIR
+ rm -r "${TMPDIR}"
+ rm -r "${TMPBUILDDIR}"
}
create_results_dirs() {
- mkdir -p $BASE/netbsd/dbd
+ mkdir -p "${BASE}/netbsd/dbd"
}
save_netbsd_results() {
- local RUN=$1
- local MACHINE=$2
- mkdir -p $TMPDIR/$RUN/${MACHINE}
- cp -pr obj/releasedir/${MACHINE} $TMPDIR/$RUN/
- find $TMPDIR/$RUN/ -name MD5 -o -name SHA512 | xargs -r rm -v
+ local RUN="${1}"; shift
+ local FULL_MACHINE="${1}"; shift
+ local MACHINE_ONLY="$(echo "${FULL_MACHINE}" | cut -f 2 -d -)"
+
+ mkdir -p "$TMPDIR/$RUN/${FULL_MACHINE}"
+ cp -pr "obj/releasedir/${MACHINE_ONLY}" "$TMPDIR/$RUN/"
+ find "${TMPDIR}/${RUN}/" \( -name MD5 -o -name SHA512 \) -exec rm {} \;
}
#
@@ -42,139 +48,150 @@ DATE=$(date -u +'%Y-%m-%d')
START=$(date +'%s')
trap cleanup_tmpdirs INT TERM EXIT
-cd $TMPBUILDDIR
-echo "============================================================================="
-echo "$(date -u) - Cloning the NetBSD git repository (which is synced with the NetBSD CVS repository)"
-echo "============================================================================="
-git clone --depth 1 https://github.com/NetBSD/src.git
-mv src netbsd
-cd netbsd
-NETBSD="$(git log -1)"
-NETBSD_VERSION=$(git describe --always)
-echo "This is netbsd $NETBSD_VERSION."
-echo
-git log -1
-
-# from $src/share/mk/bsd.README:
-# MKREPRO If "yes", create reproducable builds. This enables
-# different switches to make two builds from the same source tree
-# result in the same build results.
-# also see http://man.netbsd.org/HEAD/usr/share/man/html5/mk.conf.html
-export MKREPRO="yes"
-# MKREPRO_TIMESTAMP is set to SOURCE_DATE_EPOCH of netbsd.git
-# see http://cvsweb.netbsd.org/bsdweb.cgi/~checkout~/src/BUILDING
-SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-MKREPRO_TIMESTAMP=${SOURCE_DATE_EPOCH}
+pushd "${TMPBUILDDIR}"
+ # Prepare sources.
+ echo "============================================================================="
+ echo "$(date -u) - Cloning the NetBSD git repository (which is synced with the NetBSD CVS repository)"
+ echo "============================================================================="
+ git clone --depth 1 https://github.com/NetBSD/src.git netbsd
+
+ # Get current top commit infos.
+ pushd netbsd-clean
+ NETBSD_TOP_COMMIT_LOG="$(git log -1)"
+ NETBSD_TOP_COMMIT_REV=$(git describe --always)
+ NETBSD_TIMESTAMP=$(git log -1 --format=%ct)
+ echo "This is NetBSD ${NETBSD_TOP_COMMIT_REV}."
+ echo
+ git log -1
+ popd
+
+ # First round of builds.
+ echo "============================================================================="
+ echo "$(date -u) - Building NetBSD ${NETBSD_TOP_COMMIT_REV} - first build run."
+ echo "============================================================================="
+ export TZ="/usr/share/zoneinfo/Etc/GMT+12"
+ for MACHINE in "${machines[@]}" ; do
+ this_subdir="b1-${MACHINE}"
+ this_arch="$(echo "${MACHINE}" | cut -f 1 -d -)"
+ this_mach="$(echo "${MACHINE}" | cut -f 2 -d -)"
+
+ # Create a pristine linked tree.
+ cp -Rl netbsd "${this_subdir}"
+
+ # Build and store results.
+ pushd "${this_subdir}"
+ ionice -c 3 ./build.sh -j "${NUM_CPU}" -P -U -u -a "${this_arch}" -m "${this_mach}" -N 1 tools release
+ save_netbsd_results b1 "${MACHINE}"
+ echo "${MACHINE} done, first time."
+ popd
+
+ # Clean up.
+ rm -rf "${this_subdir}"
+ done
-echo "============================================================================="
-echo "$(date -u) - Building NetBSD ${NETBSD_VERSION} - first build run."
-echo "============================================================================="
-export TZ="/usr/share/zoneinfo/Etc/GMT+12"
-# actually build everything
-for MACHINE in $MACHINES ; do
- ionice -c 3 \
- ./build.sh -j $NUM_CPU -V MKREPRO_TIMESTAMP=$MKREPRO_TIMESTAMP -U -u -m ${MACHINE} -N 1 release
- # save results in b1
- save_netbsd_results b1 ${MACHINE}
- # cleanup and explicitly delete old tooldir to force re-creation for the next $MACHINE type
- ./build.sh -U -m ${MACHINE} -N 1 cleandir
- rm obj/tooldir.* -rf
- echo "${MACHINE} done, first time."
-done
+ # Second round of builds.
+ echo "============================================================================="
+ echo "$(date -u) - Building NetBSD - second build run."
+ echo "============================================================================="
+ export TZ="/usr/share/zoneinfo/Etc/GMT-14"
+ export LANG="fr_CH.UTF-8"
+ export LC_ALL="fr_CH.UTF-8"
+ export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path"
+ export CAPTURE_ENVIRONMENT="I capture the environment"
+ umask 0002
+ NEW_NUM_CPU=$(( ${NUM_CPU} - 1 ))
+ for MACHINE in "${machines[@]}"; do
+ this_subdir="b2-${MACHINE}"
+ this_arch="$(echo "${MACHINE}" | cut -f 1 -d -)"
+ this_mach="$(echo "${MACHINE}" | cut -f 2 -d -)"
+
+ # Create a pristine linked tree.
+ cp -Rl netbsd "${this_subdir}"
+
+ # Build and store results.
+ pushd "${this_subdir}"
+ ionice -c 3 \
+ linux64 --uname-2.6 \
+ ./build.sh -j "${NEW_NUM_CPU}" -P -U -u -a "${this_arch}" -m "${this_mach}" -N 1 tools release
+ save_netbsd_results b2 "${MACHINE}"
+ echo "${MACHINE} done, second time."
+ popd
+
+ # Clean up.
+ rm -rf "${this_subdir}"
+ done
+popd
-echo "============================================================================="
-echo "$(date -u) - Building NetBSD ${NETBSD_VERSION} - cleaning up between builds."
-echo "============================================================================="
-rm obj/releasedir -r
-rm obj/destdir.* -r
-# we keep the toolchain(s)
-echo "============================================================================="
-echo "$(date -u) - Building NetBSD - second build run."
-echo "============================================================================="
-export TZ="/usr/share/zoneinfo/Etc/GMT-14"
-export LANG="fr_CH.UTF-8"
-export LC_ALL="fr_CH.UTF-8"
-export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path"
-export CAPTURE_ENVIRONMENT="I capture the environment"
-umask 0002
-# use allmost all cores for second build
-NEW_NUM_CPU=$(echo $NUM_CPU-1|bc)
-for MACHINE in $MACHINES ; do
- ionice -c 3 \
- linux64 --uname-2.6 \
- ./build.sh -j $NEW_NUM_CPU -V MKREPRO_TIMESTAMP=$MKREPRO_TIMESTAMP -U -u -m ${MACHINE} -N 1 release
- # save results in b2
- save_netbsd_results b2 ${MACHINE}
- # cleanup and explicitly delete old tooldir to force re-creation for the next $MACHINE type
- ./build.sh -U -m ${MACHINE} -N 1 cleandir
- rm obj/tooldir.* -r
- echo "${MACHINE} done, second time."
-done
-# reset environment to default values again
+# Reset environment to default values again.
export LANG="en_GB.UTF-8"
unset LC_ALL
export TZ="/usr/share/zoneinfo/UTC"
export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:"
umask 0022
-# clean up builddir to save space on tmpfs
-rm -r $TMPBUILDDIR/netbsd
+# Delete sources.
+rm -r "${TMPBUILDDIR}/netbsd"
-# temporarily enable more debug
+# Temporarily enable more debug output.
set -x
-# run diffoscope on the results
-TIMEOUT="30m"
-DIFFOSCOPE="$(schroot --directory /tmp -c chroot:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
+# Run diffoscope on the results.
+DIFFOSCOPE="$(schroot --directory /tmp -c "chroot:jenkins-reproducible-${DBDSUITE}-diffoscope" diffoscope -- --version 2>&1)"
echo "============================================================================="
echo "$(date -u) - Running $DIFFOSCOPE on NetBSD build results..."
echo "============================================================================="
-FILES_HTML=$(mktemp --tmpdir=$TMPDIR)
-GOOD_FILES_HTML=$(mktemp --tmpdir=$TMPDIR)
-BAD_FILES_HTML=$(mktemp --tmpdir=$TMPDIR)
-GOOD_SECTION_HTML=$(mktemp --tmpdir=$TMPDIR)
-BAD_SECTION_HTML=$(mktemp --tmpdir=$TMPDIR)
+FILES_HTML=$(mktemp --tmpdir="${TMPDIR}")
+GOOD_FILES_HTML=$(mktemp --tmpdir="${TMPDIR}")
+BAD_FILES_HTML=$(mktemp --tmpdir="${TMPDIR}")
+GOOD_SECTION_HTML=$(mktemp --tmpdir="${TMPDIR}")
+BAD_SECTION_HTML=$(mktemp --tmpdir="${TMPDIR}")
GOOD_FILES=0
ALL_FILES=0
SIZE=""
create_results_dirs
-cd $TMPDIR/b1
+cd "${TMPDIR}/b1"
tree .
-for i in * ; do
- cd $i
- for j in $(find * -type f |sort -u ) ; do
- let ALL_FILES+=1
- call_diffoscope $i $j
- get_filesize $j
- if [ -f $TMPDIR/$i/$j.html ] ; then
- mkdir -p $BASE/netbsd/dbd/$i/$(dirname $j)
- mv $TMPDIR/$i/$j.html $BASE/netbsd/dbd/$i/$j.html
- echo " <tr><td><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($SIZE) is unreproducible.</td></tr>" >> $BAD_FILES_HTML
- else
- SHASUM=$(sha256sum $j|cut -d " " -f1)
- echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $SIZE) is reproducible.</td></tr>" >> $GOOD_FILES_HTML
- let GOOD_FILES+=1
- rm -f $BASE/netbsd/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed
- fi
- done
- cd ..
- if [ -s $GOOD_FILES_HTML ] ; then
- echo " <table><tr><th>Reproducible artifacts for <code>$i</code></th></tr>" >> $GOOD_SECTION_HTML
- cat $GOOD_FILES_HTML >> $GOOD_SECTION_HTML
- echo " </table>" >> $GOOD_SECTION_HTML
- rm $GOOD_FILES_HTML
+for i in *; do
+ pushd "${i}"
+ for j in $(find * -type f |sort -u ) ; do
+ let ALL_FILES+=1
+ call_diffoscope "${i}" "${j}"
+ get_filesize "${j}"
+ if [ -f "${TMPDIR}/${i}/${j}.html" ]; then
+ mkdir -p "${BASE}/netbsd/dbd/${i}/$(dirname "${j}")"
+ mv "${TMPDIR}/${i}/${j}.html" "${BASE}/netbsd/dbd/${i}/${j}.html"
+ echo " <tr><td><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($SIZE) is unreproducible.</td></tr>" >> "${BAD_FILES_HTML}"
+ else
+ SHASUM=$(sha256sum "${j}" | cut -d " " -f1)
+ echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $SIZE) is reproducible.</td></tr>" >> "${GOOD_FILES_HTML}"
+ let GOOD_FILES+=1
+ rm -f "${BASE}/netbsd/dbd/${i}/${j}.html" # cleanup from previous (unreproducible) tests - if needed
+ fi
+ done
+ popd
+
+ if [ -s "${GOOD_FILES_HTML}" ]; then
+ {
+ echo " <table><tr><th>Reproducible artifacts for <code>$i</code></th></tr>"
+ cat "${GOOD_FILES_HTML}"
+ echo " </table>"
+ } >> "${GOOD_SECTION_HTML}"
fi
- if [ -s $BAD_FILES_HTML ] ; then
- echo " <table><tr><th>Unreproducible artifacts for <code>$i</code></th></tr>" >> $BAD_SECTION_HTML
- cat $BAD_FILES_HTML >> $BAD_SECTION_HTML
- echo " </table>" >> $BAD_SECTION_HTML
- rm $BAD_FILES_HTML
+ rm "${GOOD_FILES_HTML}"
+
+ if [ -s "${BAD_FILES_HTML}" ]; then
+ {
+ echo " <table><tr><th>Unreproducible artifacts for <code>$i</code></th></tr>"
+ cat "${BAD_FILES_HTML}"
+ echo " </table>"
+ } >> "${BAD_SECTION_HTML}"
fi
+ rm "${BAD_FILES_HTML}"
done
GOOD_PERCENT=$(echo "scale=1 ; ($GOOD_FILES*100/$ALL_FILES)" | bc)
-# are we there yet?
+# Are we there yet?
if [ "$GOOD_PERCENT" = "100.0" ] ; then
MAGIC_SIGN="!"
else
@@ -184,9 +201,9 @@ fi
#
# finally create the webpage
#
-cd $TMPDIR ; mkdir netbsd
-PAGE=netbsd/netbsd.html
-cat > $PAGE <<- EOF
+pushd "${TMPDIR}"
+ mkdir netbsd
+ cat > "${PAGE}" <<- EOF
<!DOCTYPE html>
<html lang="en-US">
<head>
@@ -203,26 +220,27 @@ cat > $PAGE <<- EOF
<div class="content">
<div class="page-content">
EOF
-write_page_intro NetBSD
-write_page " <p>$GOOD_FILES ($GOOD_PERCENT%) out of $ALL_FILES built NetBSD files were reproducible in our test setup"
-if [ "$GOOD_PERCENT" = "100.0" ] ; then
- write_page "!"
-else
- write_page "."
-fi
-write_page " These tests were last run on $DATE for version ${NETBSD_VERSION} with MKREPRO=yes and MKREPRO_TIMESTAMP=$MKREPRO_TIMESTAMP and were compared using ${DIFFOSCOPE}.</p>"
-write_variation_table NetBSD
-cat $BAD_SECTION_HTML >> $PAGE
-cat $GOOD_SECTION_HTML >> $PAGE
-write_page " <p><pre>"
-echo -n "$NETBSD" >> $PAGE
-write_page " </pre></p>"
-write_page " </div></div>"
-write_page_footer NetBSD
-publish_page
-rm -f $FILES_HTML $GOOD_FILES_HTML $BAD_FILES_HTML $GOOD_SECTION_HTML $BAD_SECTION_HTML
-
-# the end
+ write_page_intro NetBSD
+ write_page " <p>$GOOD_FILES ($GOOD_PERCENT%) out of $ALL_FILES built NetBSD files were reproducible in our test setup"
+ if [ "$GOOD_PERCENT" = "100.0" ] ; then
+ write_page "!"
+ else
+ write_page "."
+ fi
+ write_page " These tests were last run on ${DATE} for rev ${NETBSD_TOP_COMMIT_REV} with -P (as of @${NETBSD_TIMESTAMP}) and were compared using ${DIFFOSCOPE}.</p>"
+ write_variation_table NetBSD
+ cat "${BAD_SECTION_HTML}" >> "${PAGE}"
+ cat "${GOOD_SECTION_HTML}" >> "${PAGE}"
+ write_page " <p><pre>"
+ echo -n "${NETBSD_TOP_COMMIT_LOG}" >> "${PAGE}"
+ write_page " </pre></p>"
+ write_page " </div></div>"
+ write_page_footer NetBSD
+ publish_page
+ rm -f "${FILES_HTML}" "${GOOD_FILES_HTML}" "${BAD_FILES_HTML}" "${GOOD_SECTION_HTML}" "${BAD_SECTION_HTML}"
+popd
+
+# The end.
calculate_build_duration
print_out_duration
irc_message reproducible-changes "$REPRODUCIBLE_URL/netbsd/ has been updated. ($GOOD_PERCENT% reproducible)"
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/52df83b240e8b47f83305ce979dd849fc36f47ff
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/52df83b240e8b47f83305ce979dd849fc36f47ff
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230616/431ac859/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list