[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] Use the rebuild.sh script from live-build
Holger Levsen (@holger)
gitlab at salsa.debian.org
Sat Apr 23 12:25:43 BST 2022
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
b6cea0e1 by Roland Clobus at 2022-04-23T13:25:31+02:00
Use the rebuild.sh script from live-build
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/reproducible_debian_live_build.sh
Changes:
=====================================
bin/reproducible_debian_live_build.sh
=====================================
@@ -2,9 +2,11 @@
# vim: set noexpandtab:
# Copyright 2021-2022 Holger Levsen <holger at layer-acht.org>
-# Copyright 2021 Roland Clobus <rclobus at rclobus.nl>
+# Copyright 2021-2022 Roland Clobus <rclobus at rclobus.nl>
# released under the GPLv2
+# Coding convention: enforced by 'shfmt'
+
DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"
@@ -12,69 +14,88 @@ common_init "$@"
# common code for tests.reproducible-builds.org
. /srv/jenkins/bin/reproducible_common.sh
set -e
-set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html
+set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html
output_echo() {
- set +x
- echo "###########################################################################################"
- echo
- echo -e "$(date -u) - $1"
- echo
- if $DEBUG ; then
- set -x
- fi
+ set +x
+ echo "###########################################################################################"
+ echo
+ echo -e "$(date -u) - $1"
+ echo
+ if $DEBUG; then
+ set -x
+ fi
}
cleanup() {
- output_echo Cleanup $1
- # Cleanup the workspace
- if [ ! -z "${BUILDDIR}" ] ; then
- sudo rm -rf --one-file-system ${BUILDDIR}
- fi
- # Cleanup the results
- if [ ! -z "${RESULTSDIR}" ] ; then
- rm -rf --one-file-system ${RESULTSDIR}
- fi
+ output_echo "Publishing results."
+
+ # Do not publish the ISO images as artifact, they would be able to consume too much disk space
+ if [ ! -z "${RESULTSDIR}" ]; then
+ rm -rf ${RESULTSDIR}/b1
+ rm -rf ${RESULTSDIR}/b2
+ fi
+
+ if [ "$1" == "success" ]; then
+ export PAGE=${CONFIGURATION}-${DEBIAN_VERSION}.txt
+ mv ${RESULTSDIR}/summary_build1.txt ${PAGE}
+ rm ${RESULTSDIR}/summary_build2.txt
+ output_echo "Info: no differences found."
+ else
+ if [ -f "${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html" ]; then
+ # Publish the output of diffoscope, there are differences
+ export PAGE=${CONFIGURATION}-${DEBIAN_VERSION}.html
+ cp -a ${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html ${PAGE}
+ output_echo "Warning: diffoscope detected differences in the images."
+ else
+ export PAGE=${CONFIGURATION}-${DEBIAN_VERSION}.txt
+ printenv >${PAGE}
+ output_echo "Error: Something went wrong."
+ fi
+ if [ ! -z "${RESULTSDIR}" ]; then
+ TMPDIR=${RESULTSDIR}
+ save_artifacts debian_live_build ${CONFIGURATION}-${DEBIAN_VERSION} https://wiki.debian.org/ReproducibleInstalls/LiveImages
+ fi
+ fi
+ publish_page debian_live_build
+
+ output_echo Cleanup $1
+ # Cleanup the workspace
+ if [ ! -z "${BUILDDIR}" ]; then
+ sudo rm -rf --one-file-system ${BUILDDIR}
+ fi
+ # Cleanup the results
+ if [ ! -z "${RESULTSDIR}" ]; then
+ rm -rf --one-file-system ${RESULTSDIR}
+ fi
}
#
# main: follow https://wiki.debian.org/ReproducibleInstalls/LiveImages
#
-# Cleanup if something goes wrong
-trap cleanup INT TERM EXIT
-
-# Validate commandline arguments
# Argument 1 = image type
-case $1 in
- "smallest-build") export INSTALLER="none"; export PACKAGES=""; ;;
- "cinnamon") export INSTALLER="live"; export PACKAGES="live-task-cinnamon"; ;;
- "gnome") export INSTALLER="live"; export PACKAGES="live-task-gnome"; ;;
- "kde") export INSTALLER="live"; export PACKAGES="live-task-kde"; ;;
- "lxde") export INSTALLER="live"; export PACKAGES="live-task-lxde"; ;;
- "lxqt") export INSTALLER="live"; export PACKAGES="live-task-lxqt"; ;;
- "mate") export INSTALLER="live"; export PACKAGES="live-task-mate"; ;;
- "standard") export INSTALLER="live"; export PACKAGES="live-task-standard"; ;;
- "xfce") export INSTALLER="live"; export PACKAGES="live-task-xfce"; ;;
- *) output_echo "Error: Bad argument 1, image type: $1"; exit 1; ;;
-esac
export CONFIGURATION="$1"
# Argument 2 = Debian version
-# No further validation required, reproducible.yaml should be correct
-if [ -z "$2" ] ; then
- output_echo "Error: Bad argument 2, Debian version: it is empty"
- exit 2
-fi
export DEBIAN_VERSION="$2"
-if $DEBUG ; then
- export WGET_OPTIONS=
+# Two arguments are required
+if [ -z "${CONFIGURATION}" -o -z "${DEBIAN_VERSION}" ]; then
+ output_echo "Error: Bad command line arguments."
+ exit 1
+fi
+
+# Cleanup if something goes wrong
+trap cleanup INT TERM EXIT
+
+if $DEBUG; then
+ export WGET_OPTIONS=
else
- export WGET_OPTIONS=--quiet
+ export WGET_OPTIONS=--quiet
fi
-# randomize start time by 1-23 sec
+# Randomize start time
delay_start
# Cleanup possible artifacts of a previous build (see reproducible_common.sh for the path)
@@ -87,166 +108,68 @@ export BUILDDIR=$(mktemp --tmpdir=/srv/workspace/live-build -d -t ${CONFIGURATIO
cd ${BUILDDIR}
export RESULTSDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t ${PROJECTNAME}-${CONFIGURATION}-${DEBIAN_VERSION}-XXXXXXXX) # accessible in schroots, used to compare results
-# Fetch and use the latest version of live build
-export LIVE_BUILD=${BUILDDIR}/latest-live-build
-git clone https://salsa.debian.org/live-team/live-build.git ${LIVE_BUILD} --single-branch --no-tags
-pushd ${LIVE_BUILD}
-output_echo "Info: using live-build from git version $(git log -1 --pretty=oneline)"
-popd
-
-export LB_OUTPUT=${RESULTSDIR}/lb_output.txt
-# Use the timestamp of the latest mirror snapshot
-wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/latest --output-document ${RESULTSDIR}/latest
-#
-# Extract the timestamp from the JSON file
-#
-# Input:
-# {
-# "_api": "0.3",
-# "_comment": "notset",
-# "result": "20210828T083909Z"
-# }
-# Output:
-# 20210828T083909Z
-#
-export SNAPSHOT_TIMESTAMP=$(cat ${RESULTSDIR}/latest | awk '/"result":/ { split($0, a, "\""); print a[4] }')
-# Convert SNAPSHOT_TIMESTAMP to Unix time (insert suitable formatting first)
-export SOURCE_DATE_EPOCH=$(date -d $(echo ${SNAPSHOT_TIMESTAMP} | awk '{ printf "%s-%s-%sT%s:%s:%sZ", substr($0,1,4), substr($0,5,2), substr($0,7,2), substr($0,10,2), substr($0,12,2), substr($0,14,2) }') +%s)
-export MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP}
-output_echo "Info: using the snapshot from ${SOURCE_DATE_EPOCH} (${SNAPSHOT_TIMESTAMP})"
-
-# Configuration for the live image:
-# - For /etc/apt/sources.list: Use the mirror from ${MIRROR}, no security, no updates
-# - Don't cache the downloaded content, re-download for the second build
-# - Explicitly use the proxy that is set by ${http_proxy} to reduce some network traffic
-output_echo "Running lb config for the 1st build."
-lb config \
- --parent-mirror-bootstrap ${MIRROR} \
- --parent-mirror-binary ${MIRROR} \
- --security false \
- --updates false \
- --distribution ${DEBIAN_VERSION} \
- --debian-installer ${INSTALLER} \
- --cache-packages false \
- --apt-http-proxy ${http_proxy} \
- 2>&1 | tee $LB_OUTPUT
-RESULT=$?
-if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: lb config failed with $RESULT."
-fi
-
-if [ ! -z "${PACKAGES}" ] ; then
- echo "${PACKAGES}" > config/package-lists/desktop.list.chroot
-fi
-
-# Add additional hooks, that work around known reproducible issues
-cp ${LIVE_BUILD}/examples/hooks/reproducible/* config/hooks/normal
+# Fetch the rebuild script (and nothing else)
+git clone https://salsa.debian.org/live-team/live-build.git rebuild_script --no-checkout --depth 1
+cd rebuild_script
+git checkout HEAD test/rebuild.sh
+cd ..
# First build
-output_echo "Running lb build for the 1st build."
-set +e # We are interested in the result of 'lb build', so do not fail on errors
-sudo lb build | tee -a $LB_OUTPUT
-RESULT=$?
+output_echo "Running the rebuild script for the 1st build."
+set +e # We are interested in the return value
+# The 3rd argument is provided to allow for local testing of this script.
+# - If SNAPSHOT_TIMESTAMP is already set, use that timestamp instead of the latest snapshot
+rebuild_script/test/rebuild.sh "$1" "$2" "${SNAPSHOT_TIMESTAMP}"
+RETURNVALUE=$?
+grep --quiet --no-messages "Build result: 0" summary.txt
+BUILD_OK_FOUND=$?
+
+# Retrieve the timestamp that was used
+export SNAPSHOT_TIMESTAMP=$(grep --no-messages "Snapshot timestamp:" summary.txt | cut -f 3 -d " ")
+
set -e
-if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: lb build failed with $RESULT. The latest snapshot might not be complete (yet). Now trying again using the previous snapshot instead."
- # Occasionally the snapshot is not complete, use the the previous snapshot instead of giving up
- # Find the snapshot that matches 1 second before the current snapshot
- wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/$(date --utc -d @$((${SOURCE_DATE_EPOCH}-1)) +%Y%m%dT%H%M%SZ) --output-document ${RESULTSDIR}/but_latest
- export SNAPSHOT_TIMESTAMP=$(cat ${RESULTSDIR}/but_latest | awk '/"result":/ { split($0, a, "\""); print a[4] }')
- # Convert SNAPSHOT_TIMESTAMP to Unix time (insert suitable formatting first)
- export SOURCE_DATE_EPOCH=$(date -d $(echo ${SNAPSHOT_TIMESTAMP} | awk '{ printf "%s-%s-%sT%s:%s:%sZ", substr($0,1,4), substr($0,5,2), substr($0,7,2), substr($0,10,2), substr($0,12,2), substr($0,14,2) }') +%s)
- export MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP}
- output_echo "Info: using the snapshot from ${SOURCE_DATE_EPOCH} (${SNAPSHOT_TIMESTAMP})."
-
- # When reconfiguring, some values that were automatically set need to be updated as well
- lb config \
- --parent-mirror-bootstrap ${MIRROR} \
- --parent-mirror-binary ${MIRROR} \
- --parent-mirror-chroot ${MIRROR} \
- --parent-mirror-debian-installer ${MIRROR} \
- 2>&1 | tee -a $LB_OUTPUT
- set +e # We are interested in the result of 'lb build', so do not fail on errors
- sudo lb build | tee -a $LB_OUTPUT
- RESULT=$?
- set -e
- if [ "$RESULT" != "0" ] ; then
- output_echo "Error: lb build failed with $RESULT, even with an older snapshot."
- exit ${RESULT}
- fi
+
+if [ ${RETURNVALUE} -ne 0 -o ${BUILD_OK_FOUND} -ne 0 ]; then
+ # Something went wrong. Perhaps an alternative timestamp is proposed
+ SNAPSHOT_TIMESTAMP=$(grep --no-messages "Alternative timestamp:" summary.txt | cut -f 3 -d " ")
+ if [ -z ${SNAPSHOT_TIMESTAMP} ]; then
+ output_echo "Error: the image could not be built, no alternative was proposed."
+ exit 1
+ fi
+ output_echo "Warning: the build failed with ${RETURNVALUE}. The latest snapshot might not be complete (yet). Now trying again using the previous snapshot instead."
+ rebuild_script/test/rebuild.sh "$1" "$2" "${SNAPSHOT_TIMESTAMP}"
fi
# Move the image away
mkdir -p ${RESULTSDIR}/b1/${PROJECTNAME}/${CONFIGURATION}
mv live-image-amd64.hybrid.iso ${RESULTSDIR}/b1/${PROJECTNAME}/${CONFIGURATION}
-
-# Clean for the second build
-output_echo "Running lb clean after the 1st build."
-sudo lb clean --purge | tee -a $LB_OUTPUT
-RESULT=$?
-if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: lb clean failed with $RESULT."
-fi
-
-# Re-activate the previous configuration
-output_echo "Running lb config for the 2nd build."
-lb config
+mv summary.txt ${RESULTSDIR}/summary_build1.txt
# Second build
-output_echo "Running lb build for the 2nd build."
-sudo lb build | tee -a $LB_OUTPUT
-RESULT=$?
-if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: lb build failed with $RESULT."
-fi
+output_echo "Running the rebuild script for the 2nd build."
+rebuild_script/test/rebuild.sh "$1" "$2" "${SNAPSHOT_TIMESTAMP}"
# Move the image away
mkdir -p ${RESULTSDIR}/b2/${PROJECTNAME}/${CONFIGURATION}
mv live-image-amd64.hybrid.iso ${RESULTSDIR}/b2/${PROJECTNAME}/${CONFIGURATION}
+mv summary.txt ${RESULTSDIR}/summary_build2.txt
# Clean up
output_echo "Running lb clean after the 2nd build."
-sudo lb clean --purge | tee -a $LB_OUTPUT
-RESULT=$?
-if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: lb clean failed with $RESULT."
-fi
+sudo lb clean --purge
# The workspace is no longer required
cd ..
-# Compare the images
+# Run diffoscope on the images
output_echo "Calling diffoscope on the results."
TIMEOUT="240m"
DIFFOSCOPE="$(schroot --directory /tmp -c chroot:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
TMPDIR=${RESULTSDIR}
call_diffoscope ${PROJECTNAME} ${CONFIGURATION}/live-image-amd64.hybrid.iso
-# Do not publish the ISO images as artifact, they would be able to consume too much disk space
-rm -rf ${RESULTSDIR}/b1
-rm -rf ${RESULTSDIR}/b2
-
-# List the content of the results directory
-PAGE=${CONFIGURATION}-${DEBIAN_VERSION}.html
-if [ -f "${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html" ] ; then
- # Publish the output of diffoscope, there are differences
- cp -a ${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html ${PAGE}
- save_artifacts debian_live_build ${CONFIGURATION}-${DEBIAN_VERSION} https://wiki.debian.org/ReproducibleInstalls/LiveImages
- output_echo "Warning: diffoscope detected differences in the images."
-else
- if [ "$RESULT" != "0" ] ; then
- echo "${DIFFOSCOPE} for ${PROJECTNAME} in configuration ${CONFIGURATION} with timestamp ${SOURCE_DATE_EPOCH} (${SNAPSHOT_TIMESTAMP}) returned error code ${RESULT}." > ${PAGE}
- save_artifacts debian_live_build ${CONFIGURATION}-${DEBIAN_VERSION} https://wiki.debian.org/ReproducibleInstalls/LiveImages
- output_echo "Warning: diffoscope returned error code {$RESULT}."
- else
- echo "${PROJECTNAME} in configuration ${CONFIGURATION} with timestamp ${SOURCE_DATE_EPOCH} (${SNAPSHOT_TIMESTAMP}) is reproducible." > ${PAGE}
- output_echo "Info: no differences found."
- fi
-fi
-output_echo "Publishing results."
-publish_page debian_live_build
-
cleanup success
+
# Turn off the trap
trap - INT TERM EXIT
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/b6cea0e1eff84338ada2ad19f4db4b0dddc3c144
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/b6cea0e1eff84338ada2ad19f4db4b0dddc3c144
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20220423/044f5e17/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list