[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 5 commits: reproducible_debian_live_build: Add a better error description for incorrect/missing arguments
Holger Levsen (@holger)
gitlab at salsa.debian.org
Thu Dec 30 16:07:22 GMT 2021
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
1de71214 by Roland Clobus at 2021-12-30T17:06:57+01:00
reproducible_debian_live_build: Add a better error description for incorrect/missing arguments
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
436248f4 by Roland Clobus at 2021-12-30T17:06:57+01:00
reproducible_debian_live_build: Turn off debugging
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
daa2baf6 by Roland Clobus at 2021-12-30T17:06:57+01:00
reproducible_debian_live_build: Use unique folders and reports for each Debian version
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
6656e25f by Roland Clobus at 2021-12-30T17:06:57+01:00
reproducible_debian_live_build: Minor: whitespace correction + comment correction
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
11f2a91b by Roland Clobus at 2021-12-30T17:06:57+01:00
reproducible_debian_live_build: If the latest snapshot is not complete yet, try to use the previous snapshot instead
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/reproducible_debian_live_build.sh
Changes:
=====================================
bin/reproducible_debian_live_build.sh
=====================================
@@ -5,7 +5,7 @@
# Copyright 2021 Roland Clobus <rclobus at rclobus.nl>
# released under the GPLv2
-DEBUG=true
+DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"
@@ -56,14 +56,14 @@ case $1 in
"mate") export INSTALLER="live"; export PACKAGES="live-task-mate"; ;;
"standard") export INSTALLER="live"; export PACKAGES="live-task-standard"; ;;
"xfce") export INSTALLER="live"; export PACKAGES="live-task-xfce"; ;;
- *) output_echo "Error: Bad argument 1: $1"; exit 1; ;;
+ *) output_echo "Error: Bad argument 1, image type: $1"; exit 1; ;;
esac
export CONFIGURATION="$1"
# Argument 2 = Debian version
# No further validation required, reproducible.yaml should be correct
if [ -z "$2" ] ; then
- output_echo "Error: Bad argument 2: it is empty"
+ output_echo "Error: Bad argument 2, Debian version: it is empty"
exit 2
fi
export DEBIAN_VERSION="$2"
@@ -73,6 +73,12 @@ if [ "${DEBIAN_VERSION}" == "sid" ] ; then
DEBIAN_VERSION="unstable"
fi
+if $DEBUG ; then
+ export WGET_OPTIONS=
+else
+ export WGET_OPTIONS=--quiet
+fi
+
# randomize start time by 1-23 sec
delay_start
@@ -82,9 +88,9 @@ rm -rf $BASE/debian_live_build/artifacts/r00t-me/${CONFIGURATION}_tmp-*
# Generate and use an isolated workspace
export PROJECTNAME="live-build"
mkdir -p /srv/workspace/live-build
-export BUILDDIR=$(mktemp --tmpdir=/srv/workspace/live-build -d -t ${CONFIGURATION}.XXXXXXXX)
+export BUILDDIR=$(mktemp --tmpdir=/srv/workspace/live-build -d -t ${CONFIGURATION}-${DEBIAN_VERSION}.XXXXXXXX)
cd ${BUILDDIR}
-export RESULTSDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t ${PROJECTNAME}-${CONFIGURATION}-XXXXXXXX) # accessible in schroots, used to compare results
+export RESULTSDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t ${PROJECTNAME}-${CONFIGURATION}-${DEBIAN_VERSION}-XXXXXXXX) # accessible in schroots, used to compare results
# Fetch and use the latest version of live build
export LIVE_BUILD=${BUILDDIR}/latest-live-build
@@ -95,7 +101,7 @@ popd
export LB_OUTPUT=${RESULTSDIR}/lb_output.txt
# Use the timestamp of the latest mirror snapshot
-wget http://snapshot.notset.fr/mr/timestamp/debian/latest --output-document ${RESULTSDIR}/latest
+wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/latest --output-document ${RESULTSDIR}/latest
#
# Extract the timestamp from the JSON file
#
@@ -114,10 +120,8 @@ export SOURCE_DATE_EPOCH=$(date -d $(echo ${SNAPSHOT_TIMESTAMP} | awk '{ printf
export MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP}
output_echo "Info: using the snapshot from ${SOURCE_DATE_EPOCH} (${SNAPSHOT_TIMESTAMP})"
-# Configuration for the smallest live image (mini, without installer)
+# Configuration for the live image:
# - For /etc/apt/sources.list: Use the mirror from ${MIRROR}, no security, no updates
-# - Version to build for: bullseye
-# - No installer
# - Don't cache the downloaded content, re-download for the second build
# - Explicitly use the proxy that is set by ${http_proxy} to reduce some network traffic
output_echo "Running lb config for the 1st build"
@@ -136,7 +140,7 @@ if [ "$RESULT" != "0" ] ; then
output_echo "Warning: lb config failed with $RESULT"
fi
-if [ ! -z "${PACKAGES}" ]; then
+if [ ! -z "${PACKAGES}" ] ; then
echo "${PACKAGES}" > config/package-lists/desktop.list.chroot
fi
@@ -335,10 +339,36 @@ EOF
# First build
output_echo "Running lb build for the 1st build"
+set +e # We are interested in the result of 'lb build', so do not fail on errors
sudo lb build | tee -a $LB_OUTPUT
RESULT=$?
+set -e
if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: lb build failed with $RESULT"
+ output_echo "Warning: lb build failed with $RESULT. The latest snapshot might not be complete (yet). Use the previous snapshot instead"
+ # Occasionally the snapshot is not complete, use the the previous snapshot instead of giving up
+ # Find the snapshot that matches 1 second before the current snapshot
+ wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/$(date --utc -d @$((${SOURCE_DATE_EPOCH}-1)) +%Y%m%dT%H%M%SZ) --output-document ${RESULTSDIR}/but_latest
+ export SNAPSHOT_TIMESTAMP=$(cat ${RESULTSDIR}/but_latest | awk '/"result":/ { split($0, a, "\""); print a[4] }')
+ # Convert SNAPSHOT_TIMESTAMP to Unix time (insert suitable formatting first)
+ export SOURCE_DATE_EPOCH=$(date -d $(echo ${SNAPSHOT_TIMESTAMP} | awk '{ printf "%s-%s-%sT%s:%s:%sZ", substr($0,1,4), substr($0,5,2), substr($0,7,2), substr($0,10,2), substr($0,12,2), substr($0,14,2) }') +%s)
+ export MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP}
+ output_echo "Info: using the snapshot from ${SOURCE_DATE_EPOCH} (${SNAPSHOT_TIMESTAMP})"
+
+ # When reconfiguring, some values that were automatically set need to be updated as well
+ lb config \
+ --parent-mirror-bootstrap ${MIRROR} \
+ --parent-mirror-binary ${MIRROR} \
+ --parent-mirror-chroot ${MIRROR} \
+ --parent-mirror-debian-installer ${MIRROR} \
+ 2>&1 | tee -a $LB_OUTPUT
+ set +e # We are interested in the result of 'lb build', so do not fail on errors
+ sudo lb build | tee -a $LB_OUTPUT
+ RESULT=$?
+ set -e
+ if [ "$RESULT" != "0" ] ; then
+ output_echo "Error: lb build failed with $RESULT, even with an older snapshot"
+ exit ${RESULT}
+ fi
fi
# Move the image away
@@ -392,7 +422,7 @@ rm -rf ${RESULTSDIR}/b1
rm -rf ${RESULTSDIR}/b2
# List the content of the results directory
-PAGE=${CONFIGURATION}.html
+PAGE=${CONFIGURATION}-${DEBIAN_VERSION}.html
if [ -f "${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html" ] ; then
# Publish the output of diffoscope, there are differences
cp -a ${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html ${PAGE}
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/78eda06255bbd4ea969a69951121122d5196752e...11f2a91b1802bb7239fdcb99d84c952ffd391dad
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/78eda06255bbd4ea969a69951121122d5196752e...11f2a91b1802bb7239fdcb99d84c952ffd391dad
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20211230/81be28b2/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list