[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] Build live images twice and compare the output
Holger Levsen (@holger)
gitlab at salsa.debian.org
Wed Jun 16 11:08:29 BST 2021
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
c2987737 by Roland Clobus at 2021-06-16T12:08:17+02:00
Build live images twice and compare the output
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/reproducible_debian_live_build.sh
Changes:
=====================================
bin/reproducible_debian_live_build.sh
=====================================
@@ -2,6 +2,7 @@
# vim: set noexpandtab:
# Copyright 2021 Holger Levsen <holger at layer-acht.org>
+# Copyright 2021 Roland Clobus <rclobus at rclobus.nl>
# released under the GPLv2
DEBUG=true
@@ -24,47 +25,215 @@ output_echo() {
# main: follow https://wiki.debian.org/ReproducibleInstalls/LiveImages
#
-# 1st build
+# Fetch and use the latest version of live build
export LIVE_BUILD=$(mktemp -d -t live-build.XXXXXXXX)
+git clone https://salsa.debian.org/live-team/live-build.git ${LIVE_BUILD} --single-branch --no-tags
export LB_OUTPUT=$(mktemp -t live-build.XXXXXXXX)
-export SOURCE_DATE_EPOCH=1609489883a # why this one
+# Use 'now' as the basic timestamp
+export SOURCE_DATE_EPOCH=$(date +%s)
-# config first
-lb config --parent-mirror-bootstrap http://deb.debian.org/debian \
- --parent-mirror-binary http://deb.debian.org/debian \
+# Configuration for the smallest live image (mini, without installer)
+# - For /etc/apt/sources.list: Use the mirror from ${MIRROR}, no security, no updates
+# - Version to build for: bullseye
+# - No installer
+# - Don't cache the downloaded content, re-download for the second build
+# - Explicitly use the proxy that is set by ${http_proxy} to reduce some network traffic
+lb config \
+ --parent-mirror-bootstrap ${MIRROR} \
+ --parent-mirror-binary ${MIRROR} \
--security false \
--updates false \
- --apt-options "--yes -o Acquire::Check-Valid-Until=false" \
--distribution bullseye \
- --debian-installer live \
+ --debian-installer none \
--cache-packages false \
+ --apt-http-proxy ${http_proxy} \
2>&1 | tee $LB_OUTPUT
RESULT=$?
if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: lb config filed with $RESULT"
+ output_echo "Warning: lb config failed with $RESULT"
fi
-# build
+# Add additional hooks, that work around known reproducible issues
+# Note: Keep the hooks in sync with https://wiki.debian.org/ReproducibleInstalls/LiveImages
+cat > config/hooks/normal/1000-reproducible-function-uuid_generate_random.hook.chroot << EOF
+#!/bin/sh
+set -e
+
+# util-linux creates random UUIDs when uuid_generate_random is called
+# Use LD_PRELOAD to replace uuid_generate_random with a less random version
+
+# Don't run if gcc is not installed
+if [ ! -e /usr/bin/cc ];
+then
+ exit 0
+fi
+
+cat > unrandomize_uuid_generate_random.c << END_OF_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+
+#define SEQUENCE_FILENAME "/var/cache/unrandomize_uuid_generate_random.sequence_number"
+
+/* https://tools.ietf.org/html/rfc4122 */
+typedef unsigned char uuid_t[16];
+
+/* Our pseudo-random version */
+void uuid_generate_random(uuid_t out)
+{
+ /* Nil UUID */
+ for (int i=0;i<16;i++) {
+ out[i] = 0x00;
+ }
+ out[6]=0x40; /* UUID version 4 means randomly generated */
+ out[8]=0x80; /* bit7=1,bit6=0 */
+
+ /* The file doesn't need to exist yet */
+ FILE *f = fopen(SEQUENCE_FILENAME, "rb");
+ if (f) {
+ fread(out+12, 4, 1, f);
+ fclose(f);
+ }
+ /* Use the next number. Endianness is not important */
+ (*(unsigned long*)(out+12))++;
+
+ unsigned long long epoch;
+ /* Use SOURCE_DATE_EPOCH when provided */
+ char *date = getenv("SOURCE_DATE_EPOCH");
+ if (date) {
+ epoch = strtoll(date, NULL, 10);
+ } else {
+ epoch = 0ll;
+ }
+ out[0] = (epoch & 0xFF000000) >> 24;
+ out[1] = (epoch & 0x00FF0000) >> 16;
+ out[2] = (epoch & 0x0000FF00) >> 8;
+ out[3] = (epoch & 0x000000FF);
+
+ /* Write the sequence number */
+ f = fopen(SEQUENCE_FILENAME, "wb");
+ if (f) {
+ fwrite(out+12, 4, 1, f);
+ fclose(f);
+ }
+}
+END_OF_SOURCE
+/usr/bin/cc -shared -fPIC unrandomize_uuid_generate_random.c -Wall --pedantic -o /usr/lib/unrandomize_uuid_generate_random.so
+rm -f unrandomize_uuid_generate_random.c
+EOF
+cat > config/hooks/normal/1001-reproducible-fontconfig.hook.chroot << EOF
+#!/bin/sh
+set -e
+
+# fontconfig creates non-reproducible files with UUIDs
+# See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=864082
+#
+# Because the UUIDs should not be deleted, the proposed work-around is:
+# * Use LD_PRELOAD to replace uuid_generate_random with a less random version
+
+# Don't run if fontconfig is not installed
+if [ ! -e /usr/bin/fc-cache ];
+then
+ exit 0
+fi
+
+# Don't run if the LD_PRELOAD module is not compiled
+if [ ! -e /usr/lib/unrandomize_uuid_generate_random.so ];
+then
+ exit 0
+fi
+
+LD_PRELOAD=/usr/lib/unrandomize_uuid_generate_random.so /usr/bin/fc-cache --force --really-force --system-only --verbose
+EOF
+cat > config/hooks/normal/1002-reproducible-mdadm.hook.chroot << EOF
+#!/bin/sh
+set -e
+
+# mkconf of mdadm creates a file with a timestamp
+# A bug report with patch is available at https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=982607
+# This script duplicates that patch
+
+# Don't run if mdadm is not installed
+if [ ! -e /usr/share/mdadm/mkconf ];
+then
+ exit 0
+fi
+
+# If mkconf already contains references to SOURCE_DATE_EPOCH, there is no need to patch the file
+if grep -q SOURCE_DATE_EPOCH /usr/share/mdadm/mkconf;
+then
+ exit 0
+fi
+sed -i -e '/# This configuration was auto-generated on/cif [ -z \$SOURCE_DATE_EPOCH ]; then\n echo "# This configuration was auto-generated on \$(date -R) by mkconf"\nelse\n echo "# This configuration was auto-generated on \$(date -R --utc -d@\$SOURCE_DATE_EPOCH) by mkconf"\nfi' /usr/share/mdadm/mkconf
+EOF
+cat > config/hooks/normal/1003-reproducible-plymouth.hook.chroot << EOF
+#!/bin/sh
+set -e
+
+# The hook of plymouth in update-initramfs calls fc-cache
+
+# Don't run if plymouth is not installed
+if [ ! -e /usr/share/initramfs-tools/hooks/plymouth ];
+then
+ exit 0
+fi
+
+# Don't patch if the LD_PRELOAD module is not compiled
+if [ ! -e /usr/lib/unrandomize_uuid_generate_random.so ];
+then
+ exit 0
+fi
+
+# If the hook already contains references to LD_PRELOAD, there is no need to patch the file
+if grep -q LD_PRELOAD /usr/share/initramfs-tools/hooks/plymouth;
+then
+ exit 0
+fi
+sed -i -e 's|fc-cache -s|LD_PRELOAD=/usr/lib/unrandomize_uuid_generate_random.so fc-cache|' /usr/share/initramfs-tools/hooks/plymouth
+EOF
+
+
+# First build
+sudo lb build | tee -a $LB_OUTPUT
+RESULT=$?
+if [ "$RESULT" != "0" ] ; then
+ output_echo "Warning: lb config failed with $RESULT"
+fi
+
+# Move the image away
+mkdir -p $TMPDIR/b1/live-build
+mv live-image-amd64.hybrid.iso $TMPDIR/b1/live-build
+
+# Clean for the second build
+lb clean --purge | tee -a $LB_OUTPUT
+RESULT=$?
+if [ "$RESULT" != "0" ] ; then
+ output_echo "Warning: lb config failed with $RESULT"
+fi
+
+# Second build
sudo lb build | tee -a $LB_OUTPUT
RESULT=$?
if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: lb config filed with $RESULT"
+ output_echo "Warning: lb config failed with $RESULT"
fi
-# finally cleanup
+# Move the image away
+mkdir -p $TMPDIR/b2/live-build
+mv live-image-amd64.hybrid.iso $TMPDIR/b2/live-build
+
+# Clean up
lb clean --purge | tee -a $LB_OUTPUT
RESULT=$?
if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: lb config filed with $RESULT"
+ output_echo "Warning: lb config failed with $RESULT"
fi
-# move the image away
-# do 2nd build
-# do 2nd build with more customisations
-# compare images
-# rm images
-# build other flavors
-# build unstable too
+# Compare the images
+call_diffoscope live-build live-image-amd64.hybrid.iso
+
+# Clean up
+rm -fr $TMPDIR/b1/live-build
+rm -fr $TMPDIR/b2/live-build
# the end
rm -f $LIVE_BUILD $LB_OUTPUT
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/c29877377068a88edd7b4232bff46c74efbc7adf
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/c29877377068a88edd7b4232bff46c74efbc7adf
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20210616/77e77963/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list