[Git][qa/jenkins.debian.net][master] 3 commits: Fight $SIZE as a global variable
Mattia Rizzolo (@mattia)
gitlab at salsa.debian.org
Sun Mar 17 17:00:06 GMT 2024
Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net
Commits:
8bd70738 by Jan-Benedict Glaw at 2024-03-17T17:48:39+01:00
Fight $SIZE as a global variable
>From a785d3e06f4c27f4ac77227d0cf818a29324fe1c Mon Sep 17 00:00:00 2001
From: Jan-Benedict Glaw <jbglaw at getslash.de>
Date: Fri, 15 Mar 2024 21:56:45 +0100
Subject: [PATCH] get_filesize: Phase out usage of global $SIZE variable
[ edited by Mattia ]
Retain some previous stylistic choices.
Prevent SC2155.
Signed-off-by: Jan-Benedict Glaw <jbglaw at lug-owl.de>
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
a46517dd by Holger Levsen at 2024-03-17T17:49:27+01:00
reproducible_build.service: run diffoscope in different slices
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
377dd705 by Mattia Rizzolo at 2024-03-17T17:59:05+01:00
reproducible debian: build: improve on the last commit
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
10 changed files:
- bin/reproducible_build.sh
- bin/reproducible_build_alpine_pkg.sh
- bin/reproducible_build_archlinux_pkg.sh
- bin/reproducible_common.sh
- bin/reproducible_coreboot.sh
- bin/reproducible_freebsd.sh
- bin/reproducible_html_repository_comparison.sh
- bin/reproducible_html_rpms.sh
- bin/reproducible_netbsd.sh
- bin/reproducible_openwrt.sh
Changes:
=====================================
bin/reproducible_build.sh
=====================================
@@ -388,7 +388,12 @@ call_diffoscope_on_changes_files() {
DIFFOSCOPE_STAMP="/var/log/reproducible-builds/diffoscope_stamp_${SRCPACKAGE}_${SUITE}_${ARCH}_$(date -u +%s)"
touch ${DIFFOSCOPE_STAMP}
RESULT=0
- ( timeout 155m nice schroot \
+ # FIXME: replace --remain-after-exit with --collect once checked that it's behavin correctly
+ ( systemd-run --user --send-sighup --remain-after-exit --pipe --wait --scope \
+ --description="difoscope for Debian's $SRCPACKAGE $VERSION in $SUITE/$ARCH" \
+ --slice=rb-build-diffoscope.slice -u "rb-diffoscope-$WORKER_NAME-$BUILD_ID" \
+ -E TMPDIR \
+ timeout 155m nice schroot \
--directory $TMPDIR \
--run-session \
-c "$session" \
=====================================
bin/reproducible_build_alpine_pkg.sh
=====================================
@@ -357,7 +357,6 @@ delay_start # randomize start times
REPOSITORY=""
SRCPACKAGE=""
VERSION=""
-SIZE=""
DISTROID=$(query_db "SELECT id FROM distributions WHERE name='alpine'")
choose_package
mkdir -p $BASE/alpine/$REPOSITORY/$SRCPACKAGE
=====================================
bin/reproducible_build_archlinux_pkg.sh
=====================================
@@ -388,7 +388,6 @@ delay_start # randomize start times
REPOSITORY=""
SRCPACKAGE=""
VERSION=""
-SIZE=""
DISTROID=$(query_db "SELECT id FROM distributions WHERE name='archlinux'")
choose_package
mkdir -p $BASE/archlinux/$REPOSITORY/$SRCPACKAGE
=====================================
bin/reproducible_common.sh
=====================================
@@ -783,15 +783,13 @@ call_diffoscope() {
}
get_filesize() {
- local BYTESIZE="$(du -h -b $1 | cut -f1)"
- # numbers below 16384K are understood and more meaningful than 16M...
- if [ $BYTESIZE -gt 16777216 ] ; then
- SIZE="$(echo $BYTESIZE/1048576|bc)M"
- elif [ $BYTESIZE -gt 1024 ] ; then
- SIZE="$(echo $BYTESIZE/1024|bc)K"
- else
- SIZE="$BYTESIZE bytes"
- fi
+ # $1 - /path/to/file
+ # --> Returns: Human-readable file size ("123B", "250KiB", ...) of that file.
+ local file size
+ file="${1}"; shift
+ size="$(stat --format='%s' "${file}")"
+
+ numfmt --to=iec-i --suffix B --format="%.0f" "${size}"
}
cleanup_pkg_files() {
@@ -1197,8 +1195,7 @@ create_pkg_html() {
if [ "$LOG" = "build2.log" ] ; then
echo " <br />" >> $HTML_BUFFER
fi
- get_filesize $ARCHLINUX_PKG_PATH/$LOG
- echo " <a href=\"/archlinux/$REPOSITORY/$SRCPACKAGE/$LOG\">$LOG</a> ($SIZE)" >> $HTML_BUFFER
+ echo " <a href=\"/archlinux/$REPOSITORY/$SRCPACKAGE/$LOG\">$LOG</a> ($(get_filesize "${ARCHLINUX_PKG_PATH}/${LOG}"))" >> $HTML_BUFFER
fi
done
echo " </td>" >> $HTML_BUFFER
@@ -1313,8 +1310,7 @@ create_alpine_pkg_html() {
if [ "$LOG" = "build2.log" ] ; then
echo " <br />" >> $HTML_BUFFER
fi
- get_filesize $ALPINE_PKG_PATH/$LOG
- echo " <a href=\"/alpine/$REPOSITORY/$SRCPACKAGE/$LOG\">$LOG</a> ($SIZE)" >> $HTML_BUFFER
+ echo " <a href=\"/alpine/$REPOSITORY/$SRCPACKAGE/$LOG\">$LOG</a> ($(get_filesize "${ALPINE_PKG_PATH}/${LOG}"))" >> $HTML_BUFFER
fi
done
echo " </td>" >> $HTML_BUFFER
=====================================
bin/reproducible_coreboot.sh
=====================================
@@ -167,20 +167,18 @@ LIST_OBJECT=$(mktemp --tmpdir=$TMPDIR)
BAD_ROMS=0
GOOD_ROMS=0
ALL_ROMS=0
-SIZE=""
create_results_dirs
cd $TMPDIR/b1
for i in $(ls -1d *| sort -u) ; do
let ALL_ROMS+=1
if [ -f $i/coreboot.rom ] && [ -f $TMPDIR/b2/$i/coreboot.rom ] ; then
call_diffoscope $i coreboot.rom
- get_filesize $i/coreboot.rom
if [ -f $TMPDIR/$i/coreboot.rom.html ] ; then
mv $TMPDIR/$i/coreboot.rom.html $BASE/coreboot/dbd/$i.html
- echo " <li><a href=\"dbd/$i.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $i</a> ($SIZE) is unreproducible.</li>" >> $ROW_FRAGMENTS_BAD_ROMS
+ echo " <li><a href=\"dbd/$i.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $i</a> ($(get_filesize "${i}/coreboot.rom")) is unreproducible.</li>" >> $ROW_FRAGMENTS_BAD_ROMS
else
SHASUM=$(sha256sum $i/coreboot.rom|cut -d " " -f1)
- echo " <li><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $i ($SHASUM, $SIZE) is reproducible.</li>" >> $ROW_FRAGMENTS_GOOD_ROMS
+ echo " <li><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $i ($SHASUM, $(get_filesize "${i}/coreboot.rom")) is reproducible.</li>" >> $ROW_FRAGMENTS_GOOD_ROMS
let GOOD_ROMS+=1
rm -f $BASE/coreboot/dbd/$i.html # cleanup from previous (unreproducible) tests - if needed
fi
=====================================
bin/reproducible_freebsd.sh
=====================================
@@ -50,7 +50,6 @@ run_diffoscope_on_results() {
#echo " <ul>" > ${FILES_HTML[$FREEBSD_TARGET]}
GOOD_FILES[$FREEBSD_TARGET]=0
ALL_FILES[$FREEBSD_TARGET]=0
- SIZE=""
create_results_dirs
echo " <table><tr><th>Artifacts for <code>$TARGET_NAME</code></th></tr>" >> ${FILES_HTML[$FREEBSD_TARGET]}
if [ ! -d $TMPDIR/b1 ] || [ ! -d $TMPDIR/b1 ] ; then
@@ -66,14 +65,13 @@ run_diffoscope_on_results() {
for j in $(find * -type l -prune -o -regex "freebsd_main_git.........\.tar\.xz" -prune -o -type f -print | sort -u) ; do
ALL_FILES[$FREEBSD_TARGET]=$(( ${ALL_FILES[$FREEBSD_TARGET]}+1 ))
call_diffoscope . $j
- get_filesize $j
if [ -f $TMPDIR/$j.html ] ; then
mkdir -p $BASE/freebsd/dbd/$(dirname $j)
mv $TMPDIR/$j.html $BASE/freebsd/dbd/$j.html
- echo " <tr><td><a href=\"dbd/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($SIZE) is unreproducible.</td></tr>" >> ${FILES_HTML[$FREEBSD_TARGET]}
+ echo " <tr><td><a href=\"dbd/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($(get_filesize "${j}")) is unreproducible.</td></tr>" >> ${FILES_HTML[$FREEBSD_TARGET]}
else
SHASUM=$(sha256sum $j|cut -d " " -f1)
- echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $SIZE) is reproducible.</td></tr>" >> ${FILES_HTML[$FREEBSD_TARGET]}
+ echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $(get_filesize "${j}")) is reproducible.</td></tr>" >> ${FILES_HTML[$FREEBSD_TARGET]}
GOOD_FILES[$FREEBSD_TARGET]=$(( ${GOOD_FILES[$FREEBSD_TARGET]}+1 ))
rm -f $BASE/freebsd/dbd/$j.html # cleanup from previous (unreproducible) tests - if needed
fi
=====================================
bin/reproducible_html_repository_comparison.sh
=====================================
@@ -28,10 +28,16 @@ write_row() {
}
custom_curl() {
- echo -n "$(date -u) - downloading $1 to $2 - "
- curl -s $1 > $2
- local SIZE=$(ls -la $2 |cut -d " " -f5)
- echo "$SIZE bytes."
+ # $1 - URL to download
+ # $2 - target filename to save data
+ local url="${1}"; shift
+ local target="${1}"; shift
+ local size
+
+ echo -n "$(date -u) - downloading ${url} to ${target} - "
+ curl -s "${url}" > "${target}"
+ size=$(stat --formar='%s' "${target}")
+ echo "${size} bytes."
}
ARCH="amd64"
=====================================
bin/reproducible_html_rpms.sh
=====================================
@@ -26,7 +26,6 @@ HTML_UNKNOWN=$(mktemp -t rhtml-rpms-XXXXXXXX)
HTML_BUFFER=$(mktemp -t rhtml-rpms-XXXXXXXX)
HTML_TARGET=""
HTML_RPM_STATS=$(mktemp -t rhtml-rpms-XXXXXXXX)
-SIZE=""
TESTED=0
NR_FTBFS=0
NR_FTBR=0
@@ -80,8 +79,7 @@ for PKG in $(find $RPMBASE/$RELEASE/$ARCH/* -maxdepth 1 -type d -exec basename {
echo " <td>$(date -u --date=@$(stat -c %Y $RPM_PKG_PATH/build1.log) +%F)</td>" >> $HTML_BUFFER
for LOG in build1.log build2.log ; do
if [ -f $RPM_PKG_PATH/$LOG ] ; then
- get_filesize $RPM_PKG_PATH/$LOG
- echo " <td><a href=\"/rpms/$RELEASE/$ARCH/$PKG/$LOG\">$LOG</a> ($SIZE)</td>" >> $HTML_BUFFER
+ echo " <td><a href=\"/rpms/$RELEASE/$ARCH/$PKG/$LOG\">$LOG</a> ($(get_filesize "${RPM_PKG_PATH}/${LOG}"))</td>" >> $HTML_BUFFER
else
echo " <td> </td>" >> $HTML_BUFFER
fi
=====================================
bin/reproducible_netbsd.sh
=====================================
@@ -175,7 +175,6 @@ GOOD_SECTION_HTML=$(mktemp --tmpdir="${TMPDIR}")
BAD_SECTION_HTML=$(mktemp --tmpdir="${TMPDIR}")
GOOD_FILES=0
ALL_FILES=0
-SIZE=""
create_results_dirs
cd "${TMPDIR}/b1"
tree .
@@ -184,14 +183,13 @@ for i in *; do
for j in $(find * -type f |sort -u ) ; do
let ALL_FILES+=1
call_diffoscope "${i}" "${j}"
- get_filesize "${j}"
if [ -f "${TMPDIR}/${i}/${j}.html" ]; then
mkdir -p "${BASE}/netbsd/dbd/${i}/$(dirname "${j}")"
mv "${TMPDIR}/${i}/${j}.html" "${BASE}/netbsd/dbd/${i}/${j}.html"
- echo " <tr><td><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($SIZE) is unreproducible.</td></tr>" >> "${BAD_FILES_HTML}"
+ echo " <tr><td><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($(get_filesize "${j}")) is unreproducible.</td></tr>" >> "${BAD_FILES_HTML}"
else
SHASUM=$(sha256sum "${j}" | cut -d " " -f1)
- echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $SIZE) is reproducible.</td></tr>" >> "${GOOD_FILES_HTML}"
+ echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $(get_filesize "${j}")) is reproducible.</td></tr>" >> "${GOOD_FILES_HTML}"
let GOOD_FILES+=1
rm -f "${BASE}/netbsd/dbd/${i}/${j}.html" # cleanup from previous (unreproducible) tests - if needed
fi
=====================================
bin/reproducible_openwrt.sh
=====================================
@@ -570,7 +570,6 @@ DBD_BAD_PKGS_HTML=$(mktemp "--tmpdir=$RESULTSDIR")
# run diffoscope on the images
GOOD_IMAGES=0
ALL_IMAGES=0
-SIZE=""
cd "$RESULTSDIR/b1/targets"
tree .
@@ -593,8 +592,8 @@ for target in * ; do
echo " <table><tr><th>Images for <code>$target/$subtarget</code></th></tr>" >> "$DBD_HTML"
for image in $(printf "%s\n%s" "$IMGS1" "$IMGS2" | sort -u ) ; do
let ALL_IMAGES+=1
- if [ ! -f "$RESULTSDIR/b1/targets/$target/$subtarget/$image" ] || [ ! -f "$RESULTSDIR/b2/targets/$target/$subtarget/$image" ] ; then
- echo " <tr><td><img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> $image (${SIZE}) failed to build.</td></tr>" >> "$DBD_HTML"
+ if [ ! -f "$RESULTSDIR/b1/targets/$target/$subtarget/$image" -o ! -f "$RESULTSDIR/b2/targets/$target/$subtarget/$image" ] ; then
+ echo " <tr><td><img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> $image failed to build.</td></tr>" >> "$DBD_HTML"
rm -f "$BASE/openwrt/dbd/targets/$target/$subtarget/$image.html" # cleanup from previous (unreproducible) tests - if needed
continue
fi
@@ -605,14 +604,13 @@ for target in * ; do
else
echo "$(date -u) - targets/$target/$subtarget/$image is reproducible!"
fi
- get_filesize "$image"
if [ -f "$RESULTSDIR/targets/$target/$subtarget/$image.html" ] ; then
mkdir -p "$BASE/openwrt/dbd/targets/$target/$subtarget"
mv "$RESULTSDIR/targets/$target/$subtarget/$image.html" "$BASE/openwrt/dbd/targets/$target/$subtarget/$image.html"
- echo " <tr><td><a href=\"dbd/targets/$target/$subtarget/$image.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $image</a> (${SIZE}) is unreproducible.</td></tr>" >> "$DBD_HTML"
+ echo " <tr><td><a href=\"dbd/targets/$target/$subtarget/$image.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $image</a> ($(get_filesize "${image}")) is unreproducible.</td></tr>" >> "$DBD_HTML"
else
SHASUM=$(sha256sum "$image" |cut -d " " -f1)
- echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $image ($SHASUM, $SIZE) is reproducible.</td></tr>" >> "$DBD_HTML"
+ echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $image ($SHASUM, $(get_filesize "${image}")) is reproducible.</td></tr>" >> "$DBD_HTML"
let GOOD_IMAGES+=1
rm -f "$BASE/openwrt/dbd/targets/$target/$subtarget/$image.html" # cleanup from previous (unreproducible) tests - if needed
fi
@@ -647,8 +645,8 @@ for i in * ; do
for j in $(printf "%s\n%s" "$PKGS1" "$PKGS2" | sort -u ) ; do
let ALL_PACKAGES+=1
- if [ ! -f "$RESULTSDIR/b1/$i/$j" ] || [ ! -f "$RESULTSDIR/b2/$i/$j" ] ; then
- echo " <tr><td><img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> $j (${SIZE}) failed to build.</td></tr>" >> "$DBD_BAD_PKGS_HTML"
+ if [ ! -f "$RESULTSDIR/b1/$i/$j" -o ! -f "$RESULTSDIR/b2/$i/$j" ] ; then
+ echo " <tr><td><img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> $j failed to build.</td></tr>" >> "$DBD_BAD_PKGS_HTML"
rm -f "$BASE/openwrt/dbd/$i/$j.html" # cleanup from previous (unreproducible) tests - if needed
continue
fi
@@ -658,14 +656,13 @@ for i in * ; do
else
echo "$(date -u) - $i/$j is reproducible!"
fi
- get_filesize "$j"
if [ -f "$RESULTSDIR/$i/$j.html" ] ; then
mkdir -p "$BASE/openwrt/dbd/$i/$(dirname "$j")"
mv "$RESULTSDIR/$i/$j.html" "$BASE/openwrt/dbd/$i/$j.html"
- echo " <tr><td><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($SIZE) is unreproducible.</td></tr>" >> "$DBD_BAD_PKGS_HTML"
+ echo " <tr><td><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($(get_filesize "${j}")) is unreproducible.</td></tr>" >> "$DBD_BAD_PKGS_HTML"
else
SHASUM=$(sha256sum "$j" |cut -d " " -f1)
- echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $SIZE) is reproducible.</td></tr>" >> "$DBD_GOOD_PKGS_HTML"
+ echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $(get_filesize "${j}")) is reproducible.</td></tr>" >> "$DBD_GOOD_PKGS_HTML"
let GOOD_PACKAGES+=1
rm -f "$BASE/openwrt/dbd/$i/$j.html" # cleanup from previous (unreproducible) tests - if needed
fi
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/9b00f037b05ec176805c439054e59aef78ace9c5...377dd7057082270874bb2657eb450ff59e2d9216
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/9b00f037b05ec176805c439054e59aef78ace9c5...377dd7057082270874bb2657eb450ff59e2d9216
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20240317/7ffbbf79/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list