[Git][qa/jenkins.debian.net][master] NetBSD: Start to clean up this messy script
Holger Levsen (@holger)
gitlab at salsa.debian.org
Fri Mar 29 13:40:09 GMT 2024
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
bd35d1bf by Jan-Benedict Glaw at 2024-03-29T14:39:49+01:00
NetBSD: Start to clean up this messy script
This is really only half the work. Calling `diffoscope` and generating
the web page is left for the second round.
Signed-off-by: Jan-Benedict Glaw <jbglaw at lug-owl.de>
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/reproducible_netbsd.sh
Changes:
=====================================
bin/reproducible_netbsd.sh
=====================================
@@ -7,32 +7,38 @@
PAGE=netbsd/netbsd.html
TIMEOUT="30m"
DEBUG=true
-export TMPDIR=/var/tmp
. /srv/jenkins/bin/common-functions.sh
-common_init "$@"
+common_init "${@}"
# common code for tests.reproducible-builds.org
. /srv/jenkins/bin/reproducible_common.sh
set -e
+set -x
# Build for these architectures/ports.
-declare -a machines # Contains <arch>-<machine> pairs.
-machines=( sparc64-sparc64 x86_64-amd64 )
+declare -a machines # Contains <machine>-<arch> (= <port>-<cpu>) pairs.
+machines+=( sparc64-sparc64 )
+#machines+=( amd64-x86_64 )
# How much Noise shall NetBSD's build process generate?
declare -a nb_noise
nb_noise=( -N 1 )
-
+# Temporary debug function.
+DG() {
+ echo "LOGJJ: $(pwd) - ${*}"
+}
cleanup_tmpdirs() {
+ DG "In cleanup_tmpdirs, TMPDIR=${TMPDIR}, TMPBUILDDIR=${TMPBUILDDIR}"
rm -r "${TMPDIR}"
rm -r "${TMPBUILDDIR}"
}
create_results_dirs() {
+ DG "In create_results_dirs"
mkdir -p "${BASE}/netbsd/dbd"
}
@@ -40,25 +46,29 @@ create_results_dirs() {
# $2 - machine ("x86_64-amd64")
# $3 - release dir
save_netbsd_results() {
- local RUN="${1}"; shift
- local FULL_MACHINE="${1}"; shift
- local RELEASE_DIR="${1}"; shift
+ local run="${1}"; shift
+ local full_machine="${1}"; shift
+ local release_dir="${1}"; shift
+
+ DG "save_netbsd_results: For build ${run} of machine ${full_machine}, copying ${release_dir} to $TMPDIR/${run}/${full_machine}"
- mkdir -p "${TMPDIR}/${RUN}/${FULL_MACHINE}"
- (cd "${RELEASE_DIR}" && tar cf - .) | ( cd "$TMPDIR/$RUN/${FULL_MACHINE}" && tar xf -)
- find "${TMPDIR}/${RUN}/${FULL_MACHINE}" \( -name MD5 -o -name SHA512 \) -exec rm {} \;
+ mkdir -p "${TMPDIR}/${run}/${full_machine}"
+ (cd "${release_dir}" && tar cf - .) | ( cd "${TMPDIR}/${run}/${full_machine}" && tar xf -)
+ find "${TMPDIR}/${run}/${full_machine}" \( -name MD5 -o -name SHA512 \) -exec rm {} \;
}
#
# main
#
-TMPBUILDDIR=$(mktemp --tmpdir=/srv/workspace/chroots/ -d -t rbuild-netbsd-build-XXXXXXXX) # used to build on tmpfs
-TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t rbuild-netbsd-results-XXXXXXXX) # accessible in schroots, used to compare results
-DATE=$(date -u +'%Y-%m-%d')
+TMPBUILDDIR=$(mktemp --tmpdir=/srv/workspace/chroots/ -d -t rbuild-netbsd-build-XXXXXXXX) # used to build on tmpfs
+TMPDIR=$( mktemp --tmpdir=/srv/reproducible-results -d -t rbuild-netbsd-results-XXXXXXXX) # accessible in schroots, used to compare results
+script_start_time=$(date -u +'%Y-%m-%d')
START=$(date +'%s')
trap cleanup_tmpdirs INT TERM EXIT
pushd "${TMPBUILDDIR}"
+ DG "main: Checking our NetBSD src GIT tree"
+
# Prepare sources.
echo "============================================================================="
echo "$(date -u) - Cloning the NetBSD git repository (which is synced with the NetBSD CVS repository)"
@@ -67,112 +77,130 @@ pushd "${TMPBUILDDIR}"
# Get current top commit infos.
pushd netbsd
- NETBSD_TOP_COMMIT_LOG="$(git log -1)"
- NETBSD_TOP_COMMIT_REV=$(git describe --always)
+ DG "main: Fetching some GIT data items"
+ netbsd_top_commit_log="$(git log -1)"
+ netbsd_top_commit_rev=$(git describe --always)
NETBSD_TIMESTAMP=$(git log -1 --format=%ct)
- echo "This is NetBSD ${NETBSD_TOP_COMMIT_REV}."
+ echo "This is NetBSD ${netbsd_top_commit_rev}."
echo
git log -1
popd
- # First round of builds.
- echo "============================================================================="
- echo "$(date -u) - Building NetBSD ${NETBSD_TOP_COMMIT_REV} - first build run."
- echo "============================================================================="
- export TZ="/usr/share/zoneinfo/Etc/GMT+12"
- for MACHINE in "${machines[@]}" ; do
- this_subdir="b1-${MACHINE}"
- this_arch="$(echo "${MACHINE}" | cut -f 1 -d -)"
- this_mach="$(echo "${MACHINE}" | cut -f 2 -d -)"
-
- # Create a pristine linked tree.
- cp -Rl netbsd "${this_subdir}"
-
- # Build and store results.
- pushd "${this_subdir}"
- release_dir=_release_
- full_release_dir="$(realpath "${release_dir}")"
- mkdir -p "${release_dir}"
-
- ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" tools || true
- ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" release || true
- ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" iso-image || true
- ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" install-image || true
- ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" live-image || true
-
- save_netbsd_results b1 "${MACHINE}" "${full_release_dir}"
- echo "${MACHINE} done, first time."
- popd
-
- # Clean up.
- rm -rf "${this_subdir}"
- done
-
- # Second round of builds.
- echo "============================================================================="
- echo "$(date -u) - Building NetBSD - second build run."
- echo "============================================================================="
- export TZ="/usr/share/zoneinfo/Etc/GMT-14"
- export LANG="fr_CH.UTF-8"
- export LC_ALL="fr_CH.UTF-8"
- export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path"
- export CAPTURE_ENVIRONMENT="I capture the environment"
- umask 0002
- NEW_NUM_CPU=$(( ${NUM_CPU} - 1 ))
- for MACHINE in "${machines[@]}"; do
- this_subdir="b2-${MACHINE}"
- this_arch="$(echo "${MACHINE}" | cut -f 1 -d -)"
- this_mach="$(echo "${MACHINE}" | cut -f 2 -d -)"
-
- # Create a pristine linked tree.
- cp -Rl netbsd "${this_subdir}"
-
- # Build and store results.
- pushd "${this_subdir}"
- release_dir=_release_
- full_release_dir="$(realpath "${release_dir}")"
- mkdir -p "${release_dir}"
-
- ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${NEW_NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" tools || true
- ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${NEW_NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" release || true
- ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${NEW_NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" iso-image || true
- ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${NEW_NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" install-image || true
- ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${NEW_NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" live-image || true
-
- save_netbsd_results b2 "${MACHINE}" "${full_release_dir}"
- echo "${MACHINE} done, second time."
- popd
-
- # Clean up.
- rm -rf "${this_subdir}"
- done
-popd
+ # First round of builds. Run in a subshell to protect environment.
+ (
+ echo "============================================================================="
+ echo "$(date -u) - Building NetBSD ${netbsd_top_commit_rev} - first build run."
+ echo "============================================================================="
+ export TZ="/usr/share/zoneinfo/Etc/GMT+12"
+ for this_machine in "${machines[@]}" ; do
+ DG "main: Building NB #1 for ${this_machine}"
+
+ this_subdir="b1-${this_machine}"
+ this_mach="$(echo "${this_machine}" | cut -f 1 -d -)"
+ this_arch="$(echo "${this_machine}" | cut -f 2 -d -)"
+
+ # Create a pristine linked tree.
+ cp -Rl netbsd "${this_subdir}"
+
+ # Build and store results.
+ pushd "${this_subdir}"
+ release_dir=_release_
+ full_release_dir="$(realpath "${release_dir}")"
+ mkdir -p "${release_dir}"
+
+ DG "main: Actually building NB #1 for ${this_machine}, results in ${release_dir}"
+ ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" tools || true
+ ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" release || true
+ ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" iso-image || true
+ ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" install-image || true
+ ionice -c 3 ./build.sh -j "${NUM_CPU}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" live-image || true
+
+ save_netbsd_results b1 "${this_machine}" "${full_release_dir}"
+ echo "${this_machine} done, first time."
+ popd
+
+ # Clean up.
+ DG "main: Cleaning up NB #1 for ${this_machine}"
+ rm -rf "${this_subdir}"
+ done
+ )
+
+ # Second round of builds. Also in a subshell to not taint our own environment.
+ (
+ echo "============================================================================="
+ echo "$(date -u) - Building NetBSD ${netbsd_top_commit_rev} - second build run."
+ echo "============================================================================="
+ export TZ="/usr/share/zoneinfo/Etc/GMT-14"
+ export LANG="fr_CH.UTF-8"
+ export LC_ALL="fr_CH.UTF-8"
+ export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path"
+ export CAPTURE_ENVIRONMENT="I capture the environment"
+ umask 0002
+ new_num_cpu=$(( ${NUM_CPU} - 1 ))
+ for this_machine in "${machines[@]}"; do
+ DG "main: Building NB #2 for ${this_machine}"
+
+ this_subdir="b2-${this_machine}"
+ this_mach="$(echo "${this_machine}" | cut -f 1 -d -)"
+ this_arch="$(echo "${this_machine}" | cut -f 2 -d -)"
+
+ # Create a pristine linked tree.
+ cp -Rl netbsd "${this_subdir}"
+
+ # Build and store results.
+ pushd "${this_subdir}"
+ release_dir=_release_
+ full_release_dir="$(realpath "${release_dir}")"
+ mkdir -p "${release_dir}"
+
+ DG "main: Actually building NB #1 for ${this_machine}, results in ${release_dir}"
+ ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${new_num_cpu}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" tools || true
+ ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${new_num_cpu}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" release || true
+ ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${new_num_cpu}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" iso-image || true
+ ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${new_num_cpu}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" install-image || true
+ ionice -c 3 linux64 --uname-2.6 ./build.sh -j "${new_num_cpu}" -R "${full_release_dir}" -P -U -u -a "${this_arch}" -m "${this_mach}" "${nb_noise[@]}" live-image || true
+
+ save_netbsd_results b2 "${this_machine}" "${full_release_dir}"
+ echo "${this_machine} done, second time."
+ popd
+
+ # Clean up.
+ DG "main: Cleaning up NB #2 for ${this_machine}"
+ rm -rf "${this_subdir}"
+ done
+ )
+ # Delete sources.
+ DG "main: Cleaning initial NetBSD GIT checkout"
+ rm -rf netbsd
+popd # from ${TMPBUILDDIR}
-# Reset environment to default values again.
-export LANG="en_GB.UTF-8"
-unset LC_ALL
-export TZ="/usr/share/zoneinfo/UTC"
-export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:"
-umask 0022
+#
+# Compare results and create a nice HTML page.
+#
-# Delete sources.
-rm -r "${TMPBUILDDIR}/netbsd"
+# Prepare a file list
+declare -a file_list
+declare -A file_list_reproducible
+declare -A file_list_nonreproducible
+tree "${TMPDIR}/b1"
+tree "${TMPDIR}/b2"
+file_list=( $( (cd "${TMPDIR}/b1" && find . -type f; cd "${TMPDIR}/b2" && find . -type f) | sort | uniq ) )
-# Temporarily enable more debug output.
-set -x
+
+# XXX This is old code. Before cleaning it up, let's try to get the initial part running.
# Run diffoscope on the results.
-DIFFOSCOPE="$(schroot --directory /tmp -c "chroot:jenkins-reproducible-${DBDSUITE}-diffoscope" diffoscope -- --version 2>&1)"
+diffoscope_version="$(schroot --directory /tmp -c "chroot:jenkins-reproducible-${DBDSUITE}-diffoscope" diffoscope -- --version 2>&1)"
echo "============================================================================="
-echo "$(date -u) - Running $DIFFOSCOPE on NetBSD build results..."
+echo "$(date -u) - Running ${diffoscope_version} on NetBSD build results..."
echo "============================================================================="
-FILES_HTML=$(mktemp --tmpdir="${TMPDIR}")
-GOOD_FILES_HTML=$(mktemp --tmpdir="${TMPDIR}")
-BAD_FILES_HTML=$(mktemp --tmpdir="${TMPDIR}")
+FILES_HTML=$( mktemp --tmpdir="${TMPDIR}")
+GOOD_FILES_HTML=$( mktemp --tmpdir="${TMPDIR}")
+BAD_FILES_HTML=$( mktemp --tmpdir="${TMPDIR}")
GOOD_SECTION_HTML=$(mktemp --tmpdir="${TMPDIR}")
-BAD_SECTION_HTML=$(mktemp --tmpdir="${TMPDIR}")
+BAD_SECTION_HTML=$( mktemp --tmpdir="${TMPDIR}")
GOOD_FILES=0
ALL_FILES=0
create_results_dirs
@@ -223,7 +251,7 @@ else
fi
#
-# finally create the webpage
+# Finally create the web page.
#
pushd "${TMPDIR}"
mkdir netbsd
@@ -251,12 +279,12 @@ EOF
else
write_page "."
fi
- write_page " These tests were last run on ${DATE} for rev ${NETBSD_TOP_COMMIT_REV} with -P (as of @${NETBSD_TIMESTAMP}) and were compared using ${DIFFOSCOPE}.</p>"
+ write_page " These tests were last run on ${script_start_time} for rev ${netbsd_top_commit_rev} with -P (as of @${NETBSD_TIMESTAMP}) and were compared using ${diffoscope_version}.</p>"
write_variation_table NetBSD
cat "${BAD_SECTION_HTML}" >> "${PAGE}"
cat "${GOOD_SECTION_HTML}" >> "${PAGE}"
write_page " <p><pre>"
- echo -n "${NETBSD_TOP_COMMIT_LOG}" >> "${PAGE}"
+ echo -n "${netbsd_top_commit_log}" >> "${PAGE}"
write_page " </pre></p>"
write_page " </div></div>"
write_page_footer NetBSD
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/bd35d1bf720c0ce0a27730cf2e87fc98d7853653
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/bd35d1bf720c0ce0a27730cf2e87fc98d7853653
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20240329/e9481c6b/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list