[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 6 commits: reproducible: Move the function save_artifacts to reproducible_common.sh
Holger Levsen (@holger)
gitlab at salsa.debian.org
Tue Aug 17 17:19:49 BST 2021
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
7bdecf34 by Roland Clobus at 2021-08-17T18:19:07+02:00
reproducible: Move the function save_artifacts to reproducible_common.sh
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
66bb1a36 by Roland Clobus at 2021-08-17T18:19:07+02:00
reproducible_html_rsync: Use an identifiable folder on master for the artifacts instead of /tmp
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
e982a5d3 by Roland Clobus at 2021-08-17T18:19:07+02:00
reproducible_debian_live_build: Use throttle category 'debian_live_build' to allow only one job running at the same time. Remove unused nagiator
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
effd095e by Roland Clobus at 2021-08-17T18:19:07+02:00
reproducible_debian_live_build: Link to the view in the sidebar
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
7d74e95c by Roland Clobus at 2021-08-17T18:19:07+02:00
reproducible_debian_live_build: Cleanup artifacts, at the beginning of the build and during the maintenance
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
7da991d1 by Roland Clobus at 2021-08-17T18:19:07+02:00
reproducible_debian_live_build: Create artifacts when a difference is detected. Move the artifacts to the Jenkins host
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
6 changed files:
- bin/reproducible_build.sh
- bin/reproducible_common.sh
- bin/reproducible_debian_live_build.sh
- bin/reproducible_html_rsync_remote_results.sh
- bin/reproducible_maintenance.sh
- job-cfg/reproducible.yaml
Changes:
=====================================
bin/reproducible_build.sh
=====================================
@@ -42,22 +42,9 @@ create_results_dirs() {
mkdir -vp $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}
}
-save_artifacts() {
- local random=$(head /dev/urandom | tr -cd '[:alnum:]'| head -c5)
- local ARTIFACTS="artifacts/r00t-me/${SRCPACKAGE}_${SUITE}_${ARCH}_tmp-${random}"
- local URL="$DEBIAN_URL/$ARTIFACTS/"
- local HEADER="$DEBIAN_BASE/$ARTIFACTS/.HEADER.html"
- mkdir -p $DEBIAN_BASE/$ARTIFACTS
- cp -r $TMPDIR/* $DEBIAN_BASE/$ARTIFACTS/ || true
- local msg="Artifacts from this build have been preserved. They will be available for 24h only, so download them now.\n"
- msg="${msg}WARNING: You shouldn't trust packages downloaded from this host, they can contain malware or the worst of your fears, packaged nicely in debian format.\n"
- msg="${msg}If you are aware of this and just want to use these artifacts to investigate why $DIFFOSCOPE had issues, you can download the artifacts from the following location: $URL\n"
- log_info "$msg"
- echo "<p>" > $HEADER
- printf "$msg" | sed 's#$#<br />#g' >> $HEADER
- echo "Package page: <a href=\"$DEBIAN_URL/${SUITE}/${ARCH}/${SRCPACKAGE}\">$DEBIAN_URL/${SUITE}/${ARCH}/${SRCPACKAGE}</a><br />" >> $HEADER
- echo "</p>" >> $HEADER
- chmod 644 $HEADER
+notification() {
+ if [ "$SAVE_ARTIFACTS" = "1" ] ; then
+ save_artifacts debian ${SUITE}_${ARCH}_${SRCPACKAGE} $REPRODUCIBLE_URL/rb-pkg/${SUITE}/${ARCH}/${SRCPACKAGE}.html
# irc message
if [ ! -z "$NOTIFY" ] ; then
local MESSAGE="Artifacts for ${SRCPACKAGE}, $STATUS in ${SUITE}/${ARCH}: $URL"
@@ -70,11 +57,6 @@ save_artifacts() {
irc_message debian-reproducible "$MESSAGE"
fi
fi
-}
-
-notification() {
- if [ "$SAVE_ARTIFACTS" = "1" ] ; then
- save_artifacts # this will also notify IRC as needed
else
case "$NOTIFY" in # the diffoscope_err case is handled by save_artifacts()
''|0) ;;
=====================================
bin/reproducible_common.sh
=====================================
@@ -595,6 +595,39 @@ write_variation_table() {
write_page "</table></p>"
}
+# Save the artifacts located in $TMPDIR
+#
+# Arguments:
+# 1 The top level
+# 2 Base folder name for the artifacts
+# 3 URL to the page with more information
+# Input variables:
+# TMPDIR The folder that contains the artifacts to save
+# DIFFOSCOPE The output of 'diffoscope --version'
+# Variables from reproducible_common:
+# REPRODUCIBLE_URL
+# BASE
+save_artifacts() {
+ local TOP_LEVEL=$1
+ local BASE_FOLDER_NAME=$2
+ local MOREINFO_URL=$3
+ local random=$(head /dev/urandom | tr -cd '[:alnum:]'| head -c5)
+ local ARTIFACTS="${TOP_LEVEL}/artifacts/r00t-me/${BASE_FOLDER_NAME}_tmp-${random}"
+ local URL="$REPRODUCIBLE_URL/$ARTIFACTS/"
+ local HEADER="$BASE/$ARTIFACTS/.HEADER.html"
+ mkdir -p $BASE/$ARTIFACTS
+ cp -r $TMPDIR/* $BASE/$ARTIFACTS/ || true
+ local msg="Artifacts from this build have been preserved. They will be available for 24h only, so download them now.\n"
+ msg="${msg}WARNING: You shouldn't trust packages downloaded from this host, they can contain malware or the worst of your fears, packaged nicely in debian format.\n"
+ msg="${msg}If you are aware of this and just want to use these artifacts to investigate why $DIFFOSCOPE had issues, you can download the artifacts from the following location: $URL\n"
+ log_info "$msg"
+ echo "<p>" > $HEADER
+ printf "$msg" | sed 's#$#<br />#g' >> $HEADER
+ echo "<a href=\"$MOREINFO_URL\">More information</a><br />" >> $HEADER
+ echo "</p>" >> $HEADER
+ chmod 644 $HEADER
+}
+
publish_page() {
if [ "$1" = "" ] ; then
TARGET=$PAGE
=====================================
bin/reproducible_debian_live_build.sh
=====================================
@@ -66,6 +66,9 @@ export DEBIAN_VERSION="$2"
# randomize start time by 1-23 sec
delay_start
+# Cleanup possible artifacts of a previous build (see reproducible_common.sh for the path)
+rm -rf $BASE/debian_live_build/artifacts/r00t-me/${CONFIGURATION}_tmp-*
+
# Generate and use an isolated workspace
export PROJECTNAME="live-build"
mkdir -p /srv/workspace/live-build
@@ -300,9 +303,11 @@ PAGE=${CONFIGURATION}.html
if [ -f "${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html" ] ; then
# Publish the output of diffoscope, there are differences
cp -a ${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html ${PAGE}
+ save_artifacts debian_live_build ${CONFIGURATION} https://wiki.debian.org/ReproducibleInstalls/LiveImages
else
if [ "$RESULT" != "0" ] ; then
echo "${DIFFOSCOPE} for ${PROJECTNAME} in configuration ${CONFIGURATION} returned error code ${RESULT}" > ${PAGE}
+ save_artifacts debian_live_build ${CONFIGURATION} https://wiki.debian.org/ReproducibleInstalls/LiveImages
else
echo "${PROJECTNAME} in configuration ${CONFIGURATION} is reproducible" > ${PAGE}
fi
=====================================
bin/reproducible_html_rsync_remote_results.sh
=====================================
@@ -15,7 +15,7 @@ rsync_remote_results() {
PROJECT=$1
NODE=$2
echo "$(date -u) - Starting to rsync results for '$PROJECT'."
- local RESULTS=$(mktemp --tmpdir=$TEMPDIR -d reproducible-rsync-XXXXXXXXX)
+ local RESULTS=$(mktemp --tmpdir=$BASE/.. -d reproducible-rsync-${BUILD_ID}-XXXXXXXXX)
# copy the new results from build node to webserver node
if rsync -r -v -e "ssh -o 'Batchmode = yes'" $NODE:$BASE/$PROJECT/ $RESULTS 2>/dev/null ; then
chmod 775 $RESULTS
=====================================
bin/reproducible_maintenance.sh
=====================================
@@ -720,6 +720,16 @@ if [ ! -z "$ARTIFACTS" ] ; then
echo
fi
+# remove artifacts from the debian live build jobs, older than a day
+echo "$(date -u) - Checking for artifacts from debian live build jobs, that are older than a day."
+ARTIFACTS=$(find $BASE/debian_live_build/artifacts/r00t-me/* -maxdepth 1 -type d -mtime +1 -exec ls -lad {} \; 2>/dev/null|| true)
+if [ ! -z "$ARTIFACTS" ] ; then
+ echo
+ echo "Removed old artifacts:"
+ find $BASE/debian_live_build/artifacts/r00t-me/* -maxdepth 1 -type d -mtime +1 -exec rm -rv --one-file-system {} \; || true
+ echo
+fi
+
# find + chmod files with bad permissions
echo "$(date -u) - Checking for files with bad permissions."
# automatically fix rbuild files with wrong permissions...
=====================================
job-cfg/reproducible.yaml
=====================================
@@ -220,6 +220,26 @@
- job-template:
defaults: reproducible
name: 'reproducible_debian_live_build_{imagetype}_{debian_version}'
+ concurrent: true
+ properties:
+ - sidebar: *sb1
+ - sidebar:
+ url: https://jenkins.debian.net/view/live
+ text: reproducible live image jobs
+ icon: /userContent/images/debian-jenkins-24x24.png
+ - sidebar: *sb3
+ - throttle:
+ max-total: 0
+ max-per-node: 1
+ enabled: true
+ option: category
+ categories:
+ - debian_live_build
+ publishers:
+ - logparser: *logparser
+ - email: *email
+ - trigger:
+ project: '{my_trigger}'
- job-template:
defaults: reproducible
@@ -943,13 +963,10 @@
name: reproducible_debian_live_build
my_recipients: 'qa-jenkins-scm at lists.alioth.debian.org'
my_node: '{my_hname}'
- my_naginator_regex: 'dummyshouldneverhappen'
my_timeout: '720'
imagetype:
- "smallest-build"
- - cinnamon:
- # run this job an hour after the others, to make this job run alone
- my_timed: '55 12 * * 0'
+ - cinnamon
- gnome
- kde
- lxde
@@ -965,4 +982,5 @@
my_timed: '55 11 * * 0'
my_hname: 'osuosl173'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh {imagetype} {debian_version}'
+ my_trigger: 'reproducible_html_rsync_remote_results'
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/e1612351ab44df3089ec9809e9a872a8d7d9b48d...7da991d1a5629525499ea6e676bff87eec742ff4
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/e1612351ab44df3089ec9809e9a872a8d7d9b48d...7da991d1a5629525499ea6e676bff87eec742ff4
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20210817/1aeb1d7d/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list