[Qa-jenkins-scm] [jenkins.debian.net] 01/05: eproducible: support for saving artifacts from the builds. use 'artifacts' as the first package when manual scheduling and packages will save files!
Holger Levsen
holger at moszumanska.debian.org
Fri Mar 20 20:15:52 UTC 2015
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit ba94d6f11d5f3e5e92755d14272aeef35b156429
Author: Mattia Rizzolo <mattia at mapreri.org>
Date: Fri Mar 20 20:22:34 2015 +0100
eproducible: support for saving artifacts from the builds. use 'artifacts' as the first package when manual scheduling and packages will save files!
---
README | 2 ++
bin/reproducible_build.sh | 15 ++++++++++++++-
bin/reproducible_common.sh | 4 +++-
bin/reproducible_maintainance.sh | 12 +++++++++++-
bin/reproducible_schedule_on_demand.sh | 11 ++++++++++-
5 files changed, 40 insertions(+), 4 deletions(-)
diff --git a/README b/README
index e07e06e..008b849 100644
--- a/README
+++ b/README
@@ -147,6 +147,8 @@ Installation tests inside chroot environments.
jenkins at jenkins:~$ /srv/jenkins/bin/reproducible_schedule_on_demand.sh $suite $package1 $package2
----
+If $package1 is "artifacts" then the build job will preserv the produced packages and publish them under reproducible.debian.net, to allow easy+deeper investigation of reproducible issues. The url is published in the build log.
+
* Blacklisting packages can be done similarily:
----
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index d76e52e..e281e89 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -24,6 +24,18 @@ create_results_dirs() {
}
cleanup_all() {
+ if [ "$SAVE_ARTIFACTS" == 1 ] ; then
+ local hash=$(tr -cd '[:alnum:]' < /dev/urandom | fold -w5 | head -1)
+ local ARTIFACTS="artifacts/r00t-me/tmp-${hash}/${SUITE}/${PACKAGE}"
+ mkdir -p "/var/lib/jenkins/userContent/$ARTIFACTS"
+ cp -r $TMPDIR/* "/var/lib/jenkins/userContent/$ARTIFACTS"
+ echo | tee -a ${RBUILDLOG}
+ echo "Artifacts from this build are preserved. They will be available for very short time so download them now if you want them." | tee -a ${RBUILDLOG}
+ echo "You shouldn't trust packages you downloaded from this host, they can contain malware or the worst of your fear, packaged in a debian format." | tee -a ${RBUILDLOG}
+ echo "If you are not afraid of facing your fears helping the world by investigating reproducible build issues, you can download the artifacts from the following url:" | tee -a ${RBUILDLOG}
+ echo "https://reproducible.debian.net/$ARTIFACTS" | tee -a ${RBUILDLOG}
+ echo | tee -a ${RBUILDLOG}
+ fi
rm -r $TMPDIR $TMPCFG
}
@@ -141,7 +153,7 @@ for i in $SUITES ; do
done
SQL_SUITES="$SQL_SUITES)"
-RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT s.suite, s.id, s.name, sch.date_scheduled FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id WHERE sch.date_build_started = '' AND s.suite IN $SQL_SUITES ORDER BY date_scheduled LIMIT 1")
+RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT s.suite, s.id, s.name, sch.date_scheduled, sch.save_artifacts FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id WHERE sch.date_build_started = '' AND s.suite IN $SQL_SUITES ORDER BY date_scheduled LIMIT 1")
if [ -z "$RESULT" ] ; then
echo "No packages scheduled, sleeping 30m."
sleep 30m
@@ -151,6 +163,7 @@ else
SRCPKGID=$(echo $RESULT|cut -d "|" -f2)
SRCPACKAGE=$(echo $RESULT|cut -d "|" -f3)
SCHEDULED_DATE=$(echo $RESULT|cut -d "|" -f4)
+ SAVE_ARTIFACTS=$(echo $RESULT|cut -d "|" -f5)
create_results_dirs
echo "============================================================================="
echo "Trying to reproducibly build ${SRCPACKAGE} in ${SUITE} on ${ARCH} now."
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index f428d6a..7f318e1 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -79,8 +79,10 @@ schedule_packages() {
# so schedule them in the past, so they are picked earlier :)
DATE="2014-10-01 00:23"
TMPFILE=$(mktemp)
+ ARTIFACTS=$1
+ shift
for PKG_ID in $@ ; do
- echo "REPLACE INTO schedule (package_id, date_scheduled, date_build_started) VALUES ('$PKG_ID', '$DATE', '');" >> $TMPFILE
+ echo "REPLACE INTO schedule (package_id, date_scheduled, date_build_started, save_artifacts) VALUES ('$PKG_ID', '$DATE', '', '$ARTIFACTS');" >> $TMPFILE
done
cat $TMPFILE | sqlite3 -init $INIT ${PACKAGES_DB}
rm $TMPFILE
diff --git a/bin/reproducible_maintainance.sh b/bin/reproducible_maintainance.sh
index 84624bb..97e588d 100755
--- a/bin/reproducible_maintainance.sh
+++ b/bin/reproducible_maintainance.sh
@@ -101,7 +101,8 @@ if [ ! -z "$FAILED_BUILDS" ] ; then
check_candidates
if [ $TOTAL -ne 0 ] ; then
echo " - in $SUITE: $CANDIDATES"
- schedule_packages $PACKAGE_IDS
+ # '0' here means the artifacts will not be saved
+ schedule_packages 0 $PACKAGE_IDS
fi
done
DIRTY=true
@@ -200,6 +201,15 @@ if [ ! -z "$OLDSTUFF" ] ; then
echo
fi
+# remove artifacts older than 3 days
+mkdir -p /var/lib/jenkins/userContent/artifacts
+ARTIFACTS=$(find /var/lib/jenkins/userContent/artifacts -maxdepth 1 -type d -mtime +3 -exec rm -rv {} \;)
+if [ ! -z "$ARTIFACTS" ] ; then
+ echo
+ echo "Removed $ARTIFACTS artifacts."
+ echo
+fi
+
if ! $DIRTY ; then
echo "Everything seems to be fine."
echo
diff --git a/bin/reproducible_schedule_on_demand.sh b/bin/reproducible_schedule_on_demand.sh
index 74321e2..c4bd987 100755
--- a/bin/reproducible_schedule_on_demand.sh
+++ b/bin/reproducible_schedule_on_demand.sh
@@ -22,6 +22,15 @@ if [ "$SUITE" = "sid" ] ; then
SUITE=unstable
fi
+ARTIFACTS=0
+if [ $1 = "artifacts" ] ; then
+ ARTIFACTS=1
+ shift
+ echo
+ echo "The artifacts of these builds will be saved. Look at the build log for the link"
+ echo
+fi
+
CANDIDATES="$@"
check_candidates
if [ ${#PACKAGE_IDS} -gt 256 ] ; then
@@ -34,7 +43,7 @@ fi
MESSAGE="$TOTAL $PACKAGES_TXT $ACTION in $SUITE: ${PACKAGES_NAMES:0:256}$BLABLABLA"
# finally
-schedule_packages $PACKAGE_IDS
+schedule_packages $ARTIFACTS $PACKAGE_IDS
echo
echo "$MESSAGE"
if [ -z "${BUILD_URL:-}" ] && [ $TOTAL -ne 0 ] ; then
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list