[Qa-jenkins-scm] [jenkins.debian.net] 01/04: reproducible: add a sane mean to notify the IRC channel when builds finish.
Holger Levsen
holger at moszumanska.debian.org
Sat Apr 25 11:09:33 UTC 2015
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 9c1c845a51249714a528133a65ba89c70469e472
Author: Mattia Rizzolo <mattia at mapreri.org>
Date: Thu Apr 23 17:26:40 2015 +0200
reproducible: add a sane mean to notify the IRC channel when builds finish.
* new column in the database, schedule table
* turn $SAVE_ARTIFACTS a true boolean, 0→no artifacts, 1→yes artifacts
* $NOTIFY in the build script is text containg a keyword:
+ if empty no notification will be sent
+ if "debbindiff" a "$DBDVERSION had trouble with these" will be sent
+ if "failure" a "no artifacts could have been saved"-like will be sent
+ any other value trigs a normal message
* adapt the maintenance script and the manual scheduler to cope with the new
column
---
bin/reproducible_build.sh | 44 +++++++++++++++++++++-------------
bin/reproducible_common.sh | 4 +---
bin/reproducible_db_maintenance.py | 4 ++++
bin/reproducible_maintenance.sh | 4 ++--
bin/reproducible_schedule_on_demand.sh | 30 ++++++++++++++++-------
5 files changed, 56 insertions(+), 30 deletions(-)
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 9a276eb..701f531 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -48,7 +48,10 @@ handle_race_condition() {
printf "$msg" | tee -a $BUILDLOG
printf "$msg" | mail -s "race condition found" qa-jenkins-scm at lists.alioth.debian.org
echo "$(date) - Terminating nicely this build..." | tee -a $RBUILDLOG
- if [ $SAVE_ARTIFACTS -eq 1 ] || [ $SAVE_ARTIFACTS -eq 3 ] ; then SAVE_ARTIFACTS=2 ; fi
+ if [ $SAVE_ARTIFACTS -eq 1 ] ; then
+ SAVE_ARTIFACTS=0
+ if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
+ fi
exit 0
}
@@ -79,19 +82,22 @@ save_artifacts() {
chmod 644 $HEADER
echo | tee -a ${RBUILDLOG}
# irc message
- local MESSAGE="$URL published"
- if [ $SAVE_ARTIFACTS -eq 3 ] ; then
- local MESSAGE="$MESSAGE, $DBDVERSION had troubles with these..."
+ if [ ! -z "$NOTIFY" ] ; then
+ local MESSAGE="$URL published"
+ if [ "$NOTIFY" = "debbindiff failure" ] ; then
+ local MESSAGE="$MESSAGE, $DBDVERSION had troubles with these..."
+ fi
+ irc_message "$MESSAGE"
fi
- irc_message "$MESSAGE"
}
cleanup_all() {
- if [ $SAVE_ARTIFACTS -eq 1 ] || [ $SAVE_ARTIFACTS -eq 3 ] ; then
- save_artifacts
- elif [ $SAVE_ARTIFACTS -eq 2 ] ; then
+ if [ $SAVE_ARTIFACTS -eq 1 ] ; then save_artifacts ; fi
+ if [ "$NOTIFY" = "failure" ] ; then
echo "No artifacts were saved for this build." | tee -a ${RBUILDLOG}
- irc_message "Check $REPRODUCIBLE_URL/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log to find out why no artifacts were saved."
+ irc_message "Check $REPRODUCIBLE_URL/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log and $BUILD_URL to find out why no artifacts were saved."
+ elif [ ! -z "$NOTIFY" ] && [ $SAVE_ARTIFACTS -eq 0 ] ; then
+ irc_message "This package just finished building: $REPRODUCIBLE_URL/$SUITE/$ARCH/$SRCPACKAGE"
fi
rm -r $TMPDIR
if ! $BAD_LOCKFILE ; then rm -f $LOCKFILE ; fi
@@ -139,7 +145,8 @@ handle_404() {
echo "Warning: Maybe there was a network problem, or ${SRCPACKAGE} is not a source package in ${SUITE}, or it was removed or renamed. Please investigate." | tee -a ${RBUILDLOG}
DURATION=''
update_db_and_html "404"
- if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=2 ; fi
+ if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
+ if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
exit 0
}
@@ -148,7 +155,8 @@ handle_not_for_us() {
echo "Package ${SRCPACKAGE} (${VERSION}) shall only be build on \"$(echo "$@" | xargs echo )\" and thus was skipped." | tee -a ${RBUILDLOG}
DURATION=''
update_db_and_html "not for us"
- if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=2 ; fi
+ if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
+ if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
exit 0
}
@@ -156,7 +164,8 @@ handle_ftbfs() {
echo "${SRCPACKAGE} failed to build from source."
calculate_build_duration
update_db_and_html "FTBFS"
- if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=2 ; fi
+ if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
+ if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
}
handle_ftbr() {
@@ -230,7 +239,8 @@ dbd_timeout() {
else
local msg="$msg, but there is still $REPRODUCIBLE_URL/dbd/$SUITE/$ARCH/$DDBREPORT"
fi
- SAVE_ARTIFACTS=3
+ SAVE_ARTIFACTS=1
+ NOTIFY="debbindiff"
handle_ftbr "$msg"
}
@@ -265,7 +275,8 @@ call_debbindiff() {
handle_ftbr "$DBDVERSION found issues, please investigate $REPRODUCIBLE_URL/dbd/${SUITE}/${ARCH}/${DBDREPORT}"
;;
2)
- SAVE_ARTIFACTS=3
+ SAVE_ARTIFACTS=1
+ NOTIFY="debbindiff"
handle_ftbr "$DBDVERSION had trouble comparing the two builds. Please investigate $REPRODUCIBLE_URL/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log"
;;
124)
@@ -280,12 +291,13 @@ call_debbindiff() {
}
choose_package () {
- local RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT s.suite, s.id, s.name, sch.date_scheduled, sch.save_artifacts FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id WHERE sch.date_build_started = '' ORDER BY date_scheduled LIMIT 1")
+ local RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT s.suite, s.id, s.name, sch.date_scheduled, sch.save_artifacts, sch.notify FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id WHERE sch.date_build_started = '' ORDER BY date_scheduled LIMIT 1")
SUITE=$(echo $RESULT|cut -d "|" -f1)
SRCPKGID=$(echo $RESULT|cut -d "|" -f2)
SRCPACKAGE=$(echo $RESULT|cut -d "|" -f3)
SCHEDULED_DATE=$(echo $RESULT|cut -d "|" -f4)
SAVE_ARTIFACTS=$(echo $RESULT|cut -d "|" -f5)
+ NOTIFY=$(echo $RESULT|cut -d "|" -f6)
if [ -z "$RESULT" ] ; then
echo "No packages scheduled, sleeping 30m."
sleep 30m
@@ -427,7 +439,7 @@ START=$(date +'%s')
RBUILDLOG=$(mktemp --tmpdir=$TMPDIR)
BAD_LOCKFILE=false
-choose_package # defines SUITE, PKGID, SRCPACKAGE, SCHEDULED_DATE, SAVE_ARTIFACTS
+choose_package # defines SUITE, PKGID, SRCPACKAGE, SCHEDULED_DATE, SAVE_ARTIFACTS, NOTIFY
# used to catch race conditions when the same package is being built by two parallel jobs
LOCKFILE="/tmp/${SUITE}-${ARCH}-${SRCPACKAGE}"
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index b39ddc0..40608f3 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -90,10 +90,8 @@ schedule_packages() {
MINS=$(date +'%M') # schedule on the full hour so we can recognize them easily
DATE=$(date +'%Y-%m-%d %H:%M' -d "$DAYS day ago - $HOURS hours - $MINS minutes")
TMPFILE=$(mktemp)
- ARTIFACTS=$1
- shift
for PKG_ID in $@ ; do
- echo "REPLACE INTO schedule (package_id, date_scheduled, date_build_started, save_artifacts) VALUES ('$PKG_ID', '$DATE', '', '$ARTIFACTS');" >> $TMPFILE
+ echo "REPLACE INTO schedule (package_id, date_scheduled, date_build_started, save_artifacts, notify) VALUES ('$PKG_ID', '$DATE', '', '$ARTIFACTS', '$NOTIFY');" >> $TMPFILE
done
cat $TMPFILE | sqlite3 -init $INIT ${PACKAGES_DB}
rm $TMPFILE
diff --git a/bin/reproducible_db_maintenance.py b/bin/reproducible_db_maintenance.py
index 003ae32..f69e83b 100755
--- a/bin/reproducible_db_maintenance.py
+++ b/bin/reproducible_db_maintenance.py
@@ -361,6 +361,10 @@ schema_updates = {
architecture TEXT NOT NULL,
PRIMARY KEY (name, suite, architecture))''',
'INSERT INTO rb_schema VALUES ("11", "' + now + '")'],
+ 12: [ # refactor the artifacts handling, splitting artifacts savind from
+ # IRC notification
+ 'ALTER TABLE schedule ADD COLUMN notify TEXT',
+ 'INSERT INTO rb_schema VALUES ("12", "' + now + '")'],
}
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index f471d13..ab5206f 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -109,8 +109,8 @@ if [ ! -z "$FAILED_BUILDS" ] ; then
check_candidates
if [ $TOTAL -ne 0 ] ; then
echo " - in $SUITE: $CANDIDATES"
- # '0' here means the artifacts will not be saved
- schedule_packages 0 $PACKAGE_IDS
+ ARTIFACTS=0
+ schedule_packages $PACKAGE_IDS
fi
done
DIRTY=true
diff --git a/bin/reproducible_schedule_on_demand.sh b/bin/reproducible_schedule_on_demand.sh
index 436c605..27c32dc 100755
--- a/bin/reproducible_schedule_on_demand.sh
+++ b/bin/reproducible_schedule_on_demand.sh
@@ -22,14 +22,24 @@ if [ "$SUITE" = "sid" ] ; then
SUITE=unstable
fi
-ARTIFACTS=0
-if [ "$1" = "artifacts" ] ; then
- ARTIFACTS=1
- shift
- echo
- echo "The artifacts of the build(s) will be saved to the location mentioned at the end of the build log(s)."
- echo
-fi
+case "$1" in
+ "artifacts")
+ ARTIFACTS=1
+ NOTIFY=true
+ shift
+ printf "\nThe artifacts of the build(s) will be saved to the location mentioned at the end of the build log(s).\n\n"
+ ;;
+ "notify")
+ ARTIFACTS=0
+ NOTIFY=true
+ shift
+ printf "\nThe IRC channel will be notify once the build(s) finished.\n\n"
+ ;;
+ *)
+ ARTIFACTS=0
+ NOTIFY=''
+ ;;
+esac
CANDIDATES="$@"
check_candidates
@@ -43,10 +53,12 @@ fi
MESSAGE="$TOTAL $PACKAGES_TXT $ACTION in $SUITE: ${PACKAGES_NAMES:0:256}$BLABLABLA"
if [ $ARTIFACTS -eq 1 ] ; then
MESSAGE="$MESSAGE - artifacts will be preserved."
+elif $NOTIFY ; then
+ MESSAGE="$MESSAGE - notification once finished."
fi
# finally
-schedule_packages $ARTIFACTS $PACKAGE_IDS
+schedule_packages $PACKAGE_IDS
echo
echo "$MESSAGE"
if [ -z "${BUILD_URL:-}" ] && [ $TOTAL -ne 0 ] ; then
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list