[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 4 commits: reproducible Archlinux: plan how to switch to using db
Holger Levsen
gitlab at salsa.debian.org
Fri Sep 21 18:54:18 BST 2018
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
049c1cff by Holger Levsen at 2018-09-21T15:20:53Z
reproducible Archlinux: plan how to switch to using db
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
d02cc93f by Holger Levsen at 2018-09-21T15:21:25Z
reproducible Archlinux: document code
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
880bb661 by Holger Levsen at 2018-09-21T16:31:30Z
reproducible Archlinux: add the beginnings of sql based scheduler
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
8acce39f by Holger Levsen at 2018-09-21T17:14:46Z
reproducible Debian|Archlinux: refactoring
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
4 changed files:
- TODO
- bin/reproducible_archlinux_scheduler.sh
- bin/reproducible_build.sh
- bin/reproducible_common.sh
Changes:
=====================================
TODO
=====================================
@@ -330,7 +330,20 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
** create page for untested packages?
* use db
-* then fix build.sh:
+** use debian tables with suite=archlinux_core etc and arch=x86_64
+** problem: we currently have more (detailed) stati in archlinux
+** extend scheduler.sh to add packages to db and schedule them in db
+*** enable FIXME line once the db has been initially populated
+*** actually schedule old packages once queue is empty
+*** also delete unknown packages from db
+*** check/make sure that updated packages are only scheduled once
+** make build.sh read & write db
+** make build respect pacman exit code
+** create new job to recreate all pkg pages
+** call pkg html function after each build
+** cleanup now useless code from scheduler
+
+* fix build.sh:
** build2.log doesnt get deleted if build1 fails
** -> rename build2.log to $pkg_$version_build2.log
=====================================
bin/reproducible_archlinux_scheduler.sh
=====================================
@@ -14,14 +14,20 @@ set -e
update_archlinux_repositories() {
echo "$(date -u) - Updating Arch Linux repositories, currently $(find $BASE/archlinux/ -name pkg.needs_build | wc -l ) packages scheduled."
+ #
+ # init
+ #
UPDATED=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
NEW=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
OLDER=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
local SESSION="archlinux-scheduler-$RANDOM"
schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
schroot --run-session -c $SESSION --directory /var/tmp -- sudo pacman -Syu --noconfirm
+
+ #
# Get a list of unique package bases. Non-split packages don't have a pkgbase set
# so we need to use the pkgname for them instead.
+ #
schroot --run-session -c $SESSION --directory /var/tmp -- expac -S '%r %e %n %v' | \
while read repo pkgbase pkgname version; do
if [[ "$pkgbase" = "(null)" ]]; then
@@ -33,7 +39,9 @@ update_archlinux_repositories() {
TOTAL=$(cat ${ARCHLINUX_PKGS}_full_pkgbase_list | wc -l)
echo "$(date -u ) - $TOTAL Arch Linux packages are known in total to us."
+ #
# remove packages which are gone (only when run between 21:00 and 23:59)
+ #
if [ $(date +'%H') -gt 21 ] ; then
REMOVED=0
REMOVE_LIST=""
@@ -55,13 +63,50 @@ update_archlinux_repositories() {
irc_message archlinux-reproducible "$MESSAGE"
fi
fi
-
+
+ #
# schedule packages
+ #
for REPO in $ARCHLINUX_REPOS ; do
TMPPKGLIST=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
echo "$(date -u ) - updating list of available packages in repository '$REPO'."
grep "^$REPO" "$ARCHLINUX_PKGS"_full_pkgbase_list | \
while read repo pkgbase version; do
+ #
+ # db based scheduler
+ #
+ PKG=$pkgbase
+ SUITE="archlinux_$repo"
+ ARCH="x86_64"
+ VERSION=$(query_db "SELECT version FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
+ DATE="$(date -u +'%Y-%m-%d %H:%M')"
+ if [ -z "$VERSION" ] ; then
+ # new package, add to db and schedule
+ query_db "INSERT into sources (name, version, suite, architecture) VALUES ('$PKG', '$VERSION', '$SUITE', '$ARCH');"
+ PKGID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
+ #FIXME: enable next line once the db has been initially populated
+ # query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('$PKGID', '$DATE');"
+ elif [ "$VERSION" != "$version" ] ; then
+ VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"
+ if [ "$VERCMP" = "1" ] ; then
+ # known package but with new version, update db and schedule
+ echo $REPO/$pkgbase >> $UPDATED
+ echo "$(date -u ) - we know $REPO/$pkgbase $VERSION, but repo has $version, so rescheduling... "
+ query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"
+ PKGID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
+ query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('$PKGID', '$DATE');"
+
+ elif [ "$VERCMP" = "-1" ] ; then
+ # our version is higher than what's in the repo because we build trunk
+ echo "$REPO/$pkgbase $VERSION > $version" >> $OLDER
+ else
+ echo "$(date -u ) - This should never happen: we know about $pkgbase $VERSION, but repo has $version. \$VERCMP=$VERCMP"
+ fi
+ fi
+
+ #
+ # old scheduling to be deleted once the db is used
+ #
if [ ! -d $BASE/archlinux/$REPO/$pkgbase ] ; then
# schedule (all) entirely new packages
echo $REPO/$pkgbase >> $NEW
@@ -100,8 +145,12 @@ update_archlinux_repositories() {
schroot --end-session -c $SESSION
echo "$(date -u) - the following packages are known to us with higher versions than the repo because we build trunk:"
cat $OLDER
+ echo
+
+ #
# schedule up to $MAX packages we already know about
# (only if less than $THRESHOLD packages are currently scheduled)
+ #
old=""
local MAX=350
local THRESHOLD=450
@@ -124,6 +173,11 @@ update_archlinux_repositories() {
fi
fi
rm $OLDER
+ echo
+
+ #
+ # output stats
+ #
total=$(find $BASE/archlinux/ -name pkg.needs_build | wc -l )
rm "$ARCHLINUX_PKGS"_full_pkgbase_list
new=$(cat $NEW | wc -l 2>/dev/null|| true)
=====================================
bin/reproducible_build.sh
=====================================
@@ -20,28 +20,6 @@ common_init "$@"
set -e
-log_info () {
- _log "I:" "$@"
-}
-
-log_error () {
- _log "E:" "$@"
-}
-
-log_warning () {
- _log "W:" "$@"
-}
-
-log_file () {
- cat $@ | tee -a $RBUILDLOG
-}
-
-_log () {
- local prefix="$1"
- shift 1
- echo -e "$(date -u) $prefix $*" | tee -a $RBUILDLOG
-}
-
exit_early_if_debian_is_broken() {
# debian is fine, thanks
if false && [ "$ARCH" = "armhf" ] ; then
@@ -63,19 +41,6 @@ create_results_dirs() {
mkdir -vp $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}
}
-handle_race_condition() {
- local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID'")
- local msg="Package ${SRCPACKAGE} (id=$SRCPKGID) in ${SUITE} on ${ARCH} is probably already building at $RESULT, while this is $BUILD_URL.\n"
- log_warning "$msg"
- printf "$(date -u) - $msg" >> /var/log/jenkins/reproducible-race-conditions.log
- log_warning "Terminating this build quickly and nicely..."
- if [ $SAVE_ARTIFACTS -eq 1 ] ; then
- SAVE_ARTIFACTS=0
- if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
- fi
- exit 0
-}
-
save_artifacts() {
local random=$(head /dev/urandom | tr -cd '[:alnum:]'| head -c5)
local ARTIFACTS="artifacts/r00t-me/${SRCPACKAGE}_${SUITE}_${ARCH}_tmp-${random}"
=====================================
bin/reproducible_common.sh
=====================================
@@ -114,6 +114,29 @@ if $DEBUG ; then
set -x
fi
+# some cmomon logging functions
+log_info () {
+ _log "I:" "$@"
+}
+
+log_error () {
+ _log "E:" "$@"
+}
+
+log_warning () {
+ _log "W:" "$@"
+}
+
+log_file () {
+ cat $@ | tee -a $RBUILDLOG
+}
+
+_log () {
+ local prefix="$1"
+ shift 1
+ echo -e "$(date -u) $prefix $*" | tee -a $RBUILDLOG
+}
+
# sleep 1-23 secs to randomize start times
delay_start() {
/bin/sleep $(echo "scale=1 ; $(shuf -i 1-230 -n 1)/10" | bc )
@@ -664,6 +687,19 @@ cleanup_pkg_files() {
rm -vf $DEBIAN_BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz}
}
+handle_race_condition() {
+ local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID'")
+ local msg="Package ${SRCPACKAGE} (id=$SRCPKGID) in ${SUITE} on ${ARCH} is probably already building at $RESULT, while this is $BUILD_URL.\n"
+ log_warning "$msg"
+ printf "$(date -u) - $msg" >> /var/log/jenkins/reproducible-race-conditions.log
+ log_warning "Terminating this build quickly and nicely..."
+ if [ $SAVE_ARTIFACTS -eq 1 ] ; then
+ SAVE_ARTIFACTS=0
+ if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
+ fi
+ exit 0
+}
+
#
# create the png (and query the db to populate a csv file...)
#
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/6b98e303c425aa4f1a33286e37f62f1c6fdfd2ed...8acce39ff0f4fd44a87b1a8e613125de1115d35a
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/6b98e303c425aa4f1a33286e37f62f1c6fdfd2ed...8acce39ff0f4fd44a87b1a8e613125de1115d35a
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20180921/5c45af00/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list