[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: reproducible Archlinux SQL scheduler: don't schedule new versions of blacklisted packages
Holger Levsen
gitlab at salsa.debian.org
Sat Sep 22 17:26:41 BST 2018
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
4db078d4 by Holger Levsen at 2018-09-22T16:12:32Z
reproducible Archlinux SQL scheduler: don't schedule new versions of blacklisted packages
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
3d46a865 by Holger Levsen at 2018-09-22T16:26:27Z
fix bashims (if ==)
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
5 changed files:
- TODO
- bin/reproducible_archlinux_scheduler.sh
- bin/reproducible_common.sh
- bin/reproducible_html_nodes_info.sh
- bin/reproducible_maintenance.sh
Changes:
=====================================
TODO
=====================================
@@ -332,10 +332,9 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
* use db
** problem: we currently have more (detailed) stati in archlinux
** extend scheduler.sh:
-*** actually schedule old packages once queue is empty
-*** also delete unknown packages from db
-*** check/make sure that updated packages are only scheduled once
-*** dont schedule blacklisted packages
+*** actually schedule old packages once queue is empty (except blacklisted packages)
+*** also delete unknown packages from db: sources and schedule, later results as well
+*** check/make sure that packages which are newer in trunk than repo are only scheduled once
** make build.sh read & write db
*** delete all pkg.needs_build files on disk
*** drop all occurrances of pkg.needs_build from the code
=====================================
bin/reproducible_archlinux_scheduler.sh
=====================================
@@ -23,6 +23,7 @@ update_archlinux_repositories() {
local SESSION="archlinux-scheduler-$RANDOM"
schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
schroot --run-session -c $SESSION --directory /var/tmp -- sudo pacman -Syu --noconfirm
+ local BLACKLIST="/($(echo $ARCHLINUX_BLACKLISTED | sed "s# #|#g"))/"
#
# Get a list of unique package bases. Non-split packages don't have a pkgbase set
@@ -90,15 +91,18 @@ update_archlinux_repositories() {
elif [ "$VERSION" != "$version" ] ; then
VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"
if [ "$VERCMP" = "1" ] ; then
- # known package but with new version, update db and schedule
+ # known package with new version, so update db and schedule
echo $REPO/$pkgbase >> $UPDATED
echo "$REPO/$pkgbase $VERSION is known in the database, but repo has $version which is newer, so rescheduling... "
echo " UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"
query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"
- PKGID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
- echo " INSERT INTO schedule (package_id, date_scheduled) VALUES ('$PKGID', '$DATE');"
- query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('$PKGID', '$DATE');"
-
+ if [ -z $(echo $PKG | egrep -v "$BLACKLIST") ] ; then
+ echo "$PKG is blacklisted, so not scheduling it."
+ else
+ PKGID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
+ echo " INSERT INTO schedule (package_id, date_scheduled) VALUES ('$PKGID', '$DATE');"
+ query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('$PKGID', '$DATE');"
+ fi
elif [ "$VERCMP" = "-1" ] ; then
# our version is higher than what's in the repo because we build trunk
echo "$REPO/$pkgbase $VERSION in db is higher than $version in repo because we build trunk."
@@ -161,7 +165,6 @@ update_archlinux_repositories() {
local THRESHOLD=450
if [ $(find $BASE/archlinux/ -name pkg.needs_build | wc -l ) -le $THRESHOLD ] ; then
rm -f $OLDER
- local BLACKLIST="/($(echo $ARCHLINUX_BLACKLISTED | sed "s# #|#g"))/"
# reschedule
for i in $( ( for REPO in $ARCHLINUX_REPOS ; do
find $BASE/archlinux/$REPO -type d -wholename "$BASE/archlinux/$REPO/*" -printf '%T+ %p\n' | egrep -v "$BLACKLIST"
=====================================
bin/reproducible_common.sh
=====================================
@@ -239,7 +239,7 @@ write_page_header() {
class=''
fi
uri="/debian/${s}/index_suite_${ARCH}_stats.html"
- if [ $comma == 1 ] ; then
+ if [ $comma = 1 ] ; then
suite_links+=", {\"s\": \"${s}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
else
suite_links+="{\"s\": \"${s}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
@@ -258,7 +258,7 @@ write_page_header() {
class=''
fi
uri="/debian/$SUITE/index_suite_${a}_stats.html"
- if [ $comma == 1 ] ; then
+ if [ $comma = 1 ] ; then
arch_links+=", {\"a\": \"${a}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
else
arch_links+="{\"a\": \"${a}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
=====================================
bin/reproducible_html_nodes_info.sh
=====================================
@@ -39,7 +39,7 @@ for ARCH in ${ARCHS} ; do
write_page "<th>pbuilder setup $SUITE</th>"
done
for SUITE in ${SUITES} ; do
- if [ "$SUITE" == "experimental" ]; then
+ if [ "$SUITE" = "experimental" ]; then
continue
fi
write_page "<th>schroot setup $SUITE</th>"
@@ -119,7 +119,7 @@ for ARCH in ${ARCHS} ; do
done
# diffoscope schroot setup
for SUITE in ${SUITES} ; do
- if [ "$SUITE" == "experimental" ]; then
+ if [ "$SUITE" = "experimental" ]; then
continue
fi
URL="https://jenkins.debian.net/view/reproducible/view/Debian_setup_${ARCH}/job/reproducible_setup_schroot_${SUITE}_diffoscope_${ARCH}_${JENKINS_NODENAME}"
=====================================
bin/reproducible_maintenance.sh
=====================================
@@ -126,14 +126,14 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
arm64) NODE="codethink-sled${NODE_ALIAS#codethink}-arm64.debian.net" ;;
armhf) NODE="${NODE_ALIAS}-armhf-rb.debian.net" ;;
esac
- if [ "$NODE" == "jenkins" ] ; then
+ if [ "$NODE" = "jenkins" ] ; then
echo 'Skipping jenkins...'
continue
fi
cd $i/builds
LAST=$(ls -rt1 | tail -1)
GOOD=$(basename $(readlink -f lastStableBuild))
- if [ "$LAST" == "$GOOD" ] ; then
+ if [ "$LAST" = "$GOOD" ] ; then
DIFF=0
else
let DIFF=$LAST-$GOOD || DIFF=-1
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/93099d634c128bf35cd25a80920c584d66eed958...3d46a8655142cd0b5421e77bbd264b81cd0e6d9f
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/93099d634c128bf35cd25a80920c584d66eed958...3d46a8655142cd0b5421e77bbd264b81cd0e6d9f
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20180922/60f86d6e/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list