[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible Archlinux: exclusivly use the db to track blacklisted packages
Holger Levsen
gitlab at salsa.debian.org
Mon Dec 3 13:40:47 GMT 2018
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
fa0996b4 by Holger Levsen at 2018-12-03T12:17:07Z
reproducible Archlinux: exclusivly use the db to track blacklisted packages
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
3 changed files:
- TODO
- bin/reproducible_archlinux_scheduler.sh
- bin/reproducible_common.sh
Changes:
=====================================
TODO
=====================================
@@ -315,7 +315,6 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
** check for archlinux schroot sessions which should not be there and delete them. complain if that fails.
* use db - see https://tests.reproducible-builds.org/reproducibledb.htm
-** blacklisting uses shell variables and db state, thats horrible
** scheduler.sh:
*** use asp to update trunk packages?
** html:
=====================================
bin/reproducible_archlinux_scheduler.sh
=====================================
@@ -23,7 +23,6 @@ update_archlinux_repositories() {
local UPDATED=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
local NEW=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
local KNOWN=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
- local BLACKLIST="/($(echo $ARCHLINUX_BLACKLISTED | sed "s# #|#g"))/"
local TOTAL=$(cat ${ARCHLINUX_PKGS}_* | wc -l)
echo "$(date -u ) - $TOTAL Arch Linux packages were previously known to Arch Linux."
query_db "SELECT suite, name, version FROM sources WHERE architecture='$ARCH';" > $KNOWN
@@ -115,12 +114,14 @@ update_archlinux_repositories() {
VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"
if [ "$VERCMP" = "1" ] ; then
# known package with new version, so update db and schedule
- echo $REPO/$pkgbase >> $UPDATED
- echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"
- if [ -z $(echo $PKG | egrep -v "$BLACKLIST") ] ; then
+ PKG_STATUS=$(query_db "SELECT status FROM results WHERE package_id='$PKG_ID';")
+ if [ "$PKG_STATUS" = "BLACKLISTED" ] ; then
echo "$PKG is blacklisted, so not scheduling it."
+ continue
else
+ echo $REPO/$pkgbase >> $UPDATED
+ echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
PKG_ID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID}';"
SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
@@ -188,12 +189,10 @@ update_archlinux_repositories() {
local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")
if [ $CURRENT -le $THRESHOLD ] ; then
echo "$(date -u ) - scheduling $MAX old packages."
- # this query schedules blacklisted packages as they dont have status blacklisted in db...
- # FIXME: remove bash string blacklisting NOW
QUERY="SELECT s.id, s.name, max(r.build_date) max_date
FROM sources AS s JOIN results AS r ON s.id = r.package_id
WHERE s.architecture='x86_64'
- AND r.status != 'blacklisted'
+ AND r.status != 'BLACKLISTED'
AND r.build_date < '$MINDATE'
AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
GROUP BY s.id, s.name
=====================================
bin/reproducible_common.sh
=====================================
@@ -58,7 +58,6 @@ USERTAGS="toolchain infrastructure timestamps fileordering buildpath username ho
# common settings for testing Arch Linux
ARCHLINUX_REPOS="core extra multilib community"
ARCHLINUX_PKGS=/srv/reproducible-results/archlinux_pkgs
-ARCHLINUX_BLACKLISTED="syslinux ltrace gnutls haskell-hakyll mitmproxy pandoc python-pyftpdlib v2ray python-urllib3 perl-net-ssleay"
ARCHBASE=$BASE/archlinux
# common settings for testing rpm based distros
@@ -989,8 +988,6 @@ create_pkg_html() {
local buffer_message
local STATE
- local blacklisted=false
-
# clear files from previous builds
cd "$ARCHLINUX_PKG_PATH"
for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
@@ -1008,21 +1005,13 @@ create_pkg_html() {
#
#
if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
- for i in $ARCHLINUX_BLACKLISTED ; do
- if [ "xxx${SRCPACKAGE}xxx" = "xxx${i}xxx" ] ; then
- blacklisted=true
- fi
- done
# this horrible if elif elif elif elif... monster is needed because
# https://lists.archlinux.org/pipermail/pacman-dev/2017-September/022156.html
# has not yet been merged yet...
# FIXME: this has been merged, see http://jlk.fjfi.cvut.cz/arch/manpages/man/makepkg
# check different states and figure out what the page should look like
- if $blacklisted ; then
- STATE=BLACKLISTED
- buffer_message='blacklisted'
- elif find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
+ if find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
STATE=DEPWAIT_0
buffer_message='could not resolve dependencies as there are conflicts'
elif find_in_buildlogs '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)'; then
@@ -1094,11 +1083,20 @@ create_pkg_html() {
STATE=FTBFS_4
buffer_message='failed to build, pkg relations contain invalid characters'
else
- STATE=UNKNOWN
- buffer_message='probably failed to build from source, please investigate'
+ STATE=query_db "SELECT r.status FROM results AS r
+ JOIN sources as s on s.id=r.package_id
+ WHERE s.architecture='x86_64'
+ AND s.name='$SRCPACKAGE'
+ AND s.suite='archlinux_$REPOSITORY';"
+ if [ "$STATE" = "BLACKLISTED" ] ; then
+ buffer_message='blacklisted'
+ else
+ STATE=UNKNOWN
+ buffer_message='probably failed to build from source, please investigate'
+ fi
fi
# print build failures
- if [[ $STATE = UNKNOWN ]]; then
+ if [ "$STATE" = "UNKNOWN" ]; then
echo " $buffer_message" >> $HTML_BUFFER
else
include_icon $STATE "$buffer_message"
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/fa0996b4dfdd280457045cab28424665f60c5285
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/fa0996b4dfdd280457045cab28424665f60c5285
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20181203/f5ba6254/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list