[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible Archlinux: exclusivly use the db to track blacklisted packages

Holger Levsen gitlab at salsa.debian.org
Mon Dec 3 13:40:47 GMT 2018


Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
fa0996b4 by Holger Levsen at 2018-12-03T12:17:07Z
reproducible Archlinux: exclusivly use the db to track blacklisted packages

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


3 changed files:

- TODO
- bin/reproducible_archlinux_scheduler.sh
- bin/reproducible_common.sh


Changes:

=====================================
TODO
=====================================
@@ -315,7 +315,6 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
 ** check for archlinux schroot sessions which should not be there and delete them. complain if that fails.
 
 * use db - see https://tests.reproducible-builds.org/reproducibledb.htm
-** blacklisting uses shell variables and db state, thats horrible
 ** scheduler.sh:
 *** use asp to update trunk packages?
 ** html:


=====================================
bin/reproducible_archlinux_scheduler.sh
=====================================
@@ -23,7 +23,6 @@ update_archlinux_repositories() {
 	local UPDATED=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
 	local NEW=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
 	local KNOWN=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
-	local BLACKLIST="/($(echo $ARCHLINUX_BLACKLISTED | sed "s# #|#g"))/"
 	local TOTAL=$(cat ${ARCHLINUX_PKGS}_* | wc -l)
 	echo "$(date -u ) - $TOTAL Arch Linux packages were previously known to Arch Linux."
 	query_db "SELECT suite, name, version FROM sources WHERE architecture='$ARCH';" > $KNOWN
@@ -115,12 +114,14 @@ update_archlinux_repositories() {
 					VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"
 					if [ "$VERCMP" = "1" ] ; then
 						# known package with new version, so update db and schedule
-						echo $REPO/$pkgbase >> $UPDATED
-						echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
 						query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"
-						if [ -z $(echo $PKG | egrep -v "$BLACKLIST") ] ; then
+						PKG_STATUS=$(query_db "SELECT status FROM results WHERE package_id='$PKG_ID';")
+						if [ "$PKG_STATUS" = "BLACKLISTED" ] ; then
 							echo "$PKG is blacklisted, so not scheduling it."
+							continue
 						else
+							echo $REPO/$pkgbase >> $UPDATED
+							echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
 							PKG_ID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
 							echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID}';"
 							SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
@@ -188,12 +189,10 @@ update_archlinux_repositories() {
 	local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")
 	if [ $CURRENT -le $THRESHOLD ] ; then
 		echo "$(date -u ) - scheduling $MAX old packages."
-		# this query schedules blacklisted packages as they dont have status blacklisted in db...
-		# FIXME: remove bash string blacklisting NOW
 		QUERY="SELECT s.id, s.name, max(r.build_date) max_date
 			FROM sources AS s JOIN results AS r ON s.id = r.package_id
 			WHERE s.architecture='x86_64'
-			AND r.status != 'blacklisted'
+			AND r.status != 'BLACKLISTED'
 			AND r.build_date < '$MINDATE'
 			AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
 			GROUP BY s.id, s.name


=====================================
bin/reproducible_common.sh
=====================================
@@ -58,7 +58,6 @@ USERTAGS="toolchain infrastructure timestamps fileordering buildpath username ho
 # common settings for testing Arch Linux
 ARCHLINUX_REPOS="core extra multilib community"
 ARCHLINUX_PKGS=/srv/reproducible-results/archlinux_pkgs
-ARCHLINUX_BLACKLISTED="syslinux ltrace gnutls haskell-hakyll mitmproxy pandoc python-pyftpdlib v2ray python-urllib3 perl-net-ssleay"
 ARCHBASE=$BASE/archlinux
 
 # common settings for testing rpm based distros
@@ -989,8 +988,6 @@ create_pkg_html() {
 	local buffer_message
 	local STATE
 
-	local blacklisted=false
-
 	# clear files from previous builds
 	cd "$ARCHLINUX_PKG_PATH"
 	for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
@@ -1008,21 +1005,13 @@ create_pkg_html() {
 	#
 	#
 	if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
-		for i in $ARCHLINUX_BLACKLISTED ; do
-			if [ "xxx${SRCPACKAGE}xxx" = "xxx${i}xxx" ] ; then
-				blacklisted=true
-			fi
-		done
 		# this horrible if elif elif elif elif...  monster is needed because
 		# https://lists.archlinux.org/pipermail/pacman-dev/2017-September/022156.html
 	        # has not yet been merged yet...
 		# FIXME: this has been merged, see http://jlk.fjfi.cvut.cz/arch/manpages/man/makepkg
 
 		# check different states and figure out what the page should look like
-		if $blacklisted ; then
-			STATE=BLACKLISTED
-			buffer_message='blacklisted'
-		elif find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
+		if find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
 			STATE=DEPWAIT_0
 			buffer_message='could not resolve dependencies as there are conflicts'
 		elif find_in_buildlogs '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)'; then
@@ -1094,11 +1083,20 @@ create_pkg_html() {
 			STATE=FTBFS_4
 			buffer_message='failed to build, pkg relations contain invalid characters'
 		else
-			STATE=UNKNOWN
-			buffer_message='probably failed to build from source, please investigate'
+			STATE=query_db "SELECT r.status FROM results AS r
+				JOIN sources as s on s.id=r.package_id
+				WHERE s.architecture='x86_64'
+				AND s.name='$SRCPACKAGE'
+				AND s.suite='archlinux_$REPOSITORY';"
+			if [ "$STATE" = "BLACKLISTED" ] ; then
+				buffer_message='blacklisted'
+			else
+				STATE=UNKNOWN
+				buffer_message='probably failed to build from source, please investigate'
+			fi
 		fi
 		# print build failures
-		if [[ $STATE = UNKNOWN ]]; then
+		if [ "$STATE" = "UNKNOWN" ]; then
 			echo "       $buffer_message" >> $HTML_BUFFER
 		else
 			include_icon $STATE "$buffer_message"



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/fa0996b4dfdd280457045cab28424665f60c5285

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/fa0996b4dfdd280457045cab28424665f60c5285
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20181203/f5ba6254/attachment-0001.html>


More information about the Qa-jenkins-scm mailing list