[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible Archlinux scheduler: clean output

Holger Levsen gitlab at salsa.debian.org
Sun Sep 23 00:20:26 BST 2018 

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
3f73de7d by Holger Levsen at 2018-09-22T23:20:00Z
reproducible Archlinux scheduler: clean output

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


2 changed files:

- TODO
- bin/reproducible_archlinux_scheduler.sh


Changes:

=====================================
TODO
=====================================
@@ -337,13 +337,11 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
 **** call pkg html function after each build
 *** create index pages
 *** create new archlinux.html - keep the old one til this works
-
-** cleanup now useless code from scheduler
 ** create json
 
 * fix build.sh:
 ** build2.log doesnt get deleted if build1 fails
-** -> rename build2.log to $pkg_$version_build2.log
+** -> rename build2.log to $version_build2.log (dont include package name...)
 
 * test against real archlinux pkgs:
 ** https://github.com/Foxboron/devtools-repro


=====================================
bin/reproducible_archlinux_scheduler.sh
=====================================
@@ -13,23 +13,28 @@ common_init "$@"
 set -e
 
 update_archlinux_repositories() {
-	local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' and sch.date_build_started is NULL;")
-	echo "$(date -u) - Updating Arch Linux repositories, currently $total packages scheduled."
 	#
 	# init
 	#
-	UPDATED=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
-	NEW=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
-	KNOWN=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
+	local UPDATED=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
+	local NEW=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
+	local KNOWN=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
+	local BLACKLIST="/($(echo $ARCHLINUX_BLACKLISTED | sed "s# #|#g"))/"
+	local TOTAL=$(cat ${ARCHLINUX_PKGS}_* | wc -l)
+	echo "$(date -u ) - $TOTAL Arch Linux packages were previously known to Arch Linux."
+	query_db "select suite, name, version FROM sources WHERE architecture='$ARCH';" > $KNOWN
+	echo "$(date -u ) - $(cat $KNOWN | wc -l) Arch Linux packages are known in our database."
+	# init session
 	local SESSION="archlinux-scheduler-$RANDOM"
 	schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
+	echo "$(date -u ) - updating pacman's knowledge of Arch Linux repositories (by running pacman -Syu --noconform')."
 	schroot --run-session -c $SESSION --directory /var/tmp -- sudo pacman -Syu --noconfirm
-	local BLACKLIST="/($(echo $ARCHLINUX_BLACKLISTED | sed "s# #|#g"))/"
 
 	#
 	# Get a list of unique package bases.  Non-split packages don't have a pkgbase set
 	# so we need to use the pkgname for them instead.
 	#
+	echo "$(date -u ) - exporting pacman's knowledge of Arch Linux repositories to the filesystem (by running 'expac -S...')."
 	schroot --run-session -c $SESSION --directory /var/tmp -- expac -S '%r %e %n %v' | \
 		while read repo pkgbase pkgname version; do
 			if [[ "$pkgbase" = "(null)" ]]; then
@@ -39,7 +44,9 @@ update_archlinux_repositories() {
 			fi
 		done | sort -u -R > "$ARCHLINUX_PKGS"_full_pkgbase_list
 	TOTAL=$(cat ${ARCHLINUX_PKGS}_full_pkgbase_list | wc -l)
-	echo "$(date -u ) - $TOTAL Arch Linux packages are known in total to us."
+	echo "$(date -u ) - $TOTAL Arch Linux packages are now known to Arch Linux."
+	local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' and sch.date_build_started is NULL;")
+	echo "$(date -u) - updating Arch Linux repositories, currently $total packages scheduled."
 
 	#
 	# remove packages which are gone (only when run between 21:00 and 23:59)
@@ -50,15 +57,18 @@ update_archlinux_repositories() {
 		REMOVED=0
 		REMOVE_LIST=""
 		for REPO in $ARCHLINUX_REPOS ; do
-			echo "$(date -u ) - Dropping removed packages from repository '$REPO':"
+			echo "$(date -u ) - dropping removed packages from filesystem in repository '$REPO':"
 			for i in $(find $BASE/archlinux/$REPO -type d -wholename "$BASE/archlinux/$REPO/*" | sort) ; do
 				PKG=$(basename $i)
 				if ! grep -q "$REPO $PKG" ${ARCHLINUX_PKGS}_full_pkgbase_list > /dev/null ; then
 					let REMOVED=$REMOVED+1
 					REMOVE_LIST="$REMOVE_LIST $REPO/$PKG"
 					rm -r --one-file-system $BASE/archlinux/$REPO/$PKG
-					# FIXME: we actually need to drop them from the db now...
 				        echo "$REPO/$PKG removed as it's gone from the Archlinux repositories."
+					# FIXME: we actually need to drop them from the db now...
+					# from results
+					# from scheduled
+					# from sources
 				fi
 			done
 		done
@@ -72,11 +82,10 @@ update_archlinux_repositories() {
 	#
 	# schedule packages
 	#
-	query_db "select suite, name, version FROM sources WHERE architecture='$ARCH';" > $KNOWN
 
 	for REPO in $ARCHLINUX_REPOS ; do
 		TMPPKGLIST=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
-		echo "$(date -u ) - updating list of available packages in repository '$REPO'."
+		echo "$(date -u ) - updating database with available packages in repository '$REPO'."
 		DATE="$(date -u +'%Y-%m-%d %H:%M')"
 		grep "^$REPO" "$ARCHLINUX_PKGS"_full_pkgbase_list | \
 			while read repo pkgbase version; do
@@ -95,7 +104,7 @@ update_archlinux_repositories() {
 					if [ "$VERCMP" = "1" ] ; then
 						# known package with new version, so update db and schedule
 						echo $REPO/$pkgbase >> $UPDATED
-						echo "$REPO/$pkgbase $VERSION is known in the database, but repo has $version which is newer, so rescheduling... "
+						echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
 						query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"
 						if [ -z $(echo $PKG | egrep -v "$BLACKLIST") ] ; then
 							echo "$PKG is blacklisted, so not scheduling it."
@@ -166,7 +175,7 @@ update_archlinux_repositories() {
 		#echo "$(date -u ) - didn't schedule any packages."
 	fi
 	rm $NEW $UPDATED $KNOWN > /dev/null
-	echo "$(date -u) - Done updating Arch Linux repositories, currently $TOTAL packages known."
+	echo "$(date -u) - done updating Arch Linux repositories and scheduling, $TOTAL packages known and $total packages scheduled."
 }
 
 ARCH="x86_64"



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/3f73de7d4ce90c45396e73ac7cd1f72368eb6bf0

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/3f73de7d4ce90c45396e73ac7cd1f72368eb6bf0
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20180922/d094df52/attachment-0001.html>

More information about the Qa-jenkins-scm mailing list