[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: reproducible Archlinux: announce summary of scheduled packages again

Holger Levsen gitlab at salsa.debian.org
Tue Sep 25 19:20:00 BST 2018 

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
41620d4a by Holger Levsen at 2018-09-25T16:57:04Z
reproducible Archlinux: announce summary of scheduled packages again

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
53666fe8 by Holger Levsen at 2018-09-25T18:19:11Z
reproducible Archlinux: re-schedule old packages again and re-enable irc notification

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


2 changed files:

- TODO
- bin/reproducible_archlinux_scheduler.sh


Changes:

=====================================
TODO
=====================================
@@ -320,10 +320,8 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
 * use db - see https://tests.reproducible-builds.org/reproducibledb.html
 ** extend scheduler.sh:
 *** comparing versions is probably needed: if its not empty it must be higher (due to repo constraints), but it can be even higher than in the repo, because we build trunk
-**** at first reschedule packages which never have been build (according to the db)
-*** also delete unknown packages from db: sources and schedule, later results as well
+*** also delete unknown packages from db: sources and schedule and results as well
 *** check/make sure that packages which are newer in trunk than repo are only scheduled once
-*** fix irc notification
 ** make _archlinux_build.sh
 *** refactor and reuse unregister_build from _build.sh
 *** we share /var/log/jenkins/reproducible-race-conditions.log with debian, is that ok? (else fix maintenance job and handle_race_condition())


=====================================
bin/reproducible_archlinux_scheduler.sh
=====================================
@@ -18,11 +18,12 @@ update_archlinux_repositories() {
 	#
 	local UPDATED=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
 	local NEW=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
+	local OLD=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
 	local KNOWN=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
 	local BLACKLIST="/($(echo $ARCHLINUX_BLACKLISTED | sed "s# #|#g"))/"
 	local TOTAL=$(cat ${ARCHLINUX_PKGS}_* | wc -l)
 	echo "$(date -u ) - $TOTAL Arch Linux packages were previously known to Arch Linux."
-	query_db "select suite, name, version FROM sources WHERE architecture='$ARCH';" > $KNOWN
+	query_db "SELECT suite, name, version FROM sources WHERE architecture='$ARCH';" > $KNOWN
 	echo "$(date -u ) - $(cat $KNOWN | wc -l) Arch Linux packages are known in our database."
 	# init session
 	local SESSION="archlinux-scheduler-$RANDOM"
@@ -45,7 +46,7 @@ update_archlinux_repositories() {
 		done | sort -u -R > "$ARCHLINUX_PKGS"_full_pkgbase_list
 	TOTAL=$(cat ${ARCHLINUX_PKGS}_full_pkgbase_list | wc -l)
 	echo "$(date -u ) - $TOTAL Arch Linux packages are now known to Arch Linux."
-	local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' and sch.date_build_started is NULL;")
+	local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")
 	echo "$(date -u) - updating Arch Linux repositories, currently $total packages scheduled."
 
 	#
@@ -53,7 +54,7 @@ update_archlinux_repositories() {
 	#
 	#if [ $(date +'%H') -gt 21 ] ; then
 	#FIXME: the next lines actually disables this code block...
-	#if [ $(date +'%H') -gt 25 ] ; then
+	if [ $(date +'%H') -gt 25 ] ; then
 		REMOVED=0
 		REMOVE_LIST=""
 		for REPO in $ARCHLINUX_REPOS ; do
@@ -77,7 +78,7 @@ update_archlinux_repositories() {
 		if [ $REMOVED -ne 0 ] ; then
 			irc_message archlinux-reproducible "$MESSAGE"
 		fi
-	#fi
+	fi
 	
 	#
 	# schedule packages
@@ -95,6 +96,7 @@ update_archlinux_repositories() {
 				VERSION=$(echo ${PKG_IN_DB} | cut -d "|" -f3)
 			        if [ -z "${PKG_IN_DB}" ] ; then
 					# new package, add to db and schedule
+					echo $REPO/$pkgbase >> $NEW
 					echo "new package found: $repo/$pkgbase $version "
 					query_db "INSERT into sources (name, version, suite, architecture) VALUES ('$PKG', '$version', '$SUITE', '$ARCH');"
 					PKG_ID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
@@ -132,9 +134,9 @@ update_archlinux_repositories() {
 				printf '%s %s\n' "$pkgbase" "$version" >> $TMPPKGLIST
 			done
 		mv $TMPPKGLIST "$ARCHLINUX_PKGS"_"$REPO"
-		#new=$(grep -c ^$REPO $NEW || true)
-		#updated=$(grep -c ^$REPO $UPDATED || true)
-		#FIXME echo "$(date -u ) - scheduled $new/$updated packages in repository '$REPO'."
+		new=$(grep -c ^$REPO $NEW || true)
+		updated=$(grep -c ^$REPO $UPDATED || true)
+		echo "$(date -u ) - scheduled $new/$updated packages in repository '$REPO'."
 	done
 	schroot --end-session -c $SESSION
 
@@ -142,18 +144,42 @@ update_archlinux_repositories() {
 	# schedule up to $MAX packages we already know about
 	# (only if less than $THRESHOLD packages are currently scheduled)
 	#
+	echo "$(date -u ) - should we schedule old packages?"
 	old=""
 	local MAX=350
 	local THRESHOLD=450
-	#FIXME add actual code here :)
+	local MINDATE=$(date -u +"%Y-%m-%d %H:%M" -d "14 days ago")
+	local SCHDATE=$(date -u +"%Y-%m-%d %H:%M" -d "7 days")
+	local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+	if [ $CURRENT -le $THRESHOLD ] ; then
+		echo "$(date -u ) - scheduling $MAX old packages."
+		QUERY="SELECT s.id, s.name, max(r.build_date) max_date
+			FROM sources AS s JOIN results AS r ON s.id = r.package_id
+			WHERE s.architecture='x86_64'
+			AND r.status != 'blacklisted'
+			AND r.build_date < '$MINDATE'
+			AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+			GROUP BY s.id, s.name
+			ORDER BY max_date
+			LIMIT $MAX;"
+		OLD=$(query_db "$QUERY")
+		for PKG_ID in $(cut -d '|' -f1 $OLD) ; do
+			QUERY="INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$SCHDATE');"
+			query_db "$QUERY_DB"
+		done
+		echo "$(date -u ) - done scheduling $MAX old packages."
+	else
+		echo "$(date -u ) - $CURRENT packages already scheduled, not scheduling more."
+	fi
 
 	#
 	# output stats
 	#
-	total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' and sch.date_build_started is NULL;")
 	rm "$ARCHLINUX_PKGS"_full_pkgbase_list
+	total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")
 	new=$(cat $NEW | wc -l 2>/dev/null|| true)
 	updated=$(cat $UPDATED 2>/dev/null| wc -l || true)
+	old=$(cat $OLD | wc -l 2>/dev/null|| true)
 	if [ $new -ne 0 ] || [ $updated -ne 0 ] || [ -n "$old" ] ; then
 		message="scheduled"
 		if [ $new -ne 0 ] ; then
@@ -165,17 +191,18 @@ update_archlinux_repositories() {
 		if [ $updated -ne 0 ] ; then
 			message="$message $updated packages with newer versions"
 		fi
-		if [ -n "$old" ] && ( [ $new -ne 0 ] || [ $updated -ne 0 ] ) ; then
-			old=", plus$old"
-		fi
+		if [ $old -ne 0 ] && ( [ $new -ne 0 ] || [ $updated -ne 0 ] ) ; then
+			old=", plus $old already tested ones"
+		elif [ $old -ne 0 ] ; then
+			old="$old already tested packages"
+	fi
 		MESSAGE="${message}$old, for $total scheduled out of $TOTAL."
-		#FIXME irc_message archlinux-reproducible "$MESSAGE"
-		#echo "$(date -u ) - $MESSAGE"
-	#else
-		#echo "$(date -u ) - didn't schedule any packages."
+		echo -n "$(date -u ) - "
+		irc_message archlinux-reproducible "$MESSAGE"
+	else
+		echo "$(date -u ) - didn't schedule any packages."
 	fi
-	rm $NEW $UPDATED $KNOWN > /dev/null
-	echo "$(date -u) - done updating Arch Linux repositories and scheduling, $TOTAL packages known and $total packages scheduled."
+	rm -f $NEW $UPDATED $OLD $KNOWN > /dev/null
 }
 
 ARCH="x86_64"



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/04d72b3e1c45c9cf88afd59de61a55fb3fabf335...53666fe84bfd55e4a5d166b22672e5148c71c70d

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/04d72b3e1c45c9cf88afd59de61a55fb3fabf335...53666fe84bfd55e4a5d166b22672e5148c71c70d
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20180925/30cb4a9b/attachment-0001.html>

More information about the Qa-jenkins-scm mailing list