[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 5 commits: reproducible alpine: properly upgrade alpine in maintainance job

Mattia Rizzolo gitlab at salsa.debian.org
Tue Dec 3 19:39:04 GMT 2019



Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net


Commits:
6c3c15e0 by kpcyrd at 2019-12-03T19:11:40Z
reproducible alpine: properly upgrade alpine in maintainance job

- - - - -
2d81fa1a by kpcyrd at 2019-12-03T19:11:40Z
reproducible alpine: force install locally built package

abuild-keygen usually configures this properly

- - - - -
30138aa1 by kpcyrd at 2019-12-03T19:23:46Z
reproducible alpine: automatically remove packages that no longer exist

- - - - -
74cc9c58 by Mattia Rizzolo at 2019-12-03T19:38:21Z
remove needless +x bit

Signed-off-by: Mattia Rizzolo <mattia at debian.org>

- - - - -
7677b378 by Mattia Rizzolo at 2019-12-03T19:38:37Z
reproducible Debian: use another lib to do version comparison

Signed-off-by: Mattia Rizzolo <mattia at debian.org>

- - - - -


11 changed files:

- bin/rblib/__init__.py
- bin/rblib/bugs.py
- bin/rblib/confparse.py
- bin/rblib/const.py
- bin/rblib/html.py
- bin/rblib/models.py
- bin/rblib/utils.py
- bin/reproducible_alpine_scheduler.sh
- bin/reproducible_maintenance.sh
- bin/reproducible_scheduler.py
- bin/reproducible_setup_alpine_schroot.sh


Changes:

=====================================
bin/rblib/__init__.py
=====================================


=====================================
bin/rblib/bugs.py
=====================================


=====================================
bin/rblib/confparse.py
=====================================


=====================================
bin/rblib/const.py
=====================================


=====================================
bin/rblib/html.py
=====================================


=====================================
bin/rblib/models.py
=====================================


=====================================
bin/rblib/utils.py
=====================================


=====================================
bin/reproducible_alpine_scheduler.sh
=====================================
@@ -53,41 +53,41 @@ update_alpine_repositories() {
 	#
 	# remove packages which are gone (only when run between 21:00 and 23:59)
 	#
-#	if [ $(date +'%H') -gt 21 ] ; then
-#		REMOVED=0
-#		REMOVE_LIST=""
-#		for REPO in $ALPINE_REPOS ; do
-#			echo "$(date -u ) - dropping removed packages from filesystem in repository '$REPO':"
-#			for i in $(find $BASE/alpine/$REPO -type d -wholename "$BASE/alpine/$REPO/*" | sort) ; do
-#				PKG=$(basename $i)
-#				if ! grep -q "$REPO $PKG" ${ALPINE_PKGS}_full_pkgbase_list > /dev/null ; then
-#					# we could check here whether a package is currently building,
-#					# and if so defer the pkg removal. (but I think this is pointless,
-#					# as we are unlikely to kill that build, so meh, let it finish
-#					# and fail to update the db, because the package is gone...)
-#					let REMOVED=$REMOVED+1
-#					REMOVE_LIST="$REMOVE_LIST $REPO/$PKG"
-#					rm -r --one-file-system $BASE/alpine/$REPO/$PKG
-#					echo "$(date -u) - $REPO/$PKG removed as it's gone from the alpine repositories."
-#					SUITE="alpine_$REPO"
-#					PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
-#					if [ -n "${PKG_ID}" ] ; then
-#						query_db "DELETE FROM results WHERE package_id='${PKG_ID}';"
-#						query_db "DELETE FROM schedule WHERE package_id='${PKG_ID}';"
-#						query_db "DELETE FROM sources WHERE id='${PKG_ID}';"
-#						echo "$(date -u) - $SUITE $PKG removed from database."
-#					else
-#						echo "$(date -u) - $SUITE $PKG not found in database."
-#					fi
-#				fi
-#			done
-#		done
-#		MESSAGE="Deleted $REMOVED packages: $REMOVE_LIST"
-#		echo -n "$(date -u ) - "
-#		if [ $REMOVED -ne 0 ] ; then
-#			irc_message alpine-reproducible "$MESSAGE"
-#		fi
-#	fi
+	if [ $(date +'%H') -gt 21 ] ; then
+		REMOVED=0
+		REMOVE_LIST=""
+		for REPO in $ALPINE_REPOS ; do
+			echo "$(date -u ) - dropping removed packages from filesystem in repository '$REPO':"
+			for i in $(find $BASE/alpine/$REPO -type d -wholename "$BASE/alpine/$REPO/*" | sort) ; do
+				PKG=$(basename $i)
+				if ! grep -q "$REPO $PKG" ${ALPINE_PKGS}_full_pkgbase_list > /dev/null ; then
+					# we could check here whether a package is currently building,
+					# and if so defer the pkg removal. (but I think this is pointless,
+					# as we are unlikely to kill that build, so meh, let it finish
+					# and fail to update the db, because the package is gone...)
+					let REMOVED=$REMOVED+1
+					REMOVE_LIST="$REMOVE_LIST $REPO/$PKG"
+					rm -r --one-file-system $BASE/alpine/$REPO/$PKG
+					echo "$(date -u) - $REPO/$PKG removed as it's gone from the alpine repositories."
+					SUITE="alpine_$REPO"
+					PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
+					if [ -n "${PKG_ID}" ] ; then
+						query_db "DELETE FROM results WHERE package_id='${PKG_ID}';"
+						query_db "DELETE FROM schedule WHERE package_id='${PKG_ID}';"
+						query_db "DELETE FROM sources WHERE id='${PKG_ID}';"
+						echo "$(date -u) - $SUITE $PKG removed from database."
+					else
+						echo "$(date -u) - $SUITE $PKG not found in database."
+					fi
+				fi
+			done
+		done
+		MESSAGE="Deleted $REMOVED packages: $REMOVE_LIST"
+		echo -n "$(date -u ) - "
+		if [ $REMOVED -ne 0 ] ; then
+			irc_message alpine-reproducible "$MESSAGE"
+		fi
+	fi
 
 	#
 	# schedule packages


=====================================
bin/reproducible_maintenance.sh
=====================================
@@ -332,6 +332,7 @@ case $HOSTNAME in
 	osuosl-build169*|osuosl-build170*|jenkins)
 		echo "$(date -u) - updating alpine schroot now."
 		schroot --directory /tmp -c source:jenkins-reproducible-alpine -u root -- apk update
+		schroot --directory /tmp -c source:jenkins-reproducible-alpine -u root -- apk upgrade
 		RESULT=$?
 		if [ $RESULT -eq 1 ] ; then
 			echo "Warning: failed to update alpine schroot."


=====================================
bin/reproducible_scheduler.py
=====================================
@@ -13,10 +13,10 @@
 import sys
 import lzma
 import deb822
-import apt_pkg
 from sqlalchemy import sql
 from urllib.request import urlopen
 from datetime import datetime, timedelta
+from debian.debian_support import Version
 
 from rblib import query_db, db_table
 from rblib.confparse import log
@@ -27,7 +27,6 @@ from reproducible_html_live_status import generate_schedule
 from reproducible_html_packages import gen_packages_html
 from reproducible_html_packages import purge_old_pages
 
-apt_pkg.init_system()
 
 """
 How the scheduler chooses which limit to apply, based on the MAXIMA
@@ -334,7 +333,7 @@ def update_sources_db(suite, arch, sources):
         pkg_id = result[0]
         old_version = result[1]
         notify_maint = int(result[2])
-        if apt_pkg.version_compare(pkg[1], old_version) > 0:
+        if Version(pkg[1]) > Version(old_version):
             log.debug('New version: ' + str(pkg) + ' (we had  ' +
                       old_version + ')')
             updated_pkgs.append({
@@ -495,7 +494,7 @@ def query_new_versions(suite, arch, limit):
     # packages in our repository != official repo,
     # so they will always be selected by the query above
     # so we only accept them if there version is greater than the already tested one
-    packages = [(x[0], x[1]) for x in pkgs if apt_pkg.version_compare(x[2], x[3]) > 0]
+    packages = [(x[0], x[1]) for x in pkgs if Version(x[2]) > Version(x[3])]
     print_schedule_result(suite, arch, criteria, packages)
     return packages
 


=====================================
bin/reproducible_setup_alpine_schroot.sh
=====================================
@@ -164,7 +164,7 @@ echo "$(date -u) - cloning aports repo"
 $USERCMD sh -c "$GIT_OPTIONS git clone --depth=1 https://git.alpinelinux.org/aports.git /var/lib/jenkins/aports"
 
 # build and install a patched abuild
-$USERCMD sh -c "cd /var/lib/jenkins/aports/main/abuild && base64 -d | git apply - && PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' abuild -r && sudo /sbin/apk add ~/packages/main/x86_64/abuild-3.5.0_rc2-r1.apk && git checkout ." <<-__END__
+$USERCMD sh -c "cd /var/lib/jenkins/aports/main/abuild && base64 -d | git apply - && PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' abuild -r && sudo /sbin/apk add --allow-untrusted ~/packages/main/x86_64/abuild-3.5.0_rc2-r1.apk && git checkout ." <<-__END__
 ZGlmZiAtLWdpdCBhL21haW4vYWJ1aWxkLzAwMDItcmVwcm8ucGF0Y2ggYi9tYWluL2FidWlsZC8w
 MDAyLXJlcHJvLnBhdGNoCm5ldyBmaWxlIG1vZGUgMTAwNjQ0CmluZGV4IDAwMDAwMDAwLi41ZmYy
 MjAxMwotLS0gL2Rldi9udWxsCisrKyBiL21haW4vYWJ1aWxkLzAwMDItcmVwcm8ucGF0Y2gKQEAg



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/d37be0c227a208f5780e18c6317e570851406dcf...7677b378ddc2ad6b24676a219e93bf33de2977d6

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/d37be0c227a208f5780e18c6317e570851406dcf...7677b378ddc2ad6b24676a219e93bf33de2977d6
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20191203/065d24c7/attachment-0001.html>


More information about the Qa-jenkins-scm mailing list