[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 4 commits: reproducible Debian: improve comments

[Holger Levsen]

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
2ca84a97 by Holger Levsen at 2019-02-20T19:07:29Z
reproducible Debian: improve comments

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
084f95eb by Holger Levsen at 2019-02-20T19:14:17Z
reproducible Debian: add support for looping in different orderings

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
198e7a07 by Holger Levsen at 2019-02-20T19:20:24Z
reproducible Debian: add support for locking

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
02d19ea6 by Holger Levsen at 2019-02-20T19:23:06Z
reproducible Debian: add two more runners

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


2 changed files:

- bin/reproducible_compare_Debian_sha1sums.sh
- job-cfg/reproducible.yaml


Changes:

=====================================
bin/reproducible_compare_Debian_sha1sums.sh
=====================================
@@ -1,9 +1,10 @@
 #!/bin/bash
 
-# as posted by Vagrant on https://lists.reproducible-builds.org/pipermail/rb-general/2018-October/001239.html
-
 # Copyright 2019 Holger Levsen <holger at layer-acht.org>
 # released under the GPLv=2+
+#
+# based on an idea by Vagrant Cascadian <vagrant at debian.org>
+# see https://lists.reproducible-builds.org/pipermail/rb-general/2018-October/001239.html
 
 DEBUG=false
 . /srv/jenkins/bin/common-functions.sh
@@ -31,6 +32,7 @@ echo 'this is an early prototype...'
 echo
 echo
 RELEASE=buster
+MODE="$1"
 
 bdn_url="https://buildinfo.debian.net/api/v1/buildinfos/checksums/sha1"
 log=$(mktemp --tmpdir=$TMPDIR sha1-comp-XXXXXXX)
@@ -39,10 +41,14 @@ SHA1DIR=/srv/reproducible-results/debian-sha1
 mkdir -p $SHA1DIR
 cd $SHA1DIR
 
-# downloading (and keeping) all the packages is also too much, but let's prototype this... (and improve later)
 PACKAGES=$(mktemp --tmpdir=$TMPDIR sha1-comp-XXXXXXX)
 schroot --directory  $SHA1DIR -c chroot:jenkins-reproducible-${RELEASE}-diffoscope cat /var/lib/apt/lists/cdn-fastly.deb.debian.org_debian_dists_${RELEASE}_main_binary-amd64_Packages > $PACKAGES
-packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | sort | xargs echo)"
+case MODE in
+	random)		SORT="sort -R";;
+	reverse)	SORT="sort -r" ;;
+	*)		SORT="sort" ;;
+esac
+packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | $SORT | xargs echo)"
 
 reproducible_packages=
 unreproducible_packages=
@@ -73,9 +79,19 @@ cleanup_all() {
 
 trap cleanup_all INT TERM EXIT
 
+rm -f *.lock	# this is a tiny bit hackish, but also an elegant way to get rid of old locks...
+		# (locks are held for 30s only anyway and there is an 3/60000th chance of a race condition only anyway)
+
 for package in $packages ; do
 	cd $SHA1DIR
 	echo
+	LOCK="$SHA1DIR/${package}.lock"
+	if [ -e $LOCK ] ; then
+		echo "$(date -u) - skipping locked package $package"
+		continue
+	else
+		touch $LOCK
+	fi
 	echo "$(date -u) - checking whether we have seen the .deb for $package before"
 	version=$(grep-dctrl -X -P ${package} -s version -n $PACKAGES)
 	arch=$(grep-dctrl -X -P ${package} -s Architecture -n $PACKAGES)
@@ -106,6 +122,7 @@ for package in $packages ; do
 	else
 		echo "$(date -u) - UNREPRODUCIBLE: $package_file: $SHA1SUM_PKG on ftp.debian.org, but nowhere else."
 	fi
+	rm -f $LOCK
 done | tee $log
 
 cleanup_all


=====================================
job-cfg/reproducible.yaml
=====================================
@@ -584,6 +584,14 @@
                 - 'compare_Debian_sha1sums':
                     my_description: 'Compare sha1sums of Debian binary packages with rebuild ones'
                     my_timed: ''
+                - 'compare_Debian_sha1sums_reverse_runner':
+                    my_description: 'Compare sha1sums of Debian binary packages (in reverse order) with rebuild ones'
+                    my_timed: ''
+                    my_shell: '/srv/jenkins/bin/reproducible_compare_Debian_sha1sums.sh reverse'
+                - 'compare_Debian_sha1sums_random_runner':
+                    my_description: 'Compare sha1sums of Debian binary packages (in random order) with rebuild ones'
+                    my_timed: ''
+                    my_shell: '/srv/jenkins/bin/reproducible_compare_Debian_sha1sums.sh random'
             my_shellext: ".sh"
             my_shell: '/srv/jenkins/bin/reproducible_{my_task}{my_shellext}'
             my_hname: ''



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/e34647a1e6e9fcb23af93c279ed3aa9514de7494...02d19ea6e605d9b48c50880818577f2891a3de42

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/e34647a1e6e9fcb23af93c279ed3aa9514de7494...02d19ea6e605d9b48c50880818577f2891a3de42
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190220/5cd8080e/attachment-0001.html>