[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 3 commits: reproducible Debian: do this systematically (for quicker results)

[Holger Levsen]

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
96998438 by Holger Levsen at 2019-02-20T13:54:03Z
reproducible Debian: do this systematically (for quicker results)

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
a91ac012 by Holger Levsen at 2019-02-20T14:00:05Z
reproducible Debian: refactor

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
85dd7f16 by Holger Levsen at 2019-02-20T14:24:06Z
reproducible Debian: refactor, throw away downloaded .deb files

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


1 changed file:

- bin/reproducible_compare_Debian_sha1sums.sh


Changes:

=====================================
bin/reproducible_compare_Debian_sha1sums.sh
=====================================
@@ -17,7 +17,6 @@ set -e
 # TODOs:
 # - ${package_file}.sha1output includes ${package_file} in the file name and contents
 # - run on osuoslXXX ? harder with using db..
-# - delete downloaded packages, keep sha1s, use them
 # - GRAPH
 # - save results in db
 # - loop through all packages known in db
@@ -40,7 +39,9 @@ mkdir -p $SHA1DIR
 cd $SHA1DIR
 
 # downloading (and keeping) all the packages is also too much, but let's prototype this... (and improve later)
-packages="$(schroot --directory  $SHA1DIR -c chroot:jenkins-reproducible-unstable-diffoscope grep ^Package: /var/lib/apt/lists/cdn-fastly.deb.debian.org_debian_dists_unstable_main_binary-amd64_Packages| awk '{print $2}' | sort -R | xargs echo)"
+PACKAGES=$(mktemp --tmpdir=$TMPDIR sha1-comp-XXXXXXX)
+schroot --directory  $SHA1DIR -c chroot:jenkins-reproducible-unstable-diffoscope cat /var/lib/apt/lists/cdn-fastly.deb.debian.org_debian_dists_unstable_main_binary-amd64_Packages > $PACKAGES
+packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | sort | xargs echo)"
 
 reproducible_packages=
 unreproducible_packages=
@@ -66,31 +67,32 @@ cleanup_all() {
 	echo
 	echo "$(du -sch $SHA1DIR)"
 	echo
-	rm $log
+	rm $log $PACKAGES
 }
 
 trap cleanup_all INT TERM EXIT
 
 for package in $packages ; do
-	echo "$(date -u) - preparing to download binary package $package"
-	schroot --directory  $SHA1DIR -c chroot:jenkins-reproducible-unstable-diffoscope apt-get download ${package} || continue
-	echo "$(date -u) - checking if more than one .deb exists locally"
-	package_file="$(ls -1 ${package}_*.deb)"
-	if [ $(echo "${package_file}" | wc -l) -ne 1 ] ; then
-		OLD_DEB=$(echo "${package_file}" | head -1)
-		echo "deleting $OLD_DEB..."
-		rm $OLD_DEB # first I thought to delete $OLD_DEB* but only deleting $OLD_DEB is better
-		package_file=$(echo "${package_file}" | tail -1 )
-	fi
-	echo "$(date -u) - gathering sha1sum"
+	echo "$(date -u) - checking whether we have seen the .deb for $package before"
+	version=$(grep-dctrl -X -P ${package} -s version -n $PACKAGES)
+	arch=$(grep-dctrl -X -P ${package} -s Architecture -n $PACKAGES)
+	package_file="${package}_$(echo $version | sed 's#:#%3a#')_${arch}.deb"
+	#pool_dir="$(dirname $filename)"
 	if [ ! -e ${package_file}.sha1output ] ; then
+		echo "$(date -u) - preparing to download $filename"
+		schroot --directory  $SHA1DIR -c chroot:jenkins-reproducible-unstable-diffoscope apt-get download ${package} || continue
+		echo "$(date -u) - calculating sha1sum"
 		SHA1SUM_PKG="$(sha1sum ${package_file} | tee ${package_file}.sha1output | awk '{print $1}' )"
+		rm ${package_file}
 	else
+		echo "$(date -u) - ${package_file} is known, gathering sha1sum"
 		SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
 	fi
-	echo "$(date -u) - downloading .json from buildinfo.debian.net"
 	if [ ! -e ${package_file}.json ]; then
+		echo "$(date -u) - downloading .json from buildinfo.debian.net"
 		wget --quiet -O ${package_file}.json ${bdn_url}/${SHA1SUM_PKG} || echo "WARNING: failed to download ${bdn_url}/${SHA1SUM_PKG}"
+	else
+		echo "$(date -u) - reusing local copy of .json from buildinfo.debian.net"
 	fi
 	echo "$(date -u) - generating result"
 	count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/e49ad391f231b8402cab619b633cff62b1046de0...85dd7f16bdcc329f34daf5beacc65ace0797019f

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/e49ad391f231b8402cab619b633cff62b1046de0...85dd7f16bdcc329f34daf5beacc65ace0797019f
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190220/934bcf56/attachment-0001.html>