[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 3 commits: reproducible Debian real-world: refactor, speed up result gatherer

Holger Levsen gitlab at salsa.debian.org
Fri Feb 22 10:19:39 GMT 2019


Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
68f63af1 by Holger Levsen at 2019-02-22T10:05:22Z
reproducible Debian real-world: refactor, speed up result gatherer

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
75e7d1cf by Holger Levsen at 2019-02-22T10:05:22Z
reproducible Debian real-world: also count packages with unknown state

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
9b6fd8e7 by Holger Levsen at 2019-02-22T10:09:16Z
reproducible Debian real-world: provide two statistics: with and without packages in unknown state

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


1 changed file:

- bin/reproducible_compare_Debian_sha1sums.sh


Changes:

=====================================
bin/reproducible_compare_Debian_sha1sums.sh
=====================================
@@ -61,6 +61,7 @@ case "$MODE" in
 	reverse)	SORT="sort -r" ;;
 	forward)	SORT="sort" ;;
 	*)		SORT="sort" ; MODE="results" ; RESULTS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*REPRODUCIBLE.buster" > $RESULTS
+			JSONS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*.json" > $JSONS ;;
 esac
 packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | $SORT | xargs echo)"
 
@@ -69,24 +70,39 @@ unreproducible_packages=
 
 cleanup_all() {
 	if [ "$MODE" = "results" ]; then
+		unknown_packages=$(awk '/ UNKNOWN: /{print $9}' $log)
+		unknown_count=$(echo $unknown_packages | wc -w)
 		reproducible_packages=$(awk '/ REPRODUCIBLE: /{print $9}' $log)
 		reproducible_count=$(echo $reproducible_packages | wc -w)
 		unreproducible_packages=$(awk '/ UNREPRODUCIBLE: /{print $9}' $log)
 		unreproducible_count=$(echo $unreproducible_packages | wc -w)
-		percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
-		percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
+		percent_unknown=$(echo "scale=4 ; $unknown_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+		percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+		percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
 		echo "-------------------------------------------------------------"
+		echo "unknown packages: $unknown_count: $unknown_packages"
+		echo
 		echo "reproducible packages: $reproducible_count: $reproducible_packages"
 		echo
 		echo "unreproducible packages: $unreproducible_count: $unreproducible_packages"
 		echo
+		echo "Statistics including packages (currently) in an unknown state"
+		echo "-------------------------------------------------------------"
+		echo "unknown packages in $RELEASE/amd64: $unknown_count: ($percent_unknown%)"
+		echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
+		echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
+		echo
+		percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
+		percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
+		echo "Statistics of packages in known state only"
+		echo "-------------------------------------------------------------"
 		echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
 		echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
 		echo
 		echo
 		echo "$(du -sch $SHA1DIR)"
 		echo
-		rm $RESULTS
+		rm $RESULTS $JSONS
 	fi
 	rm $log $PACKAGES
 }
@@ -96,32 +112,53 @@ trap cleanup_all INT TERM EXIT
 rm -f $SHA1DIR/*.lock	# this is a tiny bit hackish, but also an elegant way to get rid of old locks...
 			# (locks are held for 30s only anyway and there is an 3/60000th chance of a race condition only anyway)
 
-for package in $packages ; do
-	if [ "$MODE" = "results" ] ; then
+if [ "$MODE" = "results" ] ; then
+	for package in $packages ; do
 		result=$(grep "/${package}_" $RESULTS || true)
 		if [ -n "$result" ] ; then
 			if $(echo $result | grep -q UNREPRODUCIBLE) ; then
-				package_file=$(echo $result | sed 's#\.deb\.UNREPRODUCIBLE\.buster$#.deb#' )
-				count=1
-				SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
-				package_file=$(basename $package_file)
-				echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
+				#package_file=$(echo $result | sed 's#\.deb\.UNREPRODUCIBLE\.buster$#.deb#' )
+				#count=1
+				#SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
+				#package_file=$(basename $package_file)
+				#echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
+				echo "$(date -u) - UNREPRODUCIBLE: $package"
 			else
-				package_file=$(echo $result | sed 's#\.deb\.REPRODUCIBLE\.buster$#.deb#' )
-				count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
-				SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
-				package_file=$(basename $package_file)
+				#package_file=$(echo $result | sed 's#\.deb\.REPRODUCIBLE\.buster$#.deb#' )
+				#count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
+				#SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
+				#package_file=$(basename $package_file)
+				#echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
+				echo "$(date -u) - REPRODUCIBLE: $package"
+			fi
+			continue
+		fi
+		json=$(grep "/${package}_" $JSONS || true)
+		if [ -n "$json" ] ; then
+			package_file=$(echo $json | sed 's#\.deb\.json$#.deb#' )
+			count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)
+			SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
+			if [ "${count}" -ge 2 ]; then
+				echo $count > ${package_file}.REPRODUCIBLE.$RELEASE
 				echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
+			else
+				echo 1 > ${package_file}.UNREPRODUCIBLE.$RELEASE
+				echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
 			fi
 			continue
 		fi
-	fi
+		echo "$(date -u) - UNKNOWN: $package"
+	done | tee $log
+	exit
+fi
+
+# only used by the runners
+for package in $packages ; do
 	LOCK="$SHA1DIR/${package}.lock"
 	if [ -e $LOCK ] ; then
 		echo "$(date -u) - skipping locked package $package"
 		continue
-	elif [ ! "$MODE" = "results" ] ; then
-		# MODE=results is read-only
+	else
 		touch $LOCK
 	fi
 	version=$(grep-dctrl -X -P ${package} -s version -n $PACKAGES)
@@ -130,29 +167,6 @@ for package in $packages ; do
 	pool_dir="$SHA1DIR/$(dirname $(grep-dctrl -X -P ${package} -s Filename -n $PACKAGES))"
 	mkdir -p $pool_dir
 	cd $pool_dir
-	if [ "$MODE" = "results" ] ; then
-		# this code block can be removed once all packages with existing results have been processed once...
-		if [ -e ${package_file}.REPRODUCIBLE.$RELEASE ] ; then
-			count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
-			SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
-			echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
-		elif [ -e ${package_file}.UNREPRODUCIBLE.$RELEASE ] ; then
-			count=1
-			SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
-			echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
-		elif [ -e ${package_file}.json ] ; then
-			count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)
-			SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
-			if [ "${count}" -ge 2 ]; then
-				echo $count > ${package_file}.REPRODUCIBLE.$RELEASE
-				echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
-			else
-				echo 1 > ${package_file}.UNREPRODUCIBLE.$RELEASE
-				echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
-			fi
-		fi
-		continue
-	fi
 	if [ ! -e ${package_file}.sha1output ] ; then
 		echo -n "$(date -u) - downloading... "
 		( schroot --directory $pool_dir -c chroot:jenkins-reproducible-${RELEASE}-diffoscope apt-get download ${package}/${RELEASE} 2>&1 |xargs echo ) || continue
@@ -180,7 +194,7 @@ for package in $packages ; do
 		echo "$(date -u) - not updating data about ${package_file}"
 	fi
 	rm -f $LOCK
-done | tee $log
+done | tee -a $log
 
 cleanup_all
 trap - INT TERM EXIT



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/395e8ad70e51f884260a04477bea2392c9f64fb2...9b6fd8e759c2739e0199a42cba75bb13b006700b

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/395e8ad70e51f884260a04477bea2392c9f64fb2...9b6fd8e759c2739e0199a42cba75bb13b006700b
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190222/991f9c3f/attachment-0001.html>


More information about the Qa-jenkins-scm mailing list