[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 3 commits: reproducible Debian real-world: refactor, speed up result gatherer
Holger Levsen
gitlab at salsa.debian.org
Fri Feb 22 10:19:39 GMT 2019
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
68f63af1 by Holger Levsen at 2019-02-22T10:05:22Z
reproducible Debian real-world: refactor, speed up result gatherer
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
75e7d1cf by Holger Levsen at 2019-02-22T10:05:22Z
reproducible Debian real-world: also count packages with unknown state
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
9b6fd8e7 by Holger Levsen at 2019-02-22T10:09:16Z
reproducible Debian real-world: provide two statistics: with and without packages in unknown state
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/reproducible_compare_Debian_sha1sums.sh
Changes:
=====================================
bin/reproducible_compare_Debian_sha1sums.sh
=====================================
@@ -61,6 +61,7 @@ case "$MODE" in
reverse) SORT="sort -r" ;;
forward) SORT="sort" ;;
*) SORT="sort" ; MODE="results" ; RESULTS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*REPRODUCIBLE.buster" > $RESULTS
+ JSONS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*.json" > $JSONS ;;
esac
packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | $SORT | xargs echo)"
@@ -69,24 +70,39 @@ unreproducible_packages=
cleanup_all() {
if [ "$MODE" = "results" ]; then
+ unknown_packages=$(awk '/ UNKNOWN: /{print $9}' $log)
+ unknown_count=$(echo $unknown_packages | wc -w)
reproducible_packages=$(awk '/ REPRODUCIBLE: /{print $9}' $log)
reproducible_count=$(echo $reproducible_packages | wc -w)
unreproducible_packages=$(awk '/ UNREPRODUCIBLE: /{print $9}' $log)
unreproducible_count=$(echo $unreproducible_packages | wc -w)
- percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
- percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
+ percent_unknown=$(echo "scale=4 ; $unknown_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+ percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+ percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
echo "-------------------------------------------------------------"
+ echo "unknown packages: $unknown_count: $unknown_packages"
+ echo
echo "reproducible packages: $reproducible_count: $reproducible_packages"
echo
echo "unreproducible packages: $unreproducible_count: $unreproducible_packages"
echo
+ echo "Statistics including packages (currently) in an unknown state"
+ echo "-------------------------------------------------------------"
+ echo "unknown packages in $RELEASE/amd64: $unknown_count: ($percent_unknown%)"
+ echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
+ echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
+ echo
+ percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
+ percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
+ echo "Statistics of packages in known state only"
+ echo "-------------------------------------------------------------"
echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
echo
echo
echo "$(du -sch $SHA1DIR)"
echo
- rm $RESULTS
+ rm $RESULTS $JSONS
fi
rm $log $PACKAGES
}
@@ -96,32 +112,53 @@ trap cleanup_all INT TERM EXIT
rm -f $SHA1DIR/*.lock # this is a tiny bit hackish, but also an elegant way to get rid of old locks...
# (locks are held for 30s only anyway and there is an 3/60000th chance of a race condition only anyway)
-for package in $packages ; do
- if [ "$MODE" = "results" ] ; then
+if [ "$MODE" = "results" ] ; then
+ for package in $packages ; do
result=$(grep "/${package}_" $RESULTS || true)
if [ -n "$result" ] ; then
if $(echo $result | grep -q UNREPRODUCIBLE) ; then
- package_file=$(echo $result | sed 's#\.deb\.UNREPRODUCIBLE\.buster$#.deb#' )
- count=1
- SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
- package_file=$(basename $package_file)
- echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
+ #package_file=$(echo $result | sed 's#\.deb\.UNREPRODUCIBLE\.buster$#.deb#' )
+ #count=1
+ #SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
+ #package_file=$(basename $package_file)
+ #echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
+ echo "$(date -u) - UNREPRODUCIBLE: $package"
else
- package_file=$(echo $result | sed 's#\.deb\.REPRODUCIBLE\.buster$#.deb#' )
- count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
- SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
- package_file=$(basename $package_file)
+ #package_file=$(echo $result | sed 's#\.deb\.REPRODUCIBLE\.buster$#.deb#' )
+ #count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
+ #SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
+ #package_file=$(basename $package_file)
+ #echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
+ echo "$(date -u) - REPRODUCIBLE: $package"
+ fi
+ continue
+ fi
+ json=$(grep "/${package}_" $JSONS || true)
+ if [ -n "$json" ] ; then
+ package_file=$(echo $json | sed 's#\.deb\.json$#.deb#' )
+ count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)
+ SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
+ if [ "${count}" -ge 2 ]; then
+ echo $count > ${package_file}.REPRODUCIBLE.$RELEASE
echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
+ else
+ echo 1 > ${package_file}.UNREPRODUCIBLE.$RELEASE
+ echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
fi
continue
fi
- fi
+ echo "$(date -u) - UNKNOWN: $package"
+ done | tee $log
+ exit
+fi
+
+# only used by the runners
+for package in $packages ; do
LOCK="$SHA1DIR/${package}.lock"
if [ -e $LOCK ] ; then
echo "$(date -u) - skipping locked package $package"
continue
- elif [ ! "$MODE" = "results" ] ; then
- # MODE=results is read-only
+ else
touch $LOCK
fi
version=$(grep-dctrl -X -P ${package} -s version -n $PACKAGES)
@@ -130,29 +167,6 @@ for package in $packages ; do
pool_dir="$SHA1DIR/$(dirname $(grep-dctrl -X -P ${package} -s Filename -n $PACKAGES))"
mkdir -p $pool_dir
cd $pool_dir
- if [ "$MODE" = "results" ] ; then
- # this code block can be removed once all packages with existing results have been processed once...
- if [ -e ${package_file}.REPRODUCIBLE.$RELEASE ] ; then
- count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
- SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
- echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
- elif [ -e ${package_file}.UNREPRODUCIBLE.$RELEASE ] ; then
- count=1
- SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
- echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
- elif [ -e ${package_file}.json ] ; then
- count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)
- SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
- if [ "${count}" -ge 2 ]; then
- echo $count > ${package_file}.REPRODUCIBLE.$RELEASE
- echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
- else
- echo 1 > ${package_file}.UNREPRODUCIBLE.$RELEASE
- echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
- fi
- fi
- continue
- fi
if [ ! -e ${package_file}.sha1output ] ; then
echo -n "$(date -u) - downloading... "
( schroot --directory $pool_dir -c chroot:jenkins-reproducible-${RELEASE}-diffoscope apt-get download ${package}/${RELEASE} 2>&1 |xargs echo ) || continue
@@ -180,7 +194,7 @@ for package in $packages ; do
echo "$(date -u) - not updating data about ${package_file}"
fi
rm -f $LOCK
-done | tee $log
+done | tee -a $log
cleanup_all
trap - INT TERM EXIT
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/395e8ad70e51f884260a04477bea2392c9f64fb2...9b6fd8e759c2739e0199a42cba75bb13b006700b
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/395e8ad70e51f884260a04477bea2392c9f64fb2...9b6fd8e759c2739e0199a42cba75bb13b006700b
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190222/991f9c3f/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list