[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 4 commits: reproducible Debian: sha1 checker: redownload b.d.n's files if they are older than 2 weeks
Holger Levsen
gitlab at salsa.debian.org
Wed Mar 6 19:03:41 GMT 2019
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
5740acdc by Holger Levsen at 2019-03-06T15:37:57Z
reproducible Debian: sha1 checker: redownload b.d.n's files if they are older than 2 weeks
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
4798f1b3 by Holger Levsen at 2019-03-06T15:37:57Z
reproducible Debian: sha1 checker: enable a 2nd download runner again
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
50272048 by Holger Levsen at 2019-03-06T15:42:31Z
reproducible Debian: sha1 checker: simplefy stats
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
05db9170 by Holger Levsen at 2019-03-06T16:12:47Z
reproducible Debian: sha1 checker: deal with packages with more than one version
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
2 changed files:
- bin/reproducible_compare_Debian_sha1sums.sh
- job-cfg/reproducible.yaml
Changes:
=====================================
bin/reproducible_compare_Debian_sha1sums.sh
=====================================
@@ -23,7 +23,6 @@ set -e
# - show results in 'normal pages'
# - store date when a package was last reproduced... (and constantly do that...)
# - throw away results (if none has been|which have not) signed with a tests.r-b.o key
-# - json files from buildinfo.d.n are never re-downloaded
# - rebuilder:
# - run on osuosl173
# - loop randomly through unreproducible packages first, then reproducible ones. do one attempt only.
@@ -51,16 +50,20 @@ bdn_url="https://buildinfo.debian.net/api/v1/buildinfos/checksums/sha1"
log=$(mktemp --tmpdir=$TMPDIR sha1-log-XXXXXXX)
echo "$(date -u) - logfile used is $log"
+FORCE_DATE=$(date -u -d "14 days ago" '+%Y-%m-%d')
+DUMMY_FILE=$(mktemp --tmpdir=$TMPDIR sha1-date-XXXXXXX)
+touch -d "$(date '+%Y-%m-%d') 00:00 UTC" $DUMMY_FILE
+
SHA1DIR=/srv/reproducible-results/debian-sha1
mkdir -p $SHA1DIR
PACKAGES=$(mktemp --tmpdir=$TMPDIR sha1-pkgs-XXXXXXX)
schroot --directory $SHA1DIR -c chroot:jenkins-reproducible-${RELEASE}-diffoscope cat /var/lib/apt/lists/cdn-fastly.deb.debian.org_debian_dists_${RELEASE}_main_binary-amd64_Packages > $PACKAGES
case "$MODE" in
- random) SORT="sort -R";;
- reverse) SORT="sort -r" ;;
- forward) SORT="sort" ;;
- *) SORT="sort" ; MODE="results" ; RESULTS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*REPRODUCIBLE.buster" > $RESULTS ;;
+ random) SORT="sort -u -R";;
+ reverse) SORT="sort -u -r" ;;
+ forward) SORT="sort -u" ;;
+ *) SORT="sort -u" ; MODE="results" ; RESULTS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*REPRODUCIBLE.buster" > $RESULTS ;;
esac
packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | $SORT | xargs echo)"
@@ -84,32 +87,21 @@ cleanup_all() {
percent_unknown=$(echo "scale=4 ; $unknown_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+ percent_binnmu_repro=$(echo "scale=4 ; $reproducible_binnmu / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+ percent_binnmu_unrepro=$(echo "scale=4 ; $unreproducible_binnmu / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+ percent_arch_all_repro=$(echo "scale=4 ; $reproducible_arch_all / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+ percent_arch_all_unrepro=$(echo "scale=4 ; $unreproducible_arch_all / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+ percent_arch_amd64_repro=$(echo "scale=4 ; $reproducible_arch_amd64 / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+ percent_arch_amd64_unrepro=$(echo "scale=4 ; $unreproducible_arch_amd64 / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
echo "-------------------------------------------------------------"
- echo "unknown packages: $unknown_count: $unknown_packages"
- echo
- echo "reproducible packages: $reproducible_count: $reproducible_packages"
- echo
- echo "unreproducible packages: $unreproducible_count: $unreproducible_packages"
echo
- echo "ftp.debian.org package reproducibility statistics including packages (currently) in an unknown state"
+ echo "ftp.debian.org package reproducibility statistics"
echo "-------------------------------------------------------------"
echo "packages in unknown reproducibility state in $RELEASE/amd64: $unknown_count: ($percent_unknown%)"
echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
echo "total number of packages in $RELEASE/amd64: $(echo $reproducible_count+$unreproducible_count+$unknown_count | bc)"
echo
- percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
- percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
- percent_binnmu_repro=$(echo "scale=4 ; $reproducible_binnmu / ($reproducible_count+$unreproducible_count) * 100" | bc)
- percent_binnmu_unrepro=$(echo "scale=4 ; $unreproducible_binnmu / ($reproducible_count+$unreproducible_count) * 100" | bc)
- percent_arch_all_repro=$(echo "scale=4 ; $reproducible_arch_all / ($reproducible_count+$unreproducible_count) * 100" | bc)
- percent_arch_all_unrepro=$(echo "scale=4 ; $unreproducible_arch_all / ($reproducible_count+$unreproducible_count) * 100" | bc)
- percent_arch_amd64_repro=$(echo "scale=4 ; $reproducible_arch_amd64 / ($reproducible_count+$unreproducible_count) * 100" | bc)
- percent_arch_amd64_unrepro=$(echo "scale=4 ; $unreproducible_arch_amd64 / ($reproducible_count+$unreproducible_count) * 100" | bc)
- echo "ftp.debian.org package reproducibility statistics of packages in known states only"
- echo "-------------------------------------------------------------"
- echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
- echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
echo
echo "reproducible binNMUs in $RELEASE/amd64: $reproducible_binnmu: ($percent_binnmu_repro%)"
echo "unreproducible binNMU in $RELEASE/amd64: $unreproducible_binnmu: ($percent_binnmu_unrepro%)"
@@ -167,10 +159,10 @@ for package in $packages ; do
else
touch $LOCK
fi
- version=$(grep-dctrl -X -P ${package} -s version -n $PACKAGES)
- arch=$(grep-dctrl -X -P ${package} -s Architecture -n $PACKAGES)
+ version=$(grep-dctrl -X -P ${package} -s Version -n $PACKAGES | head -1)
+ arch=$(grep-dctrl -X -P ${package} -s Architecture -n $PACKAGES | head -1)
package_file="${package}_$(echo $version | sed 's#:#%3a#')_${arch}.deb"
- pool_dir="$SHA1DIR/$(dirname $(grep-dctrl -X -P ${package} -s Filename -n $PACKAGES))"
+ pool_dir="$SHA1DIR/$(dirname $(grep-dctrl -X -P ${package} -s Filename -n $PACKAGES | head -1))"
mkdir -p $pool_dir
cd $pool_dir
if [ ! -e ${package_file}.sha1output ] ; then
@@ -184,7 +176,8 @@ for package in $packages ; do
rm ${package}_*REPRODUCIBLE.$RELEASE
fi
fi
- if [ ! -e ${package_file}.json ]; then
+ # download .json if non existing or older than $DUMMY_FILE
+ if [ ! -e ${package_file}.json ] || [ ${package_file}.json -ot $DUMMY_FILE ]; then
echo "$(date -u) - downloading .json for ${package_file} (${SHA1SUM_PKG}) from buildinfo.debian.net"
wget --quiet -O ${package_file}.json ${bdn_url}/${SHA1SUM_PKG} || echo "WARNING: failed to download ${bdn_url}/${SHA1SUM_PKG}"
count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)
=====================================
job-cfg/reproducible.yaml
=====================================
@@ -606,7 +606,6 @@
my_timed: '56 * * * *'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_hname: 'osuosl173'
- my_disabled: true
- 'compare_Debian_sha1sums_random_runner_1':
my_description: 'Fetch .buildinfo files of Debian binary packages (in random order)'
my_timed: '56 * * * *'
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/4d614051323028334c88a2d98eb105f96dc197ed...05db9170755e047f8119c0344b29e5f579c6ca64
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/4d614051323028334c88a2d98eb105f96dc197ed...05db9170755e047f8119c0344b29e5f579c6ca64
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190306/322edfa4/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list