[Qa-jenkins-scm] [jenkins.debian.net] 01/01: reproducible fedora rpms: build a lot more packages (700 instead of 150) and choose them via some searchterms

Thu Dec 17 17:15:11 UTC 2015

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch master
in repository jenkins.debian.net.

commit 26d5cac524be12c1dcd5a181844a4626c92a4785
Author: Holger Levsen <holger at layer-acht.org>
Date:   Thu Dec 17 18:14:49 2015 +0100

    reproducible fedora rpms: build a lot more packages (700 instead of 150) and choose them via some searchterms
---
 bin/reproducible_build_rpm.sh | 15 +++++++++++----
 bin/reproducible_common.sh    |  3 ++-
 bin/reproducible_html_rpms.sh |  2 +-
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/bin/reproducible_build_rpm.sh b/bin/reproducible_build_rpm.sh
index 3b66fc8..936c284 100755
--- a/bin/reproducible_build_rpm.sh
+++ b/bin/reproducible_build_rpm.sh
@@ -59,12 +59,19 @@ download_package() {
 }
 
 choose_package() {
-	echo "$(date -u ) - choosing package to be build."
 	local MIN_AGE=7
 	touch -d "$(date -u -d "$MIN_AGE days ago" '+%Y-%m-%d %H:%M') UTC" $DUMMY
-	# instead of hardcoding the list of packages we can also use something like this to get a list of all packages:
-	# yumdownloader --urls --source --releasever=23 '*'
-	for PKG in $(echo audit-libs basesystem bash binutils bzip2 bzip2-libs ca-certificates chkconfig coreutils cpio cpp cracklib cracklib-dicts crypto-policies curl cyrus-sasl-lib diffutils dwz elfutils elfutils-default-yama-scope elfutils-libelf elfutils-libs emacs-filesystem expat fedora-release fedora-repos file file-libs filesystem findutils gawk gc gcc gcc-c++ gdb gdbm ghc-srpm-macros glib2 glibc glibc-common glibc-devel glibc-headers gmp gnat-srpm-macros gnupg2 gnutls go-srpm-macros g [...]
+	if [ ! -f ${RPM_PKGS}_$RELEASE ] || [ $DUMMY -nt ${RPM_PKGS}_$RELEASE ] ; then
+		echo "$(date -u ) - updating list of available packages for $RELEASE"
+		SEARCHTERMS="apache2 bash fedora gcc gnome gnu ipa kde linux mock openssl redhat rpm ssh system-config systemd xorg yum"
+		echo "$(date -u ) - instead of building everything only packages matching these searchterms are build: $SEARCHTERMS"
+		local i=""
+		( for i in $SEARCHTERMS ; do repoquery --qf "%{name}" "*$i*" ; done ) | sort -u > ${RPM_PKGS}_$RELEASE
+		cat ${RPM_PKGS}_$RELEASE
+	fi
+	echo "$(date -u ) - choosing package to be build."
+	local PKG=""
+	for PKG in $(cat ${RPM_PKGS}_$RELEASE) ; do
 		# build package if it has never build or at least $MIN_AGE days ago
 		if [ ! -d $BASE/rpms/$RELEASE/$ARCH/$PKG ] || [ $DUMMY -nt $BASE/rpms/$RELEASE/$ARCH/$PKG ] ; then
 			SRCPACKAGE=$PKG
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index b17cf3b..3aee08c 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -59,6 +59,7 @@ ARCHLINUX_PKGS=/srv/reproducible-results/.archlinux_pkgs
 
 # common settings for testing rpm based distros
 RPM_BUILD_NODE=profitbricks-build3-amd64
+RPM_PKGS=/srv/reproducible-results/.rpm_pkgs
 
 # number of cores to be used
 NUM_CPU=$(grep -c '^processor' /proc/cpuinfo)
@@ -316,7 +317,7 @@ write_page_intro() {
 	elif [ "$1" = "fedora-23" ] ; then
 		local PROJECTNAME="Fedora 23"
 		write_page "        <p><em>Reproducible $PROJECTNAME</em> is an effort to apply this to $PROJECTNAME. Thus $PROJECTNAME packages are build twice, with a few varitations added and then the resulting packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
-		write_page "        <p>Please note that this set up is as new as December 12th, so quite some things are still lacking, eg. https://github.com/kholia/ReproducibleBuilds is not followed at all yet and there are no variations introduced for the 2nd build. Also only 151 source packages from the <em>@buildsys-build</em> package group are currently being tested. OTOH this setup is mature enough that it requires very few trivial changes to build all 17080 source packages in $PROJECTNAME, if  [...]
+		write_page "        <p>Please note that this set up is as new as December 12th, so quite some things are still lacking, eg. https://github.com/kholia/ReproducibleBuilds is not followed at all yet and there are no variations introduced for the 2nd build. Also only about 700 source packages are currently being tested. OTOH this setup is mature enough that it requires very few trivial changes to build all 17080 source packages in $PROJECTNAME, if it were sensible. Which it isn't right now [...]
 	fi
 	if [ "$1" != "Arch Linux" ] && [ "$1" != "fedora-23" ] ; then
 		write_page "       <p>There is a weekly run <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_$PROJECTNAME/\">jenkins job</a> to test the <code>$BRANCH</code> branch of <a href=\"$PROJECTURL\">$PROJECTNAME.git</a>. The jenkins job is running <a href=\"http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/tree/bin/reproducible_$PROJECTNAME.sh\">reproducible_$PROJECTNAME.sh</a>$BUILD_ENVIRONMENT and this script is solely responsible for creating this page. Feel i [...]
diff --git a/bin/reproducible_html_rpms.sh b/bin/reproducible_html_rpms.sh
index cc3bb3b..f826bd2 100755
--- a/bin/reproducible_html_rpms.sh
+++ b/bin/reproducible_html_rpms.sh
@@ -28,7 +28,7 @@ HTML_TARGET=""
 HTML_RPM_STATS=$(mktemp)
 
 echo "$(date -u) - starting to analyse build results for '$RELEASE' ($ARCH)."
-TOTAL=151 # FIXME: calculate this like $(cat ${ARCHLINUX_PKGS}_$RELEASE | sed -s "s# #\n#g" | wc -l)
+TOTAL=$(cat ${RPM_PKGS}_$RELEASE | sed -s "s# #\n#g" | wc -l)
 TESTED=0
 NR_FTBFS=0
 NR_FTBR=0

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git

