[Qa-jenkins-scm] [jenkins.debian.net] 11/12: reproducible: refactor, do pkg sets for all suites except experimental
Holger Levsen
holger at moszumanska.debian.org
Thu Mar 12 18:20:39 UTC 2015
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit b50232e473c6c269dea47eade6c04de658107277
Author: Holger Levsen <holger at layer-acht.org>
Date: Thu Mar 12 19:14:02 2015 +0100
reproducible: refactor, do pkg sets for all suites except experimental
---
bin/reproducible_common.sh | 4 +-
bin/reproducible_create_meta_pkg_sets.sh | 297 ++++++++++++++++---------------
bin/reproducible_html_graphs.sh | 8 +-
3 files changed, 158 insertions(+), 151 deletions(-)
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index fec3f11..c2d7409 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -143,8 +143,8 @@ write_page_header() {
write_page "</li>"
done
for TARGET in $ALLVIEWS ; do
- if [ "$TARGET" = "pkg_sets" ] && [ "$SUITE" != "sid" ] ; then
- # FIXME: no pkg_sets are tested in experimental
+ if [ "$TARGET" = "pkg_sets" ] && [ "$SUITE" = "experimental" ] ; then
+ # no pkg_sets are tested in experimental
continue
fi
SPOKEN_TARGET=${SPOKENTARGET[$TARGET]}
diff --git a/bin/reproducible_create_meta_pkg_sets.sh b/bin/reproducible_create_meta_pkg_sets.sh
index 7052db3..f08a453 100755
--- a/bin/reproducible_create_meta_pkg_sets.sh
+++ b/bin/reproducible_create_meta_pkg_sets.sh
@@ -11,26 +11,6 @@ common_init "$@"
. /srv/jenkins/bin/reproducible_common.sh
ARCH=amd64
-SUITE=sid
-
-DISTNAME="$SUITE-$ARCH"
-TPATH=/srv/reproducible-results/meta_pkgsets-$SUITE
-CHPATH=/srv/reproducible-results/chdist-$SUITE
-mkdir -p $TPATH $CHPATH
-
-# delete possibly existing dist
-cd $CHPATH
-rm -rf $DISTNAME
-cd -
-
-# the "[arch=$ARCH]" is a workaround until #774685 is fixed
-chdist --data-dir=$CHPATH --arch=$ARCH create $DISTNAME "[arch=$ARCH]" $MIRROR $SUITE main
-chdist --data-dir=$CHPATH --arch=$ARCH apt-get $DISTNAME update
-
-PACKAGES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_binary-${ARCH}_Packages)
-SOURCES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_source_Sources)
-TMPFILE=$(mktemp)
-TMPFILE2=$(mktemp)
# helper functions
convert_into_source_packages_only() {
@@ -50,7 +30,6 @@ convert_into_source_packages_only() {
cut -d " " -f1 ${TMPFILE2} | cut -d ":" -f1 | sort -u > $TMPFILE
rm ${TMPFILE2}
}
-
convert_from_deb822_into_source_packages_only() {
# given a Packages file in deb822 format on standard input, the
# following perl "oneliner" outputs the associated (unversioned)
@@ -88,135 +67,163 @@ update_if_similar() {
fi
}
+update_pkg_sets() {
+ # the essential package set
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[1]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[3]}.pkgset ] ; then
+ chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X -FEssential yes > $TMPFILE
+ convert_from_deb822_into_source_packages_only
+ update_if_similar ${META_PKGSET[1]}.pkgset
+ fi
-#
-# main
-#
-
-# the essential package set
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[1]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[3]}.pkgset ] ; then
- chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X -FEssential yes > $TMPFILE
- convert_from_deb822_into_source_packages_only
- update_if_similar ${META_PKGSET[1]}.pkgset
-fi
-
-# the required package set
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[2]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[2]}.pkgset ] ; then
- chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X -FPriority required > $TMPFILE
- convert_from_deb822_into_source_packages_only
- update_if_similar ${META_PKGSET[2]}.pkgset
-fi
-
-# build-essential
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[3]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[3]}.pkgset ] ; then
- chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FBuild-Essential yes --or -FPackage build-essential \) > ${TMPFILE2}
- # here we want the installable set:
- schroot --directory /tmp -c source:jenkins-dpkg-jessie -- dose-deb-coinstall --deb-native-arch=$ARCH --bg=$PACKAGES --fg=${TMPFILE2} > $TMPFILE
- convert_from_deb822_into_source_packages_only
- update_if_similar ${META_PKGSET[3]}.pkgset
-fi
-
-# popcon top 1337 installed sources
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[4]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[4]}.pkgset ] ; then
- SQL_QUERY="SELECT popcon_src.source FROM popcon_src ORDER BY popcon_src.insts DESC LIMIT 1337;"
- PGPASSWORD=public-udd-mirror \
- psql -U public-udd-mirror \
- -h public-udd-mirror.xvm.mit.edu -p 5432 \
- -t \
- udd -c"${SQL_QUERY}" > $TMPFILE
- update_if_similar ${META_PKGSET[4]}.pkgset
-fi
-
-# installed on one or more .debian.org machines
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[5]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[5]}.pkgset ] ; then
- # one day we will get a proper data provider from DSA...
- # (so far it was a manual "dpkg --get-selections" on all machines
- # converted into a list of source packages...)
- cat /srv/jenkins/bin/reproducible_installed_on_debian.org > $TMPFILE
- update_if_similar ${META_PKGSET[5]}.pkgset
-fi
-
-# packages which had a DSA
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[6]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[6]}.pkgset ] ; then
- svn export svn://svn.debian.org/svn/secure-testing/data/DSA/list ${TMPFILE2}
- grep "^\[" ${TMPFILE2} | grep "DSA-" | cut -d " " -f5|sort -u > $TMPFILE
- convert_into_source_packages_only
- update_if_similar ${META_PKGSET[6]}.pkgset
-fi
-
-# gnome and everything it depends on
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[7]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[7]}.pkgset ] ; then
- chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FPriority required --or -FPackage gnome \) > ${TMPFILE2}
- schroot --directory /tmp -c source:jenkins-dpkg-jessie -- dose-deb-coinstall --deb-native-arch=$ARCH --bg=$PACKAGES --fg=${TMPFILE2} > $TMPFILE
- convert_from_deb822_into_source_packages_only
- update_if_similar ${META_PKGSET[7]}.pkgset
-fi
-
-# The build-depends of X tasks can be solved once dose-ceve is able to read
-# Debian source packages (possible in dose3 git but needs a new dose3 release
-# and upload to unstable)
-#
-# Ignoring parsing issues, the current method is unable to resolve virtual
-# build dependencies
-#
-# The current method also ignores Build-Depends-Indep and Build-Depends-Arch
-
-# all build depends of gnome
-rm -f $TMPFILE
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[8]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[8]}.pkgset ] ; then
- for PKG in $(cat $TPATH/${META_PKGSET[7]}.pkgset) ; do
- grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE
- done
- convert_into_source_packages_only
- update_if_similar ${META_PKGSET[8]}.pkgset
-fi
-
-# tails
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[9]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[9]}.pkgset ] ; then
- curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.binpkgs > $TMPFILE
- curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.srcpkgs >> $TMPFILE
- convert_into_source_packages_only
- update_if_similar ${META_PKGSET[9]}.pkgset
-fi
-
-# all build depends of tails
-rm -f $TMPFILE
-if [ -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[10]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[10]}.pkgset ] ; then
- for PKG in $(cat $TPATH/${META_PKGSET[9]}.pkgset) ; do
- grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE
- done
- convert_into_source_packages_only
- update_if_similar ${META_PKGSET[10]}.pkgset
-fi
-
-# grml
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[11]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[11]}.pkgset ] ; then
- curl http://grml.org/files/grml64-full_latest/dpkg.selections | cut -f1 > $TMPFILE
- if ! grep '<title>404 Not Found</title>' $TMPFILE ; then
+ # the required package set
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[2]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[2]}.pkgset ] ; then
+ chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X -FPriority required > $TMPFILE
+ convert_from_deb822_into_source_packages_only
+ update_if_similar ${META_PKGSET[2]}.pkgset
+ fi
+
+ # build-essential
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[3]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[3]}.pkgset ] ; then
+ chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FBuild-Essential yes --or -FPackage build-essential \) > ${TMPFILE2}
+ # here we want the installable set:
+ schroot --directory /tmp -c source:jenkins-dpkg-jessie -- dose-deb-coinstall --deb-native-arch=$ARCH --bg=$PACKAGES --fg=${TMPFILE2} > $TMPFILE
+ convert_from_deb822_into_source_packages_only
+ update_if_similar ${META_PKGSET[3]}.pkgset
+ fi
+
+ # popcon top 1337 installed sources
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[4]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[4]}.pkgset ] ; then
+ SQL_QUERY="SELECT popcon_src.source FROM popcon_src ORDER BY popcon_src.insts DESC LIMIT 1337;"
+ PGPASSWORD=public-udd-mirror \
+ psql -U public-udd-mirror \
+ -h public-udd-mirror.xvm.mit.edu -p 5432 \
+ -t \
+ udd -c"${SQL_QUERY}" > $TMPFILE
+ update_if_similar ${META_PKGSET[4]}.pkgset
+ fi
+
+ # installed on one or more .debian.org machines
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[5]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[5]}.pkgset ] ; then
+ # one day we will get a proper data provider from DSA...
+ # (so far it was a manual "dpkg --get-selections" on all machines
+ # converted into a list of source packages...)
+ cat /srv/jenkins/bin/reproducible_installed_on_debian.org > $TMPFILE
+ update_if_similar ${META_PKGSET[5]}.pkgset
+ fi
+
+ # packages which had a DSA
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[6]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[6]}.pkgset ] ; then
+ svn export svn://svn.debian.org/svn/secure-testing/data/DSA/list ${TMPFILE2}
+ grep "^\[" ${TMPFILE2} | grep "DSA-" | cut -d " " -f5|sort -u > $TMPFILE
convert_into_source_packages_only
- update_if_similar ${META_PKGSET[11]}.pkgset
- else
- echo "Warning: could not download grml's latest dpkg.selections file, skipping pkg set..."
+ update_if_similar ${META_PKGSET[6]}.pkgset
fi
-fi
-# all build depends of grml
-rm -f $TMPFILE
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[12]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[12]}.pkgset ] ; then
- for PKG in $(cat $TPATH/${META_PKGSET[11]}.pkgset) ; do
- grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE
- done
- convert_into_source_packages_only
- update_if_similar ${META_PKGSET[12]}.pkgset
-fi
+ # gnome and everything it depends on
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[7]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[7]}.pkgset ] ; then
+ chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FPriority required --or -FPackage gnome \) > ${TMPFILE2}
+ schroot --directory /tmp -c source:jenkins-dpkg-jessie -- dose-deb-coinstall --deb-native-arch=$ARCH --bg=$PACKAGES --fg=${TMPFILE2} > $TMPFILE
+ convert_from_deb822_into_source_packages_only
+ update_if_similar ${META_PKGSET[7]}.pkgset
+ fi
+
+ # The build-depends of X tasks can be solved once dose-ceve is able to read
+ # Debian source packages (possible in dose3 git but needs a new dose3 release
+ # and upload to unstable)
+ #
+ # Ignoring parsing issues, the current method is unable to resolve virtual
+ # build dependencies
+ #
+ # The current method also ignores Build-Depends-Indep and Build-Depends-Arch
+
+ # all build depends of gnome
+ rm -f $TMPFILE
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[8]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[8]}.pkgset ] ; then
+ for PKG in $(cat $TPATH/${META_PKGSET[7]}.pkgset) ; do
+ grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE
+ done
+ convert_into_source_packages_only
+ update_if_similar ${META_PKGSET[8]}.pkgset
+ fi
+
+ # tails
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[9]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[9]}.pkgset ] ; then
+ curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.binpkgs > $TMPFILE
+ curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.srcpkgs >> $TMPFILE
+ convert_into_source_packages_only
+ update_if_similar ${META_PKGSET[9]}.pkgset
+ fi
+
+ # all build depends of tails
+ rm -f $TMPFILE
+ if [ -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[10]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[10]}.pkgset ] ; then
+ for PKG in $(cat $TPATH/${META_PKGSET[9]}.pkgset) ; do
+ grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE
+ done
+ convert_into_source_packages_only
+ update_if_similar ${META_PKGSET[10]}.pkgset
+ fi
+
+ # grml
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[11]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[11]}.pkgset ] ; then
+ curl http://grml.org/files/grml64-full_latest/dpkg.selections | cut -f1 > $TMPFILE
+ if ! grep '<title>404 Not Found</title>' $TMPFILE ; then
+ convert_into_source_packages_only
+ update_if_similar ${META_PKGSET[11]}.pkgset
+ else
+ echo "Warning: could not download grml's latest dpkg.selections file, skipping pkg set..."
+ fi
+ fi
+
+ # all build depends of grml
+ rm -f $TMPFILE
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[12]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[12]}.pkgset ] ; then
+ for PKG in $(cat $TPATH/${META_PKGSET[11]}.pkgset) ; do
+ grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE
+ done
+ convert_into_source_packages_only
+ update_if_similar ${META_PKGSET[12]}.pkgset
+ fi
+
+ # pkg-perl-maintainers
+ if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[13]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[13]}.pkgset ] ; then
+ grep-dctrl -sPackage -n -FMaintainer pkg-perl-maintainers at lists.alioth.debian.org $SOURCES > $TMPFILE
+ update_if_similar ${META_PKGSET[13]}.pkgset
+ fi
+}
+
+TMPFILE=$(mktemp)
+TMPFILE2=$(mktemp)
+for SUITE in $SUITES ; do
+ if [ "$SUITE" = "experimental" ] ; then
+ # no pkg sets in experimental
+ continue
+ fi
+ echo
+ echo "$(date) - Creating meta package sets for $SUITE now."
+ echo
+
+ DISTNAME="$SUITE-$ARCH"
+ TPATH=/srv/reproducible-results/meta_pkgsets-$SUITE
+ CHPATH=/srv/reproducible-results/chdist-$SUITE
+ mkdir -p $TPATH $CHPATH
+
+ # delete possibly existing dist
+ cd $CHPATH
+ rm -rf $DISTNAME
+ cd -
+
+ # the "[arch=$ARCH]" is a workaround until #774685 is fixed
+ chdist --data-dir=$CHPATH --arch=$ARCH create $DISTNAME "[arch=$ARCH]" $MIRROR $SUITE main
+ chdist --data-dir=$CHPATH --arch=$ARCH apt-get $DISTNAME update
-# pkg-perl-maintainers
-if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[13]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[13]}.pkgset ] ; then
- grep-dctrl -sPackage -n -FMaintainer pkg-perl-maintainers at lists.alioth.debian.org $SOURCES > $TMPFILE
- update_if_similar ${META_PKGSET[13]}.pkgset
-fi
+ PACKAGES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_binary-${ARCH}_Packages)
+ SOURCES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_source_Sources)
+ # finally
+ update_pkg_sets
+ echo "$(date) - All meta package sets for $SUITE created successfully."
+done
-# finally
rm -f $TMPFILE ${TMPFILE2}
-echo "All meta package sets created successfully."
+echo
diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh
index 05116ea..c8dba35 100755
--- a/bin/reproducible_html_graphs.sh
+++ b/bin/reproducible_html_graphs.sh
@@ -478,8 +478,8 @@ create_main_stats_page() {
write_page "</p><p>"
# write meta pkg graphs per suite
for SUITE in $SUITES ; do
- if [ "$SUITE" != "sid" ] ; then
- # FIXME: no pkg sets in experimental
+ if [ "$SUITE" = "experimental" ] ; then
+ # no pkg sets in experimental
continue
fi
for i in $(seq 1 ${#META_PKGSET[@]}) ; do
@@ -536,8 +536,8 @@ for SUITE in $SUITES ; do
update_suite_stats
gather_suite_stats
create_suite_stats_page
- if [ "$SUITE" != "sid" ] ; then
- # FIXME: should be: no pkg sets in experimental
+ if [ "$SUITE" = "experimental" ] ; then
+ # no pkg sets in experimental
continue
fi
update_meta_pkg_stats
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list