[Pkg-haskell-commits] [DHG_packages] 01/01: Add mass-build.sh
Joachim Breitner
nomeata at moszumanska.debian.org
Mon Jul 13 17:57:33 UTC 2015
This is an automated email from the git hooks/post-receive script.
nomeata pushed a commit to branch master
in repository DHG_packages.
commit 06437ce77901d375254ebed68ddc1b065e678047
Author: Joachim Breitner <mail at joachim-breitner.de>
Date: Mon Jul 13 19:56:44 2015 +0200
Add mass-build.sh
with minimal changes so far. It would work better if #704924 was fixed,
and the reversal of release marker code is still darcs-specific.
---
debian2dsc.sh | 115 ++++++++++++++++++++
mass-build.sh | 314 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
order-sources.pl | 91 ++++++++++++++++
3 files changed, 520 insertions(+)
diff --git a/debian2dsc.sh b/debian2dsc.sh
new file mode 100755
index 0000000..6b23eda
--- /dev/null
+++ b/debian2dsc.sh
@@ -0,0 +1,115 @@
+#!/bin/sh
+
+set -e
+
+until [ -z "$1" ]
+do
+ case "$1" in
+ *)
+ DIRS="$DIRS $1"
+ ;;
+ esac
+ shift
+done
+
+if [ -z "$DIRS" ]
+then
+ cat <<__END__
+Usage: $0 [dir ...]
+
+Expects directories containing the contents of the debian/ directory of a
+Debian source package (i.e. control, changelog, watch, ...). Uses the watch
+file to retrieve the upstream tarball (unless it already exists in the local
+directory), creates a debian.tar.xz tarball from the given directory (ignoring
+the ususal suspects like _darcs and *~) and creates a corresponding .dsc file;
+all without unpacking the upstream tarball.
+__END__
+ exit 1
+fi
+
+for DIR in $DIRS
+do
+ if [ ! -e $DIR/control -o ! -e $DIR/changelog ]
+ then
+ echo "Did not find $DIR/control or $DIR/changelog."
+ echo "Is the repository in the debian/-only format?"
+ exit 1
+ fi
+
+ VERSION=`dpkg-parsechangelog -l$DIR/changelog -c1 -SVersion`
+ PACKAGE=`dpkg-parsechangelog -l$DIR/changelog -c1 -SSource`
+ UPSTREAM=`echo $VERSION | cut -d- -f1` # this could be improved
+ if echo $UPSTREAM | fgrep -q : ; then
+ UPSTREAM=`echo $UPSTREAM | cut -d: -f2-`
+ VERSION=`echo $VERSION | cut -d: -f2-`
+ fi
+ TARBALL_GZ=${PACKAGE}_$UPSTREAM.orig.tar.gz
+ TARBALL_BZ2=${PACKAGE}_$UPSTREAM.orig.tar.bz2
+ TARBALL_XZ=${PACKAGE}_$UPSTREAM.orig.tar.xz
+ # see 375138 for why this doesn't work as well as it could. Fall back to apt-get source
+ # as a last resort.
+ [ ! -e $TARBALL_GZ -a ! -e $TARBALL_BZ2 -a ! -e $TARBALL_XZ ] && \
+ ( uscan \
+ --rename \
+ --force-download \
+ --package "$PACKAGE" \
+ --download \
+ --watchfile $DIR/watch \
+ --copyright-file $DIR/copyright \
+ --download-version "$UPSTREAM" \
+ --upstream-version "$UPSTREAM" \
+ --destdir . \
+ --rename ||
+ apt-get source "$PACKAGE" --tar-only )
+
+ if [ ! -e $TARBALL_GZ -a ! -e $TARBALL_BZ2 -a ! -e $TARBALL_XZ ]
+ then
+ echo "Couldn't download tarball with uscan or apt-get source. See above for errors"
+ exit 1
+ fi
+
+ TARBALL=""
+
+ if [ -e $TARBALL_GZ ]
+ then
+ TARBALL="$TARBALL_GZ"
+ else
+ if [ -e $TARBALL_XZ ]
+ then
+ TARBALL="$TARBALL_XZ"
+ else
+ if [ -e $TARBALL_BZ2 ]
+ then
+ TARBALL="$TARBALL_BZ2"
+ else
+ echo "Unreachable code"
+ exit 1
+ fi
+ fi
+ fi
+
+ DEBIAN_TARBALL=${PACKAGE}_${VERSION}.debian.tar.xz
+ # -I line taken from "man dpkg-source"
+ tar --create \
+ --xz \
+ --transform s,^.,debian, \
+ --force-local \
+ --file $DEBIAN_TARBALL \
+ --directory $DIR \
+ "--exclude=*.a" "--exclude=*.la" "--exclude=*.o" "--exclude=*.so" "--exclude=.*.sw?" "--exclude=*/*~" "--exclude=,,*" "--exclude=.[#~]*" "--exclude=.arch-ids" "--exclude=.arch-inventory" "--exclude=.be" "--exclude=.bzr" "--exclude=.bzr.backup" "--exclude=.bzr.tags" "--exclude=.bzrignore" "--exclude=.cvsignore" "--exclude=.deps" "--exclude=.git" "--exclude=.gitattributes" "--exclude=.gitignore" "--exclude=.gitmodules" "--exclude=.hg" "--exclude=.hgignore" "--exclude=.hgsigs" "--exclude=. [...]
+ .
+ dpkg-source \
+ -c$DIR/control -l$DIR/changelog \
+ --format="3.0 (custom)" --target-format="3.0 (quilt)" \
+ -b / \
+ $DEBIAN_TARBALL \
+ $TARBALL
+
+ DSC=${PACKAGE}_${VERSION}.dsc
+ if [ -e $DSC ]
+ then
+ echo "Successfully created $DSC."
+ else
+ echo "Failed to create $DSC."
+ fi
+done
diff --git a/mass-build.sh b/mass-build.sh
new file mode 100755
index 0000000..075cd4f
--- /dev/null
+++ b/mass-build.sh
@@ -0,0 +1,314 @@
+#!/bin/bash
+
+schroot=$1
+shift
+
+result=$1
+shift
+
+if [ -z "$schroot" -o -z "$result" -o "$schroot" = '--help' ]
+then
+ echo "Usage: $0 <chroot> <results/> dirs..."
+ echo
+ echo "Check each directory whether it is marked for release (distribution set) but"
+ echo "not tagged with the latest version. Those packages will be built with sbuild"
+ echo "inside the given chroot. The result will be stored in the second directory"
+ echo "passed. Also, all packages in this directory are made available as a repository"
+ echo "inside the chroot."
+ echo "If the build is sucessful, it runs \"debcommit -r\" in the repository to create"
+ echo "the correct tag."
+ echo
+ echo "Until http://bugs.debian.org/608840 is fixed you have to ensure that results/"
+ echo "is set up as an apt source in the source schroot. This script will however"
+ echo "keep Packages and Sources up-to-date. For this to work the schroot should have"
+ echo " Apt::Get::AllowUnauthenticated 1;"
+ echo "set, e.g. in a file named /etc/apt/apt.conf.d/unauthenticated."
+ exit
+fi
+
+set -e
+
+# unlock sudo password
+sudo true
+
+tobuild=""
+nottobuild=""
+released=""
+
+DEBIAN2DSC=debian2dsc
+if ! which $DEBIAN2DSC >/dev/null
+then
+ pushd $(dirname $0) >/dev/null
+ DEBIAN2DSC=$PWD/debian2dsc.sh
+ popd >/dev/null
+fi
+if ! which $DEBIAN2DSC >/dev/null
+then
+ echo "Could not find debian2dsc ($DEBIAN2DSC)."
+ exit 1
+fi
+
+ORDER_SOURCES=order-sources
+if ! which $ORDER_SOURCES >/dev/null
+then
+ pushd $(dirname $0) >/dev/null
+ ORDER_SOURCES=$PWD/order-sources.pl
+ popd >/dev/null
+fi
+if ! which $ORDER_SOURCES >/dev/null
+then
+ echo "Could not find order-sources ($ORDER_SOURCES)."
+ exit 1
+fi
+
+arch="$(schroot -c $schroot -- dpkg-architecture -qDEB_BUILD_ARCH)"
+echo "The schroot $schroot builds $arch packages."
+
+echo "Finding out what to build..."
+while [ -n "$1" ]
+do
+ dir=$1
+ shift
+
+ if ! pushd "$dir" >/dev/null
+ then
+ echo "Failed to switch to \"$dir\""
+ continue
+ fi
+
+ if [ ! -e debian/changelog ]
+ then
+ echo "No changelog file found, skipping $dir"
+ popd >/dev/null
+ continue
+ fi
+
+ set -e
+ dist=$(dpkg-parsechangelog -ldebian/changelog -c1 -SDistribution)
+ set +e
+ if [ "$dist" = "UNRELEASED" ]
+ then
+ nottobuild="$nottobuild $dir"
+ else
+ package=$(dpkg-parsechangelog -ldebian/changelog -c1 -SVersion)
+ version=$(dpkg-parsechangelog -ldebian/changelog -c1 -SSource)
+ version_fixed="$(echo $version|tr ':~' '_')"
+ if git tag -l "$package_v$version" | grep -q .
+ then
+ released="$released $dir"
+ else
+ tobuild="$tobuild $dir"
+ fi
+ fi
+ popd >/dev/null
+done
+
+echo "Figuring out build order..."
+if [ -n "$tobuild" ]
+then
+ tobuild="$($ORDER_SOURCES $tobuild)"
+fi
+
+
+echo "Creating source packages..."
+tmpdir="$(mktemp -d --tmpdir=$result mass-build-XXXXX)"
+tobuild2=""
+for dir in $tobuild
+do
+ if ! pushd "$dir" >/dev/null
+ then
+ echo "Failed to switch to \"$dir\""
+ continue
+ fi
+ repodir=$PWD
+ popd >/dev/null
+
+ if ! pushd "$result" >/dev/null
+ then
+ echo "Failed to switch to $result"
+ exit
+ fi
+
+ PACKAGE=`dpkg-parsechangelog -l$repodir/debian/changelog -c1 -SSource`
+ VERSION=`dpkg-parsechangelog -l$repodir/debian/changelog -c1 -SVersion`
+ UPSTREAM=`echo $VERSION | cut -d- -f1` # this could be improved
+ if echo $UPSTREAM | fgrep -q : ; then
+ UPSTREAM=`echo $UPSTREAM | cut -d: -f2-`
+ VERSION=`echo $VERSION | cut -d: -f2-`
+ fi
+ DISTRIBUTION=$(dpkg-parsechangelog -l$repodir/debian/changelog -c1 -SDistribution)
+ DSC=${PACKAGE}_${VERSION}.dsc
+
+ if ! $DEBIAN2DSC $repodir/debian && [ -e $DSC ]
+ then
+ echo "Failed to create $DSC for $dir."
+ buildfailed="$buildfailed $dir"
+ popd >/dev/null
+ continue
+ fi
+
+ popd >/dev/null
+
+ if ! pushd "$tmpdir" >/dev/null
+ then
+ echo "Failed to switch to $tmpdir"
+ exit
+ fi
+
+ if ! ln -s "../$DSC"
+ then
+ echo "Failed to link $DSC"
+ buildfailed="$buildfailed $dir"
+ popd >/dev/null
+ continue
+ fi
+
+ nextround="$nextround $dir"
+ popd >/dev/null
+done
+
+retry=yes
+
+while [ "$retry" = yes ]
+do
+
+retry=no
+thisround="$nextround"
+nextround=""
+
+echo "Updating Packages"
+if ! pushd "$result" >/dev/null
+then
+ echo "Failed to switch to $result"
+ exit
+fi
+dpkg-scanpackages . > Packages
+
+echo "Upgrading schroot"
+sudo sbuild-update --update --upgrade $schroot
+popd >/dev/null
+
+
+echo "Checking build-installability..."
+if ! pushd "$tmpdir" >/dev/null
+then
+ echo "Failed to switch to $tmpdir"
+ exit
+fi
+
+dpkg-scansources . > Sources
+schroot -c $schroot -- bash -c "cat /var/lib/apt/lists/*Packages" > Packages
+dose-builddebcheck --latest --explain --failures --deb-native-arch=$arch Packages Sources > ../edos-failures.log
+installable="$(dose-builddebcheck --latest --successes --deb-native-arch=$arch Packages Sources|perl -ne 'print "$1\n" if /package: src:(.*)/')"
+popd >/dev/null
+
+#echo "The following $(echo $uninstallable|wc -w) packages are uninstallable:"
+#echo $uninstallable
+
+echo "Starting builds..."
+for dir in $thisround
+do
+ echo "Building $dir..."
+ if ! pushd "$dir" >/dev/null
+ then
+ echo "Failed to switch to \"$dir\""
+ continue
+ fi
+ repodir=$PWD
+ popd >/dev/null
+
+ PACKAGE=`dpkg-parsechangelog -l$repodir/debian/changelog -c1 -SSource`
+
+ if ! echo "$installable" | fgrep -xq "$PACKAGE"
+ then
+ echo "Package was found to be uninstallable, skipping.."
+ nextround="$nextround $dir"
+ continue
+ fi
+
+ VERSION=`dpkg-parsechangelog -l$repodir/debian/changelog -c1 -SVersion`
+ UPSTREAM=`echo $VERSION | cut -d- -f1` # this could be improved
+ if echo $UPSTREAM | fgrep -q : ; then
+ UPSTREAM=`echo $UPSTREAM | cut -d: -f2-`
+ VERSION=`echo $VERSION | cut -d: -f2-`
+ fi
+ DISTRIBUTION=$(dpkg-parsechangelog -l$repodir/debian/changelog -c1 -SDistribution)
+ DSC=${PACKAGE}_${VERSION}.dsc
+
+
+ pushd $result >/dev/null
+
+ dpkg-scanpackages . > Packages
+ sudo sbuild-update --update $schroot
+
+ if [ ! -e $DSC ]
+ then
+ echo "File $DSC not found, should have been created earlier."
+ buildfailed="$buildfailed $dir"
+ popd >/dev/null
+ continue
+ fi
+
+
+# Does not work, see http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=608840
+# cat > setup.sh <<__END__
+##!/bin/bash
+#echo deb file://$PWD ./ > /etc/apt/sources.list.d/mass-build.list'
+#echo deb-src file://$PWD ./ > /etc/apt/sources.list.d/mass-build.list'
+#__END__
+# chmod +x setup.sh
+
+ if sbuild --dist "$DISTRIBUTION" \
+ --chroot="$schroot" \
+ --build-dep-resolver=aptitude \
+ --arch=$arch \
+ --run-lintian \
+ --arch-all \
+ --no-apt-update \
+ --no-apt-upgrade \
+ --source \
+ "$DSC"
+# --chroot-setup-commands=$PWD/setup.sh \
+ then
+ #rm -f setup.sh
+ buildok="$buildok $dir"
+ popd >/dev/null
+ pushd $dir >/dev/null
+ debcommit -r
+ popd >/dev/null
+ echo "Building $dir was successful"
+ retry=yes
+ else
+ #rm -f setup.sh
+ buildfailed="$buildfailed $dir"
+ popd >/dev/null
+ pushd $dir >/dev/null
+ if darcs diff|diffstat|tail -n1|fgrep -q '1 file changed, 2 insertions(+), 2 deletions(-)'
+ then
+ echo "Reverting release marker."
+ darcs revert -a
+ fi
+ if darcs diff|diffstat|tail -n1|fgrep -q '1 file changed, 1 insertion(+), 1 deletion(-)'
+ then
+ echo "Reverting release marker."
+ darcs revert -a
+ fi
+ popd >/dev/null
+ echo "Building $dir failed"
+ fi
+done
+
+done
+rm -rf "$tmpdir"
+
+echo "Summary:"
+echo "Packages not to build:"
+echo $nottobuild
+echo "Packages already released:"
+echo "($(echo $released |wc -w) packages)"
+echo "Packages not yet buildable:"
+echo $nextround
+echo "Packages successfully build:"
+echo $buildok
+echo "Packages failed to build:"
+echo $buildfailed
diff --git a/order-sources.pl b/order-sources.pl
new file mode 100755
index 0000000..1d6a0c0
--- /dev/null
+++ b/order-sources.pl
@@ -0,0 +1,91 @@
+#!/usr/bin/perl
+
+use strict;
+use Dpkg::Control;
+use Dpkg::Control::Info;
+use Data::Dump qw/dump/;
+
+
+unless (scalar @ARGV > 0) {
+ print <<__END__;
+Usage: $0 <dir / control / dsc>...
+
+Each argument is expected to be a Debian source package directory; the debian
+directory in a Debian source package directory the control file or the .dsc
+file of a Debian source. These will be ordered by “obvious” build-dependencies
+and printed out again.
+__END__
+ exit;
+}
+
+my %builds;
+my %depends;
+
+sub parse_control {
+ my $dir=shift;
+ my $control=shift;
+
+ my $data = new Dpkg::Control::Info ($control);
+ my $src = $data->get_source();
+ $depends{$dir} = [];
+ push @{$depends{$dir}}, $_
+ for $src->{"Build-Depends"} =~ /([\w\d-_]+)(?:\s+\(.*?\))?\s*,?/sg;
+
+ $builds{$_->{Package}} = $dir
+ for $data->get_packages();
+}
+
+sub parse_dsc {
+ my $dir=shift;
+ my $dsc=shift;
+
+ my $data = new Dpkg::Control ((type => CTRL_PKG_SRC));
+ $data->load($dsc);
+
+ $depends{$dir} = [];
+ push @{$depends{$dir}}, $_
+ for $data->{"Build-Depends"} =~ /([\w\d-_]+)(?:\s+\(.*?\))?\s*,?/sg;
+
+ $builds{$_} = $dir
+ for $data->{"Binary"} =~ /([\w\d-_]+)\s*,?/sg;
+}
+
+for my $dir (@ARGV) {
+ if (-f "$dir/debian/control") {
+ parse_control($dir, "$dir/debian/control");
+ next;
+ }
+ if (-f "$dir/control" ) {
+ parse_control($dir, "$dir/control");
+ next;
+ }
+ if ($dir =~ m/control$/ and -f $dir) {
+ parse_control($dir, $dir);
+ next;
+ }
+ if ($dir =~ m/\.dsc$/ and -f $dir) {
+ parse_dsc($dir, $dir);
+ next;
+ }
+ warn "Could not find a control file for argument $dir\n";
+}
+
+my %waiting;
+my %done;
+
+sub do_pkg {
+ my $dir = shift;
+ return if $done{$dir};
+ return if $waiting{$dir}; # loop breaker
+ $waiting{$dir}++;
+ for my $deb (@{$depends{$dir}}) {
+ if (exists $builds{$deb}) {
+ do_pkg ($builds{$deb});
+ }
+ }
+ print "$dir\n";
+ delete $waiting{$dir};
+ $done{$dir} ++;
+}
+
+do_pkg ($_) for @ARGV;
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-haskell/DHG_packages.git
More information about the Pkg-haskell-commits
mailing list