[Blends-commit] [SCM] website branch, master, updated. f08ee18e5fd1fa1e0b086954b5b3edc19b9e4d7a
Andreas Tille
tille at debian.org
Tue Nov 6 13:59:07 UTC 2012
The following commit has been merged in the master branch:
commit f08ee18e5fd1fa1e0b086954b5b3edc19b9e4d7a
Author: Andreas Tille <tille at debian.org>
Date: Tue Nov 6 14:58:40 2012 +0100
Move script that fetches machine readable data from team VCSes from SVN to Git
diff --git a/misc/machine_readable/fetch-machine-readable b/misc/machine_readable/fetch-machine-readable
new file mode 100755
index 0000000..9d396db
--- /dev/null
+++ b/misc/machine_readable/fetch-machine-readable
@@ -0,0 +1,163 @@
+#!/bin/sh
+# This script should be called on Alioth to gather all machine readable files
+# of VCSes mentioned in SVNDIRS and GITDIRS
+#
+# TODO: store file <pkg>.vcs with the Vcs URL where the files were found
+
+TDNAME=machine-readable
+MACHINEREADABLEARCHIVE=/srv/home/groups/blends/htdocs/${TDNAME}/${TDNAME}.tar.bz2
+TARGETDIR=${HOME}/$TDNAME
+mkdir -p $TARGETDIR
+
+ERRLOG=${HOME}/${TDNAME}.log
+rm -f $ERRLOG
+
+SVNDIRS="debian-med/trunk/packages
+ debian-science/packages
+ pkg-scicomp
+ debichem/unstable
+ debichem/wnpp
+ debichem/experimental
+ pkg-games/packages
+ pkg-grass/packages
+ pkg-multimedia/unstable
+ demudi"
+
+GITDIRS="debian-med
+ debian-science/packages
+ pkg-scicomp
+ pkg-exppsy
+ pkg-games
+ pkg-grass
+ pkg-multimedia
+ pkg-perl/packages
+ demudi"
+
+svn_checkout_machine_readable () {
+ curdir=`pwd`
+ cd $TARGETDIR
+ TMPLIST=`mktemp`
+ svn list --verbose svn://localhost/$1 --recursive | \
+ grep -v -e '/tags/' -e '/branches/' -e '/patches/' | \
+ grep -e "/control$" -e "/changelog$" -e "/copyright$" -e "/upstream$" | \
+ sed 's/^.*[[:space:]]\([^[:space:]]\+\)/\1/' \
+ > $TMPLIST
+ # debug
+ # cp -a $TMPLIST ~/_svnfiles
+ PKGLIST=`mktemp`
+ # Regard different directory layout options
+ if grep -q "/trunk/debian/" $TMPLIST ; then
+ # for Debian Med and Debian Science layout
+ svndir=`echo $1 | sed 's?^[^/]*/??'`
+ sed -e 's?debian/.*??' -e 's?trunk/.*??' $TMPLIST | sed -e "s?^$svndir/??" -e 's+/$++' -e 's#^[^/]\+/\([^/]\+\)/.*#\1#' -e 's#^[^/]\+/\([^/]\+\)$#\1#' | sort | uniq > $PKGLIST
+ trunklayout="/trunk"
+ else
+ # for DebiChemn layout
+ sed 's?/debian/.*??' $TMPLIST | sort | uniq > $PKGLIST
+ trunklayout=""
+ fi
+# set -x
+ for pkgdir in `cat $PKGLIST` ; do
+ pkg=`echo $pkgdir | sed -e 's?^.*/\([^/]\+\)?\1?'`
+ chlog=`grep -e "/$pkg/[^/]\+/debian/changelog$" -e "^$pkg/trunk/debian/changelog$" -e "^$pkg/debian/changelog$" -e "/$pkg/debian/changelog$" -e "^[a-z]\+/$pkg/[a-z]\+/trunk/debian/changelog$" $TMPLIST | sort | tail -n 1 2>/dev/null`
+ if [ "" != "$chlog" ] ; then
+ svn export svn://localhost/$1/$chlog >/dev/null
+ srcname=`dpkg-parsechangelog -lchangelog 2>/dev/null | sed -n 's/^Source: //p'`
+ ## DEBUG
+ #echo "$pkgdir -> $pkg ($chlog) -> $srcname"
+ if [ "" != "$srcname" ] ; then
+ vcslocation=`echo $chlog | sed 's?debian/changelog$??'`
+ if ! echo $vcslocation | grep -q tags ; then # sometimes some tags end up here but these seem to be packages removed from Debian
+ trunk="trunk" # at least for Debian Med and Debian Science
+ firstletter=`echo $srcname | sed "s/^\(.\).*/\1/"`
+ mkdir -p $TARGETDIR/$firstletter
+ mv changelog $TARGETDIR/$firstletter/${srcname}.changelog
+ echo "Vcs-Svn: svn://svn.debian.org/$1/$vcslocation" > $TARGETDIR/$firstletter/${srcname}.vcs
+ echo "Vcs-Browser: http://svn.debian.org/wsvn/$1/$vcslocation" >> $TARGETDIR/$firstletter/${srcname}.vcs
+ echo "Blend: `echo $1 | sed 's?/.*??'`" >> $TARGETDIR/$firstletter/${srcname}.vcs
+ for file in control copyright upstream ; do
+ getfile=`grep -e "/$pkg/trunk/debian/$file$" -e "^$pkg/trunk/debian/$file$" -e "^$pkg/debian/$file$" -e "trunk/$pkg/debian/$file$" -e "/$pkg/[a-z]\+/trunk/debian/$file$" $TMPLIST 2>/dev/null`
+ if [ "" != "$getfile" ] ; then
+ svn export svn://localhost/$1/$getfile >/dev/null
+ mv $file $TARGETDIR/$firstletter/${srcname}.$file
+ else
+ if ! `echo $vcslocation | grep -q trunk` ; then
+ if [ "$file" != "upstream" ] ; then
+ echo "Package $pkg is lacking trunk directory in vcslocation ${vcslocation}. Try to find file $file anyway." >> $ERRLOG
+ getfile=`grep -e "$vcslocation/debian/$file$" $TMPLIST 2>/dev/null`
+ if [ "" != "$getfile" ] ; then
+ svn export svn://localhost/$1/$getfile >/dev/null
+ mv $file $TARGETDIR/$firstletter/${srcname}.$file
+ else
+ if [ "$file" != "upstream" ] ; then
+ echo "Did not found $file for package $pkg (`grep "$pkg" $TMPLIST | grep "$file"`)" >> $ERRLOG
+ fi
+ fi
+ fi
+ fi
+ fi
+ done
+ fi
+ else
+ echo "Can not parse changelog for source name of $pkg" >> $ERRLOG
+ mv changelog ${pkg}.broken.changelog
+ fi
+ else
+ echo "No changelog found for $pkg" >> $ERRLOG
+ cp -a $TMPLIST ~/_changelog_for_${pkg}_missing
+ cp -a $PKGLIST ~/_pkglist_for_${pkg}_missing
+ fi
+ done
+ rm $TMPLIST $PKGLIST
+ cd $curdir
+}
+
+git_checkout_machine_readable () {
+ curdir=`pwd`
+ cd $1
+ TMPCHLOG=`mktemp`
+ if git show HEAD:debian/changelog 2>/dev/null > $TMPCHLOG ; then
+ srcname=`dpkg-parsechangelog -l$TMPCHLOG 2>/dev/null | sed -n 's/^Source: //p'`
+ firstletter=`echo $srcname | sed "s/^\(.\).*/\1/"`
+ mkdir -p $TARGETDIR/$firstletter
+ mv $TMPCHLOG $TARGETDIR/$firstletter/${srcname}.changelog
+ echo "Vcs-Git: git://git.debian.org$1" > $TARGETDIR/$firstletter/${srcname}.vcs
+ echo "Vcs-Browser: http://git.debian.org"`echo $1 | sed 's+^/git/+/?p=+'` >> $TARGETDIR/$firstletter/${srcname}.vcs
+ echo "Blend: `echo $2 | sed 's?/.*??'`" >> $TARGETDIR/$firstletter/${srcname}.vcs
+ for file in `git ls-tree HEAD debian/ 2>/dev/null | grep -e "/control$" -e "/changelog$" -e "/copyright$" -e "/upstream$" | sed 's/^[0-9]\+[[:space:]]\+blob[[:space:]]\+[0-9a-f]\+[[:space:]]\+//'` ; do
+ target=$TARGETDIR/$firstletter/${srcname}.`echo $file | sed 's?debian/??'`
+ git show HEAD:$file > $target
+ done
+ else
+ echo "Can not find debian/changelog in $1" >> $ERRLOG
+ fi
+ cd $curdir
+}
+
+mkdir -p $TARGETDIR
+rm -rf $TARGETDIR/*
+
+for svndir in $SVNDIRS ; do
+ echo $svndir >> $ERRLOG
+ svn_checkout_machine_readable $svndir
+done
+
+for gitdir in $GITDIRS ; do
+ echo $gitdir >> $ERRLOG
+ for gitrepo in `find /git/$gitdir -mindepth 1 -maxdepth 1 -type d -name "*.git"` ; do
+ # echo $gitrepo
+ if [ "$gitrepo" != ".git" ] ; then
+ git_checkout_machine_readable $gitrepo $gitdir
+ fi
+ done
+done
+
+FOUNDDIRS=`ls $TDNAME | wc -l`
+if [ $FOUNDDIRS -lt 10 ] ; then
+ echo "Found way to less packages in $TDNAME" >> $ERRLOG
+ exit
+fi
+rm -f $MACHINEREADABLEARCHIVE
+cd $TARGETDIR
+cd ..
+tar -cjf $MACHINEREADABLEARCHIVE $TDNAME
--
Static and dynamic websites for Debian Pure Blends
More information about the Blends-commit
mailing list