[Blends-commit] [SCM] website branch, master, updated. 17c762e952d7fa4c4e1499afdb9250e5530ccf47

[Andreas Tille]

The following commit has been merged in the master branch:
commit 17c762e952d7fa4c4e1499afdb9250e5530ccf47
Author: Andreas Tille <tille at debian.org>
Date:   Sun Feb 11 17:53:51 2018 +0100

    Start working on fetching machine readable data from salsa

diff --git a/misc/machine_readable/fetch-machine-readable_salsa b/misc/machine_readable/fetch-machine-readable_salsa
new file mode 100755
index 0000000..4f244d3
--- /dev/null
+++ b/misc/machine_readable/fetch-machine-readable_salsa
@@ -0,0 +1,92 @@
+#!/bin/sh
+# This script should be called on Alioth to gather all machine readable files
+# of VCSes mentioned in SVNDIRS and GITDIRS
+#
+# TODO: store file <pkg>.vcs with the Vcs URL where the files were found
+
+. ./salsarc
+
+TDNAME=machine-readable
+MACHINEREADABLEARCHIVE=/srv/blends.debian.org/www/_${TDNAME}/${TDNAME}.tar.bz2
+READMEDEBIANARCHIVE=/srv/blends.debian.org/www/_${TDNAME}/README.Debian.tar.bz2
+TARGETDIR=${HOME}/$TDNAME
+mkdir -p $TARGETDIR
+
+ERRLOG=${HOME}/${TDNAME}.log
+rm -f $ERRLOG
+
+GITTEAMS="debichem-team
+          med-team
+          r-pkg-team
+          science-team"
+
+ANONSCM_DICT="debichem-team:debichem/packages\n
+              med-team:debian-med\n
+              r-pkg-team:debian-science/packages\n
+              science-team:debian-science/packages"
+
+get_one_page () {
+    curl --silent -D $HEADERS_FILE "$SALSA_URL/groups/$SALSA_GROUP_ID/projects?private_token=$SALSA_TOKEN&simple=true&per_page=$PER_PAGE&page=$PAGENO" | jq --raw-output ".[] | .path"
+}
+
+list_salsa_projects () {
+  SALSA_GROUP="$1"
+  PER_PAGE=100
+  HEADERS_FILE=$(mktemp)
+  trap "rm -f $HEADERS_FILE" EXIT
+
+  SALSA_GROUP_ID=$(curl --silent -f -XGET --header "PRIVATE-TOKEN: $SALSA_TOKEN" "$SALSA_URL/groups?all_available=false" | jq ".[] | select(.path == \"$SALSA_GROUP\") | .id")
+
+  PAGENO=1
+
+  get_one_page
+
+  TOTAL_PAGES=$(grep ^X-Total-Pages: $HEADERS_FILE)
+  TOTAL_PAGES=${TOTAL_PAGES#*: }
+
+  while [ $PAGENO -lt $TOTAL_PAGES ]
+  do
+    PAGENO=$((PAGENO+1))
+    get_one_page
+  done
+}
+
+mkdir -p $TARGETDIR
+rm -rf $TARGETDIR/*
+
+for team in $GITTEAMS ; do
+  echo $team >> $ERRLOG
+  for gitrepo in `list_salsa_projects $team` ; do
+    firstletter=`echo $gitrepo | sed "s/^\(.\).*/\1/"`
+    mkdir -p $TARGETDIR/$firstletter
+    echo "Vcs-Git: git at salsa.debian.org:$team/$gitrepo.git" > $TARGETDIR/$firstletter/${gitrepo}.vcs
+    # Hint: Debian Med and Debichem R packages do not match this but who cares ;-)
+    #       Also some packages that never were on Alioth Git do not have a valid Vcs-Browser via that method
+    anonscmdir=`echo $ANONSCM_DICT | grep $team | head -n 1 | sed "s/^ *${team}://"`
+    echo "Vcs-Browser: https://anonscm.debian.org/cgit/$anonscmdir/$gitrepo.git" >> $TARGETDIR/$firstletter/${gitrepo}.vcs
+    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/changelog -O $TARGETDIR/$firstletter/${gitrepo}.changelog
+    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/control -O $TARGETDIR/$firstletter/${gitrepo}.control
+    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/copyright -O $TARGETDIR/$firstletter/${gitrepo}.copyright
+    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/upstream/metadata -O $TARGETDIR/$firstletter/${gitrepo}.upstream
+    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/upstream/edam -O $TARGETDIR/$firstletter/${gitrepo}.edam
+  done
+done
+
+FOUNDDIRS=`ls $TDNAME | wc -l`
+if [ $FOUNDDIRS -lt 10 ] ; then
+  echo "Found way to less packages in $TDNAME" >>  $ERRLOG
+  exit
+fi
+rm -f $MACHINEREADABLEARCHIVE
+cd $TARGETDIR
+cd ..
+find $TDNAME -maxdepth 1 -mindepth 1 -name ".*" -delete
+# Remove empty files
+find $TDNAME -type f -empty -delete
+tar --exclude=*README.Debian -cjf $MACHINEREADABLEARCHIVE $TDNAME
+tar --exclude=*.control --exclude=*.changelog --exclude=*.upstream --exclude=*.edam --exclude=*.vcs --exclude=*.copyright -cjf $READMEDEBIANARCHIVE $TDNAME
+
+# Check for remaining tempfiles
+if ls -l /tmp/tmp.* 2>/dev/null | grep `whoami` > /dev/null ; then
+    echo "There are tempfiles remaining!\n" | mailx -s "$0 has left some remaining files in /tmp" tille at debian.org
+fi

-- 
Static and dynamic websites for Debian Pure Blends