[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible Debian: scripts that make up buildinfos.debian.net
Holger Levsen
gitlab at salsa.debian.org
Wed Apr 17 13:02:57 BST 2019
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
1012c656 by Holger Levsen at 2019-04-17T12:02:40Z
reproducible Debian: scripts that make up buildinfos.debian.net
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
2 changed files:
- + bin/pool_buildinfos.sh
- + bin/rsync2buildinfos.debian.net
Changes:
=====================================
bin/pool_buildinfos.sh
=====================================
@@ -0,0 +1,122 @@
+#!/bin/bash
+# vim: set noexpandtab:
+
+# Copyright 2019 Holger Levsen <holger at layer-acht.org>
+# released under the GPLv=2
+
+###################################################################
+### ###
+### /srv/ftp-master.debian.org/buildinfo/ on coccia.debian.org ###
+### is not a pool structure, but rather by year/month/day ###
+### this scripts creates links turning this into an alternate ###
+### pool structure. ###
+### Both are accessable via https://buildinfos.debian.net ###
+### ###
+###################################################################
+
+set -e
+BASEPATH=~jenkins/userContent/reproducible/debian
+FTPPATH=$BASEPATH/ftp-master.debian.org/buildinfo
+POOLPATH=$BASEPATH/buildinfo-pool
+
+PROBLEMS=$(mktemp -t poolize.XXXXXXXX)
+
+mkdir -p $POOLPATH
+
+YEAR="$(date -u +%Y)"
+MONTH="$(date -u +%m)"
+DAY="$(date -u +%d)"
+
+do_day(){
+ MONTHPATH=$FTPPATH/$YEAR/$MONTH
+ if [ ! -d $MONTHPATH ] ; then
+ echo "$MONTHPATH does not exist, next."
+ return
+ fi
+ cd $MONTHPATH
+
+ if [ ! -d $DAY ] ; then
+ echo "$MONTHPATH/$DAY does not exist, next."
+ return
+ fi
+ cd $DAY
+ echo "Working on $YEAR/$MONTH/$DAY"
+ for FILE in * ; do
+ # echo $FILE
+ PACKAGE=$(echo $FILE | cut -d '_' -f1)
+ if [ "${PACKAGE:0:3}" = "lib" ] ; then
+ POOLDIR="${PACKAGE:0:4}"
+ else
+ POOLDIR="${PACKAGE:0:1}"
+ fi
+ TARGETPATH="../../../../../buildinfo-pool/$POOLDIR/$PACKAGE"
+ mkdir -p $TARGETPATH
+ VERSION=$(grep ^Version: $FILE | head -1 | cut -d ' ' -f2)
+ if $(echo $VERSION | grep -q ":") ; then
+ #echo -n $VERSION
+ VERSION=$(echo $VERSION | cut -d ':' -f2)
+ #echo " becomes $VERSION"
+ fi
+ ARCHITECTURE=$(grep ^Architecture: $FILE | cut -d ' ' -f2-|sed 's# #-#g')
+ ARCHSUFFIX=$(echo $FILE | cut -d '_' -f3)
+ if [ "${ARCHITECTURE}.buildinfo" != "$ARCHSUFFIX" ] ; then
+ ARCHSUFFIX="${ARCHITECTURE}.buildinfo"
+ #echo $FILE is really for $ARCHITECTURE
+ fi
+ FULLTARGET="$TARGETPATH/${PACKAGE}_${VERSION}_${ARCHSUFFIX}"
+ if [ "$(readlink -f $FULLTARGET)" = "$MONTHPATH/$DAY/$FILE" ] ; then
+ #echo "$FULLTARGET already points to $MONTHPATH/$DAY/$FILE thus ignoring this...."
+ :
+ elif [ ! -e "$FULLTARGET" ] && [ -e "$MONTHPATH/$DAY/$FILE" ] ; then
+ ln -s $MONTHPATH/$DAY/$FILE $FULLTARGET
+ # echo "$MONTHPATH/$DAY/$FILE linked from $FULLTARGET"
+ elif [ ! -e $MONTHPATH/$DAY/$FILE ] ; then
+ echo "on no $MONTHPATH/$DAY/$FILE does not exist, exiting."
+ exit 1
+ elif [ -e $FULLTARGET ] ; then
+
+ if [ ! -e "$FULLTARGET.0" ] ; then
+ ln -s $MONTHPATH/$DAY/$FILE $FULLTARGET.0
+ echo "$MONTHPATH/$DAY/$FILE linked from $FULLTARGET.0"
+ elif [ "$(readlink -f $FULLTARGET.0)" = "$MONTHPATH/$DAY/$FILE" ] ; then
+ # also ignoring this
+ :
+ else
+ echo "oh no $FULLTARGET.0 also exists and thus we don't know what to do, thus ignoring." >> $PROBLEMS
+ echo "$MONTHPATH/$DAY/$FILE is the source of the problem" >> $PROBLEMS
+ ls -l $FULLTARGET >> $PROBLEMS
+ ls -l $FULLTARGET.0 >> $PROBLEMS
+ echo >> $PROBLEMS
+ #exit 1
+ fi
+ fi
+ done
+ cd ..
+}
+
+loop_through_all(){
+ for YEAR in $(seq 2019 -1 2016) ; do
+ for MONTH in $(seq -w 12 -1 01) ; do
+ for DAY in $(seq -w 31 -1 01) ; do
+ do_day
+ done
+ done
+ done
+}
+
+if [ -n "$1" ] ; then
+ loop_through_all
+else
+ do_day
+ YEAR="$(date -u -d '1 day ago' +%Y)"
+ MONTH="$(date -u -d '1 day ago' +%m)"
+ DAY="$(date -u -d '1 day ago' +%d)"
+ do_day
+fi
+
+if [ -s $PROBLEMS ] ; then
+ echo problems stored in $PROBLEMS
+else
+ rm $PROBLEMS
+fi
+
=====================================
bin/rsync2buildinfos.debian.net
=====================================
@@ -0,0 +1,53 @@
+#!/bin/bash
+# vim: set noexpandtab:
+
+# Copyright 2019 Holger Levsen <holger at layer-acht.org>
+# released under the GPLv=2
+
+###################################################################
+### ###
+### this is running on coccia.debian.org via holger's crontab: ###
+### ###
+### # m h dom mon dow command ###
+### 0 1,7,13,19 * * * ~holger/rsync2buildinfos.debian.net ###
+### ###
+###################################################################
+
+YEAR="$(date -u +%Y)"
+MONTH="$(date -u +%m)"
+DAY="$(date -u +%d)"
+cd /srv/ftp-master.debian.org/buildinfo/
+
+rsync_day(){
+ rsync -e ssh -av $YEAR/$MONTH/$DAY jenkins at profitbricks-build7-amd64.debian.net://var/lib/jenkins/userContent/reproducible/debian/ftp-master.debian.org/buildinfo/$YEAR/$MONTH/
+}
+
+rsync_month(){
+ rsync -e ssh -av $YEAR/$MONTH jenkins at profitbricks-build7-amd64.debian.net://var/lib/jenkins/userContent/reproducible/debian/ftp-master.debian.org/buildinfo/$YEAR/
+}
+
+rsync_year(){
+ rsync -e ssh -av $YEAR jenkins at profitbricks-build7-amd64.debian.net://var/lib/jenkins/userContent/reproducible/debian/ftp-master.debian.org/buildinfo/
+}
+
+if [ -n "$1" ] ; then
+ for i in $(seq 2016 $YEAR) ; do
+ rsync_year
+ done
+else
+ if [ "$DAY" = "01" ] && [ "$MONTH" = "01" ] ; then
+ # even though we do this several times a day
+ # it's ok, because the year is young :)
+ rsync_year
+ elif [ "$DAY" = "01" ] ; then
+ # see comment above
+ rsync_month
+ else
+ rsync_day
+ fi
+ # rsync yesterday, always
+ YEAR="$(date -u -d '1 day ago' +%Y)"
+ MONTH="$(date -u -d '1 day ago' +%m)"
+ DAY="$(date -u -d '1 day ago' +%d)"
+ rsync_day
+fi
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/1012c656889f19952cffe94b7e161416267bde69
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/1012c656889f19952cffe94b7e161416267bde69
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190417/d308ab27/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list