[Qa-jenkins-scm] [jenkins.debian.net] 02/02: reproducible archlinux: refactor to support multiple repositories
Holger Levsen
holger at moszumanska.debian.org
Fri Dec 11 12:54:21 UTC 2015
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 8a0a4485edff9d21fffb02f763c65afd14c97769
Author: Holger Levsen <holger at layer-acht.org>
Date: Fri Dec 11 13:53:55 2015 +0100
reproducible archlinux: refactor to support multiple repositories
---
bin/reproducible_build_archlinux_pkg.sh | 36 ++++++++--------
bin/reproducible_html_archlinux.sh | 73 +++++++++++++++++----------------
2 files changed, 57 insertions(+), 52 deletions(-)
diff --git a/bin/reproducible_build_archlinux_pkg.sh b/bin/reproducible_build_archlinux_pkg.sh
index 64c49f4..17cddcd 100755
--- a/bin/reproducible_build_archlinux_pkg.sh
+++ b/bin/reproducible_build_archlinux_pkg.sh
@@ -44,23 +44,24 @@ choose_package() {
# every 2 days we check for new archlinux packages
touch -d "$(date -d '2 days ago' '+%Y-%m-%d') 00:00 UTC" $DUMMY
if [ ! -f $ARCHLINUX_PKGS ] || [ $DUMMY -nt $ARCHLINUX_PKGS ] ; then
- echo "$(date -u ) - updating list of available packages."
+ REPOSITORY="core"
+ echo "$(date -u ) - updating list of available packages in repository '$REPOSITORY'."
local SESSION="archlinux-scheduler-$RANDOM"
schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
- schroot --run-session -c $SESSION --directory /var/abs/core -- ls -1|sort -R|xargs echo > $ARCHLINUX_PKGS
+ schroot --run-session -c $SESSION --directory /var/abs/$REPOSITORY -- ls -1|sort -R|xargs echo > $ARCHLINUX_PKGS
schroot --end-session -c $SESSION
fi
rm $DUMMY > /dev/null
- echo "$(date -u ) - these packages are known to us:"
+ echo "$(date -u ) - these packages in repository '$REPOSITORY' are known to us:"
cat $ARCHLINUX_PKGS
for PKG in $(cat $ARCHLINUX_PKGS) ; do
# build package if it has never build or at least a week ago
- if [ ! -d $BASE/archlinux/$PKG ] || [ ! -z $(find $BASE/archlinux/ -name $PKG -mtime +6) ] ; then
+ if [ ! -d $BASE/archlinux/$REPOSITORY/$PKG ] || [ ! -z $(find $BASE/archlinux/$REPOSITORY/ -name $PKG -mtime +6) ] ; then
SRCPACKAGE=$PKG
- echo "$(date -u ) - building package $PKG now..."
+ echo "$(date -u ) - building package $PKG from '$REPOSITORY' now..."
# very simple locking…
- mkdir -p $BASE/archlinux/$PKG
- touch $BASE/archlinux/$PKG
+ mkdir -p $BASE/archlinux/$REPOSITORY/$PKG
+ touch $BASE/archlinux/$REPOSITORY/$PKG
break
fi
done
@@ -77,7 +78,7 @@ choose_package() {
first_build() {
echo "============================================================================="
- echo "Building ${SRCPACKAGE} for Arch Linux on $(hostname -f) now."
+ echo "Building ${SRCPACKAGE} from repository '$REPOSITORY' for Arch Linux on $(hostname -f) now."
echo "Date: $(date)"
echo "Date UTC: $(date -u)"
echo "============================================================================="
@@ -88,7 +89,7 @@ first_build() {
schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
echo "MAKEFLAGS=-j$NUM_CPU" | schroot --run-session -c $SESSION --directory /tmp -u root -- tee -a /etc/makepkg.conf
schroot --run-session -c $SESSION --directory /tmp -- mkdir $BUILDDIR
- schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/core/$SRCPACKAGE $BUILDDIR/
+ schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/$REPOSITORY/$SRCPACKAGE $BUILDDIR/
# just set timezone in the 1st build
echo 'export TZ="/usr/share/zoneinfo/Etc/GMT+12"' | schroot --run-session -c $SESSION --directory /tmp -- tee -a /var/lib/jenkins/.bashrc
# nicely run makepkg with a timeout of 4h
@@ -104,7 +105,7 @@ first_build() {
second_build() {
echo "============================================================================="
- echo "Re-Building ${SRCPACKAGE} for Arch Linux on $(hostname -f) now."
+ echo "Re-Building ${SRCPACKAGE} from repository '$REPOSITORY' for Arch Linux on $(hostname -f) now."
echo "Date: $(date)"
echo "Date UTC: $(date -u)"
echo "============================================================================="
@@ -116,7 +117,7 @@ second_build() {
schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
echo "MAKEFLAGS=-j$NEW_NUM_CPU" | schroot --run-session -c $SESSION --directory /tmp -u root -- tee -a /etc/makepkg.conf
schroot --run-session -c $SESSION --directory /tmp -- mkdir $BUILDDIR
- schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/core/$SRCPACKAGE $BUILDDIR/
+ schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/$REPOSITORY/$SRCPACKAGE $BUILDDIR/
# add more variations in the 2nd build: TZ, LANG, LC_ALL, umask
schroot --run-session -c $SESSION --directory /tmp -- tee -a /var/lib/jenkins/.bashrc <<-__END__
export TZ="/usr/share/zoneinfo/Etc/GMT-14"
@@ -213,13 +214,14 @@ fi
# main - only used in master-mode
#
delay_start # randomize start times
-# first, we need to choose a package…
+# first, we need to choose a package from a repository…
+REPOSITORY=""
SRCPACKAGE=""
choose_package
# build package twice
mkdir b1 b2
remote_build 1
-# only do the 2nd build if the 1st produced some results
+# only do the 2nd build if the 1st produced results
if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.pkg.tar.xz 2>/dev/null|| true)" ] ; then
remote_build 2
# run diffoscope on the results
@@ -232,15 +234,15 @@ if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.pkg.tar.xz 2>/dev/null|| true)" ] ; the
call_diffoscope $SRCPACKAGE $ARTIFACT
# publish page
if [ -f $TMPDIR/$SRCPACKAGE/$ARTIFACT.html ] ; then
- cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/archlinux/$SRCPACKAGE/
+ cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/
fi
done
fi
# publish logs
cd $TMPDIR/b1/$SRCPACKAGE
-cp build1.log $BASE/archlinux/$SRCPACKAGE/
-[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/archlinux/$SRCPACKAGE/
-echo "$(date -u) - $REPRODUCIBLE_URL/archlinux/$SRCPACKAGE/ updated."
+cp build1.log $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/
+[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/
+echo "$(date -u) - $REPRODUCIBLE_URL/archlinux/$REPOSITORY/$SRCPACKAGE/ updated."
cd
cleanup_all
diff --git a/bin/reproducible_html_archlinux.sh b/bin/reproducible_html_archlinux.sh
index aa948f5..0e92e0e 100755
--- a/bin/reproducible_html_archlinux.sh
+++ b/bin/reproducible_html_archlinux.sh
@@ -37,45 +37,48 @@ cat > $PAGE <<- EOF
EOF
write_page_intro 'Arch Linux'
write_explaination_table 'Arch Linux'
-write_page " <table><tr><th>source package</th><th>test result</th><th>test date</th><th>1st build log</th><th>2nd build log</th></tr>"
-for PKG in $(find $ARCHBASE/* -maxdepth 1 -type d -exec basename {} \;) ; do
- write_page " <tr>"
- write_page " <td>$PKG</td>"
- write_page " <td>"
- if [ -z "$(cd $ARCHBASE/$PKG/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
- if [ ! -z "$(grep '==> ERROR: Could not resolve all dependencies' $ARCHBASE/$PKG/build1.log)" ] ; then
- write_page " <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies"
- elif [ ! -z "$(egrep '==> ERROR: .pacman. failed to install missing dependencies.' $ARCHBASE/$PKG/build1.log)" ] ; then
- write_page " <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> failed to install dependencies"
- elif [ ! -z "$(egrep '==> ERROR: A failure occurred in (build|package)' $ARCHBASE/$PKG/build1.log)" ] ; then
- write_page " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source"
- elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHBASE/$PKG/build1.log)" ] ; then
- write_page " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source, while running tests"
- elif [ ! -z "$(egrep '==> ERROR: Failure while downloading' $ARCHBASE/$PKG/build1.log)" ] ; then
- write_page " <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> failed to download source"
- elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHBASE/$PKG/build1.log)" ] ; then
- write_page " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to verify source"
- elif [ ! -z "$(egrep 'makepkg was killed by timeout after 4h' $ARCHBASE/$PKG/build1.log)" ] ; then
- write_page " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, killed by timeout after 4h"
+write_page " <table><tr><th>repository</th><th>source package</th><th>test result</th><th>test date</th><th>1st build log</th><th>2nd build log</th></tr>"
+for REPOSITORY in "core" ; do
+ for PKG in $(find $ARCHBASE/$REPOSITORY/* -maxdepth 1 -type d -exec basename {} \;) ; do
+ write_page " <tr>"
+ write_page " <td>$REPOSITORY</td>"
+ write_page " <td>$PKG</td>"
+ write_page " <td>"
+ if [ -z "$(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
+ if [ ! -z "$(grep '==> ERROR: Could not resolve all dependencies' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+ write_page " <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies"
+ elif [ ! -z "$(egrep '==> ERROR: .pacman. failed to install missing dependencies.' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+ write_page " <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> failed to install dependencies"
+ elif [ ! -z "$(egrep '==> ERROR: A failure occurred in (build|package)' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+ write_page " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source"
+ elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+ write_page " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source, while running tests"
+ elif [ ! -z "$(egrep '==> ERROR: Failure while downloading' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+ write_page " <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> failed to download source"
+ elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+ write_page " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to verify source"
+ elif [ ! -z "$(egrep 'makepkg was killed by timeout after 4h' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+ write_page " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, killed by timeout after 4h"
+ else
+ write_page " probably failed to build from source, please investigate"
+ # or is it reproducible???
+ fi
else
- write_page " probably failed to build from source, please investigate"
- # or is it reproducible???
+ for ARTIFACT in $(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html) ; do
+ write_page " <img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> <a href=\"/archlinux/$REPOSITORY/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible<br />"
+ done
fi
- else
- for ARTIFACT in $(cd $ARCHBASE/$PKG/ ; ls *.pkg.tar.xz.html) ; do
- write_page " <img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> <a href=\"/archlinux/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible<br />"
+ write_page " </td>"
+ write_page " <td>$(LANG=C TZ=UTC ls --full-time $ARCHBASE/$REPOSITORY/$PKG/build1.log | cut -d ' ' -f6 )</td>"
+ for LOG in build1.log build2.log ; do
+ if [ -f $ARCHBASE/$REPOSITORY/$PKG/$LOG ] ; then
+ write_page " <td><a href=\"/archlinux/$REPOSITORY/$PKG/$LOG\">$LOG</a></td>"
+ else
+ write_page " <td> </td>"
+ fi
done
- fi
- write_page " </td>"
- write_page " <td>$(LANG=C TZ=UTC ls --full-time $ARCHBASE/$PKG/build1.log | cut -d ' ' -f6 )</td>"
- for LOG in build1.log build2.log ; do
- if [ -f $ARCHBASE/$PKG/$LOG ] ; then
- write_page " <td><a href=\"/archlinux/$PKG/$LOG\">$LOG</a></td>"
- else
- write_page " <td> </td>"
- fi
+ write_page " </tr>"
done
- write_page " </tr>"
done
write_page " </table>"
write_page "</div></div>"
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list