[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: reproducible Archlinux: refactor
Holger Levsen
gitlab at salsa.debian.org
Mon Sep 24 14:38:48 BST 2018
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
887bc4b9 by Holger Levsen at 2018-09-24T13:23:52Z
reproducible Archlinux: refactor
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
ad98eedd by Holger Levsen at 2018-09-24T13:38:29Z
reproducible Archlinux: less verbose output now that those packages have been added
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
4 changed files:
- TODO
- bin/reproducible_archlinux_scheduler.sh
- bin/reproducible_build_archlinux_pkg.sh
- bin/reproducible_html_archlinux.sh
Changes:
=====================================
TODO
=====================================
@@ -324,8 +324,9 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
*** also delete unknown packages from db: sources and schedule, later results as well
*** check/make sure that packages which are newer in trunk than repo are only scheduled once
*** fix irc notification
-** make build.sh
+** make _archlinux_build.sh
*** write db
+*** refactor and reuse unregister_build from _build.sh
*** we share /var/log/jenkins/reproducible-race-conditions.log with debian, is that ok? (else fix maintenance job and handle_race_condition())
*** make build respect pacman exit code, see FIXME in _html_.sh
** html
=====================================
bin/reproducible_archlinux_scheduler.sh
=====================================
@@ -92,8 +92,6 @@ update_archlinux_repositories() {
rm pkg.build_duration pkg.state pkg.version
elif [ -f pkg.build_duration ] || [ -f pkg.state ] || [ -f pkg.version ] ; then
echo "$REPO/$PKG: one or more of pkg.build_duration, pkg.state and pkg.version does not exist, ignoring."
- else
- echo "$REPO/$PKG has been added to the db (or never been build), ignoring."
fi
cd -
fi
=====================================
bin/reproducible_build_archlinux_pkg.sh
=====================================
@@ -40,6 +40,213 @@ handle_remote_error() {
exit 0
}
+create_pkg_state_and_html() {
+ local ARCHLINUX_PKG_PATH=$ARCHBASE/$REPOSITORY/$PKG
+ local REPO=$1
+ local PKG=$2
+ local blacklisted=false
+ local VERSION="undetermined"
+
+ if [ -z "$(cd $ARCHLINUX_PKG_PATH ; ls)" ] ; then
+ # directory exists but is empty: package is building…
+ echo "$(date -u ) - ignoring $PKG from '$REPOSITORY' which is building in $ARCHLINUX_PKG_PATH since $(date -u --date=@$(stat -c %Y $ARCHLINUX_PKG_PATH) +'%F %R %Z')"
+ return
+ fi
+
+ # clear files from previous builds
+ pushd "$ARCHLINUX_PKG_PATH"
+ for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
+ if [ -f $file ] && [ pkg.build_duration -nt $file ] ; then
+ rm $file
+ echo "$ARCHLINUX_PKG_PATH/$file older than $ARCHLINUX_PKG_PATH/pkg.build_duration, thus deleting it."
+ fi
+ done
+ popd
+
+ if [ -f $ARCHLINUX_PKG_PATH/pkg.version ] ; then
+ VERSION=$(cat $ARCHLINUX_PKG_PATH/pkg.version)
+ elif [ -f $ARCHLINUX_PKG_PATH/build1.version ] ; then
+ VERSION=$(cat $ARCHLINUX_PKG_PATH/build1.version)
+ if [ -f $ARCHLINUX_PKG_PATH/build2.log ] ; then
+ if [ ! -f $ARCHLINUX_PKG_PATH/build2.version ] ; then
+ echo "$(date -u ) - $ARCHLINUX_PKG_PATH/build2.version does not exist, so the 2nd build fails. This happens."
+ elif ! diff -q $ARCHLINUX_PKG_PATH/build1.version $ARCHLINUX_PKG_PATH/build2.version ; then
+ echo "$(date -u ) - $ARCHLINUX_PKG_PATH/build1.version and $ARCHLINUX_PKG_PATH/build2.version differ, this should not happen. Please tell h01ger."
+ VERSION="$VERSION or $(cat $ARCHLINUX_PKG_PATH/build2.version)"
+ fi
+ fi
+ elif [ $(ls $ARCHLINUX_PKG_PATH/*.pkg.tar.xz.html 2>/dev/null | wc -l) -eq 1 ] ; then
+ # only determine version if there is exactly one artifact...
+ # else it's too error prone and in future the version will
+ # be determined during build anyway...
+ ARTIFACT="$(ls $ARCHLINUX_PKG_PATH/*.pkg.tar.xz.html 2>/dev/null)"
+ VERSION=$( basename $ARTIFACT | sed -s "s#$PKG-##" | sed -E -s "s#-(x86_64|any).pkg.tar.xz.html##" )
+ else
+ for i in $ARCHLINUX_BLACKLISTED ; do
+ if [ "$PKG" = "$i" ] ; then
+ blacklisted=true
+ fi
+ done
+ if ! $blacklisted ; then
+ echo "$(date -u ) - cannot determine state of $PKG from '$REPOSITORY', please check $ARCHLINUX_PKG_PATH yourself."
+ fi
+ fi
+ if [ "$VERSION" != "undetermined" ] || $blacklisted ; then
+ echo $VERSION > $ARCHLINUX_PKG_PATH/pkg.version
+ fi
+ echo " <tr>" >> $HTML_BUFFER
+ echo " <td>$REPOSITORY</td>" >> $HTML_BUFFER
+ echo " <td>$PKG</td>" >> $HTML_BUFFER
+ echo " <td>$VERSION</td>" >> $HTML_BUFFER
+ echo " <td>" >> $HTML_BUFFER
+ #
+ #
+ if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
+ for i in $ARCHLINUX_BLACKLISTED ; do
+ if [ "$PKG" = "$i" ] ; then
+ blacklisted=true
+ fi
+ done
+ # this horrible if elif elif elif elif... monster is needed because
+ # https://lists.archlinux.org/pipermail/pacman-dev/2017-September/022156.html
+ # has not yet been merged yet...
+ # FIXME: this has been merged, see http://jlk.fjfi.cvut.cz/arch/manpages/man/makepkg
+
+ if $blacklisted ; then
+ echo BLACKLISTED > $ARCHLINUX_PKG_PATH/pkg.state
+ echo " <img src=\"/userContent/static/error.png\" alt=\"blacklisted icon\" /> blacklisted" >> $HTML_BUFFER
+ elif [ ! -z "$(egrep '^error: failed to prepare transaction \(conflicting dependencies\)' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo DEPWAIT_= > $ARCHLINUX_PKG_PATH/pkg.state
+ echo " <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies as there are conflicts" >> $HTML_BUFFER
+ elif [ ! -z "$(egrep '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo DEPWAIT_1 > $ARCHLINUX_PKG_PATH/pkg.state
+ echo " <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies" >> $HTML_BUFFER
+ elif [ ! -z "$(egrep '^error: unknown package: ' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_0 > $ARCHLINUX_PKG_PATH/pkg.state
+ echo " <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> unknown package" >> $HTML_BUFFER
+ elif [ ! -z "$(egrep '(==> ERROR: Failure while downloading|==> ERROR: One or more PGP signatures could not be verified|==> ERROR: One or more files did not pass the validity check|==> ERROR: Integrity checks \(.*\) differ in size from the source array|==> ERROR: Failure while branching|==> ERROR: Failure while creating working copy|Failed to source PKGBUILD.*PKGBUILD)' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ REASON="download failed"
+ EXTRA_REASON=""
+ echo 404_0 > $ARCHLINUX_PKG_PATH/pkg.state
+ if [ ! -z "$(grep 'FAILED (unknown public key' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_6 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="to verify source with PGP due to unknown public key"
+ elif [ ! -z "$(grep 'The requested URL returned error: 403' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_2 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with 403 - forbidden"
+ elif [ ! -z "$(grep 'The requested URL returned error: 500' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_4 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with 500 - internal server error"
+ elif [ ! -z "$(grep 'The requested URL returned error: 503' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_5 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with 503 - service unavailable"
+ elif [ ! -z "$(egrep '==> ERROR: One or more PGP signatures could not be verified' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_7 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="to verify source with PGP signatures"
+ elif [ ! -z "$(egrep '(SSL certificate problem: unable to get local issuer certificate|^bzr: ERROR: .SSL: CERTIFICATE_VERIFY_FAILED)' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_1 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with SSL problem"
+ elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_8 > $ARCHLINUX_PKG_PATH/pkg.state
+ REASON="downloaded ok but failed to verify source"
+ elif [ ! -z "$(egrep '==> ERROR: Integrity checks \(.*\) differ in size from the source array' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_9 > $ARCHLINUX_PKG_PATH/pkg.state
+ REASON="Integrity checks differ in size from the source array"
+ elif [ ! -z "$(grep 'The requested URL returned error: 404' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_3 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with 404 - file not found"
+ elif [ ! -z "$(egrep 'fatal: the remote end hung up unexpectedly' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo 404_A > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="could not clone git repository"
+ fi
+ echo " <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> $REASON $EXTRA_REASON" >> $HTML_BUFFER
+ elif [ ! -z "$(egrep '==> ERROR: (install file .* does not exist or is not a regular file|The download program wget is not installed)' $ARCHLINUX_PKG_PATH/build1.log 2>/dev/null)" ] ; then
+ echo FTBFS_0 > $ARCHLINUX_PKG_PATH/pkg.state
+ echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, requirements not met" >> $HTML_BUFFER
+ elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo FTBFS_1 > $ARCHLINUX_PKG_PATH/pkg.state
+ echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build while running tests" >> $HTML_BUFFER
+ elif [ ! -z "$(egrep '==> ERROR: (An unknown error has occurred|A failure occurred in (build|package|prepare))' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo FTBFS_2 > $ARCHLINUX_PKG_PATH/pkg.state
+ echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build" >> $HTML_BUFFER
+ elif [ ! -z "$(egrep 'makepkg was killed by timeout after' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo FTBFS_3 > $ARCHLINUX_PKG_PATH/pkg.state
+ echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, killed by timeout" >> $HTML_BUFFER
+ elif [ ! -z "$(egrep '==> ERROR: .* contains invalid characters:' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
+ echo FTBFS_4 > $ARCHLINUX_PKG_PATH/pkg.state
+ echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, pkg relations contain invalid characters" >> $HTML_BUFFER
+ else
+ echo " probably failed to build from source, please investigate" >> $HTML_BUFFER
+ echo UNKNOWN > $ARCHLINUX_PKG_PATH/pkg.state
+ fi
+ else
+ local STATE=GOOD
+ local SOME_GOOD=false
+ for ARTIFACT in $(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html) ; do
+ if [ -z "$(echo $ARTIFACT | grep $VERSION)" ] ; then
+ echo "deleting $ARTIFACT as version is not $VERSION"
+ rm -f $ARTIFACT
+ continue
+ elif [ ! -z "$(grep 'build reproducible in our test framework' $ARCHLINUX_PKG_PATH/$ARTIFACT)" ] ; then
+ SOME_GOOD=true
+ echo " <img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> <a href=\"/archlinux/$REPOSITORY/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is reproducible in our current test framework<br />" >> $HTML_BUFFER
+ else
+ # change $STATE unless we have found .buildinfo differences already...
+ if [ "$STATE" != "FTBR_0" ] ; then
+ STATE=FTBR_1
+ fi
+ # this shouldnt happen, but (for now) it does, so lets mark them…
+ EXTRA_REASON=""
+ if [ ! -z "$(grep 'class="source">.BUILDINFO' $ARCHLINUX_PKG_PATH/$ARTIFACT)" ] ; then
+ STATE=FTBR_0
+ EXTRA_REASON=" with variations in .BUILDINFO"
+ fi
+ echo " <img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> <a href=\"/archlinux/$REPOSITORY/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible$EXTRA_REASON<br />" >> $HTML_BUFFER
+ fi
+ done
+ # we only count source packages…
+ case $STATE in
+ GOOD) echo GOOD > $ARCHLINUX_PKG_PATH/pkg.state ;;
+ FTBR_0) echo FTBR_0 > $ARCHLINUX_PKG_PATH/pkg.state ;;
+ FTBR_1) if $SOME_GOOD ; then
+ echo FTBR_1 > $ARCHLINUX_PKG_PATH/pkg.state
+ else
+ echo FTBR_2 > $ARCHLINUX_PKG_PATH/pkg.state
+ fi
+ ;;
+ *) ;;
+ esac
+ fi
+ echo " </td>" >> $HTML_BUFFER
+ local BUILD_DATE="$(date -u --date=@$(stat -c %Y $ARCHLINUX_PKG_PATH/build1.log) +'%F %R %Z')"
+ echo " <td>$BUILD_DATE" >> $HTML_BUFFER
+ local DURATION=$(cat $ARCHLINUX_PKG_PATH/pkg.build_duration 2>/dev/null || true)
+ if [ -n "$DURATION" ]; then
+ local HOUR=$(echo "$DURATION/3600"|bc)
+ local MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
+ local SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
+ BUILD_DURATION="<br />${HOUR}h:${MIN}m:${SEC}s"
+ else
+ BUILD_DURATION=" "
+ fi
+ echo " $BUILD_DURATION</td>" >> $HTML_BUFFER
+
+ echo " <td>" >> $HTML_BUFFER
+ for LOG in build1.log build2.log ; do
+ if [ -f $ARCHLINUX_PKG_PATH/$LOG ] ; then
+ if [ "$LOG" = "build2.log" ] ; then
+ echo " <br />" >> $HTML_BUFFER
+ fi
+ get_filesize $ARCHLINUX_PKG_PATH/$LOG
+ echo " <a href=\"/archlinux/$REPOSITORY/$PKG/$LOG\">$LOG</a> ($SIZE)" >> $HTML_BUFFER
+ fi
+ done
+ echo " </td>" >> $HTML_BUFFER
+ echo " </tr>" >> $HTML_BUFFER
+ mv $HTML_BUFFER $ARCHLINUX_PKG_PATH/pkg.html
+ chmod 644 $ARCHLINUX_PKG_PATH/pkg.html
+}
+
choose_package() {
echo "$(date -u ) - choosing package to be build."
ARCH="x86_64"
@@ -447,6 +654,7 @@ fi
echo "$(date -u) - $REPRODUCIBLE_URL/archlinux/$REPOSITORY/$SRCPACKAGE/ updated."
# force update of HTML snipplet in reproducible_html_archlinux.sh
[ ! -f $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/pkg.state ] || rm $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/pkg.state
+create_pkg_state_and_html $REPOSITORY $PKG
cd
cleanup_all
=====================================
bin/reproducible_html_archlinux.sh
=====================================
@@ -11,213 +11,6 @@ common_init "$@"
# common code
. /srv/jenkins/bin/reproducible_common.sh
-create_pkg_state_and_html() {
- local ARCHLINUX_PKG_PATH=$ARCHBASE/$REPOSITORY/$PKG
- local REPO=$1
- local PKG=$2
- local blacklisted=false
- local VERSION="undetermined"
-
- if [ -z "$(cd $ARCHLINUX_PKG_PATH ; ls)" ] ; then
- # directory exists but is empty: package is building…
- echo "$(date -u ) - ignoring $PKG from '$REPOSITORY' which is building in $ARCHLINUX_PKG_PATH since $(date -u --date=@$(stat -c %Y $ARCHLINUX_PKG_PATH) +'%F %R %Z')"
- return
- fi
-
- # clear files from previous builds
- pushd "$ARCHLINUX_PKG_PATH"
- for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
- if [ -f $file ] && [ pkg.build_duration -nt $file ] ; then
- rm $file
- echo "$ARCHLINUX_PKG_PATH/$file older than $ARCHLINUX_PKG_PATH/pkg.build_duration, thus deleting it."
- fi
- done
- popd
-
- if [ -f $ARCHLINUX_PKG_PATH/pkg.version ] ; then
- VERSION=$(cat $ARCHLINUX_PKG_PATH/pkg.version)
- elif [ -f $ARCHLINUX_PKG_PATH/build1.version ] ; then
- VERSION=$(cat $ARCHLINUX_PKG_PATH/build1.version)
- if [ -f $ARCHLINUX_PKG_PATH/build2.log ] ; then
- if [ ! -f $ARCHLINUX_PKG_PATH/build2.version ] ; then
- echo "$(date -u ) - $ARCHLINUX_PKG_PATH/build2.version does not exist, so the 2nd build fails. This happens."
- elif ! diff -q $ARCHLINUX_PKG_PATH/build1.version $ARCHLINUX_PKG_PATH/build2.version ; then
- echo "$(date -u ) - $ARCHLINUX_PKG_PATH/build1.version and $ARCHLINUX_PKG_PATH/build2.version differ, this should not happen. Please tell h01ger."
- VERSION="$VERSION or $(cat $ARCHLINUX_PKG_PATH/build2.version)"
- fi
- fi
- elif [ $(ls $ARCHLINUX_PKG_PATH/*.pkg.tar.xz.html 2>/dev/null | wc -l) -eq 1 ] ; then
- # only determine version if there is exactly one artifact...
- # else it's too error prone and in future the version will
- # be determined during build anyway...
- ARTIFACT="$(ls $ARCHLINUX_PKG_PATH/*.pkg.tar.xz.html 2>/dev/null)"
- VERSION=$( basename $ARTIFACT | sed -s "s#$PKG-##" | sed -E -s "s#-(x86_64|any).pkg.tar.xz.html##" )
- else
- for i in $ARCHLINUX_BLACKLISTED ; do
- if [ "$PKG" = "$i" ] ; then
- blacklisted=true
- fi
- done
- if ! $blacklisted ; then
- echo "$(date -u ) - cannot determine state of $PKG from '$REPOSITORY', please check $ARCHLINUX_PKG_PATH yourself."
- fi
- fi
- if [ "$VERSION" != "undetermined" ] || $blacklisted ; then
- echo $VERSION > $ARCHLINUX_PKG_PATH/pkg.version
- fi
- echo " <tr>" >> $HTML_BUFFER
- echo " <td>$REPOSITORY</td>" >> $HTML_BUFFER
- echo " <td>$PKG</td>" >> $HTML_BUFFER
- echo " <td>$VERSION</td>" >> $HTML_BUFFER
- echo " <td>" >> $HTML_BUFFER
- #
- #
- if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
- for i in $ARCHLINUX_BLACKLISTED ; do
- if [ "$PKG" = "$i" ] ; then
- blacklisted=true
- fi
- done
- # this horrible if elif elif elif elif... monster is needed because
- # https://lists.archlinux.org/pipermail/pacman-dev/2017-September/022156.html
- # has not yet been merged yet...
- # FIXME: this has been merged, see http://jlk.fjfi.cvut.cz/arch/manpages/man/makepkg
-
- if $blacklisted ; then
- echo BLACKLISTED > $ARCHLINUX_PKG_PATH/pkg.state
- echo " <img src=\"/userContent/static/error.png\" alt=\"blacklisted icon\" /> blacklisted" >> $HTML_BUFFER
- elif [ ! -z "$(egrep '^error: failed to prepare transaction \(conflicting dependencies\)' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo DEPWAIT_= > $ARCHLINUX_PKG_PATH/pkg.state
- echo " <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies as there are conflicts" >> $HTML_BUFFER
- elif [ ! -z "$(egrep '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo DEPWAIT_1 > $ARCHLINUX_PKG_PATH/pkg.state
- echo " <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies" >> $HTML_BUFFER
- elif [ ! -z "$(egrep '^error: unknown package: ' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_0 > $ARCHLINUX_PKG_PATH/pkg.state
- echo " <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> unknown package" >> $HTML_BUFFER
- elif [ ! -z "$(egrep '(==> ERROR: Failure while downloading|==> ERROR: One or more PGP signatures could not be verified|==> ERROR: One or more files did not pass the validity check|==> ERROR: Integrity checks \(.*\) differ in size from the source array|==> ERROR: Failure while branching|==> ERROR: Failure while creating working copy|Failed to source PKGBUILD.*PKGBUILD)' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- REASON="download failed"
- EXTRA_REASON=""
- echo 404_0 > $ARCHLINUX_PKG_PATH/pkg.state
- if [ ! -z "$(grep 'FAILED (unknown public key' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_6 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="to verify source with PGP due to unknown public key"
- elif [ ! -z "$(grep 'The requested URL returned error: 403' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_2 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with 403 - forbidden"
- elif [ ! -z "$(grep 'The requested URL returned error: 500' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_4 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with 500 - internal server error"
- elif [ ! -z "$(grep 'The requested URL returned error: 503' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_5 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with 503 - service unavailable"
- elif [ ! -z "$(egrep '==> ERROR: One or more PGP signatures could not be verified' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_7 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="to verify source with PGP signatures"
- elif [ ! -z "$(egrep '(SSL certificate problem: unable to get local issuer certificate|^bzr: ERROR: .SSL: CERTIFICATE_VERIFY_FAILED)' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_1 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with SSL problem"
- elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_8 > $ARCHLINUX_PKG_PATH/pkg.state
- REASON="downloaded ok but failed to verify source"
- elif [ ! -z "$(egrep '==> ERROR: Integrity checks \(.*\) differ in size from the source array' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_9 > $ARCHLINUX_PKG_PATH/pkg.state
- REASON="Integrity checks differ in size from the source array"
- elif [ ! -z "$(grep 'The requested URL returned error: 404' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_3 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with 404 - file not found"
- elif [ ! -z "$(egrep 'fatal: the remote end hung up unexpectedly' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo 404_A > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="could not clone git repository"
- fi
- echo " <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> $REASON $EXTRA_REASON" >> $HTML_BUFFER
- elif [ ! -z "$(egrep '==> ERROR: (install file .* does not exist or is not a regular file|The download program wget is not installed)' $ARCHLINUX_PKG_PATH/build1.log 2>/dev/null)" ] ; then
- echo FTBFS_0 > $ARCHLINUX_PKG_PATH/pkg.state
- echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, requirements not met" >> $HTML_BUFFER
- elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo FTBFS_1 > $ARCHLINUX_PKG_PATH/pkg.state
- echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build while running tests" >> $HTML_BUFFER
- elif [ ! -z "$(egrep '==> ERROR: (An unknown error has occurred|A failure occurred in (build|package|prepare))' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo FTBFS_2 > $ARCHLINUX_PKG_PATH/pkg.state
- echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build" >> $HTML_BUFFER
- elif [ ! -z "$(egrep 'makepkg was killed by timeout after' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo FTBFS_3 > $ARCHLINUX_PKG_PATH/pkg.state
- echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, killed by timeout" >> $HTML_BUFFER
- elif [ ! -z "$(egrep '==> ERROR: .* contains invalid characters:' $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null)" ] ; then
- echo FTBFS_4 > $ARCHLINUX_PKG_PATH/pkg.state
- echo " <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, pkg relations contain invalid characters" >> $HTML_BUFFER
- else
- echo " probably failed to build from source, please investigate" >> $HTML_BUFFER
- echo UNKNOWN > $ARCHLINUX_PKG_PATH/pkg.state
- fi
- else
- local STATE=GOOD
- local SOME_GOOD=false
- for ARTIFACT in $(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html) ; do
- if [ -z "$(echo $ARTIFACT | grep $VERSION)" ] ; then
- echo "deleting $ARTIFACT as version is not $VERSION"
- rm -f $ARTIFACT
- continue
- elif [ ! -z "$(grep 'build reproducible in our test framework' $ARCHLINUX_PKG_PATH/$ARTIFACT)" ] ; then
- SOME_GOOD=true
- echo " <img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> <a href=\"/archlinux/$REPOSITORY/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is reproducible in our current test framework<br />" >> $HTML_BUFFER
- else
- # change $STATE unless we have found .buildinfo differences already...
- if [ "$STATE" != "FTBR_0" ] ; then
- STATE=FTBR_1
- fi
- # this shouldnt happen, but (for now) it does, so lets mark them…
- EXTRA_REASON=""
- if [ ! -z "$(grep 'class="source">.BUILDINFO' $ARCHLINUX_PKG_PATH/$ARTIFACT)" ] ; then
- STATE=FTBR_0
- EXTRA_REASON=" with variations in .BUILDINFO"
- fi
- echo " <img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> <a href=\"/archlinux/$REPOSITORY/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible$EXTRA_REASON<br />" >> $HTML_BUFFER
- fi
- done
- # we only count source packages…
- case $STATE in
- GOOD) echo GOOD > $ARCHLINUX_PKG_PATH/pkg.state ;;
- FTBR_0) echo FTBR_0 > $ARCHLINUX_PKG_PATH/pkg.state ;;
- FTBR_1) if $SOME_GOOD ; then
- echo FTBR_1 > $ARCHLINUX_PKG_PATH/pkg.state
- else
- echo FTBR_2 > $ARCHLINUX_PKG_PATH/pkg.state
- fi
- ;;
- *) ;;
- esac
- fi
- echo " </td>" >> $HTML_BUFFER
- local BUILD_DATE="$(date -u --date=@$(stat -c %Y $ARCHLINUX_PKG_PATH/build1.log) +'%F %R %Z')"
- echo " <td>$BUILD_DATE" >> $HTML_BUFFER
- local DURATION=$(cat $ARCHLINUX_PKG_PATH/pkg.build_duration 2>/dev/null || true)
- if [ -n "$DURATION" ]; then
- local HOUR=$(echo "$DURATION/3600"|bc)
- local MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
- local SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
- BUILD_DURATION="<br />${HOUR}h:${MIN}m:${SEC}s"
- else
- BUILD_DURATION=" "
- fi
- echo " $BUILD_DURATION</td>" >> $HTML_BUFFER
-
- echo " <td>" >> $HTML_BUFFER
- for LOG in build1.log build2.log ; do
- if [ -f $ARCHLINUX_PKG_PATH/$LOG ] ; then
- if [ "$LOG" = "build2.log" ] ; then
- echo " <br />" >> $HTML_BUFFER
- fi
- get_filesize $ARCHLINUX_PKG_PATH/$LOG
- echo " <a href=\"/archlinux/$REPOSITORY/$PKG/$LOG\">$LOG</a> ($SIZE)" >> $HTML_BUFFER
- fi
- done
- echo " </td>" >> $HTML_BUFFER
- echo " </tr>" >> $HTML_BUFFER
- mv $HTML_BUFFER $ARCHLINUX_PKG_PATH/pkg.html
- chmod 644 $ARCHLINUX_PKG_PATH/pkg.html
-}
-
#
# analyse results to create the webpage
#
@@ -244,13 +37,6 @@ WIDTH=1920
HEIGHT=960
for REPOSITORY in $ARCHLINUX_REPOS ; do
echo "$(date -u) - starting to analyse build results for '$REPOSITORY'."
- # create pkg.html for packages which need it
- for PKG in $(find $ARCHBASE/$REPOSITORY/* -maxdepth 1 -type d -exec basename {} \;|sort -u -f) ; do
- if [ ! -f $ARCHBASE/$REPOSITORY/$PKG/pkg.state ] ; then
- create_pkg_state_and_html $REPOSITORY $PKG
- fi
-
- done
# prepare stats per repository
TOTAL=$(cat ${ARCHLINUX_PKGS}_$REPOSITORY | wc -l)
TESTED=$(cat $ARCHBASE/$REPOSITORY/*/pkg.state | grep -c ^ || true )
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/992afbc3e4d9d76972251a172835f75883a202e9...ad98eedd2268759f412e0b7b2b95072e59da21a8
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/992afbc3e4d9d76972251a172835f75883a202e9...ad98eedd2268759f412e0b7b2b95072e59da21a8
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20180924/c9e8f3eb/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list