[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 6 commits: reproducible Archlinux: add comment explaining why we (rarely) might delete…
Holger Levsen
gitlab at salsa.debian.org
Wed Oct 3 20:04:22 BST 2018
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
62f1c71e by Holger Levsen at 2018-10-03T13:14:21Z
reproducible Archlinux: add comment explaining why we (rarely) might delete packages which are building
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
2c669fa5 by Holger Levsen at 2018-10-03T13:17:29Z
reproducible Archlinux: refactoring
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
e5c7b932 by Holger Levsen at 2018-10-03T13:25:54Z
reproducible Archlinux: further refactoring, prepare future changes
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
bfdb66b1 by Holger Levsen at 2018-10-03T13:36:36Z
reproducible Archlinux: obsoleted
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
6ddd6bb3 by Holger Levsen at 2018-10-03T13:44:37Z
reproducible Archlinux: rename PKG variable to SRCPACKAGE
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
25a12992 by Holger Levsen at 2018-10-03T19:04:01Z
reproducible Archlinux: further refactoring
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
5 changed files:
- TODO
- bin/reproducible_archlinux_scheduler.sh
- bin/reproducible_build_archlinux_pkg.sh
- bin/reproducible_common.sh
- bin/reproducible_html_archlinux.sh
Changes:
=====================================
TODO
=====================================
@@ -319,7 +319,6 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
* use db - see https://tests.reproducible-builds.org/reproducibledb.html
** scheduler.sh:
-*** dont delete packages which are building?
*** use asp to update trunk packages?
** html:
*** leave all pkg.html files, delete them (much) later, then check all pkg.* files are gone
=====================================
bin/reproducible_archlinux_scheduler.sh
=====================================
@@ -59,6 +59,10 @@ update_archlinux_repositories() {
for i in $(find $BASE/archlinux/$REPO -type d -wholename "$BASE/archlinux/$REPO/*" | sort) ; do
PKG=$(basename $i)
if ! grep -q "$REPO $PKG" ${ARCHLINUX_PKGS}_full_pkgbase_list > /dev/null ; then
+ # we could check here whether a package is currently building,
+ # and if so defer the pkg removal. (but I think this is pointless,
+ # as we are unlikely to kill that build, so meh, let it finish
+ # and fail to update the db, because the package is gone...)
let REMOVED=$REMOVED+1
REMOVE_LIST="$REMOVE_LIST $REPO/$PKG"
rm -r --one-file-system $BASE/archlinux/$REPO/$PKG
=====================================
bin/reproducible_build_archlinux_pkg.sh
=====================================
@@ -60,203 +60,6 @@ update_pkg_in_db() {
}
-find_in_buildlogs() {
- egrep -q "$1" $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null
-}
-
-include_icon(){
- local PNG="$1"
- local TEXT="$2"
- local ALT=""
- case $PNG in
- error) ALT="blacklisted" ;;
- weather-snow) ALT="depwait" ;;
- weather-severe-alert) ALT="404" ;;
- weather-storm) ALT="ftbfs" ;;
- weather-clear) ALT="reproducible" ;;
- weather-showers-scattered) ALT="unreproducible" ;;
- esac
- echo " <img src=\"/userContent/static/$PNG.png\" alt=\"$ALT icon\" /> $TEXT" >> $HTML_BUFFER
-}
-
-create_pkg_html() {
- local ARCHLINUX_PKG_PATH=$ARCHBASE/$REPOSITORY/$SRCPACKAGE
- HTML_BUFFER=$(mktemp -t archlinuxrb-html-XXXXXXXX)
-
- local blacklisted=false
-
- # clear files from previous builds
- cd "$ARCHLINUX_PKG_PATH"
- for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
- if [ -f $file ] && [ pkg.build_duration -nt $file ] ; then
- rm $file
- echo "$ARCHLINUX_PKG_PATH/$file older than $ARCHLINUX_PKG_PATH/pkg.build_duration, thus deleting it."
- fi
- done
-
- echo " <tr>" >> $HTML_BUFFER
- echo " <td>$REPOSITORY</td>" >> $HTML_BUFFER
- echo " <td>$SRCPACKAGE</td>" >> $HTML_BUFFER
- echo " <td>$VERSION</td>" >> $HTML_BUFFER
- echo " <td>" >> $HTML_BUFFER
- #
- #
- if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
- for i in $ARCHLINUX_BLACKLISTED ; do
- if [ "$SRCPACKAGE" = "$i" ] ; then
- blacklisted=true
- fi
- done
- # this horrible if elif elif elif elif... monster is needed because
- # https://lists.archlinux.org/pipermail/pacman-dev/2017-September/022156.html
- # has not yet been merged yet...
- # FIXME: this has been merged, see http://jlk.fjfi.cvut.cz/arch/manpages/man/makepkg
-
- if $blacklisted ; then
- echo BLACKLISTED > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon error blacklisted
- elif find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
- echo DEPWAIT_0 > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon weather-snow "could not resolve dependencies as there are conflicts"
- elif find_in_buildlogs '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)'; then
- if find_in_buildlogs 'error: failed to init transaction \(unable to lock database\)'; then
- echo DEPWAIT_2 > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon weather-snow "pacman could not lock database"
- else
- echo DEPWAIT_1 > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon weather-snow "could not resolve dependencies"
- fi
- elif find_in_buildlogs '^error: unknown package: '; then
- echo 404_0 > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon weather-severe-alert "unknown package"
- elif find_in_buildlogs '(==> ERROR: Failure while downloading|==> ERROR: One or more PGP signatures could not be verified|==> ERROR: One or more files did not pass the validity check|==> ERROR: Integrity checks \(.*\) differ in size from the source array|==> ERROR: Failure while branching|==> ERROR: Failure while creating working copy|Failed to source PKGBUILD.*PKGBUILD)'; then
- REASON="download failed"
- EXTRA_REASON=""
- echo 404_0 > $ARCHLINUX_PKG_PATH/pkg.state
- if find_in_buildlogs 'FAILED \(unknown public key'; then
- echo 404_6 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="to verify source with PGP due to unknown public key"
- elif find_in_buildlogs 'The requested URL returned error: 403'; then
- echo 404_2 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with 403 - forbidden"
- elif find_in_buildlogs 'The requested URL returned error: 500'; then
- echo 404_4 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with 500 - internal server error"
- elif find_in_buildlogs 'The requested URL returned error: 503'; then
- echo 404_5 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with 503 - service unavailable"
- elif find_in_buildlogs '==> ERROR: One or more PGP signatures could not be verified'; then
- echo 404_7 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="to verify source with PGP signatures"
- elif find_in_buildlogs '(SSL certificate problem: unable to get local issuer certificate|^bzr: ERROR: .SSL: CERTIFICATE_VERIFY_FAILED)'; then
- echo 404_1 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with SSL problem"
- elif find_in_buildlogs '==> ERROR: One or more files did not pass the validity check'; then
- echo 404_8 > $ARCHLINUX_PKG_PATH/pkg.state
- REASON="downloaded ok but failed to verify source"
- elif find_in_buildlogs '==> ERROR: Integrity checks \(.*\) differ in size from the source array'; then
- echo 404_9 > $ARCHLINUX_PKG_PATH/pkg.state
- REASON="Integrity checks differ in size from the source array"
- elif find_in_buildlogs 'The requested URL returned error: 404'; then
- echo 404_3 > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with 404 - file not found"
- elif find_in_buildlogs 'fatal: the remote end hung up unexpectedly'; then
- echo 404_A > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="could not clone git repository"
- elif find_in_buildlogs 'The requested URL returned error: 504'; then
- echo 404_B > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="with 504 - gateway timeout"
- elif find_in_buildlogs '==> ERROR: Failure while downloading .* git repo'; then
- echo 404_C > $ARCHLINUX_PKG_PATH/pkg.state
- EXTRA_REASON="from git repo"
- fi
- include_icon weather-severe-alert "$REASON $EXTRA_REASON"
- elif find_in_buildlogs '==> ERROR: (install file .* does not exist or is not a regular file|The download program wget is not installed)'; then
- echo FTBFS_0 > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon weather-storm "failed to build, requirements not met"
- elif find_in_buildlogs '==> ERROR: A failure occurred in check'; then
- echo FTBFS_1 > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon weather-storm "failed to build while running tests"
- elif find_in_buildlogs '==> ERROR: (An unknown error has occurred|A failure occurred in (build|package|prepare))'; then
- echo FTBFS_2 > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon weather-storm "failed to build"
- elif find_in_buildlogs 'makepkg was killed by timeout after'; then
- echo FTBFS_3 > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon weather-storm "failed to build, killed by timeout"
- elif find_in_buildlogs '==> ERROR: .* contains invalid characters:'; then
- echo FTBFS_4 > $ARCHLINUX_PKG_PATH/pkg.state
- include_icon weather-storm "failed to build, pkg relations contain invalid characters"
- else
- echo " probably failed to build from source, please investigate" >> $HTML_BUFFER
- echo UNKNOWN > $ARCHLINUX_PKG_PATH/pkg.state
- fi
- else
- local STATE=GOOD
- local SOME_GOOD=false
- for ARTIFACT in $(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html) ; do
- if [ -z "$(echo $ARTIFACT | grep $VERSION)" ] ; then
- echo "deleting $ARTIFACT as version is not $VERSION"
- rm -f $ARTIFACT
- continue
- elif [ ! -z "$(grep 'build reproducible in our test framework' $ARCHLINUX_PKG_PATH/$ARTIFACT)" ] ; then
- SOME_GOOD=true
- include_icon weather-clear "<a href=\"/archlinux/$REPOSITORY/$SRCPACKAGE/$ARTIFACT\">${ARTIFACT:0:-5}</a> is reproducible in our current test framework<br />"
- else
- # change $STATE unless we have found .buildinfo differences already...
- if [ "$STATE" != "FTBR_0" ] ; then
- STATE=FTBR_1
- fi
- # this shouldnt happen, but (for now) it does, so lets mark them…
- EXTRA_REASON=""
- if [ ! -z "$(grep 'class="source">.BUILDINFO' $ARCHLINUX_PKG_PATH/$ARTIFACT)" ] ; then
- STATE=FTBR_0
- EXTRA_REASON=" with variations in .BUILDINFO"
- fi
- include_icon weather-showers-scattered "<a href=\"/archlinux/$REPOSITORY/$SRCPACKAGE/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible$EXTRA_REASON<br />"
- fi
- done
- # we only count source packages…
- case $STATE in
- GOOD) echo GOOD > $ARCHLINUX_PKG_PATH/pkg.state ;;
- FTBR_0) echo FTBR_0 > $ARCHLINUX_PKG_PATH/pkg.state ;;
- FTBR_1) if $SOME_GOOD ; then
- echo FTBR_1 > $ARCHLINUX_PKG_PATH/pkg.state
- else
- echo FTBR_2 > $ARCHLINUX_PKG_PATH/pkg.state
- fi
- ;;
- *) ;;
- esac
- fi
- echo " </td>" >> $HTML_BUFFER
- echo " <td>$DATE" >> $HTML_BUFFER
- local DURATION=$(cat $ARCHLINUX_PKG_PATH/pkg.build_duration 2>/dev/null || true)
- if [ -n "$DURATION" ]; then
- local HOUR=$(echo "$DURATION/3600"|bc)
- local MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
- local SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
- BUILD_DURATION="<br />${HOUR}h:${MIN}m:${SEC}s"
- else
- BUILD_DURATION=" "
- fi
- echo " $BUILD_DURATION</td>" >> $HTML_BUFFER
-
- echo " <td>" >> $HTML_BUFFER
- for LOG in build1.log build2.log ; do
- if [ -f $ARCHLINUX_PKG_PATH/$LOG ] ; then
- if [ "$LOG" = "build2.log" ] ; then
- echo " <br />" >> $HTML_BUFFER
- fi
- get_filesize $ARCHLINUX_PKG_PATH/$LOG
- echo " <a href=\"/archlinux/$REPOSITORY/$SRCPACKAGE/$LOG\">$LOG</a> ($SIZE)" >> $HTML_BUFFER
- fi
- done
- echo " </td>" >> $HTML_BUFFER
- echo " </tr>" >> $HTML_BUFFER
- mv $HTML_BUFFER $ARCHLINUX_PKG_PATH/pkg.html
- chmod 644 $ARCHLINUX_PKG_PATH/pkg.html
-}
-
choose_package() {
echo "$(date -u ) - choosing package to be build."
ARCH="x86_64"
=====================================
bin/reproducible_common.sh
=====================================
@@ -859,3 +859,202 @@ create_png_from_table() {
fi
rm ${TABLE[$1]}.csv
}
+
+find_in_buildlogs() {
+ egrep -q "$1" $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null
+}
+
+include_icon(){
+ local PNG="$1"
+ local TEXT="$2"
+ local ALT=""
+ case $PNG in
+ error) ALT="blacklisted" ;;
+ weather-snow) ALT="depwait" ;;
+ weather-severe-alert) ALT="404" ;;
+ weather-storm) ALT="ftbfs" ;;
+ weather-clear) ALT="reproducible" ;;
+ weather-showers-scattered) ALT="unreproducible" ;;
+ esac
+ echo " <img src=\"/userContent/static/$PNG.png\" alt=\"$ALT icon\" /> $TEXT" >> $HTML_BUFFER
+}
+
+create_pkg_html() {
+ local ARCHLINUX_PKG_PATH=$ARCHBASE/$REPOSITORY/$SRCPACKAGE
+ HTML_BUFFER=$(mktemp -t archlinuxrb-html-XXXXXXXX)
+
+ local blacklisted=false
+
+ # clear files from previous builds
+ cd "$ARCHLINUX_PKG_PATH"
+ for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
+ if [ -f $file ] && [ pkg.build_duration -nt $file ] ; then
+ rm $file
+ echo "$ARCHLINUX_PKG_PATH/$file older than $ARCHLINUX_PKG_PATH/pkg.build_duration, thus deleting it."
+ fi
+ done
+
+ echo " <tr>" >> $HTML_BUFFER
+ echo " <td>$REPOSITORY</td>" >> $HTML_BUFFER
+ echo " <td>$SRCPACKAGE</td>" >> $HTML_BUFFER
+ echo " <td>$VERSION</td>" >> $HTML_BUFFER
+ echo " <td>" >> $HTML_BUFFER
+ #
+ #
+ if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
+ for i in $ARCHLINUX_BLACKLISTED ; do
+ if [ "$SRCPACKAGE" = "$i" ] ; then
+ blacklisted=true
+ fi
+ done
+ # this horrible if elif elif elif elif... monster is needed because
+ # https://lists.archlinux.org/pipermail/pacman-dev/2017-September/022156.html
+ # has not yet been merged yet...
+ # FIXME: this has been merged, see http://jlk.fjfi.cvut.cz/arch/manpages/man/makepkg
+
+ if $blacklisted ; then
+ echo BLACKLISTED > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon error blacklisted
+ elif find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
+ echo DEPWAIT_0 > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon weather-snow "could not resolve dependencies as there are conflicts"
+ elif find_in_buildlogs '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)'; then
+ if find_in_buildlogs 'error: failed to init transaction \(unable to lock database\)'; then
+ echo DEPWAIT_2 > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon weather-snow "pacman could not lock database"
+ else
+ echo DEPWAIT_1 > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon weather-snow "could not resolve dependencies"
+ fi
+ elif find_in_buildlogs '^error: unknown package: '; then
+ echo 404_0 > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon weather-severe-alert "unknown package"
+ elif find_in_buildlogs '(==> ERROR: Failure while downloading|==> ERROR: One or more PGP signatures could not be verified|==> ERROR: One or more files did not pass the validity check|==> ERROR: Integrity checks \(.*\) differ in size from the source array|==> ERROR: Failure while branching|==> ERROR: Failure while creating working copy|Failed to source PKGBUILD.*PKGBUILD)'; then
+ REASON="download failed"
+ EXTRA_REASON=""
+ echo 404_0 > $ARCHLINUX_PKG_PATH/pkg.state
+ if find_in_buildlogs 'FAILED \(unknown public key'; then
+ echo 404_6 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="to verify source with PGP due to unknown public key"
+ elif find_in_buildlogs 'The requested URL returned error: 403'; then
+ echo 404_2 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with 403 - forbidden"
+ elif find_in_buildlogs 'The requested URL returned error: 500'; then
+ echo 404_4 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with 500 - internal server error"
+ elif find_in_buildlogs 'The requested URL returned error: 503'; then
+ echo 404_5 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with 503 - service unavailable"
+ elif find_in_buildlogs '==> ERROR: One or more PGP signatures could not be verified'; then
+ echo 404_7 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="to verify source with PGP signatures"
+ elif find_in_buildlogs '(SSL certificate problem: unable to get local issuer certificate|^bzr: ERROR: .SSL: CERTIFICATE_VERIFY_FAILED)'; then
+ echo 404_1 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with SSL problem"
+ elif find_in_buildlogs '==> ERROR: One or more files did not pass the validity check'; then
+ echo 404_8 > $ARCHLINUX_PKG_PATH/pkg.state
+ REASON="downloaded ok but failed to verify source"
+ elif find_in_buildlogs '==> ERROR: Integrity checks \(.*\) differ in size from the source array'; then
+ echo 404_9 > $ARCHLINUX_PKG_PATH/pkg.state
+ REASON="Integrity checks differ in size from the source array"
+ elif find_in_buildlogs 'The requested URL returned error: 404'; then
+ echo 404_3 > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with 404 - file not found"
+ elif find_in_buildlogs 'fatal: the remote end hung up unexpectedly'; then
+ echo 404_A > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="could not clone git repository"
+ elif find_in_buildlogs 'The requested URL returned error: 504'; then
+ echo 404_B > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="with 504 - gateway timeout"
+ elif find_in_buildlogs '==> ERROR: Failure while downloading .* git repo'; then
+ echo 404_C > $ARCHLINUX_PKG_PATH/pkg.state
+ EXTRA_REASON="from git repo"
+ fi
+ include_icon weather-severe-alert "$REASON $EXTRA_REASON"
+ elif find_in_buildlogs '==> ERROR: (install file .* does not exist or is not a regular file|The download program wget is not installed)'; then
+ echo FTBFS_0 > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon weather-storm "failed to build, requirements not met"
+ elif find_in_buildlogs '==> ERROR: A failure occurred in check'; then
+ echo FTBFS_1 > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon weather-storm "failed to build while running tests"
+ elif find_in_buildlogs '==> ERROR: (An unknown error has occurred|A failure occurred in (build|package|prepare))'; then
+ echo FTBFS_2 > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon weather-storm "failed to build"
+ elif find_in_buildlogs 'makepkg was killed by timeout after'; then
+ echo FTBFS_3 > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon weather-storm "failed to build, killed by timeout"
+ elif find_in_buildlogs '==> ERROR: .* contains invalid characters:'; then
+ echo FTBFS_4 > $ARCHLINUX_PKG_PATH/pkg.state
+ include_icon weather-storm "failed to build, pkg relations contain invalid characters"
+ else
+ echo " probably failed to build from source, please investigate" >> $HTML_BUFFER
+ echo UNKNOWN > $ARCHLINUX_PKG_PATH/pkg.state
+ fi
+ else
+ local STATE=GOOD
+ local SOME_GOOD=false
+ for ARTIFACT in $(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html) ; do
+ if [ -z "$(echo $ARTIFACT | grep $VERSION)" ] ; then
+ echo "deleting $ARTIFACT as version is not $VERSION"
+ rm -f $ARTIFACT
+ continue
+ elif [ ! -z "$(grep 'build reproducible in our test framework' $ARCHLINUX_PKG_PATH/$ARTIFACT)" ] ; then
+ SOME_GOOD=true
+ include_icon weather-clear "<a href=\"/archlinux/$REPOSITORY/$SRCPACKAGE/$ARTIFACT\">${ARTIFACT:0:-5}</a> is reproducible in our current test framework<br />"
+ else
+ # change $STATE unless we have found .buildinfo differences already...
+ if [ "$STATE" != "FTBR_0" ] ; then
+ STATE=FTBR_1
+ fi
+ # this shouldnt happen, but (for now) it does, so lets mark them…
+ EXTRA_REASON=""
+ if [ ! -z "$(grep 'class="source">.BUILDINFO' $ARCHLINUX_PKG_PATH/$ARTIFACT)" ] ; then
+ STATE=FTBR_0
+ EXTRA_REASON=" with variations in .BUILDINFO"
+ fi
+ include_icon weather-showers-scattered "<a href=\"/archlinux/$REPOSITORY/$SRCPACKAGE/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible$EXTRA_REASON<br />"
+ fi
+ done
+ # we only count source packages…
+ case $STATE in
+ GOOD) echo GOOD > $ARCHLINUX_PKG_PATH/pkg.state ;;
+ FTBR_0) echo FTBR_0 > $ARCHLINUX_PKG_PATH/pkg.state ;;
+ FTBR_1) if $SOME_GOOD ; then
+ echo FTBR_1 > $ARCHLINUX_PKG_PATH/pkg.state
+ else
+ echo FTBR_2 > $ARCHLINUX_PKG_PATH/pkg.state
+ fi
+ ;;
+ *) ;;
+ esac
+ fi
+ echo " </td>" >> $HTML_BUFFER
+ echo " <td>$DATE" >> $HTML_BUFFER
+ local DURATION=$(cat $ARCHLINUX_PKG_PATH/pkg.build_duration 2>/dev/null || true)
+ if [ -n "$DURATION" ]; then
+ local HOUR=$(echo "$DURATION/3600"|bc)
+ local MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
+ local SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
+ BUILD_DURATION="<br />${HOUR}h:${MIN}m:${SEC}s"
+ else
+ BUILD_DURATION=" "
+ fi
+ echo " $BUILD_DURATION</td>" >> $HTML_BUFFER
+
+ echo " <td>" >> $HTML_BUFFER
+ for LOG in build1.log build2.log ; do
+ if [ -f $ARCHLINUX_PKG_PATH/$LOG ] ; then
+ if [ "$LOG" = "build2.log" ] ; then
+ echo " <br />" >> $HTML_BUFFER
+ fi
+ get_filesize $ARCHLINUX_PKG_PATH/$LOG
+ echo " <a href=\"/archlinux/$REPOSITORY/$SRCPACKAGE/$LOG\">$LOG</a> ($SIZE)" >> $HTML_BUFFER
+ fi
+ done
+ echo " </td>" >> $HTML_BUFFER
+ echo " </tr>" >> $HTML_BUFFER
+ mv $HTML_BUFFER $ARCHLINUX_PKG_PATH/pkg.html
+ chmod 644 $ARCHLINUX_PKG_PATH/pkg.html
+}
+
+
=====================================
bin/reproducible_html_archlinux.sh
=====================================
@@ -12,34 +12,16 @@ common_init "$@"
. /srv/jenkins/bin/reproducible_common.sh
#
-# analyse results to create the webpage
#
-echo "$(date -u) - starting."
-DATE=$(date -u +'%Y-%m-%d')
-YESTERDAY=$(date '+%Y-%m-%d' -d "-1 day")
-MEMBERS_FTBFS="0 1 2 3 4"
-MEMBERS_DEPWAIT="0 1 2"
-MEMBERS_404="0 1 2 3 4 5 6 7 8 9 A B C"
-MEMBERS_FTBR="0 1 2"
-HTML_BUFFER=$(mktemp -t archlinuxrb-html-XXXXXXXX)
-HTML_REPOSTATS=$(mktemp -t archlinuxrb-html-XXXXXXXX)
-ARCHLINUX_TOTAL=0
-ARCHLINUX_TESTED=0
-ARCHLINUX_NR_FTBFS=0
-ARCHLINUX_NR_FTBR=0
-ARCHLINUX_NR_DEPWAIT=0
-ARCHLINUX_NR_404=0
-ARCHLINUX_NR_GOOD=0
-ARCHLINUX_NR_BLACKLISTED=0
-ARCHLINUX_NR_UNKNOWN=0
-WIDTH=1920
-HEIGHT=960
-PAGE=""
-TITLE=""
-STATE=""
-REPOSITORY=""
-PKG=""
+# reproducible_html_archlinux.sh can be called in two ways:
+# - without params, then it will build all the index and dashboard pages
+# - with exactly two params, $REPOSITORY and $SRCPACKAGE, in which case that packages html page will be created
+#
+
+#
+# helper functions
+#
get_state_from_counter() {
local counter=$1
case $counter in
@@ -55,7 +37,7 @@ get_state_from_counter() {
include_pkg_html_in_page(){
- cat $ARCHBASE/$REPOSITORY/$PKG/pkg.html >> $PAGE 2>/dev/null || true
+ cat $ARCHBASE/$REPOSITORY/$SRCPACKAGE/pkg.html >> $PAGE 2>/dev/null || true
}
include_pkg_table_header_in_page(){
@@ -248,7 +230,7 @@ repository_pages(){
include_pkg_table_header_in_page
REPO_PKGS=$(query_db "SELECT s.name FROM sources AS s JOIN results AS r ON s.id=r.package_id
WHERE s.architecture='x86_64' AND s.suite='$SUITE' ORDER BY r.status,s.name")
- for PKG in $REPO_PKGS ; do
+ for SRCPACKAGE in $REPO_PKGS ; do
include_pkg_html_in_page
done
write_page " </table>"
@@ -279,14 +261,14 @@ state_pages(){
SUITE="archlinux_$REPOSITORY"
STATE_PKGS=$(query_db "SELECT s.name FROM sources AS s JOIN results AS r ON s.id=r.package_id
WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE '$STATE%' ORDER BY r.status,s.name")
- for PKG in ${STATE_PKGS} ; do
+ for SRCPACKAGE in ${STATE_PKGS} ; do
include_pkg_html_in_page
done
if [ "$STATE" = "UNKNOWN" ] ; then
# untested packages are also state UNKNOWN...
STATE_PKGS=$(query_db "SELECT s.name FROM sources AS s
WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND s.id NOT IN (SELECT package_id FROM results) ORDER BY s.name")
- for PKG in ${STATE_PKGS} ; do
+ for SRCPACKAGE in ${STATE_PKGS} ; do
include_pkg_html_in_page
done
fi
@@ -319,14 +301,14 @@ repository_state_pages(){
include_pkg_table_header_in_page
STATE_PKGS=$(query_db "SELECT s.name FROM sources AS s JOIN results AS r ON s.id=r.package_id
WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE '$STATE%' ORDER BY r.status,s.name")
- for PKG in ${STATE_PKGS} ; do
+ for SRCPACKAGE in ${STATE_PKGS} ; do
include_pkg_html_in_page
done
if [ "$STATE" = "UNKNOWN" ] ; then
# untested packages are also state UNKNOWN...
STATE_PKGS=$(query_db "SELECT s.name FROM sources AS s
WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND s.id NOT IN (SELECT package_id FROM results) ORDER BY s.name")
- for PKG in ${STATE_PKGS} ; do
+ for SRCPACKAGE in ${STATE_PKGS} ; do
include_pkg_html_in_page
done
fi
@@ -336,12 +318,57 @@ repository_state_pages(){
done
}
-repostats
-single_main_page
-repository_pages
-state_pages
-repository_state_pages
-rm $HTML_REPOSTATS > /dev/null
+#
+# main
+#
+echo "$(date -u) - starting."
+DATE=$(date -u +'%Y-%m-%d')
+YESTERDAY=$(date '+%Y-%m-%d' -d "-1 day")
+PAGE=""
+TITLE=""
+STATE=""
+REPOSITORY=""
+SRCPACKAGE=""
+
+if [ -z "$1" ] ; then
+ MEMBERS_FTBFS="0 1 2 3 4"
+ MEMBERS_DEPWAIT="0 1 2"
+ MEMBERS_404="0 1 2 3 4 5 6 7 8 9 A B C"
+ MEMBERS_FTBR="0 1 2"
+ HTML_REPOSTATS=$(mktemp -t archlinuxrb-html-XXXXXXXX)
+ ARCHLINUX_TOTAL=0
+ ARCHLINUX_TESTED=0
+ ARCHLINUX_NR_FTBFS=0
+ ARCHLINUX_NR_FTBR=0
+ ARCHLINUX_NR_DEPWAIT=0
+ ARCHLINUX_NR_404=0
+ ARCHLINUX_NR_GOOD=0
+ ARCHLINUX_NR_BLACKLISTED=0
+ ARCHLINUX_NR_UNKNOWN=0
+ WIDTH=1920
+ HEIGHT=960
+
+ repostats
+ single_main_page
+ repository_pages
+ state_pages
+ repository_state_pages
+
+ rm $HTML_REPOSTATS > /dev/null
+elif [ -z "$2" ] ; then
+ echo "$(date -u) - $0 needs two params or none, exiting."
+ #FIXME: in future also except all core extra multilib community
+ exit 1
+else
+ REPOSITORY=$1
+ SRCPACKAGE=$2
+ if [ ! -d $ARCHBASE/$REPOSITORY/$SRCPACKAGE ] ; then
+ echo "$(date -u) - $ARCHBASE/$REPOSITORY/$SRCPACKAGE does not exist, exiting."
+ exit 1
+ fi
+ HTML_BUFFER=''
+ create_pkg_html
+fi
echo "$(date -u) - all done."
# vim: set sw=0 noet :
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/3d88820d9588bd9dc89fa57e6b7b3e9374a48f20...25a129920abd2e7f4ba33a4d3f4c42ea4e888c68
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/3d88820d9588bd9dc89fa57e6b7b3e9374a48f20...25a129920abd2e7f4ba33a4d3f4c42ea4e888c68
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20181003/fba2e7bf/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list