[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] more stabbing after re-reading https://jenkins.io/blog/2016/11/21/gc-tuning/
Holger Levsen
gitlab at salsa.debian.org
Tue Oct 30 21:08:25 GMT 2018
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
45894bb4 by Holger Levsen at 2018-10-30T21:08:15Z
more stabbing after re-reading https://jenkins.io/blog/2016/11/21/gc-tuning/
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
3 changed files:
- bin/reproducible_common.sh
- bin/reproducible_html_archlinux.sh
- hosts/jenkins/etc/default/jenkins
Changes:
=====================================
bin/reproducible_common.sh
=====================================
@@ -857,6 +857,104 @@ create_debian_png_from_table() {
rm ${TABLE[$1]}.csv
}
+#
+# create the png (and query the db to populate a csv file...) for Arch Linux
+#
+create_archlinux_png_from_table() {
+ echo "Checking whether to update $2..."
+ # $1 = id of the stats table
+ # $2 = image file name
+ echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
+ # prepare query
+ WHERE_EXTRA="WHERE suite = '$SUITE'"
+ if [ $1 -eq 0 ] || [ $1 -eq 2 ] ; then
+ # TABLE[0+2] have a architecture column:
+ WHERE_EXTRA="$WHERE_EXTRA AND architecture = '$ARCH'"
+ fi
+ # run query
+ if [ $1 -eq 1 ] ; then
+ # not sure if it's worth to generate the following query...
+ WHERE_EXTRA="AND architecture='$ARCH'"
+
+ # This query became much more obnoxious when gaining
+ # compatibility with postgres
+ query_to_csv "SELECT stats.datum,
+ COALESCE(reproducible_stretch,0) AS reproducible_stretch,
+ COALESCE(reproducible_buster,0) AS reproducible_buster,
+ COALESCE(reproducible_unstable,0) AS reproducible_unstable,
+ COALESCE(reproducible_experimental,0) AS reproducible_experimental,
+ COALESCE(FTBR_stretch,0) AS FTBR_stretch,
+ COALESCE(FTBR_buster,0) AS FTBR_buster,
+ COALESCE(FTBR_unstable,0) AS FTBR_unstable,
+ COALESCE(FTBR_experimental,0) AS FTBR_experimental,
+ COALESCE(FTBFS_stretch,0) AS FTBFS_stretch,
+ COALESCE(FTBFS_buster,0) AS FTBFS_buster,
+ COALESCE(FTBFS_unstable,0) AS FTBFS_unstable,
+ COALESCE(FTBFS_experimental,0) AS FTBFS_experimental,
+ COALESCE(other_stretch,0) AS other_stretch,
+ COALESCE(other_buster,0) AS other_buster,
+ COALESCE(other_unstable,0) AS other_unstable,
+ COALESCE(other_experimental,0) AS other_experimental
+ FROM (SELECT s.datum,
+ COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA),0) AS reproducible_stretch,
+ COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA),0) AS reproducible_buster,
+ COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable,
+ COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental,
+ (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBR_stretch,
+ (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBR_buster,
+ (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBR_unstable,
+ (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBR_experimental,
+ (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBFS_stretch,
+ (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBFS_buster,
+ (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable,
+ (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBFS_experimental,
+ (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS other_stretch,
+ (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS other_buster,
+ (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS other_unstable,
+ (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS other_experimental
+ FROM stats_builds_per_day AS s GROUP BY s.datum) as stats
+ ORDER BY datum" >> ${TABLE[$1]}.csv
+ elif [ $1 -eq 2 ] ; then
+ # just make a graph of the oldest reproducible build (ignore FTBFS and FTBR)
+ query_to_csv "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
+ else
+ query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
+ fi
+ # this is a gross hack: normally we take the number of colors a table should have...
+ # for the builds_age table we only want one color, but different ones, so this hack:
+ COLORS=${COLOR[$1]}
+ if [ $1 -eq 2 ] ; then
+ case "$SUITE" in
+ stretch) COLORS=40 ;;
+ buster) COLORS=41 ;;
+ unstable) COLORS=42 ;;
+ experimental) COLORS=43 ;;
+ esac
+ fi
+ local WIDTH=1920
+ local HEIGHT=960
+ # only generate graph if the query returned data
+ if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then
+ echo "Updating $2..."
+ DIR=$(dirname $2)
+ mkdir -p $DIR
+ echo "Generating $2."
+ /srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT
+ mv $2 $ARCHBASE/$DIR
+ [ "$DIR" = "." ] || rmdir $(dirname $2)
+ # create empty dummy png if there havent been any results ever
+ elif [ ! -f $ARCHBASE/$DIR/$(basename $2) ] ; then
+ DIR=$(dirname $2)
+ mkdir -p $DIR
+ echo "Creating $2 dummy."
+ convert -size 1920x960 xc:#aaaaaa -depth 8 $2
+ mv $2 $ARCHBASE/$DIR
+ [ "$DIR" = "." ] || rmdir $(dirname $2)
+ fi
+ rm ${TABLE[$1]}.csv
+}
+
+
find_in_buildlogs() {
egrep -q "$1" $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null
}
=====================================
bin/reproducible_html_archlinux.sh
=====================================
@@ -437,6 +437,15 @@ if [ -z "$1" ] ; then
ARCHLINUX_NR_UNKNOWN=0
WIDTH=1920
HEIGHT=960
+ # variables related to the stats we update
+ # FIELDS[0]="datum, reproducible, FTBR, FTBFS, other, untested" # FIXME: for this Arch Linux still uses a .csv file...
+ FIELDS[1]="datum"
+ for i in reproducible FTBR FTBFS other ; do
+ for j in $SUITES ; do
+ FIELDS[1]="${FIELDS[1]}, ${i}_${j}"
+ done
+ done
+ FIELDS[2]="datum, oldest"
repostats
dashboard_page
=====================================
hosts/jenkins/etc/default/jenkins
=====================================
@@ -17,7 +17,7 @@ JAVA_ARGS="-Djava.awt.headless=true"
#JAVA_ARGS="-Xmx256m"
#JAVA_ARGS="-Djava.net.preferIPv4Stack=true" # make jenkins listen on IPv4 address
# See some recommendation from https://jenkins.io/blog/2016/11/21/gc-tuning/
-JAVA_ARGS="$JAVA_ARGS -Xms4G -Xmx8G" # heap settings
+JAVA_ARGS="$JAVA_ARGS -Xms8G -Xmx24G" # heap settings
JAVA_ARGS="$JAVA_ARGS -server -XX:+AlwaysPreTouch"
# store heap dumps in case of OOM, to investigate it later
JAVA_ARGS="$JAVA_ARGS -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$JENKINS_HOME/heapDumps/"
@@ -30,8 +30,8 @@ JAVA_ARGS="$JAVA_ARGS -XX:+PrintGCCause -XX:+PrintTenuringDistribution -XX:+Prin
JAVA_ARGS="$JAVA_ARGS -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication"
JAVA_ARGS="$JAVA_ARGS -XX:+UnlockExperimentalVMOptions -XX:G1NewSizePercent=20 -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1"
# h01ger trying stuff
-JAVA_ARGS="$JAVA_ARGS -XX:+UseCompressedOops"
-MALLOC_ARENA_MAX=2
+JAVA_ARGS="$JAVA_ARGS -XX:+UseCompressedOops -XX:+UseCompressedClassPointers"
+export MALLOC_ARENA_MAX=2
# make jenkins listen on IPv4 address
#JAVA_ARGS="-Djava.net.preferIPv4Stack=true"
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/45894bb4752e606173f5e8dd74218c31523ca49c
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/45894bb4752e606173f5e8dd74218c31523ca49c
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20181030/6473bf58/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list