[Qa-jenkins-scm] [jenkins.debian.net] 05/06: reproducible: further refactoring, move main code into four functions
Holger Levsen
holger at moszumanska.debian.org
Mon Mar 9 19:12:53 UTC 2015
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit e65eecc920ce097610e23fa811503b3468551787
Author: Holger Levsen <holger at layer-acht.org>
Date: Mon Mar 9 19:52:50 2015 +0100
reproducible: further refactoring, move main code into four functions
---
bin/reproducible_html_graphs.sh | 215 +++++++++++++++++++++-------------------
1 file changed, 114 insertions(+), 101 deletions(-)
diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh
index fb5dfc4..aae369c 100755
--- a/bin/reproducible_html_graphs.sh
+++ b/bin/reproducible_html_graphs.sh
@@ -71,71 +71,75 @@ YLABEL[5]="Amount of issues"
#
# update package + build stats
#
-RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"")
-if [ -z $RESULT ] ; then
- echo "Updating packages and builds stats for $SUITE on $DATE."
- ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) FROM sources WHERE suite='${SUITE}'")
- GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE';")
- GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)='$DATE';")
- BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE';")
- BAAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)='$DATE';")
- UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE';")
- UGLDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)='$DATE';")
- REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)<='$DATE';")
- RESDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)='$DATE';")
- OLDESTG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status = 'reproducible' AND s.suite='$SUITE' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
- OLDESTB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
- OLDESTU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
- DIFFG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTG');")
- if [ -z $DIFFG ] ; then DIFFG=0 ; fi
- DIFFB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTB');")
- if [ -z $DIFFB ] ; then DIFFB=0 ; fi
- DIFFU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTU');")
- if [ -z $DIFFU ] ; then DIFFU=0 ; fi
- let "TOTAL=GOOD+BAD+UGLY+REST" || true # let FOO=0+0 returns error in bash...
- if [ "$ALL" != "$TOTAL" ] ; then
- let "UNTESTED=ALL-TOTAL"
- else
- UNTESTED=0
- fi
- sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)"
- sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)"
- sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")"
- # we do 3 later and 6 is special anyway...
- for i in 0 1 2 4 5 ; do
- PREFIX=""
- if [ $i -eq 0 ] ; then
- PREFIX=$SUITE
- fi
- # force regeneration of the image if it exists
- if [ -f /var/lib/jenkins/userContent/$PREFIX/${TABLE[$i]}.png ] ; then
- echo "Touching $PREFIX/${TABLE[$i]}.png..."
- touch -d "$FORCE_DATE 00:00" /var/lib/jenkins/userContent/$PREFIX/${TABLE[$i]}.png
+update_suite_stats() {
+ RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"")
+ if [ -z $RESULT ] ; then
+ echo "Updating packages and builds stats for $SUITE on $DATE."
+ ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) FROM sources WHERE suite='${SUITE}'")
+ GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE';")
+ GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)='$DATE';")
+ BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE';")
+ BAAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)='$DATE';")
+ UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE';")
+ UGLDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)='$DATE';")
+ REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)<='$DATE';")
+ RESDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)='$DATE';")
+ OLDESTG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status = 'reproducible' AND s.suite='$SUITE' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
+ OLDESTB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
+ OLDESTU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
+ DIFFG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTG');")
+ if [ -z $DIFFG ] ; then DIFFG=0 ; fi
+ DIFFB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTB');")
+ if [ -z $DIFFB ] ; then DIFFB=0 ; fi
+ DIFFU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTU');")
+ if [ -z $DIFFU ] ; then DIFFU=0 ; fi
+ let "TOTAL=GOOD+BAD+UGLY+REST" || true # let FOO=0+0 returns error in bash...
+ if [ "$ALL" != "$TOTAL" ] ; then
+ let "UNTESTED=ALL-TOTAL"
+ else
+ UNTESTED=0
fi
- done
-fi
+ sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)"
+ sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)"
+ sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")"
+ # we do 3 later and 6 is special anyway...
+ for i in 0 1 2 4 5 ; do
+ PREFIX=""
+ if [ $i -eq 0 ] ; then
+ PREFIX=$SUITE
+ fi
+ # force regeneration of the image if it exists
+ if [ -f /var/lib/jenkins/userContent/$PREFIX/${TABLE[$i]}.png ] ; then
+ echo "Touching $PREFIX/${TABLE[$i]}.png..."
+ touch -d "$FORCE_DATE 00:00" /var/lib/jenkins/userContent/$PREFIX/${TABLE[$i]}.png
+ fi
+ done
+ fi
+}
#
# update notes stats
#
-RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum from ${TABLE[4]} WHERE datum = \"$DATE\"")
-if [ -z $RESULT ] ; then
- echo "Updating notes stats for $DATE."
- NOTES_GIT_PATH="/var/lib/jenkins/jobs/reproducible_html_notes/workspace"
- if [ ! -d ${NOTES_GIT_PATH} ] ; then
- echo "Warning: ${NOTES_GIT_PATH} does not exist, has the job been renamed???"
- echo "Please investigate and fix!"
- exit 1
- elif [ ! -f ${NOTES_GIT_PATH}/packages.yml ] || [ ! -f ${NOTES_GIT_PATH}/issues.yml ] ; then
- echo "Warning: ${NOTES_GIT_PATH}/packages.yml or issues.yml does not exist, something has changed in notes.git it seems."
- echo "Please investigate and fix!"
- exit 1
+update_notes_stats() {
+ RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum from ${TABLE[4]} WHERE datum = \"$DATE\"")
+ if [ -z $RESULT ] ; then
+ echo "Updating notes stats for $DATE."
+ NOTES_GIT_PATH="/var/lib/jenkins/jobs/reproducible_html_notes/workspace"
+ if [ ! -d ${NOTES_GIT_PATH} ] ; then
+ echo "Warning: ${NOTES_GIT_PATH} does not exist, has the job been renamed???"
+ echo "Please investigate and fix!"
+ exit 1
+ elif [ ! -f ${NOTES_GIT_PATH}/packages.yml ] || [ ! -f ${NOTES_GIT_PATH}/issues.yml ] ; then
+ echo "Warning: ${NOTES_GIT_PATH}/packages.yml or issues.yml does not exist, something has changed in notes.git it seems."
+ echo "Please investigate and fix!"
+ exit 1
+ fi
+ NOTES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/packages.yml)
+ sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[4]} VALUES (\"$DATE\", \"$NOTES\")"
+ ISSUES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/issues.yml)
+ sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[5]} VALUES (\"$DATE\", \"$ISSUES\")"
fi
- NOTES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/packages.yml)
- sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[4]} VALUES (\"$DATE\", \"$NOTES\")"
- ISSUES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/issues.yml)
- sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[5]} VALUES (\"$DATE\", \"$ISSUES\")"
-fi
+}
#
# gather suite stats
@@ -197,52 +201,56 @@ gather_meta_stats() {
#
# update meta pkg stats
#
-if [ "$SUITE" != "experimental" ] ; then
- # no meta pkg sets in experimental
- for i in $(seq 1 ${#META_PKGSET[@]}) ; do
- RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKGSET[$i]}\"")
- if [ -z $RESULT ] ; then
- META_RESULT=true
- gather_meta_stats $i
- if $META_RESULT ; then
- sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)"
- echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE on $DATE."
+update_meta_pkg_stats() {
+ if [ "$SUITE" != "experimental" ] ; then
+ # no meta pkg sets in experimental
+ for i in $(seq 1 ${#META_PKGSET[@]}) ; do
+ RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKGSET[$i]}\"")
+ if [ -z $RESULT ] ; then
+ META_RESULT=true
+ gather_meta_stats $i
+ if $META_RESULT ; then
+ sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)"
+ echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE on $DATE."
+ fi
+ echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..."
+ touch -d "$FORCE_DATE 00:00" /var/lib/jenkins/userContent/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png
fi
- echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..."
- touch -d "$FORCE_DATE 00:00" /var/lib/jenkins/userContent/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png
- fi
- done
-fi
+ done
+ fi
+}
#
-# update bugs stats
+# update bug stats
#
-RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"")
-if [ -z $RESULT ] ; then
- echo "Updating bug stats for $DATE."
- declare -a DONE
- declare -a OPEN
- SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" "
- for TAG in $USERTAGS ; do
- OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds at lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l)
- DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds at lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l)
- # test if both values are integers
- if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then
- echo "Non-integers value detected, exiting."
- echo "Usertag: $TAG"
- echo "Open: ${OPEN[$TAG]}"
- echo "Done: ${DONE[$TAG]}"
- exit 1
- fi
- SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}"
- done
- SQL="$SQL)"
- echo $SQL
- sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL"
- # force regeneration of the image
- echo "Touching ${TABLE[3]}.png..."
- touch -d "$FORCE_DATE 00:00" /var/lib/jenkins/userContent/${TABLE[3]}.png
-fi
+update_bug_stats() {
+ RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"")
+ if [ -z $RESULT ] ; then
+ echo "Updating bug stats for $DATE."
+ declare -a DONE
+ declare -a OPEN
+ SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" "
+ for TAG in $USERTAGS ; do
+ OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds at lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l)
+ DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds at lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l)
+ # test if both values are integers
+ if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then
+ echo "Non-integers value detected, exiting."
+ echo "Usertag: $TAG"
+ echo "Open: ${OPEN[$TAG]}"
+ echo "Done: ${DONE[$TAG]}"
+ exit 1
+ fi
+ SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}"
+ done
+ SQL="$SQL)"
+ echo $SQL
+ sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL"
+ # force regeneration of the image
+ echo "Touching ${TABLE[3]}.png..."
+ touch -d "$FORCE_DATE 00:00" /var/lib/jenkins/userContent/${TABLE[3]}.png
+ fi
+}
#
# create the png (and query the db to populate a csv file...)
@@ -320,6 +328,11 @@ write_usertag_table() {
fi
}
+update_bug_stats
+update_notes_stats
+update_suite_stats
+update_meta_pkg_stats
+
gather_suite_stats
VIEW=suite_stats
PAGE=index_${VIEW}.html
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list