[Qa-jenkins-scm] [jenkins.debian.net] 01/01: reproducible debian: rewrite reproducible_html_pkg_sets in python
Holger Levsen
holger at layer-acht.org
Sun Jul 17 13:08:23 UTC 2016
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 6610b45c60bfe958fb1ce2a351bd4d34fc57cad3
Author: Valerie R Young <spectranaut at riseup.net>
Date: Wed Jun 22 17:39:11 2016 -0400
reproducible debian: rewrite reproducible_html_pkg_sets in python
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
bin/reproducible_common.py | 25 ++-
bin/reproducible_common.sh | 6 -
bin/reproducible_html_pkg_sets.py | 299 +++++++++++++++++++++++++++++++
bin/reproducible_html_pkg_sets.sh | 219 ----------------------
bin/templates/basic_page.mustache | 4 +
bin/templates/main_navigation.mustache | 4 +-
bin/templates/pkgset_details.mustache | 17 ++
bin/templates/pkgset_navigation.mustache | 14 ++
job-cfg/reproducible.yaml | 1 +
9 files changed, 353 insertions(+), 236 deletions(-)
diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py
index 28b4add..95b652e 100755
--- a/bin/reproducible_common.py
+++ b/bin/reproducible_common.py
@@ -24,8 +24,8 @@ import pystache
import psycopg2
import html as HTML
from string import Template
-from subprocess import call
from traceback import print_exception
+from subprocess import call, check_call
from tempfile import NamedTemporaryFile
from datetime import datetime, timedelta
@@ -49,6 +49,7 @@ BIN_PATH = '/srv/jenkins/bin'
BASE = '/var/lib/jenkins/userContent/reproducible'
DEBIAN_BASE = '/var/lib/jenkins/userContent/reproducible/debian'
TEMPLATE_PATH = BIN_PATH + '/templates'
+PKGSET_DEF_PATH = '/srv/reproducible-results'
TEMP_PATH="/tmp/reproducible"
REPRODUCIBLE_JSON = BASE + '/reproducible.json'
@@ -143,7 +144,7 @@ if args.ignore_missing_files:
tab = ' '
# Templates used for creating package pages
-renderer = pystache.Renderer();
+renderer = pystache.Renderer()
status_icon_link_template = renderer.load_template(
TEMPLATE_PATH + '/status_icon_link')
default_page_footer_template = renderer.load_template(
@@ -260,11 +261,13 @@ def create_main_navigation(page_title, suite, arch, displayed_page=None):
'suite': suite,
'arch': arch,
'project_links_html': renderer.render(project_links_template),
- 'experimental': False,
'suite_list': [{'s': s} for s in SUITES],
'arch_list': [{'a': a} for a in ARCHS],
'debian_url': DEBIAN_URL,
}
+ if suite != 'experimental':
+ # there are not package sets in experimental
+ context['include_pkgset_link'] = True
# this argument controls which of the main page navigation items will be
# highlighted.
if displayed_page:
@@ -289,15 +292,18 @@ def write_html_page(title, body, destfile, suite=defaultsuite, arch=defaultarch,
arch=arch,
displayed_page=displayed_page,
)
-
+ # Add the 'mainbody' div only if including a header
+ html += "<div class='mainbody'>"
html += body
if style_note:
html += renderer.render(pkg_legend_template, {})
if not noendpage:
html += create_default_page_footer(now)
- html += '</body>\n</html>'
- else:
- html += '</body>\n</html>'
+ if not noheader:
+ # If the header was included, we need to end the 'mainbody' div after
+ # the 'mainbody' content
+ html += '</div>'
+ html += '</body>\n</html>'
try:
os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True)
except OSError as e:
@@ -414,8 +420,9 @@ def link_package(package, suite, arch, bugs={}, popcon=None, is_popular=None):
return html
-def link_packages(packages, suite, arch):
- bugs = get_bugs()
+def link_packages(packages, suite, arch, bugs=None):
+ if bugs is None:
+ bugs = get_bugs()
html = ''
for pkg in packages:
html += link_package(pkg, suite, arch, bugs)
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 00e7add..202b4f2 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -400,12 +400,6 @@ write_page_footer() {
write_page "</body></html>"
}
-# A mustache equivalent of this function has been created. See TEMPLATE_PATH/pkg_list_key.mustache
-write_page_meta_sign() {
- write_page "<p style=\"font-size:0.9em;\">A package name displayed with a <span style=\"font-weight: bold;\">bold font</span> is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.</br>"
- write_page "A <code><span class=\"bug\">#</span></code> sign after the name of a package indicates that a bug is filed against it. Likewise, a <code><span class=\"bug-patch\">+</span></code> sign indicates there is a patch available, a <code><span class="bug-pending">P</span></code> means a pending bug while <code><span class=\"bug-done\">#</span></code> indicates a closed bug. In cases of several bugs, the symbol is repeated.</p>"
-}
-
write_variation_table() {
write_page "<p style=\"clear:both;\">"
if [ "$1" = "fedora-23" ] ; then
diff --git a/bin/reproducible_html_pkg_sets.py b/bin/reproducible_html_pkg_sets.py
new file mode 100755
index 0000000..8072c73
--- /dev/null
+++ b/bin/reproducible_html_pkg_sets.py
@@ -0,0 +1,299 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2016 Valerie Young <spectranaut at riseup.net>
+# Based on reproducible_html_pkg_sets.sh, which was
+# © 2014-2016 Holger Levsen <holger at layer-acht.org>
+# © 2015 Mattia Rizzolo <mattia at debian.org>
+# Licensed under GPL-2
+#
+# Depends: python3, reproducible_common, time, sqlite3, pystache, csv
+#
+# Build rb-pkg pages (the pages that describe the package status)
+
+from reproducible_common import *
+
+import csv
+import time
+import sqlite3
+import pystache
+
+# Templates used for creating package pages
+renderer = pystache.Renderer()
+pkgset_navigation_template = renderer.load_template(
+ os.path.join(TEMPLATE_PATH, 'pkgset_navigation'))
+pkgset_details_template = renderer.load_template(
+ os.path.join(TEMPLATE_PATH, 'pkgset_details'))
+basic_page_template = renderer.load_template(
+ os.path.join(TEMPLATE_PATH, 'basic_page'))
+pkg_legend_template = renderer.load_template(
+ os.path.join(TEMPLATE_PATH, 'pkg_symbol_legend'))
+
+# we only do stats up until yesterday
+YESTERDAY = (datetime.now()-timedelta(days=1)).strftime('%Y-%m-%d')
+
+def gather_meta_stats(suite, arch, pkgset_name):
+ pkgset_file = os.path.join(PKGSET_DEF_PATH, 'meta_pkgsets-' + suite,
+ pkgset_name + '.pkgset')
+
+ try:
+ with open(pkgset_file) as f:
+ pkgset_list = [s.strip() for s in f.readlines()]
+ except FileNotFoundError:
+ log.warning('No meta package set information exists at ' + pkgset_file)
+ return {}
+
+ if not pkgset_list:
+ log.warning('No packages listed for package set: ' + pkgset_name)
+ return {}
+
+ package_where = "s.name in ('" + ("', '").join(pkgset_list) + "')"
+ root_query = """
+ SELECT s.name
+ FROM results AS r
+ JOIN sources AS s ON r.package_id=s.id
+ WHERE s.suite='{suite}'
+ AND s.architecture='{arch}'
+ AND date(r.build_date)<='{date}'
+ AND {package_where}
+ """.format(suite=suite, arch=arch, date=YESTERDAY,
+ package_where=package_where)
+
+ stats = {}
+ good = query_db(root_query + "AND r.status = 'reproducible' " +
+ "ORDER BY s.name;")
+ stats['good'] = [t[0] for t in good]
+ stats['count_good'] = len(stats['good'])
+
+ bad = query_db(root_query + "AND r.status = 'unreproducible'" +
+ "ORDER BY r.build_date;")
+ stats['bad'] = [t[0] for t in bad]
+ stats['count_bad'] = len(stats['bad'])
+
+ ugly = query_db(root_query + "AND r.status = 'FTBFS'" +
+ "ORDER BY r.build_date;")
+ stats['ugly'] = [t[0] for t in ugly]
+ stats['count_ugly'] = len(stats['ugly'])
+
+ rest = query_db(root_query + "AND (r.status != 'FTBFS' AND " +
+ "r.status != 'unreproducible' AND " +
+ "r.status != 'reproducible') ORDER BY r.build_date;")
+ stats['rest'] = [t[0] for t in rest]
+ stats['count_rest'] = len(stats['rest'])
+
+ stats['count_all'] = (stats['count_good'] + stats['count_bad'] +
+ stats['count_ugly'] + stats['count_rest'])
+ stats['count_all'] = stats['count_all'] if stats['count_all'] else 1
+ stats['percent_good'] = percent(stats['count_good'], stats['count_all'])
+ stats['percent_bad'] = percent(stats['count_bad'], stats['count_all'])
+ stats['percent_ugly'] = percent(stats['count_ugly'], stats['count_all'])
+ stats['percent_rest'] = percent(stats['count_rest'], stats['count_all'])
+ return stats
+
+
+def update_stats(suite, arch, stats, pkgset_name):
+ result = query_db("""
+ SELECT datum, meta_pkg, suite
+ FROM stats_meta_pkg_state
+ WHERE datum = '{date}' AND suite = '{suite}'
+ AND architecture = '{arch}' AND meta_pkg = '{name}'
+ """.format(date=YESTERDAY, suite=suite, arch=arch, name=pkgset_name))
+
+ # if there is not a result for this day, add one
+ if not result:
+ insert = "INSERT INTO stats_meta_pkg_state VALUES ('{date}', " + \
+ "'{suite}', '{arch}', '{pkgset_name}', '{count_good}', " + \
+ "'{count_bad}', '{count_ugly}', '{count_rest}')"
+ query_db(insert.format(date=YESTERDAY, suite=suite, arch=arch,
+ pkgset_name=pkgset_name, count_good=stats['count_good'],
+ count_bad=stats['count_bad'], count_ugly=stats['count_ugly'],
+ count_rest=stats['count_rest']))
+ log.info("Updating meta pkgset stats for %s in %s/%s on %s.",
+ pkgset_name, suite, arch, YESTERDAY)
+ else:
+ log.info("Stats for meta pkgset %s in %s/%s on %s exist - not updating.",
+ pkgset_name, suite, arch, YESTERDAY)
+
+
+def create_pkgset_navigation(suite, arch, view=None):
+ context = {
+ 'package_sets': [],
+ 'suite': suite,
+ 'arch': arch,
+ }
+
+ for index in range(1, len(META_PKGSET)+1):
+ pkgset_name = META_PKGSET[index]
+ pkgset = {
+ 'class': "active" if pkgset_name == view else "",
+ 'pkgset_name': pkgset_name,
+ }
+
+ thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name)
+ # if the graph image doesn't exist, don't include it in the context
+ if os.access(thumb_file, os.R_OK):
+ pkgset['thumb'] = thumb_href
+ context['package_sets'].append(pkgset)
+
+ return renderer.render(pkgset_navigation_template, context)
+
+
+def create_index_page(suite, arch):
+ title = 'Package sets in %s/%s for Reproducible Builds' % (suite, arch)
+ now = datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC')
+ page_context = {
+ 'main_html': create_pkgset_navigation(suite, arch) +
+ create_default_page_footer(now),
+ 'main_navigation_html': create_main_navigation(title, suite, arch,
+ 'pkg_set'),
+ }
+ body = renderer.render(basic_page_template, page_context)
+ destfile = os.path.join(DEBIAN_BASE, suite, arch,
+ "index_pkg_sets.html")
+ write_html_page(title=title, body=body, destfile=destfile,
+ noheader=True, noendpage=True)
+
+
+def gen_other_arch_context(archs, suite, pkgset_name):
+ context = []
+ page = "pkg_set_" + pkgset_name + ".html"
+ for arch in archs:
+ context.append({
+ 'arch': arch,
+ 'link': "/".join([DEBIAN_BASE, suite, arch, page])
+ })
+ return context
+
+
+def stats_png_file_href(suite, arch, pkgset_name):
+ return (os.path.join(DEBIAN_BASE, suite, arch, 'stats_meta_pkg_state_' +
+ pkgset_name + '.png'),
+ "/".join(["/debian", suite, arch, 'stats_meta_pkg_state_' +
+ pkgset_name + '.png'])
+ )
+
+
+def stats_thumb_file_href(suite, arch, pkgset_name):
+ return (os.path.join(DEBIAN_BASE, suite, arch, 'stats_meta_pkg_state_' +
+ pkgset_name + '-thumbnail.png'),
+ "/".join(["/debian", suite, arch, 'stats_meta_pkg_state_' +
+ pkgset_name + '-thumbnail.png'])
+ )
+
+
+def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name):
+ html = ""
+ html += create_pkgset_navigation(suite, arch, pkgset_name)
+ pkgset_context = ({
+ 'pkgset_name': pkgset_name,
+ 'suite': suite,
+ 'arch': arch,
+ 'pkg_symbol_legend_html':
+ renderer.render(pkg_legend_template, {}),
+ })
+
+ png_file, png_href = stats_png_file_href(suite, arch, pkgset_name)
+ thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name)
+ yesterday_timestamp = (datetime.now()-timedelta(days=1)).timestamp()
+
+ if ( not os.access(png_file, os.R_OK) or
+ os.stat(png_file).st_mtime < yesterday_timestamp ):
+ create_pkgset_graph(png_file, suite, arch, pkgset_name)
+ check_call(['convert', png_file, '-adaptive-resize', '160x80',
+ thumb_file])
+
+ pkgset_context['png'] = png_href
+ other_archs = [a for a in ARCHS if a != arch]
+ pkgset_context['other_archs']= \
+ gen_other_arch_context(other_archs, suite, pkgset_name)
+
+ pkgset_context['status_details'] = []
+
+ status_cutename_descriptions = [
+ ('unreproducible', 'bad', 'failed to build reproducibly'),
+ ('FTBFS', 'ugly', 'failed to build from souce'),
+ ('rest', 'rest',
+ 'are either blacklisted, not for us or cannot be downloaded'),
+ ('reproducible', 'good', 'successfully build reproducibly'),
+ ]
+
+ for (status, cutename, description) in status_cutename_descriptions:
+ icon_html = ''
+ if status == 'rest':
+ for s in ['not_for_us', 'blacklisted', '404']:
+ s, icon, spokenstatus = get_status_icon(s)
+ icon_html += gen_status_link_icon(s, None, icon, suite, arch)
+ else:
+ status, icon, spokenstatus = get_status_icon(status)
+ icon_html = gen_status_link_icon(status, None, icon, suite, arch)
+
+ details_context = {
+ 'icon_html': icon_html,
+ 'description': description,
+ 'package_list_html': link_packages(stats[cutename], suite, arch, bugs),
+ 'status_count': stats["count_" + cutename],
+ 'status_percent': stats["percent_" + cutename],
+ }
+
+ if (status in ('reproducible', 'unreproducible') or
+ stats["count_" + cutename] != 0):
+ pkgset_context['status_details'].append(details_context)
+
+ html += renderer.render(pkgset_details_template, pkgset_context)
+ now = datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC')
+ html += create_default_page_footer(now)
+ title = 'Reproducible Builds %s package set for %s/%s for' % \
+ (pkgset_name, suite, arch)
+
+ body = renderer.render(basic_page_template, {
+ 'main_html': html,
+ 'main_navigation_html': create_main_navigation(title, suite, arch,
+ 'pkg_set'),
+ })
+ page = "pkg_set_" + pkgset_name + ".html"
+ destfile = os.path.join(DEBIAN_BASE, suite, arch, page)
+ write_html_page(title=title, body=body, destfile=destfile,
+ noheader=True, noendpage=True)
+
+
+def create_pkgset_graph(png_file, suite, arch, pkgset_name):
+ table = "stats_meta_pkg_state"
+ columns = ["datum", "reproducible", "unreproducible", "FTBFS", "other"]
+ where = "WHERE suite = '%s' AND architecture = '%s' AND meta_pkg = '%s'" % \
+ (suite, arch, pkgset_name)
+ if arch == 'i386':
+ # i386 only has pkg sets since later to make nicer graphs
+ # (date added in commit 7f2525f7)
+ where += " AND datum >= '2016-05-06'"
+ query = "SELECT {fields} FROM {table} {where} ORDER BY datum".format(
+ fields=", ".join(columns), table=table, where=where)
+ result = query_db(query)
+ result_rearranged = [dict(zip(columns, row)) for row in result]
+
+ with create_temp_file(mode='w') as f:
+ csv_tmp_file = f.name
+ csv_writer = csv.DictWriter(f, columns)
+ csv_writer.writeheader()
+ csv_writer.writerows(result_rearranged)
+ f.flush()
+
+ graph_command = os.path.join(BIN_PATH, "make_graph.py")
+ main_label = "Reproducibility status for packages in " + suite + \
+ " from " + pkgset_name
+ y_label = "Amount (" + pkgset_name + " packages)"
+ check_call([graph_command, csv_tmp_file, png_file, '4', main_label,
+ y_label, '1920', '960'])
+
+
+bugs = get_bugs()
+for arch in ARCHS:
+ for suite in SUITES:
+ if suite == 'experimental':
+ continue
+ create_index_page(suite, arch)
+ for index in META_PKGSET:
+ pkgset_name = META_PKGSET[index]
+ stats = gather_meta_stats(suite, arch, pkgset_name)
+ if (stats):
+ update_stats(suite, arch, stats, pkgset_name)
+ create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name)
diff --git a/bin/reproducible_html_pkg_sets.sh b/bin/reproducible_html_pkg_sets.sh
deleted file mode 100755
index bc875fa..0000000
--- a/bin/reproducible_html_pkg_sets.sh
+++ /dev/null
@@ -1,219 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014-2016 Holger Levsen <holger at layer-acht.org>
-# © 2015 Mattia Rizzolo <mattia at mapreri.org>
-# released under the GPLv=2
-
-DEBUG=false
-. /srv/jenkins/bin/common-functions.sh
-common_init "$@"
-
-# common code defining db access
-. /srv/jenkins/bin/reproducible_common.sh
-
-#
-# init some variables
-#
-# we only do stats up until yesterday... we also could do today too but not update the db yet...
-DATE=$(date -d "1 day ago" '+%Y-%m-%d')
-FORCE_DATE=$(date -d "3 days ago" '+%Y-%m-%d')
-
-# variables related to the stats we update
-FIELDS[6]="datum, reproducible, unreproducible, FTBFS, other"
-COLOR[6]=4
-
-#
-# gather meta pkg stats
-#
-gather_meta_stats() {
- PKGSET_PATH=/srv/reproducible-results/meta_pkgsets-$SUITE/${META_PKGSET[$1]}.pkgset
- if [ -f $PKGSET_PATH ] ; then
- META_LIST=$(cat $PKGSET_PATH)
- if [ ! -z "$META_LIST" ] ; then
- META_WHERE=""
- # gather data about all packages we know about
- # as a result, unknown packages in the package set
- # are silently ignored
- set +x
- for PKG in $META_LIST ; do
- if [ -z "$META_WHERE" ] ; then
- META_WHERE="s.name in ('$PKG'"
- else
- META_WHERE="$META_WHERE, '$PKG'"
- fi
- done
- if "$DEBUG" ; then set -x ; fi
- META_WHERE="$META_WHERE)"
- else
- META_WHERE="name = 'meta-name-does-not-exist'"
- fi
- COUNT_META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;")
- COUNT_META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;")
- COUNT_META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE;")
- COUNT_META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE;")
- let META_ALL=COUNT_META_GOOD+COUNT_META_BAD+COUNT_META_UGLY+COUNT_META_REST || META_ALL=1
- PERCENT_META_GOOD=$(echo "scale=1 ; ($COUNT_META_GOOD*100/$META_ALL)" | bc)
- PERCENT_META_BAD=$(echo "scale=1 ; ($COUNT_META_BAD*100/$META_ALL)" | bc)
- PERCENT_META_UGLY=$(echo "scale=1 ; ($COUNT_META_UGLY*100/$META_ALL)" | bc)
- PERCENT_META_REST=$(echo "scale=1 ; ($COUNT_META_REST*100/$META_ALL)" | bc)
- # order reproducible packages by name, the rest by build_date
- META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY s.name;")
- META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;")
- META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;")
- META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name AS NAME FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;")
- else
- META_RESULT=false
- fi
-}
-
-#
-# update meta pkg stats
-#
-update_meta_pkg_stats() {
- for i in $(seq 1 ${#META_PKGSET[@]}) ; do
- RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND architecture = \"$ARCH\" AND meta_pkg = \"${META_PKGSET[$i]}\"")
- if [ -z $RESULT ] ; then
- META_RESULT=true
- gather_meta_stats $i
- if $META_RESULT ; then
- sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"$ARCH\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)"
- echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE/$ARCH on $DATE."
- fi
- echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..."
- touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png
- fi
- done
-}
-
-#
-# create pkg set navigation
-#
-create_pkg_sets_navigation() {
- local i
- local view=$1
- local CLASS=""
- write_page "<ul><li>Tracked package sets in $SUITE/$ARCH: </li>"
- write_page "<br />"
- for i in $(seq 1 ${#META_PKGSET[@]}) ; do
- case $i in
- 7|10|18|26) write_page "<br />"
- ;;
- *) ;;
- esac
- if [ "$i" = "$view" ] ; then
- CLASS=" class=\"active\""
- else
- CLASS=""
- fi
-
- if [ -f $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then
- THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png"
- LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'"
- write_page "<li>"
- write_page "<a href=\"/debian/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"$CLASS>${META_PKGSET[$i]}</a>"
- write_page "<a href=\"/debian/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"><img src=\"/debian/$SUITE/$ARCH/$THUMB\" class=\"setview\" alt=\"$LABEL\" title=\"${META_PKGSET[$i]}\" name=\"${META_PKGSET[$i]}\"></a>"
- write_page "</li>"
- fi
- done
- write_page "</ul>"
-}
-
-#
-# create pkg sets pages
-#
-create_pkg_sets_pages() {
- #
- # create index page
- #
- VIEW=pkg_sets
- PAGE=index_${VIEW}.html
- echo "$(date -u) - starting to write $PAGE page."
- write_page_header $VIEW "Package sets in $SUITE/$ARCH for Reproducible builds"
- create_pkg_sets_navigation index
- write_page_footer
- publish_page debian/$SUITE/$ARCH
- #
- # create individual pages for all the sets
- #
- local i
- for i in $(seq 1 ${#META_PKGSET[@]}) ; do
- PAGE="pkg_set_${META_PKGSET[$i]}.html"
- echo "$(date -u) - starting to write $PAGE page."
- write_page_header $VIEW "Reproducible builds ${META_PKGSET[$i]} package set in $SUITE/$ARCH"
- create_pkg_sets_navigation $i
- write_page "<hr />"
- META_RESULT=true
- gather_meta_stats $i
- if $META_RESULT ; then
- MAINLABEL[6]="Reproducibility status for packages in $SUITE from '${META_PKGSET[$i]}'"
- YLABEL[6]="Amount (${META_PKGSET[$i]} packages)"
- PNG=${TABLE[6]}_${META_PKGSET[$i]}.png
- THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png"
- # redo pngs once a day
- if [ ! -f $DEBIAN_BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $DEBIAN_BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then
- create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]}
- convert $DEBIAN_BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $DEBIAN_BASE/$SUITE/$ARCH/$THUMB
- fi
- LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH"
- write_page "<p><a href=\"/debian/$SUITE/$ARCH/$PNG\"><img src=\"/debian/$SUITE/$ARCH/$PNG\" class=\"overview\" alt=\"$LABEL\"></a>"
- write_page "<br />The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of:"
- write_page " (this set on "
- for LINKARCH in ${ARCHS} ; do
- if [ "$LINKARCH" = "$ARCH" ] ; then
- continue
- else
- write_page "<a href=\"/debian/$SUITE/$LINKARCH/pkg_set_${META_PKGSET[$i]}.html\">$LINKARCH</a> "
- fi
- done
- write_page ") <br /> <br />"
- set_icon unreproducible
- write_icon
- write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to build reproducibly:"
- link_packages $META_BAD
- write_page "<br />"
- if [ $COUNT_META_UGLY -gt 0 ] ; then
- set_icon FTBFS
- write_icon
- write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:"
- link_packages $META_UGLY
- write_page "<br />"
- fi
- if [ $COUNT_META_REST -gt 0 ] ; then
- set_icon not_for_us
- write_icon
- set_icon blacklisted
- write_icon
- set_icon 404
- write_icon
- write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:"
- link_packages $META_REST
- write_page "<br />"
- fi
- write_page " <br />"
- set_icon reproducible
- write_icon
- write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:"
- link_packages $META_GOOD
- write_page "<br />"
- write_page "</p>"
- write_page_meta_sign
- fi
- write_page_footer
- publish_page debian/$SUITE/$ARCH
- done
-}
-
-#
-# main
-#
-for ARCH in $ARCHS ; do
- for SUITE in $SUITES ; do
- if [ "$SUITE" = "experimental" ] ; then
- # no pkg sets in experimental
- continue
- fi
- update_meta_pkg_stats
- create_pkg_sets_pages
- done
-done
-
diff --git a/bin/templates/basic_page.mustache b/bin/templates/basic_page.mustache
new file mode 100644
index 0000000..99fee20
--- /dev/null
+++ b/bin/templates/basic_page.mustache
@@ -0,0 +1,4 @@
+{{{main_navigation_html}}}
+<div class="mainbody">
+{{{main_html}}}
+</div>
diff --git a/bin/templates/main_navigation.mustache b/bin/templates/main_navigation.mustache
index 776a120..6dab169 100644
--- a/bin/templates/main_navigation.mustache
+++ b/bin/templates/main_navigation.mustache
@@ -39,13 +39,13 @@
</a>
</li>
</ul></li>
- {{^experimental}}
+ {{#include_pkgset_link}}
<li {{#pkg_set}}class='active'{{/pkg_set}}>
<a href="/debian/{{suite}}/{{arch}}/index_pkg_sets.html">
package sets
</a>
</li>
- {{/experimental}}
+ {{/include_pkgset_link}}
<li>Recently tested:<ul class="children">
<li {{#last_24h}}class='active'{{/last_24h}}>
<a href="/debian/{{suite}}/{{arch}}/index_last_24h.html">
diff --git a/bin/templates/pkgset_details.mustache b/bin/templates/pkgset_details.mustache
new file mode 100644
index 0000000..ac72174
--- /dev/null
+++ b/bin/templates/pkgset_details.mustache
@@ -0,0 +1,17 @@
+<p>
+{{#png}}
+ <a href="{{png}}"><img src="{{png}}" class="overview" alt="package set {{pkgset_name}} in {{suite}}/{{arch}}"></a>"
+{{/png}}
+
+ <br />
+ The package set '{{pkgset_name}}' in {{suite}}/{{arch}} consists of:
+ (see this package set on {{#other_archs}}<a href="{{link}}" title="See this packageset in unstable/{{arch}}" target="_parent">{{arch}}</a>
+{{/other_archs}})
+
+ <br />
+ {{#status_details}}
+ {{{icon_html}}} {{status_count}} ({{status_percent}}%) packages {{description}}: {{{package_list_html}}}
+ <br />
+ {{/status_details}}
+ {{{pkg_symbol_legend_html}}}
+</op>
diff --git a/bin/templates/pkgset_navigation.mustache b/bin/templates/pkgset_navigation.mustache
new file mode 100644
index 0000000..40d19c1
--- /dev/null
+++ b/bin/templates/pkgset_navigation.mustache
@@ -0,0 +1,14 @@
+<ul>
+ <li>Tracked package sets in {{suite}}/{{arch}}: </li>
+<br/>
+{{#package_sets}}
+ <li class="{{class}}">
+ <a href="/debian/{{suite}}/{{arch}}/pkg_set_{{pkgset_name}}.html">{{pkgset_name}}</a>
+ {{#thumb}}
+ <a href="/debian/{{suite}}/{{arch}}/pkg_set_{{pkgset_name}}.html">
+ <img src="{{thumb}}" class="setview" alt="Reproducibility status for packages un {{suite}}/{{arch}} from '{{pkg_name}}'" title="{{pkgset_name}}" name="{{pkgset_name}}">
+ </a>
+ {{/thumb}}
+ </li>
+{{/package_sets}}
+</ul>
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 355f027..4ea6a0d 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -426,6 +426,7 @@
- 'html_pkg_sets':
my_description: 'Generate HTML results (package sets with graphs) for reproducible builds.'
my_timed: '0 */4 * * *'
+ my_shellext: ".py"
- 'html_indexes':
my_description: 'Generate HTML results (indexes) for reproducible builds.'
my_timed: '*/15 * * * *'
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list