[Blends-commit] [SCM] website branch, master, updated. 06364a5ad6fe357fc9f5ef7f9d33fea4b9d907ef
Andreas Tille
tille at debian.org
Wed Jul 17 19:12:15 UTC 2013
The following commit has been merged in the master branch:
commit 06364a5ad6fe357fc9f5ef7f9d33fea4b9d907ef
Author: Andreas Tille <tille at debian.org>
Date: Wed Jul 17 21:06:50 2013 +0200
Gather information for bugs web sentinel pages from UDD (needs stripping of code from thermometer that was used as example)
diff --git a/webtools/bugs_udd.py b/webtools/bugs_udd.py
new file mode 100755
index 0000000..b47dfb9
--- /dev/null
+++ b/webtools/bugs_udd.py
@@ -0,0 +1,259 @@
+#!/usr/bin/python
+# Copyright 2013: Andreas Tille <tille at debian.org>
+# License: GPL
+
+#PORT=5441
+UDDPORT=5452
+PORT=UDDPORT
+DEFAULTPORT=5432
+
+from sys import argv, stderr, exit
+import os
+import psycopg2
+import json
+import re
+import time
+from datetime import datetime
+from email.Utils import formatdate
+import gettext
+
+from genshi.template import TemplateLoader
+from genshi import Markup
+from genshi.template.eval import UndefinedError
+
+from blendsunicode import to_unicode
+from blendstasktools import ReadConfig, RowDictionaries, CheckOrCreateOutputDir
+
+###########################################################################################
+# Define several prepared statements to query UDD
+try:
+ conn = psycopg2.connect(host="localhost",port=PORT,user="guest",database="udd")
+except psycopg2.OperationalError, err:
+ try:
+ conn = psycopg2.connect(host="udd.debian.org",port=UDDPORT,user="guest",database="udd")
+ except psycopg2.OperationalError, err:
+ # logger not known at this state: logger.warning
+ print >>stderr, "PostgreSQL does not seem to run on port %i .. trying default port %i.\n\tMessage: %s" % (PORT, DEFAULTPORT, str(err))
+ try:
+ conn = psycopg2.connect(host="localhost",port=DEFAULTPORT,user="guest",database="udd")
+ except psycopg2.OperationalError:
+ # Hmmm, I observed a really strange behaviour on one of my machines where connecting to
+ # localhost does not work but 127.0.0.1 works fine. No idea why ... but this should
+ # do the trick for the moment
+ conn = psycopg2.connect(host="127.0.0.1",port=DEFAULTPORT,user="guest",database="udd")
+
+curs = conn.cursor()
+# uddlog = open('logs/uddquery.log', 'w')
+
+def _execute_udd_query(query):
+ try:
+ curs.execute(query)
+ except psycopg2.ProgrammingError, err:
+ print >>stderr, "Problem with query\n%s" % (query)
+ print >>stderr, err
+ exit(-1)
+ except psycopg2.DataError, err:
+ print >>stderr, "%s; query was\n%s" % (err, query)
+
+
+def main():
+
+ if len(argv) <= 1:
+ print >>stderr, "Usage: %s <Blend name>" % argv[0]
+ exit(-1)
+
+ blendname = argv[1]
+ config = ReadConfig(blendname)
+
+ query = """PREPARE query_bug_packages (text) AS
+ SELECT distinct sources.source, tasks.tasks, CASE WHEN dependency = 'd' AND component = 'main' THEN 'depends' ELSE 'suggests' END AS status, homepage, vcs_browser, maintainer
+ FROM (
+ SELECT s.source, b.dependency, b.component, s.homepage, s.vcs_browser, s.maintainer, s.version, row_number() OVER (PARTITION BY s.source ORDER BY s.version DESC)
+ FROM blends_dependencies b
+ JOIN packages p ON p.package = b.package
+ JOIN bugs bu ON bu.source = p.source
+ JOIN sources s ON s.source = p.source
+ WHERE blend = $1 AND b.distribution = 'debian'
+ GROUP BY s.source, b.dependency, b.component, s.homepage, s.vcs_browser, s.maintainer, s.version
+ ) sources
+ LEFT OUTER JOIN (
+ SELECT source, array_agg(task) AS tasks FROM (
+ SELECT DISTINCT p.source, b.task
+ FROM packages p
+ JOIN releases r ON p.release = r.release
+ JOIN blends_dependencies b ON b.package = p.package
+ JOIN sources s ON p.source = s.source AND p.release = s.release
+ WHERE b.blend = $1
+ ) tmp
+ GROUP BY source
+ ) tasks ON sources.source = tasks.source
+ WHERE row_number = 1
+ ORDER BY source;
+ """
+ _execute_udd_query(query)
+
+ query = """PREPARE query_bugs (text) AS
+ SELECT distinct bu.source, bu.id, bu.title, bu.status, bu.done
+ FROM blends_dependencies b
+ JOIN packages p ON p.package = b.package
+ JOIN bugs bu ON bu.source = p.source
+ WHERE blend = $1 AND b.distribution = 'debian'
+ ORDER BY source, id;
+ """
+ _execute_udd_query(query)
+
+ query = """PREPARE query_get_tasks (text) AS
+ SELECT task FROM blends_tasks WHERE blend = $1 ORDER BY task;
+ """
+ _execute_udd_query(query)
+
+ _execute_udd_query( "EXECUTE query_get_tasks('%s')" % blendname)
+
+
+ states = ('depends', 'suggests', 'done')
+
+ bugs_data = {}
+ if curs.rowcount > 0:
+ for task in curs.fetchall():
+ bugs_data[task[0]] = {}
+ for status in states:
+ bugs_data[task[0]][status + '_l'] = [] # enable sorting
+ bugs_data[task[0]][status] = {}
+ else:
+ print >>stderr, "No tasks metadata received for Blend", blendname
+ exit(1)
+
+ _execute_udd_query( "EXECUTE query_bugs('%s')" % blendname)
+
+ bugs = {}
+ if curs.rowcount > 0:
+ for bug in RowDictionaries(curs):
+ if not bugs.has_key(bug['source']):
+ bugs[bug['source']] = {}
+ bugs[bug['source']]['open'] = []
+ bugs[bug['source']]['done'] = []
+ b = {}
+ for k in bug.keys():
+ if k in ('source', 'status') :
+ continue
+ b[k] = bug[k]
+ if bug['status'] == 'done':
+ bugs[bug['source']]['done'].append(b)
+ else:
+ bugs[bug['source']]['open'].append(b)
+ else:
+ print >>stderr, "No bug data received for Blend", blendname
+ exit(1)
+
+ _execute_udd_query( "EXECUTE query_bug_packages('%s')" % blendname)
+ if curs.rowcount > 0:
+ for pkg in RowDictionaries(curs):
+ for task in pkg['tasks']:
+ if not bugs_data[task][pkg['status']].has_key(pkg['source']):
+ bugs_data[task][pkg['status']][pkg['source']] = {}
+ bugs_data[task][pkg['status']][pkg['source']]['homepage'] = pkg['homepage']
+ bugs_data[task][pkg['status']][pkg['source']]['vcs_browser'] = pkg['vcs_browser']
+ bugs_data[task][pkg['status']][pkg['source']]['maintainer'] = pkg['maintainer']
+ if pkg['status'] == 'depends':
+ bugs_data[task][pkg['status']][pkg['source']]['bugs'] = bugs[pkg['source']]['open']
+ bugs_data[task][pkg['status']+'_l'].append(pkg['source'])
+ elif pkg['status'] == 'suggests':
+ bugs_data[task][pkg['status']][pkg['source']]['bugs'] = bugs[pkg['source']]['open']
+ bugs_data[task][pkg['status']+'_l'].append(pkg['source'])
+ else:
+ print >>stderr, "%s: Wrong status %s in task %s for source %s" % (blendname, pkg['status'], task, pkg['source'])
+ exit(1)
+ if bugs[pkg['source']]['done']:
+ if not bugs_data[task]['done'].has_key(pkg['source']):
+ bugs_data[task]['done'][pkg['source']] = {}
+ bugs_data[task]['done'][pkg['source']]['bugs'] = bugs[pkg['source']]['done']
+ bugs_data[task]['done_l'].append(pkg['source'])
+ else:
+ print >>stderr, "No information about buggy packages received for Blend", blendname
+ exit(1)
+
+
+ f = open(blendname+'_bugs.json', 'w')
+ for task in bugs_data:
+ print >>f, task
+ for status in states:
+ if bugs_data[task].has_key(status):
+ print >>f, status
+ print >>f, json.dumps(bugs_data[task][status])
+ f.close()
+ exit(1)
+ # Define directories used
+ current_dir = os.path.dirname(__file__)
+ # locale_dir = os.path.join(current_dir, 'locale')
+ template_dir = os.path.join(current_dir, 'templates')
+
+ # initialize gensi
+ loader = TemplateLoader([template_dir], auto_reload=True)
+
+ outputdir = CheckOrCreateOutputDir(config['outputdir'],'thermometer')
+ if outputdir == None:
+ exit(-1)
+
+ t = datetime.now()
+
+ # Initialize i18n
+ domain = 'blends-webtools'
+ gettext.install(domain)
+
+ data={}
+ data['projectname'] = blendname
+ data['blend_data'] = blend_data
+ if config.has_key('advertising') and config['advertising'] != None:
+ # we have to remove the gettext _() call which was inserted into the config
+ # file to enable easy input for config file editors - but the call has to
+ # be made explicitely in the python code
+ advertising = re.sub('_\(\W(.+)\W\)', '\\1', config['advertising'])
+ # gettext needs to escape '"' thus we need to remove the escape character '\'
+ data['projectadvertising'] = Markup(to_unicode(re.sub('\\\\"', '"', advertising)))
+ else:
+ data['projectadvertising'] = None
+
+ legend = [
+ ['upToDate', 'Up to date'],
+ ['debianOutOfDate', 'Debian stable behind unstable'],
+ ['ubuntuOutOfDate', 'Ubuntu behind Debian unstable'],
+ ['unpackaged', 'Not packaged'],
+ ['obsolete', 'Obsolete'],
+ ['newer-in-debian', 'Upstream behind unstable'],
+ ['uptodate', 'Unstable fits upstream'],
+ ['outdated', 'Unstable behind upstream'],
+ ]
+ data['legend'] = legend
+ data['summary'] = to_unicode(_('Summary'))
+ data['idxsummary'] = to_unicode(_("""A %sDebian Pure Blend%s is a Debian internal project which assembles
+a set of packages that might help users to solve certain tasks of their work. The list on
+the right shows the tasks of %s.""" ) \
+ % ('<a href="http://blends.alioth.debian.org/blends/">', '</a>', data['projectname']))
+ data['idxsummary'] = Markup(to_unicode(data['idxsummary']))
+
+ for key in ('homepage', 'projecturl', 'projectname', 'logourl', 'ubuntuhome', 'projectubuntu'):
+ data[key] = config[key]
+ data['updatetimestamp'] = to_unicode(_('Last update:')) + ' ' + formatdate(time.mktime(t.timetuple()))
+
+ data['thermometer'] = blendname + '_thermometer.html'
+ os.system("ln -sf %s %s/index.html" % (data['thermometer'], outputdir))
+ data['uthermometer'] = 'ubuntu_' + blendname + '_thermometer.html'
+ outputfile = outputdir + '/' + data['thermometer']
+ uoutputfile = outputdir + '/' + data['uthermometer']
+ try:
+ os.unlink(outputfile)
+ os.unlink(uoutputfile)
+ except: # simply continue if file does not exist
+ pass
+
+ f = open(outputfile, 'w')
+ template = loader.load('thermometer.xhtml')
+ print >> f, template.generate(**data).render('xhtml')
+ f.close()
+ f = open(uoutputfile, 'w')
+ utemplate = loader.load('uthermometer.xhtml')
+ print >> f, utemplate.generate(**data).render('xhtml')
+ f.close()
+
+if __name__ == '__main__':
+ main()
--
Static and dynamic websites for Debian Pure Blends
More information about the Blends-commit
mailing list