[Blends-commit] [SCM] website branch, master, updated. 23dc4d57ff044c3cdd947eefc73f5325b1f86aa7
Akshita Jha
akshita-guest at users.alioth.debian.org
Mon Jul 13 09:30:31 UTC 2015
The following commit has been merged in the master branch:
commit 23dc4d57ff044c3cdd947eefc73f5325b1f86aa7
Author: Akshita Jha <akshita-guest at users.alioth.debian.org>
Date: Mon Jul 13 14:59:48 2015 +0530
Add blendstasktools_udd.py and tasks_udd.py - testing
diff --git a/webtools/blendstasktools_udd.py b/webtools/blendstasktools_udd.py
new file mode 100644
index 0000000..de18f86
--- /dev/null
+++ b/webtools/blendstasktools_udd.py
@@ -0,0 +1,1229 @@
+#PORT=5441
+UDDPORT=5452
+PORT=UDDPORT
+DEFAULTPORT=5432
+
+from sys import stderr, exit
+from subprocess import Popen, PIPE
+import os
+import grp
+import stat
+import urllib
+import StringIO
+import gzip
+import bz2
+import re
+import email.Utils
+
+import psycopg2
+import gettext
+
+from genshi import Markup
+# According to http://genshi.edgewall.org/wiki/GenshiFaq#HowcanIincludeliteralXMLintemplateoutput
+# there are different options to prevent escaping '<' / '>' but HTML does not work ...
+# from genshi.input import HTML
+from blendsmarkdown import SplitDescription, MarkupString, render_longdesc
+
+try:
+ from debian import deb822
+except:
+ from debian_bundle import deb822
+from blendsunicode import to_unicode
+from blendslanguages import languages
+
+import logging
+import logging.handlers
+logger = logging.getLogger('blends')
+logger.setLevel(logging.INFO)
+# logger.setLevel(logging.DEBUG)
+
+# Seems to have problems on 17.04.2009
+# BASEURL = 'http://ftp.debian.org/debian'
+BASEURL = 'http://ftp.de.debian.org/debian'
+KEYSTOIGNORE = ( 'Architecture', 'Comment', 'Leaf', 'NeedConfig', 'Note', 'Section',
+ 'Needconfig', 'DontAvoid',
+ 'Enhances', 'Test-always-lang', 'Metapackage')
+
+CONFDIR = 'webconf'
+
+COMPRESSIONEXTENSION='bz2'
+# COMPRESSIONEXTENSION='gz' # Translations are only available as bz2 since April 2009
+
+HOMEPAGENONE = '#'
+HOMEPAGENONEFIELDS = ('homepage',
+ 'pkg-url', # Link to packages.debian.org search interface with exact
+ # package matches or URL to inofficial package
+ 'vcs-browser', # Browser-URL to packaging stuff in Vcs
+ )
+
+PKGURLMASK = 'http://packages.debian.org/search?keywords=%s%%26searchon=names%%26exact=1%%26suite=all%%26section=all'
+
+DEPENDENT = 0
+SUGGESTED = 1
+DONE = 2
+BUGLISTCAT = (DEPENDENT, SUGGESTED, DONE )
+
+# FIXME: Obtain releases from UDD table releases (is this used at all???)
+releases = {'oldstable' : ('wheezy', 'wheezy-proposed-updates', 'wheezy-security'),
+ 'stable' : ('jessie', 'jessie-proposed-updates', 'jessie-security'),
+ 'testing' : ('stretch'),
+ 'unstable' : ('sid'),
+ 'experimental' : ('experimental')
+ }
+
+pkgstatus = {'official_high' : # official package with high priority dependency
+ { 'releases' : ('oldstable', 'stable', 'testing', 'unstable'),
+ 'components' : ('main', ),
+ 'dependencies' : ('Depends', 'Recommends'),
+ 'fields-set' : (),
+ 'colorcode' : 'Green: The project is <a href="#%s">available as an official Debian package and has high relevance</a>',
+ 'order' : 1
+ },
+ 'official_low' : # official package with low priority dependency
+ { 'releases' : ('oldstable', 'stable', 'testing', 'unstable'),
+ 'components' : ('main', ),
+ 'dependencies' : ('Suggests', ),
+ 'fields-set' : (),
+ 'colorcode' : 'Green: The project is <a href="#%s">available as an official Debian package but has lower relevance</a>',
+ 'order' : 2
+ },
+ 'non-free' : # package in contrib or non-free, priority decreased to Suggests in any case
+ { 'releases' : ('oldstable', 'stable', 'testing', 'unstable'),
+ 'component' : ('contrib', 'non-free'),
+ 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
+ 'fields-set' : (),
+ 'colorcode' : 'Green: The project is <a href="#%s">available in Debian packaging pool but is not in Debian main</a>',
+ 'order' : 3
+ },
+ 'experimental' : # package which exists only in experimental
+ { 'releases' : ('experimental', ),
+ 'component' : ('main', 'contrib', 'non-free'),
+ 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
+ 'fields-set' : (),
+ 'colorcode' : 'Yellow: The project is <a href="#%s">available in Debian packaging pool but is regarded as experimental</a>',
+ 'order' : 4
+ },
+ 'new' : # package in new queue
+ { 'releases' : ('new', ),
+ 'component' : ('main', 'contrib', 'non-free'),
+ 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
+ 'fields-set' : (),
+ 'colorcode' : 'Yellow: A package of project is <a href="#%s">is in Debian New queue and hopefully available soon</a>',
+ 'order' : 5
+ },
+ 'pkgvcs' : # Not yet packaged but packaging code in Vcs
+ { 'releases' : (),
+ 'component' : (),
+ 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
+ 'fields-set' : ('vcs-svn', 'vcs-git', 'vcs-browser'),
+ 'colorcode' : 'Yellow: The packaging of project is <a href="#%s">has started and a developer might try the packaging code in VCS or help packaging.</a>',
+ 'order' : 6
+ },
+ 'unofficial' : # unofficial packages outside Debian
+ { 'releases' : (),
+ 'component' : (),
+ 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
+ 'fields-set' : ('pkg-url', ),
+ 'colorcode' : 'Yellow: There exists an <a href="#%s">unofficial package</a> of the project',
+ 'order' : 7
+ },
+ 'wnpp' : # project which has at least a WNPP bug filed
+ { 'releases' : (),
+ 'component' : (),
+ 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
+ 'fields-set' : ('wnpp', ),
+ 'colorcode' : 'Red: The project is <a href="#%s">not (yet) available as a Debian package</a> but there is some record of interest (WNPP bug).',
+ 'order' : 8
+ },
+ 'prospective' : # projects which might be interesting for a Blend but no work is done yet
+ { 'releases' : (),
+ 'component' : (),
+ 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
+ 'fields-set' : ('homepage', ), # TODO: a description should be set as well ...
+ 'colorcode' : 'Red: The project is <a href="#%s">not (yet) available as a Debian package</a>.',
+ 'order' : 9
+ },
+ 'ignore' : # Package inside Debian which is "under observation"
+ { 'releases' : (releases.keys()),
+ 'component' : ('main', 'contrib', 'non-free'),
+ 'dependencies' : ('Ignore', ),
+ 'fields-set' : (),
+ 'colorcode' : '%s',
+ 'order' : 10
+ },
+ 'avoid' : # Package inside Debian which should not go to a install medium of the Blend
+ { 'releases' : (releases.keys()),
+ 'component' : ('main', 'contrib', 'non-free'),
+ 'dependencies' : ('Avoid', ),
+ 'fields-set' : (),
+ 'colorcode' : '%s',
+ 'order' : 11
+ },
+ 'unknown' : # Everything else
+ { 'releases' : (),
+ 'component' : (),
+ 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
+ 'fields-set' : (),
+ 'colorcode' : 'White: The project has an %s status.',
+ 'order' : 100
+ },
+ }
+
+# http://wiki.python.org/moin/HowTo/Sorting#Sortingbykeys
+_tmplist=[]
+for key in pkgstatus.keys():
+ _tmplist.append((key,pkgstatus[key]['order']))
+_tmpsorted = sorted(_tmplist, key=lambda x:(x[1], x[0]))
+pkgstatus_sortedkeys = []
+for _tmp in _tmpsorted:
+ pkgstatus_sortedkeys.append(_tmp[0])
+
+dep_strength_keys = []
+for pkgstat in pkgstatus:
+ for dep in pkgstatus[pkgstat]['dependencies']:
+ if dep not in dep_strength_keys:
+ dep_strength_keys.append(dep)
+
+rmpub = open('remove-publications-from-tasks-files.dat','w+')
+
+license_in_component = {'main' : 'DFSG free',
+ 'contrib' : 'DFSG free, but needs non-free components',
+ 'non-free' : 'non-free'
+ }
+
+try:
+ import psutil
+ has_psutils=True
+except ImportError:
+ has_psutils=False
+
+def SetFilePermissions(usefile):
+ try:
+ blendsgid = grp.getgrnam("blends").gr_gid
+ os.chown(usefile, -1, blendsgid)
+ os.chmod(usefile, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH )
+ # os.system("ls -l %s" % usefile)
+ except KeyError:
+ # if groups 'blends' does not exist on the machine we are simply testing and setting group permissions is not needed
+ pass
+
+LOCKFILE='/var/lock/blends.lock'
+def LockBlendsTools():
+ """Locking mechanism to make sure the scripts will not run in parallel
+ which happened because of IO problems on udd.debian.org"""
+ if not has_psutils:
+ logger.warning("Package python-psutil is missing. No locking support available")
+ return
+ if os.path.exists(LOCKFILE):
+ try:
+ lf = open(LOCKFILE, 'r')
+ pid = int(lf.readline())
+ lf.close()
+ if pid in psutil.get_pid_list():
+ logger.error("Another process rebuilding web sentinel pages with PID %i is running. Exit." % pid)
+ exit()
+ else:
+ logger.warning("Process with PID %i is not running any more but lockfile remained. Removing %s ..." % (pid, LOCKFILE))
+ os.unlink(LOCKFILE)
+ except IOError as e:
+ pass
+ pid = os.getpid()
+ lf = open(LOCKFILE, 'w')
+ print >>lf, pid
+ lf.close()
+ SetFilePermissions(LOCKFILE)
+
+def UnlockBlendsTools():
+ """Unlock previousely locked file"""
+ if os.path.exists(LOCKFILE):
+ os.unlink(LOCKFILE)
+
+def GetDependencies2Use(dependencystatus=[], max_order='prospective'):
+ # Create a list of status of dependencies out of pkgstatus dictionary
+ use_dependencystatus = []
+ if dependencystatus == []:
+ for pkgstat in pkgstatus_sortedkeys:
+ # per default = if no expliczite dependencystatus are given, we are only interested in
+ # dependencies which are at least of order experimental
+ if pkgstatus[pkgstat]['order'] > pkgstatus[max_order]['order']:
+ continue
+ use_dependencystatus.append(pkgstat)
+ else:
+ # verify correctly given dependencies
+ for pkgstat in dependencystatus:
+ if pkgstat in pkgstatus.keys():
+ use_dependencystatus.append(pkgstat)
+ else:
+ logger.error("Unknown dependencystatus %s" % pkgstat)
+ if use_dependencystatus == []:
+ logger.error("No valid dependencystatus in", dependencystatus)
+ return use_dependencystatus
+
+###########################################################################################
+# Define several prepared statements to query UDD
+try:
+ conn = psycopg2.connect(host="localhost",port=PORT,user="guest",database="udd")
+except psycopg2.OperationalError, err:
+ try:
+ conn = psycopg2.connect("service=udd")
+ except psycopg2.OperationalError, err:
+ # logger not known at this state: logger.warning
+ print >>stderr, "Service=udd seems not to be installed on this host.\tMessage: %s" % (str(err))
+ try:
+ conn = psycopg2.connect(host="localhost",port=DEFAULTPORT,user="guest",database="udd")
+ except psycopg2.OperationalError:
+ # Hmmm, I observed a really strange behaviour on one of my machines where connecting to
+ # localhost does not work but 127.0.0.1 works fine. No idea why ... but this should
+ # do the trick for the moment
+ conn = psycopg2.connect(host="127.0.0.1",port=DEFAULTPORT,user="guest",database="udd")
+
+curs = conn.cursor()
+# uddlog = open('logs/uddquery.log', 'w')
+
+def _execute_udd_query(query):
+ try:
+ curs.execute(query)
+ logger.debug(query)
+ except psycopg2.ProgrammingError, err:
+ print >>stderr, "Problem with query\n%s" % (to_unicode(query))
+ print >>stderr, err
+ exit(-1)
+ except psycopg2.DataError, err:
+ print >>stderr, "%s; query was\n%s" % (err, query)
+
+
+query = """PREPARE query_pkgs (text[],text[]) AS
+ SELECT * FROM blends_query_packages($1,$2) AS (
+ package text, distribution text, release text, component text, version debversion,
+ maintainer text,
+ source text, section text, task text, homepage text,
+ maintainer_name text, maintainer_email text,
+ "vcs-type" text, "vcs-url" text, "vcs-browser" text,
+ changed_by text,
+ enhanced text[],
+ releases text[], versions text[], architectures text[],
+ unstable_upstream text, unstable_parsed_version text, unstable_status text,
+ vote int, recent int, -- popcon
+ debtags text[],
+ screenshot_versions text[], image text[], icon text[],
+ year text,
+ title text,
+ authors text,
+ doi text,
+ pubmed text,
+ url text,
+ journal text,
+ volume text,
+ number text,
+ pages text,
+ eprint text,
+ description_en text, long_description_en text,
+ description_cs text, long_description_cs text,
+ description_da text, long_description_da text,
+ description_de text, long_description_de text,
+ description_es text, long_description_es text,
+ description_fi text, long_description_fi text,
+ description_fr text, long_description_fr text,
+ description_hu text, long_description_hu text,
+ description_it text, long_description_it text,
+ description_ja text, long_description_ja text,
+ description_ko text, long_description_ko text,
+ description_nl text, long_description_nl text,
+ description_pl text, long_description_pl text,
+ "description_pt_BR" text, "long_description_pt_BR" text,
+ description_ru text, long_description_ru text,
+ description_sk text, long_description_sk text,
+ description_sr text, long_description_sr text,
+ description_sv text, long_description_sv text,
+ description_uk text, long_description_uk text,
+ "description_zh_CN" text, "long_description_zh_CN" text,
+ "description_zh_TW" text, "long_description_zh_TW" text
+ )"""
+_execute_udd_query(query)
+
+query = """PREPARE query_new (text[]) AS SELECT
+ p.package,
+ p.distribution, p.component, p.version, p.architecture, p.maintainer,
+ p.source, p.section, p.distribution, 'new' AS release, p.component, p.homepage,
+ s.changed_by,
+ description AS description_en, long_description AS long_description_en,
+ bibyear.value AS "year",
+ bibtitle.value AS "title",
+ bibauthor.value AS "authors",
+ bibdoi.value AS "doi",
+ bibpmid.value AS "pubmed",
+ biburl.value AS "url",
+ bibjournal.value AS "journal",
+ bibvolume.value AS "volume",
+ bibnumber.value AS "number",
+ bibpages.value AS "pages",
+ bibeprint.value AS "eprint"
+ FROM new_packages p
+ JOIN new_sources s ON p.source = s.source AND p.version = s.version
+ LEFT OUTER JOIN bibref bibyear ON p.source = bibyear.source AND bibyear.rank = 0 AND bibyear.key = 'year' AND bibyear.package = ''
+ LEFT OUTER JOIN bibref bibtitle ON p.source = bibtitle.source AND bibtitle.rank = 0 AND bibtitle.key = 'title' AND bibtitle.package = ''
+ LEFT OUTER JOIN bibref bibauthor ON p.source = bibauthor.source AND bibauthor.rank = 0 AND bibauthor.key = 'author' AND bibauthor.package = ''
+ LEFT OUTER JOIN bibref bibdoi ON p.source = bibdoi.source AND bibdoi.rank = 0 AND bibdoi.key = 'doi' AND bibdoi.package = ''
+ LEFT OUTER JOIN bibref bibpmid ON p.source = bibpmid.source AND bibpmid.rank = 0 AND bibpmid.key = 'pmid' AND bibpmid.package = ''
+ LEFT OUTER JOIN bibref biburl ON p.source = biburl.source AND biburl.rank = 0 AND biburl.key = 'url' AND biburl.package = ''
+ LEFT OUTER JOIN bibref bibjournal ON p.source = bibjournal.source AND bibjournal.rank = 0 AND bibjournal.key = 'journal' AND bibjournal.package = ''
+ LEFT OUTER JOIN bibref bibvolume ON p.source = bibvolume.source AND bibvolume.rank = 0 AND bibvolume.key = 'volume' AND bibvolume.package = ''
+ LEFT OUTER JOIN bibref bibnumber ON p.source = bibnumber.source AND bibnumber.rank = 0 AND bibnumber.key = 'number' AND bibnumber.package = ''
+ LEFT OUTER JOIN bibref bibpages ON p.source = bibpages.source AND bibpages.rank = 0 AND bibpages.key = 'pages' AND bibpages.package = ''
+ LEFT OUTER JOIN bibref bibeprint ON p.source = bibeprint.source AND bibeprint.rank = 0 AND bibeprint.key = 'eprint' AND bibeprint.package = ''
+ WHERE (p.package, p.version) IN
+ (SELECT package, max(version) FROM
+ new_packages WHERE package = ANY ($1) GROUP BY package)"""
+_execute_udd_query(query)
+
+query = """PREPARE query_vcs (text[]) AS SELECT
+ p.package,
+ p.component, p.maintainer,
+ p.source, p.section, 'vcs' AS release, p.component, p.homepage,
+ p.changed_by, p.chlog_version AS version,
+ description AS description_en, long_description AS long_description_en,
+ p.vcs_url AS "vcs-url",
+ p.vcs_type AS "vcs-type",
+ p.vcs_browser AS "vcs-browser",
+ p.blend,
+ p.license,
+ p.wnpp,
+ bibyear.value AS "year",
+ bibtitle.value AS "title",
+ bibauthor.value AS "authors",
+ bibdoi.value AS "doi",
+ bibpmid.value AS "pubmed",
+ biburl.value AS "url",
+ bibjournal.value AS "journal",
+ bibvolume.value AS "volume",
+ bibnumber.value AS "number",
+ bibpages.value AS "pages",
+ bibeprint.value AS "eprint"
+ FROM blends_prospectivepackages p
+ LEFT OUTER JOIN bibref bibyear ON p.source = bibyear.source AND bibyear.rank = 0 AND bibyear.key = 'year' AND bibyear.package = ''
+ LEFT OUTER JOIN bibref bibtitle ON p.source = bibtitle.source AND bibtitle.rank = 0 AND bibtitle.key = 'title' AND bibtitle.package = ''
+ LEFT OUTER JOIN bibref bibauthor ON p.source = bibauthor.source AND bibauthor.rank = 0 AND bibauthor.key = 'author' AND bibauthor.package = ''
+ LEFT OUTER JOIN bibref bibdoi ON p.source = bibdoi.source AND bibdoi.rank = 0 AND bibdoi.key = 'doi' AND bibdoi.package = ''
+ LEFT OUTER JOIN bibref bibpmid ON p.source = bibpmid.source AND bibpmid.rank = 0 AND bibpmid.key = 'pmid' AND bibpmid.package = ''
+ LEFT OUTER JOIN bibref biburl ON p.source = biburl.source AND biburl.rank = 0 AND biburl.key = 'url' AND biburl.package = ''
+ LEFT OUTER JOIN bibref bibjournal ON p.source = bibjournal.source AND bibjournal.rank = 0 AND bibjournal.key = 'journal' AND bibjournal.package = ''
+ LEFT OUTER JOIN bibref bibvolume ON p.source = bibvolume.source AND bibvolume.rank = 0 AND bibvolume.key = 'volume' AND bibvolume.package = ''
+ LEFT OUTER JOIN bibref bibnumber ON p.source = bibnumber.source AND bibnumber.rank = 0 AND bibnumber.key = 'number' AND bibnumber.package = ''
+ LEFT OUTER JOIN bibref bibpages ON p.source = bibpages.source AND bibpages.rank = 0 AND bibpages.key = 'pages' AND bibpages.package = ''
+ LEFT OUTER JOIN bibref bibeprint ON p.source = bibeprint.source AND bibeprint.rank = 0 AND bibeprint.key = 'eprint' AND bibeprint.package = ''
+ WHERE p.package = ANY ($1)"""
+_execute_udd_query(query)
+
+# This prepared statement is called only once but it makes sense to mention it in the
+# header to keep all stuff in one place which needs to be changed in case a new language
+# will be added
+query = """PREPARE query_metapkg_trans (text[]) AS
+ SELECT * FROM blends_metapackage_translations($1) AS (
+ package text,
+ description_en text, long_description_en text,
+ description_cs text, long_description_cs text,
+ description_da text, long_description_da text,
+ description_de text, long_description_de text,
+ description_es text, long_description_es text,
+ description_fi text, long_description_fi text,
+ description_fr text, long_description_fr text,
+ description_hu text, long_description_hu text,
+ description_it text, long_description_it text,
+ description_ja text, long_description_ja text,
+ description_ko text, long_description_ko text,
+ description_nl text, long_description_nl text,
+ description_pl text, long_description_pl text,
+ "description_pt_BR" text, "long_description_pt_BR" text,
+ description_ru text, long_description_ru text,
+ description_sk text, long_description_sk text,
+ description_sr text, long_description_sr text,
+ description_sv text, long_description_sv text,
+ description_uk text, long_description_uk text,
+ "description_zh_CN" text, "long_description_zh_CN" text,
+ "description_zh_TW" text, "long_description_zh_TW" text
+ )"""
+_execute_udd_query(query)
+
+# Sometimes the tasks file contains dependencies from virtual packages and we have to
+# obtain the real packages which provide this dependency.
+# First check whether there are such packages (only names)
+query = """PREPARE query_provides (text[]) AS
+ SELECT DISTINCT package, provides FROM packages WHERE provides IS NOT NULL AND package = ANY($1) ;"""
+_execute_udd_query(query)
+
+# Obtain more detailed information about packages that might provide a dependency
+#query = """PREPARE query_provides_version_release (text) AS
+# SELECT package, version, release FROM packages WHERE provides = $1
+# GROUP BY version, package, release ORDER BY version DESC;"""
+#_execute_udd_query(query)
+
+# Obtain the releases featuring a certain package, in case a package might show up in different components when
+# considering different releases we apply a preference for main over contrib over non-free. If this is the case
+# we mention only the releases of the selected component
+query = "PREPARE pkg_releases (text, text) AS SELECT release FROM packages WHERE package = $1 AND component = $2 GROUP BY release "
+_execute_udd_query(query)
+
+query = """PREPARE pkg_versions_stable_testing (text) AS
+ SELECT release, regexp_replace(regexp_replace(debversion, '-.*', ''), '[.+~]dfsg.*', '') AS version, debversion
+ FROM (SELECT r AS release, MAX(version) AS debversion
+ FROM versions_archs_component($1) AS (r text, version text, archs text, component text)
+ WHERE r IN ('jessie', 'stretch') GROUP BY r) AS zw;""" # FIXME: Do not hardcode names here!!!
+_execute_udd_query(query)
+
+# Number of all submissions
+query = "PREPARE popcon_submissions AS SELECT vote FROM popcon WHERE package = '_submissions'"
+_execute_udd_query(query)
+
+
+def List2PgArray(list):
+ # turn a list of strings into the syntax for a PostgreSQL array:
+ # {"string1","string2",...,"stringN"}
+ if not list:
+ return '{}'
+ komma='{'
+ PgArray=''
+ for s in list:
+ PgArray=PgArray+komma+'"'+s+'"'
+ komma=','
+ return PgArray+'}'
+
+def List2PgSimilarArray(list):
+ # turn a list of strings surrounded by '%' into the syntax for a PostgreSQL array to enable LIKE conditions:
+ # {"%string1%","%string2%",...,"%stringN%"}
+ if not list:
+ return '{}'
+ komma='{'
+ PgSimArray=''
+ for s in list:
+ PgSimArray=PgSimArray+komma+'"%'+s+'%"'
+ komma=','
+ return PgSimArray+'}'
+
+def ReadConfig(blendname=''):
+ # Try to read config file CONFDIR/<blendname>.conf
+ conffile = CONFDIR + '/' + blendname + '.conf'
+ if not os.access(conffile, os.R_OK):
+ # if config file can not be found in local dir, try /etc/blends/webconf as fallback
+ conffile_default = '/etc/blends/webconf/' + blendname + '.conf'
+ if not os.access(conffile_default, os.R_OK):
+ logger.error("Unable to open config file %s or %s." % (conffile, conffile_default))
+ exit(-1)
+ conffile = conffile_default
+ f = open(conffile, 'r')
+ ret = { 'Blend' : '',
+ 'projectname' : '',
+ 'projecturl' : '',
+ 'homepage' : '',
+ 'aliothurl' : '',
+ 'projectlist' : '',
+ 'pkglist' : '',
+ 'logourl' : '',
+ 'css' : '',
+ 'outputdir' : '',
+ 'datadir' : '',
+ 'advertising' : None, # headline with advertising string is optional
+ 'ubuntuhome' : None,
+ 'projectubuntu':None,
+ }
+ for stanza in deb822.Sources.iter_paragraphs(f, shared_storage=False):
+ ret['Blend'] = stanza['blend'] # short name of the project
+ ret['projectname'] = stanza['projectname'] # Printed name of the project
+ ret['projecturl'] = stanza['projecturl'] # Link to the developer page with dynamic content
+ # like for instance these tasks pages
+ ret['homepage'] = stanza['homepage'] # Homepage with general information about the project
+ # for instance at www.debian.org or wiki.debian.org
+ ret['aliothurl'] = stanza['aliothurl'] # Link to the Alioth page of the project
+ ret['projectlist'] = stanza['projectlist'] # Mailinglist of the project
+ if stanza.has_key('pkglist'):
+ ret['pkglist'] = stanza['pkglist'] # Packaging Mailinglist = Maintainer of group maintained packages
+ if stanza.has_key('logourl'):
+ ret['logourl'] = stanza['logourl'] # URL to logo image (might be missing
+ ret['css'] = stanza['css'] # (relative) URL to CSS file
+ ret['outputdir'] = stanza['outputdir'] # Dir for storing output HTML files
+ ret['datadir'] = stanza['datadir'] # Dir for storing SVN information about project
+ ret['vcsdir'] = stanza['vcsdir'] # Path to Blend information files at svn.debian.org
+ if stanza.has_key('advertising'):
+ # we have to remove the gettext _() call which was inserted into the config
+ # file to enable easy input for config file editors - but the call has to
+ # be made explicitely in the python code
+ advertising = re.sub('_\(\W(.+)\W\)', '\\1', stanza['advertising'])
+ # gettext needs to escape '"' thus we need to remove the escape character '\'
+ ret['advertising'] = re.sub('\\\\"', '"', advertising)
+ if stanza.has_key('ubuntuhome'):
+ ret['ubuntuhome'] = stanza['ubuntuhome']
+ if stanza.has_key('projectubuntu'):
+ ret['projectubuntu'] = stanza['projectubuntu']
+
+ return ret
+
+def CheckOrCreateOutputDir(maindir, subdir):
+ outputdir = maindir + '/' + subdir
+ if not os.access(outputdir, os.W_OK):
+ try:
+ os.mkdir(outputdir)
+ except:
+ # if more than one dir in the tree has to be crated just use mkdir -p ...
+ try:
+ os.system("mkdir -p %s" % outputdir)
+ except:
+ logger.error("Unable to create output dir " + outputdir)
+ return None
+ return outputdir
+
+def RowDictionaries(cursor):
+ """Return a list of dictionaries which specify the values by their column names"""
+
+ description = cursor.description
+ if not description:
+ # even if there are no data sets to return the description should contain the table structure. If not something went
+ # wrong and we return NULL as to represent a problem
+ return NULL
+ if cursor.rowcount <= 0:
+ # if there are no rows in the cursor we return an empty list
+ return []
+
+ data = cursor.fetchall()
+ result = []
+
+ for row in data:
+ resultrow = {}
+ i = 0
+ for dd in description:
+ resultrow[dd[0]] = row[i]
+ i += 1
+ result.append(resultrow)
+ return result
+
+def BrowserFromVcsURL(vcs_type, vcs_url):
+ # Guess Vcs-Browser URL from VCS URL
+ if not vcs_type:
+ return HOMEPAGENONE
+ if vcs_type.lower().startswith('svn'):
+ ret_url = re.sub('^svn:', 'http:', vcs_url)
+ ret_url = re.sub('/svn/', '/wsvn/', ret_url)
+ ret_url = re.sub('$', '?rev=0&sc=0', ret_url)
+ elif vcs_type.lower().startswith('git'):
+ ret_url = re.sub('^git:', 'http:', vcs_url)
+ ret_url = re.sub('/git/', '/?p=', ret_url)
+ elif vcs_type.lower().startswith('hg'):
+ # Seems that vcs_browser = vcs_url in Mercurial
+ return vcs_url
+ elif vcs_type.lower().startswith('bzr') or vcs_type.lower().startswith('cvs'):
+ logger.warning("No idea how to guess vcs_browser for %s URLS" % vcs_type)
+ return
+ else:
+ logger.warning("Unknown VCS for " + vcs_url)
+ return HOMEPAGENONE
+ if vcs_url.startswith('https://github.com') and vcs_url.endswith('.git'):
+ ret_url = re.sub('.git$', '', vcs_url)
+ elif vcs_url.startswith('http:'):
+ return vcs_url
+ if ret_url == vcs_url:
+ logger.warning("Unable to obtain Vcs-Browser from " + vcs_url)
+ return HOMEPAGENONE
+ return ret_url
+
+detect_vcs_cvs_re = re.compile("://.*cvs")
+detect_vcs_svn_re = re.compile("://.*svn")
+detect_vcs_git_re = re.compile("://.*git")
+
+def VcsTypeFromBrowserURL(vcs_browser):
+ # Guess Vcs-Type from vcs_browser
+ if detect_vcs_cvs_re.search(vcs_browser):
+ return 'Cvs'
+ if detect_vcs_svn_re.search(vcs_browser):
+ return 'Svn'
+ if detect_vcs_git_re.search(vcs_browser):
+ return 'Git'
+ return 'Unknown VCS'
+
+
+# The following keys will be mostly used for programs that
+# are not yet existing in Debian and will go to our todo list
+PROPERTIES=('homepage', # Homepage of program
+ 'section', # Section of package in the Debian hierarchy
+ 'source', # Keep the source package name which is needed for ddpo subscription
+ )
+
+###################################################################################################################
+#=================================================================================================================#
+###################################################################################################################
+
+#######################################################
+
+
+def GetPkgstat(strength):
+ if strength == 'd' or strength == 'r':
+ pkgstat = 'official_high'
+ elif strength == 's':
+ pkgstat = 'official_low'
+ elif strength == 'i':
+ pkgstat = 'ignore'
+ elif strength == 'a':
+ pkgstat = 'avoid'
+ else:
+ pkgstat = 'unknown'
+ return pkgstat
+
+
+class DependantPackage:
+ # Hold information about a package that is in dependency list
+
+ def __init__(self, blendname=None, taskname=None):
+ self.blendname = blendname # Blend that includes the package in dependency list
+ self.taskname = taskname # Task which includes the Dependency
+ self.pkg = None # Name of dependant package
+ self.source = None # Name of source package of the dependency
+ self.PrintedName = None # Only for Meta package names - no use for a real dependant package
+ # FIXME -> object model
+ self.pkgstatus = 'unknown' # global pkgstatus: characterizes status of dependency, release, packaging status
+ self.releases = [] # List of releases a package might be in
+ self.component = None # One of: 'main', 'contrib', 'non-free', if a package shows up in several components which
+ # might happen over different releases, just prefer main over contrib over non-free
+ self.why = None # basically used as comment
+
+ self.properties = {}
+ self.properties['license'] = 'unknown'
+ for field in HOMEPAGENONEFIELDS:
+ self.properties[field] = HOMEPAGENONE
+ self.properties['Enhances'] = {} # Dictionary Enhancing pkg name as key, Link to package information as value; empty in most cases
+ # because Enhances relations are quite seldom
+ self.properties['stable_testing_version'] = [] # (release, version) tuples where release is codename for stable and testing
+ self.vcs_found = 0 # we need a flag to store the fact whether Vcs information of a package is in UDD
+ self.version = [] # list of {'release', 'version', 'archs'} dictionary containing version and architecture information
+ self.outdated = {} # If not empty directory then release='upstream' and package is outdated
+ self.popcon = {} # dictionary containing vote and recnt values of popcon information
+ self.popconsubmit = 0 # number of popcon submissions - in case popcon import into UDD might be broken this remains 0
+ self.debtags = [] # list of {'tag', 'value'} dictionary containing debtag information
+ self.screenshots = [] # list of {'version', 'url'} dictionary containing screenshot information
+ self.icon = None # URL of small screenshot icon
+ self.screenshot_url = None # URL to screenshots.debian.net
+ self.responsible = None # E-Mail address of issuer of ITP or some person
+ # who volunteered to care for this program
+ self.filename = None # Filename of package in the Debian pool
+ self.desc = {} # Prospective packages should have a description ...
+ # ... which could be copied to (or from if exists)
+ # WNPP bug and finally can be used for packaging
+ self.desc['en'] = {} # An English description should be available in any case
+ self.experimental = 0 # Set to 1 if package *only* in experimental but not in unstable/testing/stable
+ self.remark = {} # Optional remark for a package
+ self.dep_strength = 0 # Type of Dependency (Depends, Recommends, Suggests, Experimental, New, Avoid, Ignore, WNPP
+
+
+ # sort these objects according to the package name
+ def __cmp__(self, other):
+ # Comparing with None object has to return something reasonable
+ if other == None:
+ return -2
+ # Sort according to package name
+ return cmp(self.pkg, other.pkg)
+
+ def __str__(self):
+ ret = "Blend: " + self.blendname + ", " \
+ "Task:" + self.taskname + ", " \
+ "pkg:" + self.pkg
+ if self.dep_strength:
+ ret += ", Dep_strength: " + self.dep_strength
+ if self.pkgstatus:
+ ret += ", Pkg_status: " + str(self.pkgstatus)
+ if self.releases:
+ ret += ", releases: " + str(self.releases)
+ if self.component:
+ ret += ", component: " + self.component
+ if self.version:
+ ret += ", versions: " + str(self.version)
+ if self.desc:
+ ret += ", desc: " + str(self.desc)
+ for prop in self.properties.keys():
+ try:
+ ret += ", %s: %s" % (prop, to_unicode(str(self.properties[prop])))
+ except UnicodeEncodeError:
+ ret += ", %s: <UnicodeEncodeError>" % (prop)
+ try:
+ ret += ", popcon = %i (%i)" % (self.popcon['vote'], self.popcon['recent'])
+ except:
+ pass
+ ret += ", debtags = " + str(self.debtags)
+ if self.icon:
+ ret += ", screenshoticon = " + self.icon
+ if self.outdated:
+ ret += ", dehsstatus = " + str(self.outdated)
+
+ # if self.desc['en']:
+ # ret += ", desc['en']:" + str(self.desc['en'])
+ return ret
+
+
+class Tasks:
+ # Provide a list of depencencies defined in metapackages
+ # This class concerns _all_ tasks of a Blend and is the most
+ # complete source of information. If only a single task
+ # should be handled by a tool that uses blendtasktools
+ # probably the class (see below) is
+ # your friend
+
+ def __init__(self, blendname):
+
+ os.system("mkdir -p logs")
+ LOG_FILENAME = 'logs/'+blendname+'.log'
+ handler = logging.handlers.RotatingFileHandler(filename=LOG_FILENAME,mode='w')
+ formatter = logging.Formatter("%(levelname)s - %(filename)s (%(lineno)d): %(message)s")
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ LockBlendsTools()
+
+ # This Instance of the Available class contains all
+ # information about packages that are avialable in Debian
+ # See below for more information about Available class
+ self.data = ReadConfig(blendname)
+ self.blendname = self.data['Blend']
+ self._InitMetapackages()
+ self.tasks = {} # Dictionary of TasksDependency objects
+ self.alldeps_in_main = [] # sorted string list of package names with all packages
+ # relevant for a Blend that are in main Debian (for use in DDTP)
+ self.alldeps_in_main_info = {} # complete dictionary with package information
+ # with all packages relevant for a Blend that are in
+ # main to easily feed DDTP translation into the structures
+ # -->
+ # self.alldeps_in_main = self.alldeps_in_main_info.keys().sort()
+
+ def _InitMetapackages(self):
+ # sorted list of metapackage names
+ self.metapackagekeys = []
+ query = "SELECT task FROM blends_tasks WHERE blend = '%s' ORDER BY task" % (self.blendname)
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ temp = curs.fetchall()
+ self.metapackagekeys = [t[0] for t in temp]
+
+
+ def GetAllDependencies(self, source=0):
+ # If we want to subscribe ddpo we need the source package names.
+ # In this case set source=1
+
+ # Obtain the prefix of the meta packages of the Blend
+ prefix = self.blendname.split('-')[1] + '-'
+
+ metapackages = []
+ for task in self.metapackagekeys:
+ metapackages.append(prefix + task)
+
+ # Verify whether there are any translations of metapackage descriptions which on
+ # one hand is a sign that this Blend has uploaded metapackages at all and on the
+ # other hand gives a clue about whether it makes sense to query for description
+ # translations
+ # to verify whether a blend has uploaded metapackage or not we can check the boolean
+ # column "metapackage" in blends_tasks table
+ query = "SELECT COUNT(*) FROM descriptions WHERE package = ANY ('%s')" % List2PgArray(metapackages)
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ hastranslations = curs.fetchone()[0]
+
+ metapkg_translations = {}
+ if hastranslations > 0:
+ query = "EXECUTE query_metapkg_trans('%s')" % List2PgArray(metapackages)
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ for row in RowDictionaries(curs):
+ metapkg_translations[row['package']] = row
+
+ for task in self.metapackagekeys:
+ td = TaskDependencies(self.blendname, task=task)
+ pkgname = prefix + task
+ translations = None
+ if metapkg_translations.has_key(pkgname):
+ translations = metapkg_translations[pkgname]
+
+ td.SetMetapackageInfo(pkgname, translations)
+ print "task : ",task
+ if td.GetTaskDependencies(source):
+ self.tasks[task] = td
+
+ else: # Kick file that is obviously no task file from metapackage list
+ self.metapackagekeys = filter(lambda name: name != task, self.metapackagekeys)
+
+ if source == 0:
+ # total number popcon submissions
+ query = "EXECUTE popcon_submissions"
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ self.popconsubmit = curs.fetchone()[0]
+ # Obtain packages that might enhance any of the packages in tasks list
+ self.LinkEnhances()
+
+ def GetNamesOnlyDict(self, dependencystatus=[]):
+ # David Paleino needs for his web tools a dictionary
+ # { taskname : [list of dependencies]}
+ # This will be prepared here from the main
+ # datastructure
+ ret = {}
+ use_dependencystatus = GetDependencies2Use(dependencystatus, 'experimental')
+
+ for task in self.metapackagekeys:
+ tdeps = self.tasks[task]
+ list = []
+ for dep in use_dependencystatus:
+ for tdep in tdeps.dependencies[dep]:
+ list.append(tdep.pkg)
+ ret[task] = list
+ return ret
+
+ def GetAllDependentPackagesOfBlend(self, dependencystatus=[]):
+ # David Paleino needs for his DDTP web tool a list of
+ # all available Dependencies.
+ # Here only those packages are returned that are in
+ # Debian main, because there are no DDTP translations
+ # for contrib and non-free available
+ if self.alldeps_in_main != []:
+ return self.alldeps_in_main
+
+ use_dependencystatus = GetDependencies2Use(dependencystatus, 'unknown')
+ for task in self.metapackagekeys:
+ tdeps = self.tasks[task]
+ for dep in use_dependencystatus:
+ for tdep in tdeps.dependencies[dep]:
+ # add only packages in main, because others do not have DDTP translations
+ if tdep.component == 'main':
+ self.alldeps_in_main.append(tdep.pkg)
+ self.alldeps_in_main_info[tdep.pkg] = tdep
+ # Also add meta package itself to make use of translated meta package description
+ self.alldeps_in_main.append(self.tasks[task].metapkg.pkg)
+ self.alldeps_in_main_info[self.tasks[task].metapkg.pkg] = self.tasks[task].metapkg
+ self.alldeps_in_main.sort()
+ return self.alldeps_in_main
+
+
+ def GetTaskDescDict(self):
+ # Return dictionary with description information of all tasks of a Blend
+ return self.tasks
+
+ def MarkupPreformatedStringsBlend(self):
+ # Genshi does not touch strings that are marked with "Markup()" - so just
+ # mark the strings that are ready formatted for the whole Blend
+ for task in self.metapackagekeys:
+ tdeps = self.tasks[task]
+ tdeps.MarkupPreformatedStrings()
+
+ def LinkEnhances(self):
+ # To provide a connection between packages enhancing other packages a set of links
+ # will be provided. The links should point to paragraphs on the tasks pages if the
+ # Enhancing package is listed in the metapackages of the blend and to packages.debian.org
+ # otherwise
+ for task in self.metapackagekeys:
+ tdeps = self.tasks[task]
+ for dependency in tdeps.dependencies.keys():
+ for dep in tdeps.dependencies[dependency]:
+ if dep.properties['Enhances'] != {}:
+ logger.debug("Package %s is enhanced by:" % dep.pkg)
+ for enh in dep.properties['Enhances'].keys():
+ # seek for Enhances on same page
+ found = 0
+ for seek_dependency in tdeps.dependencies.keys():
+ for enhdep in tdeps.dependencies[seek_dependency]:
+ if enh == enhdep.pkg:
+ dep.properties['Enhances'][enh] = '#'+enh
+ found = 1 # found enhances in same task
+ break
+ if found == 0: # If not found seek in other tasks
+ for enhtask in self.metapackagekeys:
+ if enhtask == task:
+ continue
+ enhtdeps = self.tasks[enhtask]
+ for seek_dependency in enhtdeps.dependencies.keys():
+ for enhdep in enhtdeps.dependencies[seek_dependency]:
+ if enh == enhdep.pkg:
+ dep.properties['Enhances'][enh] = './' + enhtask + '#' + enh
+ found = 1 # found enhances in other task
+ break
+ if found == 1:
+ break
+ logger.debug(" %s -> %s" % (enh, dep.properties['Enhances'][enh]))
+
+
+class TaskDependencies:
+ # List of depencencies defined in one metapackage
+ def __init__(self, blendname, task):
+
+ self.data = ReadConfig(blendname)
+ self.blendname = self.data['Blend']
+ self.task = task
+
+ # Dictionary with status of dependencies as key and list of DependantPackage
+ # instances
+ self.dependencies = {}
+ for pkgstat in pkgstatus:
+ self.dependencies[pkgstat] = []
+
+ # If a Blend just bases on the meta package of an other Blend (this is the
+ # case in Debian Science which bases on med-bio for biology and gis-workstation
+ # for geography it makes no sense to build an own sentinel page but read
+ # meta package information of other meta packages and include the content
+ # of these while enabling to add further Dependencies as well
+ #
+ # metadepends should be a SVN URL
+ #
+ # This is NOT YET implemented
+ self.metadepends = None
+
+
+ def SetMetapackageInfo(self, pkgname, ddtptranslations=None):
+ # Gather information (specifically description translations if exists) about metapackage itself
+ self.metapkg = DependantPackage(self.blendname, self.task)
+ self.metapkg.pkg = pkgname
+ self.metapkg.source = self.blendname
+ if not ddtptranslations:
+ return
+ for lang in languages:
+ if ddtptranslations['description_'+lang]:
+ self.metapkg.desc[lang] = {}
+ try:
+ short = to_unicode(ddtptranslations['description_'+lang])
+ self.metapkg.desc[lang]['short'] = MarkupString(short, self.metapkg.pkg, 'taskShortDesc', lang)
+
+ except UnicodeEncodeError, err:
+ logger.error("===> UnicodeDecodeError in metapackage %s (lang='%s'): '%s'; ErrTxt: %s" % \
+ (self.metapkg.pkg, lang, ddtptranslations['description_'+lang], err))
+ short = to_unicode(ddtptranslations['description_'+lang],'latin1')
+ self.metapkg.desc[lang]['short'] = MarkupString(short, self.metapkg.pkg, 'taskShortDesc' + lang)
+
+ try:
+ self.metapkg.desc[lang]['long'] = Markup(render_longdesc(ddtptranslations['long_description_'+lang].splitlines()))
+
+ except UnicodeDecodeError, err:
+ logger.error("===> UnicodeDecodeError in metapackage long %s (lang='%s'): '%s'; ErrTxt: %s" % \
+ (self.metapkg.pkg, lang, ddtptranslations['long_description_'+lang], err))
+ self.metapkg.desc[lang]['long'] = 'UnicodeDecodeError'
+
+ except AttributeError, err:
+ logger.error("===> AttributeError in metapackage long %s (lang='%s'): '%s'; ErrTxt: %s" % \
+ (self.metapkg.pkg, lang, ddtptranslations['long_description_'+lang], err))
+ self.metapkg.desc[lang]['long'] = 'Missing long description'
+
+ def GetDepInfo(self, curs, dependencies):
+
+ for row in RowDictionaries(curs):
+ # seek for package name in list of packages mentioned in tasks file
+ found = False
+ for dep in dependencies:
+ if dep.pkg == row['package']:
+ found = True
+ break
+
+ # Now set the information for the package found in the database
+ # Debian Edu contains packages from main/debian-installer - that's why we use startswith here
+ if row['component'].startswith('main'):
+ dep.component = 'main'
+
+ if dep.dep_strength == 'd' or dep.dep_strength == 'r':
+ dep.pkgstatus = 'official_high'
+ elif dep.dep_strength == 's':
+ dep.pkgstatus = 'official_low'
+ else:
+ dep.component = row['component']
+ # If a package is not found in main its status can be maximum non-free
+ dep.pkgstatus = 'non-free'
+
+ # if a package is released *only* in experimental decrease package status
+ if row['release'] == 'experimental':
+ dep.pkgstatus = 'experimental'
+
+ # dep.properties['license'] is already correct
+ for prop in PROPERTIES:
+ dep.properties[prop] = row[prop]
+
+ for prop in ('vcs-type', 'vcs-url'):
+ dep.properties[prop] = row[prop]
+
+ if row['vcs-browser']:
+ dep.properties['vcs-browser'] = row['vcs-browser']
+ else:
+ if dep.properties['vcs-browser'] == HOMEPAGENONE:
+ dep.properties['vcs-browser'] = BrowserFromVcsURL(dep.properties['vcs-type'], dep.properties['vcs-url'])
+
+ # enhances to be written
+ # if row['enhanced']:
+ # for pkg in row['enhanced']:
+ # dep.properties['Enhances'][pkg] = PKGURLMASK % pkg
+ # enhancing_pkgs.append(pkg)
+
+ for i in range(len(row['releases'])):
+ dep.version.append({'release':row['releases'][i], 'version': row['versions'][i], 'archs':row['architectures'][i]})
+
+
+ dep.popcon['vote'] = row['vote']
+ dep.popcon['recent'] = row['recent']
+
+
+ # Debtags as sorted list of dict fields
+ if row['debtags']:
+ if dep.debtags: # there is no reasonable way that debtags was set before - so something is wrong here and a warning should be issued
+ logger.warning("Debtags for package '%s' was just set. A duplicated result from database query is suspected. Please check the result!" % dep.pkg)
+ tagdict = {}
+ taglist = []
+ for debtag in row['debtags']:
+ (tag,value) = debtag.split('::')
+ if tagdict.has_key(tag):
+ tagdict[tag] += ', ' + value
+ else:
+ tagdict[tag] = value
+ taglist.append(tag)
+ if taglist:
+ taglist.sort()
+
+ for tag in taglist:
+ dep.debtags.append({'tag':tag, 'value':tagdict[tag]})
+
+ # screenshots
+ if row['icon']:
+ dep.icon = row['icon'][0]
+ dep.image = row['image'][0]
+ dep.screenshot_url = 'http://screenshots.debian.net/package/' + dep.pkg
+ for i in range(1,len(row['image'])):
+ dep.screenshots.append({'version':row['screenshot_versions'][i], 'url':row['image'][i]})
+
+ # it might be that the new upstream goes to experimental - this should be ignored here
+ if row['unstable_parsed_version']:
+ dep.outdated['release'] = 'upstream'
+ dep.outdated['version'] = row['unstable_upstream']
+ dep.outdated['architectures'] = ''
+
+ if row['changed_by']:
+ try:
+ changed = to_unicode(row['changed_by'])
+ except TypeError, err:
+ changed = None
+ logger.warning("Encoding problem for last uploader of package '%s' in task %s (%s)" % (dep.pkg, dep.taskname, err))
+ if changed:
+ try:
+ (_name, _url) = email.Utils.parseaddr(changed)
+ changed = '<a href="mailto:%s">%s</a>' % (_url, _name)
+ dep.properties['changed_by'] = MarkupString(changed, dep.pkg, 'changed_by')
+ dep.properties['last_uploader'] = to_unicode(changed)
+ dep.properties['last_uploader_simple'] = to_unicode('%s <%s>' % (_name, _url))
+ except UnicodeDecodeError, err:
+ logger.error("Encoding problem for last uploader - assume same as maintainer for package %s (%s)", dep.pkg, err)
+
+ # link to packages.debian.org search page to see overview about all
+ # package versions in all releases
+ # dep.properties['pkg-url'] = PKGURLMASK % dep.pkg
+
+ for l in languages:
+ if row['description_'+l]:
+ dep.desc[l] = {}
+ dep.desc[l]['short'] = MarkupString(to_unicode(row['description_'+l]), dep.pkg, 'ShortDesc')
+ if row['long_description_'+l]:
+ dep.desc[l]['long'] = Markup(render_longdesc(row['long_description_'+l].splitlines()))
+ if not dep.desc['en'].has_key('short'):
+ logger.error("Dep has no English short description: %s", dep.pkg)
+ dep.desc['en']['short'] = "??? missing short description for package %s :-(" % dep.pkg
+
+ (_name, _url) = email.Utils.parseaddr(row['maintainer'])
+ dep.properties['maintainer'] = to_unicode(row['maintainer'])
+ dep.responsible = '<a href="mailto:%s">%s</a>' % (_url, to_unicode(_name))
+
+
+ def GetTaskDependencies(self, source=0):
+ count = 0
+ global dep_strength_keys
+ dep = None
+ alldepends = [] # contains the names of all the dependencies in a task file
+ dependencies = []
+
+ # Get the title and description of each task in a blend
+ # Title, Description
+ query = "SELECT title, description, long_description FROM blends_tasks WHERE task='%s' AND blend='%s'" % (self.task, self.blendname)
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ self.metapkg.PrintedName, short, long = curs.fetchone()
+ # Markup strings to enable verbatim output of preformatted text
+ self.metapkg.desc['en']['short'] = MarkupString(short.encode('utf-8'), self.metapkg.PrintedName, 'taskShortDesc')
+ self.metapkg.desc['en']['long'] = MarkupString(long.encode('utf-8'), self.metapkg.PrintedName, 'taskLongDesc')
+
+
+ # Get the package_name, license, dependency of the official dependencies of the task
+ query = "SELECT DISTINCT b.package, b.dependency FROM blends_dependencies b JOIN packages p ON p.package=b.package \
+ WHERE b.blend='%s' AND b.task='%s'" % (self.blendname, self.task)
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ dep_info = curs.fetchall()
+
+ for each_dep in dep_info:
+ # class DependantPackage - This class contains all the possible information about a package
+ # Create an object of class DependantPackage
+ dep = DependantPackage(self.blendname, self.task)
+
+ # Information about the dependant packages
+ dep.pkg = each_dep[0] # Name
+ # dep.license
+ dep.dep_strength = each_dep[1] # Depends: 'd' or Suggests: 's'
+
+ alldepends.append(dep.pkg)
+ dependencies.append(dep)
+ #self.dependencies[dep.pkgstat].append(dep)
+
+ count = count +1
+ query = "EXECUTE query_pkgs ('%s', '%s')" % (List2PgArray(alldepends), List2PgSimilarArray(alldepends))
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ self.GetDepInfo(curs, dependencies)
+
+
+ print "+++++++++++++++++++++", count, self.task
+
+
+ # Get the package_name, license, dependency of the prospective dependencies of the task
+ query = "SELECT DISTINCT bp.package, bp.license, b.dependency, bp.component, bp.homepage, bp.section, \
+ bp.source, bp.vcs_type, bp.vcs_url, bp.vcs_browser, bp.changed_by, \
+ bp.uploaders, bp.maintainer, pop.vote, pop.recent, tags.debtags, bp.description, bp.long_description \
+ FROM blends_prospectivepackages bp JOIN blends_dependencies b ON b.blend=bp.blend \
+ LEFT OUTER JOIN popcon pop ON pop.package=bp.package \
+ LEFT OUTER JOIN ( \
+ SELECT package, array_agg(tag) AS debtags \
+ FROM debtags \
+ WHERE tag NOT LIKE 'implemented-in::%%' \
+ AND tag NOT LIKE 'protocol::%%' \
+ AND tag NOT LIKE '%%::TODO' \
+ AND tag NOT LIKE '%%not-yet-tagged%%' \
+ GROUP BY package \
+ ) tags ON tags.package = bp.package \
+ WHERE bp.blend='%s' and b.task='%s'" % (self.blendname, self.task)
+ _execute_udd_query(query)
+
+ alldepends = []
+ dependencies = []
+ if curs.rowcount > 0:
+ pros_info = curs.fetchall()
+ #print pros_info
+
+ for each_pros in pros_info:
+ dep = DependantPackage(self.blendname, self.task)
+
+ dep.pkg = each_pros[0] # Name
+ dep.license = each_pros[1]
+ dep.dep_strength = each_pros[2]
+
+ alldepends.append(dep.pkg)
+ dependencies.append(dep)
+
+ self.GetDepInfo(curs, dependencies)
+ print "-----------------", count, self.task
+
+
+ # Get information about new dependencies of the task
+ query = "SELECT DISTINCT new.package FROM new_packages new JOIN blends_dependencies b ON b.package=new.package \
+ WHERE b.blend='%s' and b.task='%s'" % (self.blendname, self.task)
+ _execute_udd_query(query)
+ pkgs_not_in_pool = []
+ if curs.rowcount > 0:
+ pkgs_not_in_pool = curs.fetchall()
+
+ query = "EXECUTE query_new('%s')" % List2PgArray(pkgs_not_in_pool)
+ _execute_udd_query(query)
+ pkgs_in_new = []
+ if curs.rowcount > 0:
+ self.GetDepInfo(curs, dependencies)
+ print "xxxxxxxxxxxxxxxxxxx", count
+
+
+
diff --git a/webtools/tasks.py b/webtools/tasks_udd.py
similarity index 98%
copy from webtools/tasks.py
copy to webtools/tasks_udd.py
index adcd938..06ae84a 100755
--- a/webtools/tasks.py
+++ b/webtools/tasks_udd.py
@@ -17,7 +17,7 @@ from genshi.template import TemplateLoader
from genshi import Markup
from genshi.template.eval import UndefinedError
-from blendstasktools import Tasks, GetDependencies2Use, pkgstatus, pkgstatus_sortedkeys, UnlockBlendsTools, CheckOrCreateOutputDir, SetFilePermissions
+from blendstasktools_udd import Tasks, GetDependencies2Use, pkgstatus, pkgstatus_sortedkeys, UnlockBlendsTools, CheckOrCreateOutputDir, SetFilePermissions
from blendsunicode import to_unicode
from blendslanguages import languages, language_dict
--
Static and dynamic websites for Debian Pure Blends
More information about the Blends-commit
mailing list