[Blends-commit] [SCM] website branch, master, updated. d8fc773ad70158f836a17e45ffad43d093c4bb89
Akshita Jha
akshita-guest at users.alioth.debian.org
Fri Jul 24 19:10:13 UTC 2015
The following commit has been merged in the master branch:
commit d8fc773ad70158f836a17e45ffad43d093c4bb89
Author: Akshita Jha <akshita-guest at users.alioth.debian.org>
Date: Sat Jul 25 00:40:03 2015 +0530
Remove blendstasktools_udd.py and tasks_udd.py from webtools
diff --git a/webtools/blendstasktools_udd.py b/webtools/blendstasktools_udd.py
deleted file mode 100644
index 135f5b8..0000000
--- a/webtools/blendstasktools_udd.py
+++ /dev/null
@@ -1,1340 +0,0 @@
-#PORT=5441
-UDDPORT=5452
-PORT=UDDPORT
-DEFAULTPORT=5432
-
-from sys import stderr, exit
-from subprocess import Popen, PIPE
-import os
-import grp
-import stat
-import urllib
-import StringIO
-import gzip
-import bz2
-import re
-import email.Utils
-
-import psycopg2
-import gettext
-
-from genshi import Markup
-# According to http://genshi.edgewall.org/wiki/GenshiFaq#HowcanIincludeliteralXMLintemplateoutput
-# there are different options to prevent escaping '<' / '>' but HTML does not work ...
-# from genshi.input import HTML
-from blendsmarkdown import SplitDescription, MarkupString, render_longdesc
-
-try:
- from debian import deb822
-except:
- from debian_bundle import deb822
-from blendsunicode import to_unicode
-from blendslanguages import languages
-
-import logging
-import logging.handlers
-logger = logging.getLogger('blends')
-logger.setLevel(logging.INFO)
-# logger.setLevel(logging.DEBUG)
-
-# Seems to have problems on 17.04.2009
-# BASEURL = 'http://ftp.debian.org/debian'
-BASEURL = 'http://ftp.de.debian.org/debian'
-KEYSTOIGNORE = ( 'Architecture', 'Comment', 'Leaf', 'NeedConfig', 'Note', 'Section',
- 'Needconfig', 'DontAvoid',
- 'Enhances', 'Test-always-lang', 'Metapackage')
-
-CONFDIR = 'webconf'
-
-COMPRESSIONEXTENSION='bz2'
-# COMPRESSIONEXTENSION='gz' # Translations are only available as bz2 since April 2009
-
-HOMEPAGENONE = '#'
-HOMEPAGENONEFIELDS = ('homepage',
- 'pkg-url', # Link to packages.debian.org search interface with exact
- # package matches or URL to inofficial package
- 'vcs-browser', # Browser-URL to packaging stuff in Vcs
- )
-
-PKGURLMASK = 'http://packages.debian.org/search?keywords=%s%%26searchon=names%%26exact=1%%26suite=all%%26section=all'
-
-DEPENDENT = 0
-SUGGESTED = 1
-DONE = 2
-BUGLISTCAT = (DEPENDENT, SUGGESTED, DONE )
-
-# FIXME: Obtain releases from UDD table releases (is this used at all???)
-releases = {'oldstable' : ('wheezy', 'wheezy-proposed-updates', 'wheezy-security'),
- 'stable' : ('jessie', 'jessie-proposed-updates', 'jessie-security'),
- 'testing' : ('stretch'),
- 'unstable' : ('sid'),
- 'experimental' : ('experimental')
- }
-
-pkgstatus = {'official_high' : # official package with high priority dependency
- { 'releases' : ('oldstable', 'stable', 'testing', 'unstable'),
- 'components' : ('main', ),
- 'dependencies' : ('Depends', 'Recommends'),
- 'fields-set' : (),
- 'colorcode' : 'Green: The project is <a href="#%s">available as an official Debian package and has high relevance</a>',
- 'order' : 1
- },
- 'official_low' : # official package with low priority dependency
- { 'releases' : ('oldstable', 'stable', 'testing', 'unstable'),
- 'components' : ('main', ),
- 'dependencies' : ('Suggests', ),
- 'fields-set' : (),
- 'colorcode' : 'Green: The project is <a href="#%s">available as an official Debian package but has lower relevance</a>',
- 'order' : 2
- },
- 'non-free' : # package in contrib or non-free, priority decreased to Suggests in any case
- { 'releases' : ('oldstable', 'stable', 'testing', 'unstable'),
- 'component' : ('contrib', 'non-free'),
- 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
- 'fields-set' : (),
- 'colorcode' : 'Green: The project is <a href="#%s">available in Debian packaging pool but is not in Debian main</a>',
- 'order' : 3
- },
- 'experimental' : # package which exists only in experimental
- { 'releases' : ('experimental', ),
- 'component' : ('main', 'contrib', 'non-free'),
- 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
- 'fields-set' : (),
- 'colorcode' : 'Yellow: The project is <a href="#%s">available in Debian packaging pool but is regarded as experimental</a>',
- 'order' : 4
- },
- 'new' : # package in new queue
- { 'releases' : ('new', ),
- 'component' : ('main', 'contrib', 'non-free'),
- 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
- 'fields-set' : (),
- 'colorcode' : 'Yellow: A package of project is <a href="#%s">is in Debian New queue and hopefully available soon</a>',
- 'order' : 5
- },
- 'pkgvcs' : # Not yet packaged but packaging code in Vcs
- { 'releases' : (),
- 'component' : (),
- 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
- 'fields-set' : ('vcs-svn', 'vcs-git', 'vcs-browser'),
- 'colorcode' : 'Yellow: The packaging of project is <a href="#%s">has started and a developer might try the packaging code in VCS or help packaging.</a>',
- 'order' : 6
- },
- 'unofficial' : # unofficial packages outside Debian
- { 'releases' : (),
- 'component' : (),
- 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
- 'fields-set' : ('pkg-url', ),
- 'colorcode' : 'Yellow: There exists an <a href="#%s">unofficial package</a> of the project',
- 'order' : 7
- },
- 'wnpp' : # project which has at least a WNPP bug filed
- { 'releases' : (),
- 'component' : (),
- 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
- 'fields-set' : ('wnpp', ),
- 'colorcode' : 'Red: The project is <a href="#%s">not (yet) available as a Debian package</a> but there is some record of interest (WNPP bug).',
- 'order' : 8
- },
- 'prospective' : # projects which might be interesting for a Blend but no work is done yet
- { 'releases' : (),
- 'component' : (),
- 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
- 'fields-set' : ('homepage', ), # TODO: a description should be set as well ...
- 'colorcode' : 'Red: The project is <a href="#%s">not (yet) available as a Debian package</a>.',
- 'order' : 9
- },
- 'ignore' : # Package inside Debian which is "under observation"
- { 'releases' : (releases.keys()),
- 'component' : ('main', 'contrib', 'non-free'),
- 'dependencies' : ('Ignore', ),
- 'fields-set' : (),
- 'colorcode' : '%s',
- 'order' : 10
- },
- 'avoid' : # Package inside Debian which should not go to a install medium of the Blend
- { 'releases' : (releases.keys()),
- 'component' : ('main', 'contrib', 'non-free'),
- 'dependencies' : ('Avoid', ),
- 'fields-set' : (),
- 'colorcode' : '%s',
- 'order' : 11
- },
- 'unknown' : # Everything else
- { 'releases' : (),
- 'component' : (),
- 'dependencies' : ('Depends', 'Recommends', 'Suggests'),
- 'fields-set' : (),
- 'colorcode' : 'White: The project has an %s status.',
- 'order' : 100
- },
- }
-
-# http://wiki.python.org/moin/HowTo/Sorting#Sortingbykeys
-_tmplist=[]
-for key in pkgstatus.keys():
- _tmplist.append((key,pkgstatus[key]['order']))
-_tmpsorted = sorted(_tmplist, key=lambda x:(x[1], x[0]))
-pkgstatus_sortedkeys = []
-for _tmp in _tmpsorted:
- pkgstatus_sortedkeys.append(_tmp[0])
-
-dep_strength_keys = []
-for pkgstat in pkgstatus:
- for dep in pkgstatus[pkgstat]['dependencies']:
- if dep not in dep_strength_keys:
- dep_strength_keys.append(dep)
-
-rmpub = open('remove-publications-from-tasks-files.dat','w+')
-
-license_in_component = {'main' : 'DFSG free',
- 'contrib' : 'DFSG free, but needs non-free components',
- 'non-free' : 'non-free'
- }
-
-try:
- import psutil
- has_psutils=True
-except ImportError:
- has_psutils=False
-
-def SetFilePermissions(usefile):
- try:
- blendsgid = grp.getgrnam("blends").gr_gid
- os.chown(usefile, -1, blendsgid)
- os.chmod(usefile, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH )
- # os.system("ls -l %s" % usefile)
- except KeyError:
- # if groups 'blends' does not exist on the machine we are simply testing and setting group permissions is not needed
- pass
-
-LOCKFILE='/var/lock/blends.lock'
-def LockBlendsTools():
- """Locking mechanism to make sure the scripts will not run in parallel
- which happened because of IO problems on udd.debian.org"""
- if not has_psutils:
- logger.warning("Package python-psutil is missing. No locking support available")
- return
- if os.path.exists(LOCKFILE):
- try:
- lf = open(LOCKFILE, 'r')
- pid = int(lf.readline())
- lf.close()
- if pid in psutil.get_pid_list():
- logger.error("Another process rebuilding web sentinel pages with PID %i is running. Exit." % pid)
- exit()
- else:
- logger.warning("Process with PID %i is not running any more but lockfile remained. Removing %s ..." % (pid, LOCKFILE))
- os.unlink(LOCKFILE)
- except IOError as e:
- pass
- pid = os.getpid()
- lf = open(LOCKFILE, 'w')
- print >>lf, pid
- lf.close()
- SetFilePermissions(LOCKFILE)
-
-def UnlockBlendsTools():
- """Unlock previousely locked file"""
- if os.path.exists(LOCKFILE):
- os.unlink(LOCKFILE)
-
-def GetDependencies2Use(dependencystatus=[], max_order='prospective'):
- # Create a list of status of dependencies out of pkgstatus dictionary
- use_dependencystatus = []
- if dependencystatus == []:
- for pkgstat in pkgstatus_sortedkeys:
- # per default = if no expliczite dependencystatus are given, we are only interested in
- # dependencies which are at least of order experimental
- if pkgstatus[pkgstat]['order'] > pkgstatus[max_order]['order']:
- continue
- use_dependencystatus.append(pkgstat)
- else:
- # verify correctly given dependencies
- for pkgstat in dependencystatus:
- if pkgstat in pkgstatus.keys():
- use_dependencystatus.append(pkgstat)
- else:
- logger.error("Unknown dependencystatus %s" % pkgstat)
- if use_dependencystatus == []:
- logger.error("No valid dependencystatus in", dependencystatus)
- return use_dependencystatus
-
-###########################################################################################
-# Define several prepared statements to query UDD
-try:
- conn = psycopg2.connect(host="localhost",port=PORT,user="guest",database="udd")
-except psycopg2.OperationalError, err:
- try:
- conn = psycopg2.connect("service=udd")
- except psycopg2.OperationalError, err:
- # logger not known at this state: logger.warning
- print >>stderr, "Service=udd seems not to be installed on this host.\tMessage: %s" % (str(err))
- try:
- conn = psycopg2.connect(host="localhost",port=DEFAULTPORT,user="guest",database="udd")
- except psycopg2.OperationalError:
- # Hmmm, I observed a really strange behaviour on one of my machines where connecting to
- # localhost does not work but 127.0.0.1 works fine. No idea why ... but this should
- # do the trick for the moment
- conn = psycopg2.connect(host="127.0.0.1",port=DEFAULTPORT,user="guest",database="udd")
-
-curs = conn.cursor()
-# uddlog = open('logs/uddquery.log', 'w')
-
-def _execute_udd_query(query):
- try:
- curs.execute(query)
- logger.debug(query)
- except psycopg2.ProgrammingError, err:
- print >>stderr, "Problem with query\n%s" % (to_unicode(query))
- print >>stderr, err
- exit(-1)
- except psycopg2.DataError, err:
- print >>stderr, "%s; query was\n%s" % (err, query)
-
-
-query = """PREPARE query_pkgs (text[],text[]) AS
- SELECT * FROM blends_query_packages($1,$2) AS (
- package text, distribution text, release text, component text, version debversion,
- maintainer text,
- source text, section text, task text, homepage text,
- maintainer_name text, maintainer_email text,
- "vcs-type" text, "vcs-url" text, "vcs-browser" text,
- changed_by text,
- enhanced text[],
- releases text[], versions text[], architectures text[],
- unstable_upstream text, unstable_parsed_version text, unstable_status text,
- vote int, recent int, -- popcon
- debtags text[],
- screenshot_versions text[], image text[], icon text[],
- year text,
- title text,
- authors text,
- doi text,
- pubmed text,
- url text,
- journal text,
- volume text,
- number text,
- pages text,
- eprint text,
- description_en text, long_description_en text,
- description_cs text, long_description_cs text,
- description_da text, long_description_da text,
- description_de text, long_description_de text,
- description_es text, long_description_es text,
- description_fi text, long_description_fi text,
- description_fr text, long_description_fr text,
- description_hu text, long_description_hu text,
- description_it text, long_description_it text,
- description_ja text, long_description_ja text,
- description_ko text, long_description_ko text,
- description_nl text, long_description_nl text,
- description_pl text, long_description_pl text,
- "description_pt_BR" text, "long_description_pt_BR" text,
- description_ru text, long_description_ru text,
- description_sk text, long_description_sk text,
- description_sr text, long_description_sr text,
- description_sv text, long_description_sv text,
- description_uk text, long_description_uk text,
- "description_zh_CN" text, "long_description_zh_CN" text,
- "description_zh_TW" text, "long_description_zh_TW" text
- )"""
-_execute_udd_query(query)
-
-query = """PREPARE query_new (text[]) AS SELECT
- p.package,
- p.distribution, p.component, p.version, p.architecture, p.maintainer,
- p.source, p.section, p.distribution, 'new' AS release, p.component, p.homepage,
- s.changed_by,
- description AS description_en, long_description AS long_description_en,
- bibyear.value AS "year",
- bibtitle.value AS "title",
- bibauthor.value AS "authors",
- bibdoi.value AS "doi",
- bibpmid.value AS "pubmed",
- biburl.value AS "url",
- bibjournal.value AS "journal",
- bibvolume.value AS "volume",
- bibnumber.value AS "number",
- bibpages.value AS "pages",
- bibeprint.value AS "eprint"
- FROM new_packages p
- JOIN new_sources s ON p.source = s.source AND p.version = s.version
- LEFT OUTER JOIN bibref bibyear ON p.source = bibyear.source AND bibyear.rank = 0 AND bibyear.key = 'year' AND bibyear.package = ''
- LEFT OUTER JOIN bibref bibtitle ON p.source = bibtitle.source AND bibtitle.rank = 0 AND bibtitle.key = 'title' AND bibtitle.package = ''
- LEFT OUTER JOIN bibref bibauthor ON p.source = bibauthor.source AND bibauthor.rank = 0 AND bibauthor.key = 'author' AND bibauthor.package = ''
- LEFT OUTER JOIN bibref bibdoi ON p.source = bibdoi.source AND bibdoi.rank = 0 AND bibdoi.key = 'doi' AND bibdoi.package = ''
- LEFT OUTER JOIN bibref bibpmid ON p.source = bibpmid.source AND bibpmid.rank = 0 AND bibpmid.key = 'pmid' AND bibpmid.package = ''
- LEFT OUTER JOIN bibref biburl ON p.source = biburl.source AND biburl.rank = 0 AND biburl.key = 'url' AND biburl.package = ''
- LEFT OUTER JOIN bibref bibjournal ON p.source = bibjournal.source AND bibjournal.rank = 0 AND bibjournal.key = 'journal' AND bibjournal.package = ''
- LEFT OUTER JOIN bibref bibvolume ON p.source = bibvolume.source AND bibvolume.rank = 0 AND bibvolume.key = 'volume' AND bibvolume.package = ''
- LEFT OUTER JOIN bibref bibnumber ON p.source = bibnumber.source AND bibnumber.rank = 0 AND bibnumber.key = 'number' AND bibnumber.package = ''
- LEFT OUTER JOIN bibref bibpages ON p.source = bibpages.source AND bibpages.rank = 0 AND bibpages.key = 'pages' AND bibpages.package = ''
- LEFT OUTER JOIN bibref bibeprint ON p.source = bibeprint.source AND bibeprint.rank = 0 AND bibeprint.key = 'eprint' AND bibeprint.package = ''
- WHERE (p.package, p.version) IN
- (SELECT package, max(version) FROM
- new_packages WHERE package = ANY ($1) GROUP BY package)"""
-_execute_udd_query(query)
-
-query = """PREPARE query_vcs (text[]) AS SELECT
- p.package,
- p.component, p.maintainer,
- p.source, p.section, 'vcs' AS release, p.component, p.homepage,
- p.changed_by, p.chlog_version AS version,
- description AS description_en, long_description AS long_description_en,
- p.vcs_url AS "vcs-url",
- p.vcs_type AS "vcs-type",
- p.vcs_browser AS "vcs-browser",
- p.blend,
- p.license,
- p.wnpp,
- bibyear.value AS "year",
- bibtitle.value AS "title",
- bibauthor.value AS "authors",
- bibdoi.value AS "doi",
- bibpmid.value AS "pubmed",
- biburl.value AS "url",
- bibjournal.value AS "journal",
- bibvolume.value AS "volume",
- bibnumber.value AS "number",
- bibpages.value AS "pages",
- bibeprint.value AS "eprint"
- FROM blends_prospectivepackages p
- LEFT OUTER JOIN bibref bibyear ON p.source = bibyear.source AND bibyear.rank = 0 AND bibyear.key = 'year' AND bibyear.package = ''
- LEFT OUTER JOIN bibref bibtitle ON p.source = bibtitle.source AND bibtitle.rank = 0 AND bibtitle.key = 'title' AND bibtitle.package = ''
- LEFT OUTER JOIN bibref bibauthor ON p.source = bibauthor.source AND bibauthor.rank = 0 AND bibauthor.key = 'author' AND bibauthor.package = ''
- LEFT OUTER JOIN bibref bibdoi ON p.source = bibdoi.source AND bibdoi.rank = 0 AND bibdoi.key = 'doi' AND bibdoi.package = ''
- LEFT OUTER JOIN bibref bibpmid ON p.source = bibpmid.source AND bibpmid.rank = 0 AND bibpmid.key = 'pmid' AND bibpmid.package = ''
- LEFT OUTER JOIN bibref biburl ON p.source = biburl.source AND biburl.rank = 0 AND biburl.key = 'url' AND biburl.package = ''
- LEFT OUTER JOIN bibref bibjournal ON p.source = bibjournal.source AND bibjournal.rank = 0 AND bibjournal.key = 'journal' AND bibjournal.package = ''
- LEFT OUTER JOIN bibref bibvolume ON p.source = bibvolume.source AND bibvolume.rank = 0 AND bibvolume.key = 'volume' AND bibvolume.package = ''
- LEFT OUTER JOIN bibref bibnumber ON p.source = bibnumber.source AND bibnumber.rank = 0 AND bibnumber.key = 'number' AND bibnumber.package = ''
- LEFT OUTER JOIN bibref bibpages ON p.source = bibpages.source AND bibpages.rank = 0 AND bibpages.key = 'pages' AND bibpages.package = ''
- LEFT OUTER JOIN bibref bibeprint ON p.source = bibeprint.source AND bibeprint.rank = 0 AND bibeprint.key = 'eprint' AND bibeprint.package = ''
- WHERE p.package = ANY ($1)"""
-_execute_udd_query(query)
-
-# This prepared statement is called only once but it makes sense to mention it in the
-# header to keep all stuff in one place which needs to be changed in case a new language
-# will be added
-query = """PREPARE query_metapkg_trans (text[]) AS
- SELECT * FROM blends_metapackage_translations($1) AS (
- package text,
- description_en text, long_description_en text,
- description_cs text, long_description_cs text,
- description_da text, long_description_da text,
- description_de text, long_description_de text,
- description_es text, long_description_es text,
- description_fi text, long_description_fi text,
- description_fr text, long_description_fr text,
- description_hu text, long_description_hu text,
- description_it text, long_description_it text,
- description_ja text, long_description_ja text,
- description_ko text, long_description_ko text,
- description_nl text, long_description_nl text,
- description_pl text, long_description_pl text,
- "description_pt_BR" text, "long_description_pt_BR" text,
- description_ru text, long_description_ru text,
- description_sk text, long_description_sk text,
- description_sr text, long_description_sr text,
- description_sv text, long_description_sv text,
- description_uk text, long_description_uk text,
- "description_zh_CN" text, "long_description_zh_CN" text,
- "description_zh_TW" text, "long_description_zh_TW" text
- )"""
-_execute_udd_query(query)
-
-# Sometimes the tasks file contains dependencies from virtual packages and we have to
-# obtain the real packages which provide this dependency.
-# First check whether there are such packages (only names)
-query = """PREPARE query_provides (text[]) AS
- SELECT DISTINCT package, provides FROM packages WHERE provides IS NOT NULL AND package = ANY($1) ;"""
-_execute_udd_query(query)
-
-# Obtain more detailed information about packages that might provide a dependency
-#query = """PREPARE query_provides_version_release (text) AS
-# SELECT package, version, release FROM packages WHERE provides = $1
-# GROUP BY version, package, release ORDER BY version DESC;"""
-#_execute_udd_query(query)
-
-# Obtain the releases featuring a certain package, in case a package might show up in different components when
-# considering different releases we apply a preference for main over contrib over non-free. If this is the case
-# we mention only the releases of the selected component
-query = "PREPARE pkg_releases (text, text) AS SELECT release FROM packages WHERE package = $1 AND component = $2 GROUP BY release "
-_execute_udd_query(query)
-
-query = """PREPARE pkg_versions_stable_testing (text) AS
- SELECT release, regexp_replace(regexp_replace(debversion, '-.*', ''), '[.+~]dfsg.*', '') AS version, debversion
- FROM (SELECT r AS release, MAX(version) AS debversion
- FROM versions_archs_component($1) AS (r text, version text, archs text, component text)
- WHERE r IN ('jessie', 'stretch') GROUP BY r) AS zw;""" # FIXME: Do not hardcode names here!!!
-_execute_udd_query(query)
-
-# Number of all submissions
-query = "PREPARE popcon_submissions AS SELECT vote FROM popcon WHERE package = '_submissions'"
-_execute_udd_query(query)
-
-
-def List2PgArray(list):
- # turn a list of strings into the syntax for a PostgreSQL array:
- # {"string1","string2",...,"stringN"}
- if not list:
- return '{}'
- komma='{'
- PgArray=''
- for s in list:
- PgArray=PgArray+komma+'"'+s+'"'
- komma=','
- return PgArray+'}'
-
-def List2PgSimilarArray(list):
- # turn a list of strings surrounded by '%' into the syntax for a PostgreSQL array to enable LIKE conditions:
- # {"%string1%","%string2%",...,"%stringN%"}
- if not list:
- return '{}'
- komma='{'
- PgSimArray=''
- for s in list:
- PgSimArray=PgSimArray+komma+'"%'+s+'%"'
- komma=','
- return PgSimArray+'}'
-
-def ReadConfig(blendname=''):
- # Try to read config file CONFDIR/<blendname>.conf
- conffile = CONFDIR + '/' + blendname + '.conf'
- if not os.access(conffile, os.R_OK):
- # if config file can not be found in local dir, try /etc/blends/webconf as fallback
- conffile_default = '/etc/blends/webconf/' + blendname + '.conf'
- if not os.access(conffile_default, os.R_OK):
- logger.error("Unable to open config file %s or %s." % (conffile, conffile_default))
- exit(-1)
- conffile = conffile_default
- f = open(conffile, 'r')
- ret = { 'Blend' : '',
- 'projectname' : '',
- 'projecturl' : '',
- 'homepage' : '',
- 'aliothurl' : '',
- 'projectlist' : '',
- 'pkglist' : '',
- 'logourl' : '',
- 'css' : '',
- 'outputdir' : '',
- 'datadir' : '',
- 'advertising' : None, # headline with advertising string is optional
- 'ubuntuhome' : None,
- 'projectubuntu':None,
- }
- for stanza in deb822.Sources.iter_paragraphs(f, shared_storage=False):
- ret['Blend'] = stanza['blend'] # short name of the project
- ret['projectname'] = stanza['projectname'] # Printed name of the project
- ret['projecturl'] = stanza['projecturl'] # Link to the developer page with dynamic content
- # like for instance these tasks pages
- ret['homepage'] = stanza['homepage'] # Homepage with general information about the project
- # for instance at www.debian.org or wiki.debian.org
- ret['aliothurl'] = stanza['aliothurl'] # Link to the Alioth page of the project
- ret['projectlist'] = stanza['projectlist'] # Mailinglist of the project
- if stanza.has_key('pkglist'):
- ret['pkglist'] = stanza['pkglist'] # Packaging Mailinglist = Maintainer of group maintained packages
- if stanza.has_key('logourl'):
- ret['logourl'] = stanza['logourl'] # URL to logo image (might be missing
- ret['css'] = stanza['css'] # (relative) URL to CSS file
- ret['outputdir'] = stanza['outputdir'] # Dir for storing output HTML files
- ret['datadir'] = stanza['datadir'] # Dir for storing SVN information about project
- ret['vcsdir'] = stanza['vcsdir'] # Path to Blend information files at svn.debian.org
- if stanza.has_key('advertising'):
- # we have to remove the gettext _() call which was inserted into the config
- # file to enable easy input for config file editors - but the call has to
- # be made explicitely in the python code
- advertising = re.sub('_\(\W(.+)\W\)', '\\1', stanza['advertising'])
- # gettext needs to escape '"' thus we need to remove the escape character '\'
- ret['advertising'] = re.sub('\\\\"', '"', advertising)
- if stanza.has_key('ubuntuhome'):
- ret['ubuntuhome'] = stanza['ubuntuhome']
- if stanza.has_key('projectubuntu'):
- ret['projectubuntu'] = stanza['projectubuntu']
-
- return ret
-
-def CheckOrCreateOutputDir(maindir, subdir):
- outputdir = maindir + '/' + subdir
- if not os.access(outputdir, os.W_OK):
- try:
- os.mkdir(outputdir)
- except:
- # if more than one dir in the tree has to be crated just use mkdir -p ...
- try:
- os.system("mkdir -p %s" % outputdir)
- except:
- logger.error("Unable to create output dir " + outputdir)
- return None
- return outputdir
-
-def RowDictionaries(cursor):
- """Return a list of dictionaries which specify the values by their column names"""
-
- description = cursor.description
- if not description:
- # even if there are no data sets to return the description should contain the table structure. If not something went
- # wrong and we return NULL as to represent a problem
- return NULL
- if cursor.rowcount <= 0:
- # if there are no rows in the cursor we return an empty list
- return []
-
- data = cursor.fetchall()
- result = []
- for row in data:
- resultrow = {}
- i = 0
- for dd in description:
- resultrow[dd[0]] = row[i]
- i += 1
- result.append(resultrow)
- return result
-
-def BrowserFromVcsURL(vcs_type, vcs_url):
- # Guess Vcs-Browser URL from VCS URL
- if not vcs_type:
- return HOMEPAGENONE
- if vcs_type.lower().startswith('svn'):
- ret_url = re.sub('^svn:', 'http:', vcs_url)
- ret_url = re.sub('/svn/', '/wsvn/', ret_url)
- ret_url = re.sub('$', '?rev=0&sc=0', ret_url)
- elif vcs_type.lower().startswith('git'):
- ret_url = re.sub('^git:', 'http:', vcs_url)
- ret_url = re.sub('/git/', '/?p=', ret_url)
- elif vcs_type.lower().startswith('hg'):
- # Seems that vcs_browser = vcs_url in Mercurial
- return vcs_url
- elif vcs_type.lower().startswith('bzr') or vcs_type.lower().startswith('cvs'):
- logger.warning("No idea how to guess vcs_browser for %s URLS" % vcs_type)
- return
- else:
- logger.warning("Unknown VCS for " + vcs_url)
- return HOMEPAGENONE
- if vcs_url.startswith('https://github.com') and vcs_url.endswith('.git'):
- ret_url = re.sub('.git$', '', vcs_url)
- elif vcs_url.startswith('http:'):
- return vcs_url
- if ret_url == vcs_url:
- logger.warning("Unable to obtain Vcs-Browser from " + vcs_url)
- return HOMEPAGENONE
- return ret_url
-
-detect_vcs_cvs_re = re.compile("://.*cvs")
-detect_vcs_svn_re = re.compile("://.*svn")
-detect_vcs_git_re = re.compile("://.*git")
-
-def VcsTypeFromBrowserURL(vcs_browser):
- # Guess Vcs-Type from vcs_browser
- if detect_vcs_cvs_re.search(vcs_browser):
- return 'Cvs'
- if detect_vcs_svn_re.search(vcs_browser):
- return 'Svn'
- if detect_vcs_git_re.search(vcs_browser):
- return 'Git'
- return 'Unknown VCS'
-
-
-# The following keys will be mostly used for programs that
-# are not yet existing in Debian and will go to our todo list
-PROPERTIES=('homepage', # Homepage of program
- 'section', # Section of package in the Debian hierarchy
- 'source', # Keep the source package name which is needed for ddpo subscription
- )
-
-###################################################################################################################
-#=================================================================================================================#
-###################################################################################################################
-
-#######################################################
-
-
-def GetPkgstat(strength):
- if strength == 'd' or strength == 'r':
- pkgstat = 'official_high'
- elif strength == 's':
- pkgstat = 'official_low'
- elif strength == 'i':
- pkgstat = 'ignore'
- elif strength == 'a':
- pkgstat = 'avoid'
- else:
- pkgstat = 'unknown'
- return pkgstat
-
-
-class DependantPackage:
- # Hold information about a package that is in dependency list
-
- def __init__(self, blendname=None, taskname=None):
- self.blendname = blendname # Blend that includes the package in dependency list
- self.taskname = taskname # Task which includes the Dependency
- self.pkg = None # Name of dependant package
- self.source = None # Name of source package of the dependency
- self.PrintedName = None # Only for Meta package names - no use for a real dependant package
- # FIXME -> object model
- self.pkgstatus = 'unknown' # global pkgstatus: characterizes status of dependency, release, packaging status
- self.releases = [] # List of releases a package might be in
- self.component = None # One of: 'main', 'contrib', 'non-free', if a package shows up in several components which
- # might happen over different releases, just prefer main over contrib over non-free
- self.why = None # basically used as comment
-
- self.properties = {}
- self.properties['license'] = 'unknown'
- for field in HOMEPAGENONEFIELDS:
- self.properties[field] = HOMEPAGENONE
- self.properties['Enhances'] = {} # Dictionary Enhancing pkg name as key, Link to package information as value; empty in most cases
- # because Enhances relations are quite seldom
- self.properties['stable_testing_version'] = [] # (release, version) tuples where release is codename for stable and testing
- self.vcs_found = 0 # we need a flag to store the fact whether Vcs information of a package is in UDD
- self.version = [] # list of {'release', 'version', 'archs'} dictionary containing version and architecture information
- self.outdated = {} # If not empty directory then release='upstream' and package is outdated
- self.popcon = {} # dictionary containing vote and recnt values of popcon information
- self.popconsubmit = 0 # number of popcon submissions - in case popcon import into UDD might be broken this remains 0
- self.debtags = [] # list of {'tag', 'value'} dictionary containing debtag information
- self.screenshots = [] # list of {'version', 'url'} dictionary containing screenshot information
- self.icon = None # URL of small screenshot icon
- self.screenshot_url = None # URL to screenshots.debian.net
- self.responsible = None # E-Mail address of issuer of ITP or some person
- # who volunteered to care for this program
- self.filename = None # Filename of package in the Debian pool
- self.desc = {} # Prospective packages should have a description ...
- # ... which could be copied to (or from if exists)
- # WNPP bug and finally can be used for packaging
- self.desc['en'] = {} # An English description should be available in any case
- self.experimental = 0 # Set to 1 if package *only* in experimental but not in unstable/testing/stable
- self.remark = {} # Optional remark for a package
- self.dep_strength = 0 # Type of Dependency (Depends, Recommends, Suggests, Experimental, New, Avoid, Ignore, WNPP
-
-
- # sort these objects according to the package name
- def __cmp__(self, other):
- # Comparing with None object has to return something reasonable
- if other == None:
- return -2
- # Sort according to package name
- return cmp(self.pkg, other.pkg)
-
- def __str__(self):
- ret = "Blend: " + self.blendname + ", " \
- "Task:" + self.taskname + ", " \
- "pkg:" + self.pkg
- if self.dep_strength:
- ret += ", Dep_strength: " + self.dep_strength
- if self.pkgstatus:
- ret += ", Pkg_status: " + str(self.pkgstatus)
- if self.releases:
- ret += ", releases: " + str(self.releases)
- if self.component:
- ret += ", component: " + self.component
- if self.version:
- ret += ", versions: " + str(self.version)
- if self.desc:
- ret += ", desc: " + str(self.desc)
- for prop in self.properties.keys():
- try:
- ret += ", %s: %s" % (prop, to_unicode(str(self.properties[prop])))
- except UnicodeEncodeError:
- ret += ", %s: <UnicodeEncodeError>" % (prop)
- try:
- ret += ", popcon = %i (%i)" % (self.popcon['vote'], self.popcon['recent'])
- except:
- pass
- ret += ", debtags = " + str(self.debtags)
- if self.icon:
- ret += ", screenshoticon = " + self.icon
- if self.outdated:
- ret += ", dehsstatus = " + str(self.outdated)
-
- # if self.desc['en']:
- # ret += ", desc['en']:" + str(self.desc['en'])
- return ret
-
-
-class Tasks:
- # Provide a list of depencencies defined in metapackages
- # This class concerns _all_ tasks of a Blend and is the most
- # complete source of information. If only a single task
- # should be handled by a tool that uses blendtasktools
- # probably the class (see below) is
- # your friend
-
- def __init__(self, blendname):
-
- os.system("mkdir -p logs")
- LOG_FILENAME = 'logs/'+blendname+'.log'
- handler = logging.handlers.RotatingFileHandler(filename=LOG_FILENAME,mode='w')
- formatter = logging.Formatter("%(levelname)s - %(filename)s (%(lineno)d): %(message)s")
- handler.setFormatter(formatter)
- logger.addHandler(handler)
-
- LockBlendsTools()
-
- # This Instance of the Available class contains all
- # information about packages that are avialable in Debian
- # See below for more information about Available class
- self.data = ReadConfig(blendname)
- self.blendname = self.data['Blend']
- self._InitMetapackages()
- self.tasks = {} # Dictionary of TasksDependency objects
- self.alldeps_in_main = [] # sorted string list of package names with all packages
- # relevant for a Blend that are in main Debian (for use in DDTP)
- self.alldeps_in_main_info = {} # complete dictionary with package information
- # with all packages relevant for a Blend that are in
- # main to easily feed DDTP translation into the structures
- # -->
- # self.alldeps_in_main = self.alldeps_in_main_info.keys().sort()
-
- def _InitMetapackages(self):
- # sorted list of metapackage names
- self.metapackagekeys = []
- query = "SELECT task FROM blends_tasks WHERE blend = '%s' ORDER BY task" % (self.blendname)
- _execute_udd_query(query)
- if curs.rowcount > 0:
- temp = curs.fetchall()
- self.metapackagekeys = [t[0] for t in temp]
-
-
- def GetAllDependencies(self, source=0):
- # If we want to subscribe ddpo we need the source package names.
- # In this case set source=1
-
- # Obtain the prefix of the meta packages of the Blend
- prefix = self.blendname.split('-')[1] + '-'
-
- metapackages = []
- for task in self.metapackagekeys:
- metapackages.append(prefix + task)
-
- # Verify whether there are any translations of metapackage descriptions which on
- # one hand is a sign that this Blend has uploaded metapackages at all and on the
- # other hand gives a clue about whether it makes sense to query for description
- # translations
- # to verify whether a blend has uploaded metapackage or not we can check the boolean
- # column "metapackage" in blends_tasks table
- query = "SELECT COUNT(*) FROM descriptions WHERE package = ANY ('%s')" % List2PgArray(metapackages)
- _execute_udd_query(query)
- if curs.rowcount > 0:
- hastranslations = curs.fetchone()[0]
-
- metapkg_translations = {}
- if hastranslations > 0:
- query = "EXECUTE query_metapkg_trans('%s')" % List2PgArray(metapackages)
- _execute_udd_query(query)
- if curs.rowcount > 0:
- for row in RowDictionaries(curs):
- metapkg_translations[row['package']] = row
-
- for task in self.metapackagekeys:
- td = TaskDependencies(self.blendname, task=task)
- pkgname = prefix + task
- translations = None
- if metapkg_translations.has_key(pkgname):
- translations = metapkg_translations[pkgname]
-
- td.SetMetapackageInfo(pkgname, translations)
- logger.debug("Task : %s " % task)
- if td.GetTaskDependencies(source):
- self.tasks[task] = td
- else: # Kick file that is obviously no task file from metapackage list
- self.metapackagekeys = filter(lambda name: name != task, self.metapackagekeys)
-
- if source == 0:
- # total number popcon submissions
- query = "EXECUTE popcon_submissions"
- _execute_udd_query(query)
- if curs.rowcount > 0:
- self.popconsubmit = curs.fetchone()[0]
- # Obtain packages that might enhance any of the packages in tasks list
- self.LinkEnhances()
-
- def GetNamesOnlyDict(self, dependencystatus=[]):
- # David Paleino needs for his web tools a dictionary
- # { taskname : [list of dependencies]}
- # This will be prepared here from the main
- # datastructure
- ret = {}
- use_dependencystatus = GetDependencies2Use(dependencystatus, 'experimental')
-
- for task in self.metapackagekeys:
- tdeps = self.tasks[task]
- list = []
- for dep in use_dependencystatus:
- for tdep in tdeps.dependencies[dep]:
- list.append(tdep.pkg)
- ret[task] = list
- return ret
-
- def GetAllDependentPackagesOfBlend(self, dependencystatus=[]):
- # David Paleino needs for his DDTP web tool a list of
- # all available Dependencies.
- # Here only those packages are returned that are in
- # Debian main, because there are no DDTP translations
- # for contrib and non-free available
- if self.alldeps_in_main != []:
- return self.alldeps_in_main
-
- use_dependencystatus = GetDependencies2Use(dependencystatus, 'unknown')
- for task in self.metapackagekeys:
- tdeps = self.tasks[task]
- for dep in use_dependencystatus:
- for tdep in tdeps.dependencies[dep]:
- # add only packages in main, because others do not have DDTP translations
- if tdep.component == 'main':
- self.alldeps_in_main.append(tdep.pkg)
- self.alldeps_in_main_info[tdep.pkg] = tdep
- # Also add meta package itself to make use of translated meta package description
- self.alldeps_in_main.append(self.tasks[task].metapkg.pkg)
- self.alldeps_in_main_info[self.tasks[task].metapkg.pkg] = self.tasks[task].metapkg
- self.alldeps_in_main.sort()
- return self.alldeps_in_main
-
-
- def GetTaskDescDict(self):
- # Return dictionary with description information of all tasks of a Blend
- return self.tasks
-
- def MarkupPreformatedStringsBlend(self):
- # Genshi does not touch strings that are marked with "Markup()" - so just
- # mark the strings that are ready formatted for the whole Blend
- for task in self.metapackagekeys:
- tdeps = self.tasks[task]
- tdeps.MarkupPreformatedStrings()
-
- def LinkEnhances(self):
- # To provide a connection between packages enhancing other packages a set of links
- # will be provided. The links should point to paragraphs on the tasks pages if the
- # Enhancing package is listed in the metapackages of the blend and to packages.debian.org
- # otherwise
- for task in self.metapackagekeys:
- tdeps = self.tasks[task]
- for dependency in tdeps.dependencies.keys():
- for dep in tdeps.dependencies[dependency]:
- if dep.properties['Enhances'] != {}:
- logger.debug("Package %s is enhanced by:" % dep.pkg)
- for enh in dep.properties['Enhances'].keys():
- # seek for Enhances on same page
- found = 0
- for seek_dependency in tdeps.dependencies.keys():
- for enhdep in tdeps.dependencies[seek_dependency]:
- if enh == enhdep.pkg:
- dep.properties['Enhances'][enh] = '#'+enh
- found = 1 # found enhances in same task
- break
- if found == 0: # If not found seek in other tasks
- for enhtask in self.metapackagekeys:
- if enhtask == task:
- continue
- enhtdeps = self.tasks[enhtask]
- for seek_dependency in enhtdeps.dependencies.keys():
- for enhdep in enhtdeps.dependencies[seek_dependency]:
- if enh == enhdep.pkg:
- dep.properties['Enhances'][enh] = './' + enhtask + '#' + enh
- found = 1 # found enhances in other task
- break
- if found == 1:
- break
- logger.debug(" %s -> %s" % (enh, dep.properties['Enhances'][enh]))
-
-
-class TaskDependencies:
- # List of depencencies defined in one metapackage
- def __init__(self, blendname, task):
-
- self.data = ReadConfig(blendname)
- self.blendname = self.data['Blend']
- self.task = task
-
- # Dictionary with status of dependencies as key and list of DependantPackage
- # instances
- self.dependencies = {}
- for pkgstat in pkgstatus:
- self.dependencies[pkgstat] = []
-
- # If a Blend just bases on the meta package of an other Blend (this is the
- # case in Debian Science which bases on med-bio for biology and gis-workstation
- # for geography it makes no sense to build an own sentinel page but read
- # meta package information of other meta packages and include the content
- # of these while enabling to add further Dependencies as well
- #
- # metadepends should be a SVN URL
- #
- # This is NOT YET implemented
- self.metadepends = None
-
-
- def SetMetapackageInfo(self, pkgname, ddtptranslations=None):
- # Gather information (specifically description translations if exists) about metapackage itself
- self.metapkg = DependantPackage(self.blendname, self.task)
- #print "self.metapkg", self.metapkg
- self.metapkg.pkg = pkgname
- self.metapkg.source = self.blendname
- if not ddtptranslations:
- return
- for lang in languages:
- if ddtptranslations['description_'+lang]:
- self.metapkg.desc[lang] = {}
- try:
- short = to_unicode(ddtptranslations['description_'+lang])
- self.metapkg.desc[lang]['short'] = MarkupString(short, self.metapkg.pkg, 'taskShortDesc', lang)
-
- except UnicodeEncodeError, err:
- logger.error("===> UnicodeDecodeError in metapackage %s (lang='%s'): '%s'; ErrTxt: %s" % \
- (self.metapkg.pkg, lang, ddtptranslations['description_'+lang], err))
- short = to_unicode(ddtptranslations['description_'+lang],'latin1')
- self.metapkg.desc[lang]['short'] = MarkupString(short, self.metapkg.pkg, 'taskShortDesc' + lang)
-
- try:
- self.metapkg.desc[lang]['long'] = Markup(render_longdesc(ddtptranslations['long_description_'+lang].splitlines()))
-
- except UnicodeDecodeError, err:
- logger.error("===> UnicodeDecodeError in metapackage long %s (lang='%s'): '%s'; ErrTxt: %s" % \
- (self.metapkg.pkg, lang, ddtptranslations['long_description_'+lang], err))
- self.metapkg.desc[lang]['long'] = 'UnicodeDecodeError'
-
- except AttributeError, err:
- logger.error("===> AttributeError in metapackage long %s (lang='%s'): '%s'; ErrTxt: %s" % \
- (self.metapkg.pkg, lang, ddtptranslations['long_description_'+lang], err))
- self.metapkg.desc[lang]['long'] = 'Missing long description'
-
-
- def _AppendDependency2List(self, dep):
- # Append dependency which was found in the tasks file if not Ignore / Avoid and
- # no dupplication in case of source depencencies
- if dep == None:
- return
- if dep.dep_strength == 'i' or dep.dep_strength == 'a':
- logger.debug("Ignore/Avoid package : %s" % dep.pkg)
- return
-
- # Solves UnicodeEncodeError because of characters present in the
- # long description of the dependecies in 'unoficial','prospective' packages
- try:
- dep.desc['en']['long'] = dep.desc['en']['long'].encode('ascii','xmlcharrefreplace')
- except:
- pass
-
- # Solves UnicodeEncodeError because of characters present in the
- # authors name of the dependencies in 'wnpp' packages
- try:
- dep.properties['published']['authors'] = dep.properties['published']['authors'].encode('ascii','xmlcharrefreplace')
- except:
- pass
-
- # Solves UnicodeEncodeError because of characters present in the title of
- # dependencies in blends debian-science, debian-pan
- try:
- dep.properties['published']['title'] = dep.properties['published']['title'].encode('ascii','xmlcharrefreplace')
- except:
- pass
-
- logger.debug("Appending package to list : %s" % dep.pkg)
- self.dependencies[dep.pkgstatus].append(dep)
- return
-
-
- def GetDepInfo(self, curs, dependencies, pkg_src):
-
- # official pacakage
- # if pkg_src == 1:
- # logger.debug("=============OFFICIAL PACKAGES=============")
- # prospective package
- # elif pkg_src == 2:
- # logger.debug("=============PROSPECTIVE PACKAGES==========")
- # new package
- # elif pkg_src == 3:
- # logger.debug("=============NEW PACKAGES==================")
- # else:
- # logger.debug("=============UNKNOWN PACKAGE===============")
-
- for row in RowDictionaries(curs):
-
- # seek for package name in list of packages mentioned in tasks file
- found = False
- for dep in dependencies:
- if dep.pkg == row['package']:
- logger.debug("Package : %s" % dep.pkg)
- found = True
- break
-
- # Now set the information for the package found in the database (not for prospective packages)
- # Debian Edu contains packages from main/debian-installer - that's why we use startswith here
- if pkg_src != 2:
- if row['component'].startswith('main'):
- dep.component = 'main'
- if dep.dep_strength == 'd' or dep.dep_strength == 'r':
- dep.pkgstatus = 'official_high'
- elif dep.dep_strength == 's':
- dep.pkgstatus = 'official_low'
- else:
- dep.pkgstatus = 'new'
- else:
- dep.component = row['component']
- # If a package is not found in main its status can be maximum non-free
- dep.pkgstatus = 'non-free'
- elif pkg_src == 2:
- dep.pkgstatus = 'pkgvcs'
-
- # if a package is released *only* in experimental decrease package status
- if 'release' in row and row['release'] == 'experimental':
- dep.pkgstatus = 'experimental'
-
- if dep.dep_strength == 'i':
- dep.pkgstatus = 'ignore'
- elif dep.dep_strength == 'a':
- dep.pkgstatus = 'avoid'
-
- # dep.properties['license'] is already correct
- for prop in PROPERTIES:
- dep.properties[prop] = row[prop]
-
- type_vcs = ''
- if 'vcs_type' in row :
- type_vcs = 'vcs_type'
- elif 'vcs-type' in row:
- type_vcs = 'vcs-type'
- if type_vcs != '':
- dep.properties['vcs-type'] = row[type_vcs]
-
- type_url = ''
- if 'vcs_url' in row :
- type_url = 'vcs_url'
- elif 'vcs-url' in row:
- type_url = 'vcs-url'
- if type_url != '':
- dep.properties['vcs-url'] = row[type_url]
-
- type_browser = ''
- if 'vcs_browser' in row :
- type_browser = 'vcs_browser'
- elif 'vcs-browser' in row:
- type_browser = 'vcs-browser'
- if type_browser != '':
- dep.properties['vcs-browser'] = row[type_browser]
- elif dep.properties['vcs-browser'] == HOMEPAGENONE and type_vcs != '' and type_url != '':
- dep.properties['vcs-browser'] = BrowserFromVcsURL(dep.properties['vcs-type'], dep.properties['vcs-url'])
-
- # enhances to be written
- # if row['enhanced']:
- # for pkg in row['enhanced']:
- # dep.properties['Enhances'][pkg] = PKGURLMASK % pkg
- # enhancing_pkgs.append(pkg)
-
-
- if 'releases' in row:
- for i in range(len(row['releases'])):
- dep.version.append({'release':row['releases'][i], 'version': row['versions'][i], 'archs':row['architectures'][i]})
-
- if 'vote' in row:
- dep.popcon['vote'] = row['vote']
- if 'recent' in row:
- dep.popcon['recent'] = row['recent']
-
- # Debtags as sorted list of dict fields
- if 'debtags' in row:
- if row['debtags']:
- if dep.debtags: # there is no reasonable way that debtags was set before - so something is wrong here and a warning should be issued
- logger.warning("Debtags for package '%s' was just set. A duplicated result from database query is suspected. Please check the result!" % dep.pkg)
- tagdict = {}
- taglist = []
- for debtag in row['debtags']:
- (tag,value) = debtag.split('::')
- if tagdict.has_key(tag):
- tagdict[tag] += ', ' + value
- else:
- tagdict[tag] = value
- taglist.append(tag)
- if taglist:
- taglist.sort()
-
- for tag in taglist:
- dep.debtags.append({'tag':tag, 'value':tagdict[tag]})
-
- # screenshots
- if 'icon' in row:
- if row['icon']:
- dep.icon = row['icon'][0]
- dep.image = row['image'][0]
- dep.screenshot_url = 'http://screenshots.debian.net/package/' + dep.pkg
- for i in range(1,len(row['image'])):
- dep.screenshots.append({'version':row['screenshot_versions'][i], 'url':row['image'][i]})
-
- # it might be that the new upstream goes to experimental - this should be ignored here
- if 'unstable_parsed_version' in row:
- if row['unstable_parsed_version']:
- dep.outdated['release'] = 'upstream'
- dep.outdated['version'] = row['unstable_upstream']
- dep.outdated['architectures'] = ''
-
- if row['changed_by']:
- try:
- changed = to_unicode(row['changed_by'])
- except TypeError, err:
- changed = None
- logger.warning("Encoding problem for last uploader of package '%s' in task %s (%s)" % (dep.pkg, dep.taskname, err))
- if changed:
- try:
- (_name, _url) = email.Utils.parseaddr(changed)
- changed = '<a href="mailto:%s">%s</a>' % (_url, _name)
- dep.properties['changed_by'] = MarkupString(changed, dep.pkg, 'changed_by')
- dep.properties['last_uploader'] = to_unicode(changed)
- dep.properties['last_uploader_simple'] = to_unicode('%s <%s>' % (_name, _url))
- except UnicodeDecodeError, err:
- logger.error("Encoding problem for last uploader - assume same as maintainer for package %s (%s)", dep.pkg, err)
-
- # link to packages.debian.org search page to see overview about all
- # package versions in all releases
- # dep.properties['pkg-url'] = PKGURLMASK % dep.pkg
-
- for l in languages:
- if 'description_'+l in row and row['description_'+l] != None:
- dep.desc[l] = {}
- dep.desc[l]['short'] = MarkupString(to_unicode(row['description_'+l]), dep.pkg, 'ShortDesc')
- if row['long_description_'+l]:
- dep.desc[l]['long'] = Markup(render_longdesc(row['long_description_'+l].splitlines()))
- if not dep.desc['en'].has_key('short'):
- logger.error("Dep has no English short description: %s", dep.pkg)
- dep.desc['en']['short'] = "??? missing short description for package %s :-(" % dep.pkg
-
- (_name, _url) = email.Utils.parseaddr(row['maintainer'])
- dep.properties['maintainer'] = to_unicode(row['maintainer'])
- dep.responsible = '<a href="mailto:%s">%s</a>' % (_url, to_unicode(_name))
-
- self._AppendDependency2List(dep)
-
- def GetTaskDependencies(self, source=0):
- count = 0
- global dep_strength_keys
- dep = None
- alldepends = [] # contains the names of all the dependencies in a task file
- dependencies = [] # contains the all the info about a dependencies in a task file
-
- # Get the title and description of each task in a blend
- # Title, Description
- query = "SELECT DISTINCT title, description, long_description FROM blends_tasks WHERE task='%s' AND blend='%s'" % (self.task, self.blendname)
- _execute_udd_query(query)
- if curs.rowcount > 0:
- self.metapkg.PrintedName, short, long = curs.fetchone()
- # Markup strings to enable verbatim output of preformatted text
- self.metapkg.desc['en']['short'] = MarkupString(short.encode('utf-8'), self.metapkg.PrintedName, 'taskShortDesc')
- long = long.decode('utf-8')
- self.metapkg.desc['en']['long'] = MarkupString(long.encode('utf-8'), self.metapkg.PrintedName, 'taskLongDesc')
-
-
- # Get the package_name, license, dependency of the official dependencies of the task
- query = "SELECT DISTINCT b.package, b.dependency FROM blends_dependencies b JOIN packages p ON p.package=b.package \
- WHERE b.blend='%s' AND b.task='%s'" % (self.blendname, self.task)
- _execute_udd_query(query)
- if curs.rowcount > 0:
- dep_info = curs.fetchall()
- for each_dep in dep_info:
- # class DependantPackage - This class contains all the possible information about a package
- # Create an object of class DependantPackage
- dep = DependantPackage(self.blendname, self.task)
-
- # Information about the dependant packages
- dep.pkg = each_dep[0] # Name
- # dep.license
- dep.dep_strength = each_dep[1] # Depends: 'd' or Suggests: 's'
-
- alldepends.append(dep.pkg)
- dependencies.append(dep)
- #self.dependencies[dep.pkgstat].append(dep)
-
- query = "EXECUTE query_pkgs ('%s', '%s')" % (List2PgArray(alldepends), List2PgSimilarArray(alldepends))
- _execute_udd_query(query)
- if curs.rowcount > 0:
- self.GetDepInfo(curs, dependencies, 1)
- logger.debug("Executed query for official packages for %s task" % self.task)
-
-
- # Get the package_name, license, dependency of the prospective dependencies of the task
- query = "SELECT DISTINCT bp.package, bp.license, b.dependency \
- FROM blends_prospectivepackages bp JOIN blends_dependencies b ON b.blend=bp.blend AND b.package=bp.package \
- WHERE bp.blend='%s' and b.task='%s' ORDER BY bp.package" % (self.blendname, self.task)
- _execute_udd_query(query)
-
- alldepends = []
- dependencies = []
- if curs.rowcount > 0:
- pros_info = curs.fetchall()
- for each_pros in pros_info:
- dep = DependantPackage(self.blendname, self.task)
-
- dep.pkg = each_pros[0] # Name
- dep.license = each_pros[1]
- dep.dep_strength = each_pros[2]
-
- alldepends.append(dep.pkg)
- dependencies.append(dep)
-
-
- query = "SELECT DISTINCT bp.package, bp.license, b.dependency, bp.component, bp.homepage, bp.section, \
- bp.source, bp.vcs_type, bp.vcs_url, bp.vcs_browser, bp.changed_by, \
- bp.uploaders, bp.maintainer, pop.vote, pop.recent, tags.debtags, bp.description AS description_en, bp.long_description AS long_description_en\
- FROM blends_prospectivepackages bp JOIN blends_dependencies b ON b.blend=bp.blend AND b.package=bp.package \
- LEFT OUTER JOIN popcon pop ON pop.package=bp.package \
- LEFT OUTER JOIN ( \
- SELECT package, array_agg(tag) AS debtags \
- FROM debtags \
- WHERE tag NOT LIKE 'implemented-in::%%' \
- AND tag NOT LIKE 'protocol::%%' \
- AND tag NOT LIKE '%%::TODO' \
- AND tag NOT LIKE '%%not-yet-tagged%%' \
- GROUP BY package \
- ) tags ON tags.package = bp.package \
- WHERE bp.blend='%s' and b.task='%s' ORDER BY bp.package" % (self.blendname, self.task)
- _execute_udd_query(query)
- if curs.rowcount > 0:
- self.GetDepInfo(curs, dependencies, 2)
- logger.debug("Executed query for prospective packages for %s task" % self.task)
-
-
- # Get information about new dependencies of the task
- query = "SELECT DISTINCT new.package FROM new_packages new JOIN blends_dependencies b ON b.package=new.package \
- WHERE b.blend='%s' and b.task='%s' ORDER BY new.package" % (self.blendname, self.task)
- _execute_udd_query(query)
- alldepends = []
- dependencies = []
- pkgs_new = []
- if curs.rowcount > 0:
- pkgs_new = curs.fetchall()
-
- for each_new_pkg in pkgs_new:
- dep = DependantPackage(self.blendname, self.task)
-
- dep.pkg = each_new_pkg[0] # Name
-
- alldepends.append(dep.pkg)
- dependencies.append(dep)
-
- query = "EXECUTE query_new('%s')" % List2PgArray(alldepends)
- _execute_udd_query(query)
- if curs.rowcount > 0:
- self.GetDepInfo(curs, dependencies, 3)
- logger.debug("Executed query for new packages for %s task" % self.task)
-
- return 1
-
- def MarkupPreformatedStrings(self):
- # Genshi does not touch strings that are marked with "Markup()" - so just
- # mark the strings that are ready formatted
-
- for dependency in self.dependencies.keys():
- for dep in self.dependencies[dependency]:
- dep.responsible = MarkupString(dep.responsible, dep.pkg, 'responsible')
- if dep.desc['en'] != {}:
- dep.desc['en']['short'] = MarkupString(dep.desc['en']['short'], dep.pkg, 'pkgShortDesc')
- try:
- dep.desc['en']['long'] = MarkupString(dep.desc['en']['long'], dep.pkg, 'pkgLongDesc')
- except KeyError:
- logger.error("Dep has no English long description: %s", dep.pkg)
- dep.desc['en']['long'] = "??? Missing long description for package %s" % dep.pkg
-
- def __str__(self):
- ret = "Blend: " + self.blendname + ", " \
- "Task:" + self.task + ", " \
- "Dependencies:" + str(self.dependencies)
- return ret
diff --git a/webtools/tasks_udd.py b/webtools/tasks_udd.py
deleted file mode 100755
index 8c398ba..0000000
--- a/webtools/tasks_udd.py
+++ /dev/null
@@ -1,309 +0,0 @@
-#!/usr/bin/python
-
-import apt
-import apt_pkg
-import apt_inst
-
-from sys import argv, exit, stderr
-import os
-import re
-import gettext
-
-import time
-from datetime import datetime
-from email.Utils import formatdate
-
-from genshi.template import TemplateLoader
-from genshi import Markup
-from genshi.template.eval import UndefinedError
-
-from blendstasktools_udd import Tasks, GetDependencies2Use, pkgstatus, pkgstatus_sortedkeys, UnlockBlendsTools, CheckOrCreateOutputDir, SetFilePermissions
-from blendsunicode import to_unicode
-from blendslanguages import languages, language_dict
-
-if len(argv) <= 1:
- print >>stderr, "Usage: %s <Blend name>\n The <Blend name> needs a matching config file webconf/<Blend name>.conf"\
- % argv[0]
- exit(-1)
-
-# LockBlendsTools() # logger handler not defined at this moment, needs rethinking ... FIXME
-tasks = Tasks(argv[1])
-tasks.GetAllDependencies()
-packages = tasks.GetNamesOnlyDict()
-#print "pacakges : \n", packages
-tasks.GetAllDependentPackagesOfBlend()
-tasks.MarkupPreformatedStringsBlend()
-
-data = tasks.data
-#print "data : \n",data
-data['tasks'] = tasks.GetTaskDescDict()
-data['taskskeys'] = tasks.metapackagekeys
-try:
- data['popconsubmit'] = tasks.popconsubmit
-except:
- data['popconsubmit'] = 'unknown'
-data['languages'] = languages
-data['language_dict'] = language_dict
-# Work around the fact that it seems to be impossible to mention '&' in the template
-data['ampandworkaround'] = Markup('&force=1')
-
-# Define directories used
-current_dir = os.path.dirname(__file__)
-locale_dir = os.path.join(current_dir, 'locale')
-template_dir = os.path.join(current_dir, 'templates')
-
-# Initialize i18n
-domain = 'blends-webtools'
-gettext.install(domain)
-l10nstring = {}
-for lang in languages:
- l10nstring[lang] = gettext.translation(domain, locale_dir, languages=[lang], fallback = True)
-
-# Translated strings regarding the categorising of dependencies need to
-# be translated and because I did not found a working solution to get
-# gettext working with genshi all are collected here even if the additional
-# attributes to blendstasktools.pkgstatus rather should go blendstasktools.py
-
-
-# initialize gensi
-loader = TemplateLoader([template_dir], auto_reload=True)
-
-outputdir = CheckOrCreateOutputDir(tasks.data['outputdir'],'tasks_udd')
-if outputdir == None:
- exit(-1)
-
-t = datetime.now()
-htaccess = outputdir + '/.htaccess'
-try:
- os.unlink(htaccess)
-except: # simply continue if file does not exist
- pass
-htafp = open(htaccess, 'w')
-print >> htafp, "DirectoryIndex index index.html\nOptions +MultiViews"
-
-detect_javascript_code_re = re.compile("onmouseover=\"Tip\(\\'")
-detect_ampersand_code_re = re.compile("href=\"http://\w+\.debian\.org/.+\?\w+=")
-
-use_dependencystatus = GetDependencies2Use()
-for lang in languages:
- print >> htafp, "AddLanguage %s .%s" % (language_dict[lang]['htaccess'], lang)
- l10nstring[lang].install()
- _ = l10nstring[lang].ugettext
- data['lang'] = lang
- data['license'] = to_unicode(_('License'))
- data['version'] = to_unicode(_('Version'))
- data['summary'] = to_unicode(_('Summary'))
- data['updatetimestamp'] = to_unicode(_('Last update:')) + ' ' + formatdate(time.mktime(t.timetuple()))
- data['nopkgavail'] = to_unicode(_('Debian package not available'))
- # Description of package in specific packges (same for Depends, Recommends, Suggests)
- data['legend'] = to_unicode(_("For a better overview of the project's availability as a Debian package, each head row has a color code according to this scheme:"))
- data['discovery'] = Markup(to_unicode(_("""If you discover a project which looks like a good candidate for %s
- to you, or if you have prepared an unofficial Debian package, please do not hesitate to
- send a description of that project to the <a href="mailto:%s">%s mailing list</a>""") % \
- (data['projectname'], data['projectlist'], data['projectname'])))
- data['description'] = to_unicode(_("The list to the right includes various software projects which are of some interest to the %s Project. Currently, only a few of them are available as Debian packages. It is our goal, however, to include all software in %s which can sensibly add to a high quality Debian Pure Blend.") % (data['projectname'], data['projectname']))
- data['gtstrTasksPage'] = to_unicode(_('Tasks page'))
- data['gtstrProject'] = to_unicode(_('Project'))
- data['gtstrThisIsAList'] = to_unicode(_('This is a list of the Tasks %s is made of:') % data['projectname'])
- data['langavail'] = to_unicode(_('This page is also available in the following languages:'))
- data['howtosetlang'] = to_unicode(_('How to set <a href="%s">the default document language</a>'))
- data['howtosetlang'] = Markup(to_unicode(data['howtosetlang'] % ('http://www.debian.org/intro/cn.%s.html' % lang)))
- data['nohomepage'] = to_unicode(_('Homepage not available'))
- data['translatedesc'] = to_unicode(_('Translate description'))
- data['fixtranslation'] = to_unicode(_('Fix translated description'))
- data['popconexplanation'] = to_unicode(_('Popularitycontest results: number of people who use this package regularly (number of people who upgraded this package recently) out of'))
- data['tableofcontents'] = to_unicode(_('Table of contents'))
- data['packagelist'] = to_unicode(_('complete packagelist'))
- if data['advertising'] != None:
- # If data['advertising'] is enclosed in _() gettext tries to ask for translations of 'advertising'
- # which makes no sense. That's why this is masked by an extra string variable
- advertising = data['advertising']
- ##data['projectadvertising'] = _(advertising) # Hopefully translation will work this way ...
- # Genshi needs explicite information that it is dealing with an UTF-8 string which should not be changed
- advertising = _(advertising)
- data['projectadvertising'] = Markup(to_unicode(advertising))
- else:
- data['projectadvertising'] = None
-
- data['packages'] = to_unicode(_('Packages'))
- data['idxsummary'] = to_unicode(_("""A %sDebian Pure Blend%s is a Debian internal project which assembles
-a set of packages that might help users to solve certain tasks of their work. The list on
-the right shows the tasks of %s.""" ) \
- % ('<a href="http://blends.alioth.debian.org/blends/">', '</a>', data['projectname']))
- data['idxsummary'] = Markup(to_unicode(data['idxsummary']))
-
-
- pkgstatus['official_high']['headline'] = to_unicode(_('Official Debian packages with high relevance'))
- # before fiddling around with unicode() read
- # http://www.red-mercury.com/blog/eclectic-tech/python-unicode-fixing-utf-8-encoded-as-latin-1-iso-8859-1/
- pkgstatus['official_high']['pdolinkname'] = to_unicode(_('Official Debian package'))
- pkgstatus['official_low'] ['headline'] = to_unicode(_('Official Debian packages with lower relevance'))
- pkgstatus['official_low'] ['pdolinkname'] = pkgstatus['official_high']['pdolinkname']
- pkgstatus['non-free'] ['headline'] = to_unicode(_('Debian packages in contrib or non-free'))
- pkgstatus['non-free'] ['pdolinkname'] = to_unicode(_('Debian package in contrib/non-free'))
- pkgstatus['experimental'] ['headline'] = to_unicode(_('Debian packages in experimental'))
- pkgstatus['experimental'] ['pdolinkname'] = to_unicode(_('Debian package in experimental'))
- pkgstatus['new'] ['headline'] = to_unicode(_('Debian packages in New queue (hopefully available soon)'))
- pkgstatus['new'] ['pdolinkname'] = to_unicode(_('New Debian package'))
- pkgstatus['pkgvcs'] ['headline'] = to_unicode(_('Packaging has started and developers might try the packaging code in VCS'))
- pkgstatus['pkgvcs'] ['pdolinkname'] = to_unicode(_('Unofficial Debian package'))
- pkgstatus['unofficial'] ['headline'] = to_unicode(_('Unofficial packages built by somebody else'))
- pkgstatus['unofficial'] ['pdolinkname'] = pkgstatus['pkgvcs']['pdolinkname']
- pkgstatus['prospective'] ['headline'] = to_unicode(_('No known packages available'))
- pkgstatus['prospective'] ['pdolinkname'] = to_unicode(_('Debian package not available'))
- pkgstatus['wnpp'] ['headline'] = to_unicode(_('No known packages available but some record of interest (WNPP bug)'))
- pkgstatus['wnpp'] ['pdolinkname'] = pkgstatus['prospective']['pdolinkname']
- # Ignore/Avoid is only relevant for blends-dev, not webtools
- pkgstatus['ignore'] ['headline'] = 'Should not show up here'
- pkgstatus['prospective'] ['pdolinkname'] = ''
- pkgstatus['avoid'] ['headline'] = 'Should not show up here'
- pkgstatus['prospective'] ['pdolinkname'] = ''
- pkgstatus['unknown'] ['headline'] = 'Should not show up here' # There must be a bug somewhere
- pkgstatus['prospective'] ['pdolinkname'] = ''
-
- # Create the index page
- # Make sure that strings that need to be rendered as they are because they might
- # contain several html tags will be marked correctly
- VERBATIM=('projectadvertising', )
- for verbatim in VERBATIM:
- if data.has_key(verbatim) and data[verbatim] != None:
- data[verbatim] = Markup(data[verbatim])
-
- template = loader.load('tasks_idx.xhtml')
-
- outputfile = outputdir + '/index'
- if lang != 'xyz': # let 'en' be a language as any other and add suffix to file name
- outputfile += '.' + language_dict[lang]['short'] + '.html'
- try:
- os.unlink(outputfile)
- except: # simply continue if file does not exist
- pass
- f = open(outputfile, 'w')
- try:
- print >> f, template.generate(**data).render('xhtml')
- except UnicodeDecodeError, errtxt:
- print >> stderr, \
- "Some critical encoding problem occured when trying to render index for lang %s.\n%s" \
- % (lang, errtxt)
-
- f.close()
- SetFilePermissions(outputfile)
-
- data['dependencies'] = {}
- data['projects'] = {}
-
- data['headline'] = {}
- data['pdolinkname'] = {}
- data['maintainer'] = {}
-
- # I18n for headlines, link description, maintainer
- for status in use_dependencystatus:
- data['headline'][status] = Markup(to_unicode(pkgstatus[status]['headline']))
- data['pdolinkname'][status]= Markup(to_unicode(pkgstatus[status]['pdolinkname']))
- if pkgstatus[status]['order'] <= pkgstatus['experimental']['order']:
- data['maintainer'][status] = to_unicode(_('Maintainer'))
- else:
- data['maintainer'][status] = to_unicode(_('Responsible'))
-
- for task in data['taskskeys']:
- data['task'] = task
- # Keep the Dependency lists per task to be able to loop over all tasks in plain package list
- data['dependencies'][task] = []
- found_status = {}
- for status in use_dependencystatus:
- if len(tasks.tasks[task].dependencies[status]) > 0:
- if status == 'unknown':
- # Just print an error message if there are packages with unknown status
- if lang == 'en':
- # ... but only once and not per language
- for dep in tasks.tasks[task].dependencies[status]:
- print >>stderr, "Warning: Dependency with unknown status:", dep.pkg
- else:
- data['dependencies'][task].append(status)
- if status in data['dependencies'][task]:
- if not found_status.has_key(status):
- found_status[status] = 1
- # Keep the Project lists per task to be able to loop over all tasks in plain package list
- data['projects'][task] = tasks.tasks[task].dependencies
- data['othertasks'] = _("Links to other tasks")
- data['indexlink'] = _("Index of all tasks")
-
- outputfile = outputdir + '/' + task
- if lang != 'xyz': # let 'en' be a language as any other and add suffix to file name
- outputfile += '.' + language_dict[lang]['short'] + '.html'
- try:
- os.unlink(outputfile)
- except: # simply continue if file does not exist
- pass
-
- template = loader.load('tasks.xhtml')
- f = open(outputfile+'_tmp', "w")
- try:
- print >> f, template.generate(**data).render('xhtml')
- except UnicodeEncodeError, errtxt:
- print "Some critical encoding problem occured when trying to render task %s for lang %s.\n%s" \
- % (task, lang, errtxt)
- except UnicodeDecodeError, errtxt:
- print >> stderr, \
- "Some critical encoding problem occured when trying to render task %s for lang %s.\n%s" \
- % (task, lang, errtxt)
- except UndefinedError, errtxt:
- print >> stderr, \
- "UndefinedError while rendering task %s for lang %s.\n%s" \
- % (task, lang, errtxt)
- f.close()
- SetFilePermissions(outputfile+'_tmp')
- # Really rude hack to get back '<' / '>' signs which actually shoul be
- # in the output
- tmp = open(outputfile+'_tmp', "r")
- f = open(outputfile, "w")
- for line in tmp.readlines():
- # We had to mask ampersand ('&') from Genshi but even if the browser shows
- # the correct character packages.debian.org gets confused - so turn it back here
- if detect_ampersand_code_re.search(line):
- line = re.sub('%26', '&', line)
- print >>f, line,
- f.close()
- tmp.close()
- os.unlink(outputfile+'_tmp')
- SetFilePermissions(outputfile)
-
- template = loader.load('packagelist.xhtml')
-
- outputfile = outputdir + '/packagelist'
- if lang != 'xyz': # let 'en' be a language as any other and add suffix to file name
- outputfile += '.' + lang + '.html'
- try:
- os.unlink(outputfile)
- except: # simply continue if file does not exist
- pass
- f = open(outputfile, 'w')
-
- data['projectsintasks'] = []
- for task in data['taskskeys']:
- data['projectsintasks'] = tasks.tasks[task].dependencies
-
- try:
- print >> f, template.generate(**data).render('xhtml')
- except UnicodeDecodeError, errtxt:
- print >> stderr, \
- "Some critical encoding problem occured when trying to render long package list for lang %s.\n%s" \
- % (lang, errtxt)
- except UndefinedError, errtxt:
- print >> stderr, \
- "UndefinedError while trying to render long package list for lang %s.\n%s" \
- % (lang, errtxt)
-
- f.close()
- SetFilePermissions(outputfile)
-
-
-print >> htafp, "LanguagePriority",
-for lang in languages:
- print >> htafp, language_dict[lang]['htaccess'] ,
-print >> htafp
-
-htafp.close()
-SetFilePermissions(htaccess)
-
-UnlockBlendsTools()
--
Static and dynamic websites for Debian Pure Blends
More information about the Blends-commit
mailing list