[Blends-commit] r2602 - in /blends/trunk/webtools: blendslanguages.py blendstasktools.py tasks.py
tille at users.alioth.debian.org
tille at users.alioth.debian.org
Mon Jan 17 20:53:37 UTC 2011
Author: tille
Date: Mon Jan 17 20:53:36 2011
New Revision: 2602
URL: http://svn.debian.org/wsvn/blends/?sc=1&rev=2602
Log:
Reduce the number of database queries from order 10000 to order 100 by fetching all features for a set of packages in three querys per task. The number of dependencies per task does not matter for the number of queries any more
Added:
blends/trunk/webtools/blendslanguages.py
Modified:
blends/trunk/webtools/blendstasktools.py
blends/trunk/webtools/tasks.py
Added: blends/trunk/webtools/blendslanguages.py
URL: http://svn.debian.org/wsvn/blends/blends/trunk/webtools/blendslanguages.py?rev=2602&op=file
==============================================================================
--- blends/trunk/webtools/blendslanguages.py (added)
+++ blends/trunk/webtools/blendslanguages.py Mon Jan 17 20:53:36 2011
@@ -1,0 +1,38 @@
+#!/usr/bin/python
+# Copyright 2010: Andreas Tille <tille at debian.org>
+# License: GPL
+
+# Define languages which are supported in web sentinel pages
+
+from genshi import Markup
+
+# The keys of the following dictionary are used to address the translation files of DDTP
+# The values contain dictionaries in turn with the following meaning:
+# 'short' = shortcut of language as extension of the output files
+# 'htaccess' = language definition in htaccess
+# 'title' = English name of the language used in title attribut of link
+# 'printed' = printed name for links
+language_dict = { 'en' : {'short':'en', 'htaccess':'en' , 'title':'English', 'printed':Markup('English')},
+ 'cs' : {'short':'cs', 'htaccess':'cs' , 'title':'Czech', 'printed':Markup('česky')},
+ 'da' : {'short':'da', 'htaccess':'da' , 'title':'Danish' , 'printed':Markup('dansk')},
+ 'de' : {'short':'de', 'htaccess':'de' , 'title':'German', 'printed':Markup('Deutsch')},
+ 'es' : {'short':'es', 'htaccess':'es' , 'title':'Spanish', 'printed':Markup('español')},
+ 'fi' : {'short':'fi', 'htaccess':'fi' , 'title':'Finnish', 'printed':Markup('suomi')},
+ 'fr' : {'short':'fr', 'htaccess':'fr' , 'title':'French', 'printed':Markup('français')},
+ 'hu' : {'short':'hu', 'htaccess':'hu' , 'title':'Hungarian', 'printed':Markup('magyar')},
+ 'it' : {'short':'it', 'htaccess':'it' , 'title':'Italian', 'printed':Markup('Italiano')},
+ 'ja' : {'short':'ja', 'htaccess':'ja' , 'title':'Japanese', 'printed':Markup('日本語 (Nihongo)')},
+ 'ko' : {'short':'ko', 'htaccess':'ko' , 'title':'Korean', 'printed':Markup('한국어 (Hangul)')},
+ 'nl' : {'short':'nl', 'htaccess':'nl' , 'title':'Dutch', 'printed':Markup('Nederlands')},
+ 'pl' : {'short':'pl', 'htaccess':'pl' , 'title':'Polish', 'printed':Markup('polski')},
+ 'pt_BR' : {'short':'pt', 'htaccess':'pt-BR', 'title':'Portuguese', 'printed':Markup('Português')},
+ 'ru' : {'short':'ru', 'htaccess':'ru' , 'title':'Russian', 'printed':Markup('Русский (Russkij)')},
+ 'uk' : {'short':'uk', 'htaccess':'uk' , 'title':'Ukrainian', 'printed':Markup("українська (ukrajins'ka)")},
+ 'sk' : {'short':'sk', 'htaccess':'sk' , 'title':'Slovenian', 'printed':Markup('slovensky')},
+ 'sv' : {'short':'sv', 'htaccess':'sv' , 'title':'Swedish', 'printed':Markup('svenska')},
+ 'zh_CN' : {'short':'zh_cn', 'htaccess':'zh-CN', 'title':'Chinese (China)', 'printed':Markup('中文(简)')},
+ 'zh_TW' : {'short':'zh_tw', 'htaccess':'zh-TW', 'title':'Chinese (Taiwan)', 'printed':Markup('中文(正)')},
+ }
+# global languages
+languages = language_dict.keys()
+languages.sort()
Modified: blends/trunk/webtools/blendstasktools.py
URL: http://svn.debian.org/wsvn/blends/blends/trunk/webtools/blendstasktools.py?rev=2602&op=diff
==============================================================================
--- blends/trunk/webtools/blendstasktools.py (original)
+++ blends/trunk/webtools/blendstasktools.py Mon Jan 17 20:53:36 2011
@@ -35,8 +35,12 @@
# from genshi.input import HTML
from blendsmarkdown import SplitDescription, MarkupString, render_longdesc
-from debian_bundle import deb822
+try:
+ from debian import deb822
+except:
+ from debian_bundle import deb822
from blendsunicode import to_unicode
+from blendslanguages import languages
import logging
import logging.handlers
@@ -237,139 +241,151 @@
conn = psycopg2.connect(host="127.0.0.1",port=DEFAULTPORT,user="guest",database="udd")
curs = conn.cursor()
-query = """PREPARE query_pkg (text) AS SELECT
- distribution, component, version, architecture, maintainer,
- source, section, task, distribution, release, component, homepage,
- description, long_description FROM packages
- WHERE package = $1 ORDER BY version"""
-curs.execute(query)
-
-query = """PREPARE query_new (text) AS SELECT
+# uddlog = open('logs/uddquery.log', 'w')
+
+def _execute_udd_query(query):
+ try:
+ curs.execute(query)
+ # print >>uddlog, query
+ except psycopg2.ProgrammingError, err:
+ try:
+ print >>stderr, "%s; query was\n%s" % (err, to_unicode(query))
+ except UnicodeDecodeError, err2:
+ print >>stderr, "%s; beginn of query was\n%s\n%s" % (err, query[0], err2)
+ exit(-1)
+ except psycopg2.DataError, err:
+ print >>stderr, "%s; query was\n%s" % (err, query)
+
+query = """PREPARE query_pkgs (text[],text[]) AS
+ SELECT * FROM blends_query_packages($1,$2) AS (
+ package text, distribution text, release text, component text, version debversion,
+ maintainer text,
+ source text, section text, task text, homepage text,
+ maintainer_name text, maintainer_email text,
+ "vcs-type" text, "vcs-url" text, "vcs-browser" text,
+ changed_by text,
+ enhanced text[],
+ releases text[], versions text[], architectures text[],
+ unstable_upstream text, unstable_parsed_version text, unstable_status dehs_status, experimental_parsed_version text, experimental_status dehs_status,
+ vote int, recent int, -- popcon
+ debtags text[],
+ screenshot_versions text[], image text[], icon text[],
+ description_en text, long_description_en text,
+ description_cs text, long_description_cs text,
+ description_da text, long_description_da text,
+ description_de text, long_description_de text,
+ description_es text, long_description_es text,
+ description_fi text, long_description_fi text,
+ description_fr text, long_description_fr text,
+ description_hu text, long_description_hu text,
+ description_it text, long_description_it text,
+ description_ja text, long_description_ja text,
+ description_ko text, long_description_ko text,
+ description_nl text, long_description_nl text,
+ description_pl text, long_description_pl text,
+ "description_pt_BR" text, "long_description_pt_BR" text,
+ description_ru text, long_description_ru text,
+ description_sk text, long_description_sk text,
+ description_sv text, long_description_sv text,
+ description_uk text, long_description_uk text,
+ "description_zh_CN" text, "long_description_zh_CN" text,
+ "description_zh_TW" text, "long_description_zh_TW" text
+ )"""
+_execute_udd_query(query)
+
+query = """PREPARE query_new (text[]) AS SELECT
+ package,
distribution, component, version, architecture, maintainer,
source, section, distribution, 'new' AS release, component, homepage,
- description, long_description FROM new_packages
- WHERE package = $1 ORDER BY version LIMIT 1"""
-curs.execute(query)
+ description AS description_en, long_description AS long_description_en
+ FROM new_packages
+ WHERE (package, version) IN
+ (SELECT package, max(version) FROM new_packages WHERE package = ANY ($1) GROUP BY package)"""
+_execute_udd_query(query)
+
+# This prepared statement is called only once but it makes sense to mention it in the
+# header to keep all stuff in one place which needs to be changed in case a new language
+# will be added
+query = """PREPARE query_metapkg_trans (text[]) AS
+ SELECT * FROM blends_metapackage_translations($1) AS (
+ package text,
+ description_cs text, long_description_cs text,
+ description_da text, long_description_da text,
+ description_de text, long_description_de text,
+ description_es text, long_description_es text,
+ description_fi text, long_description_fi text,
+ description_fr text, long_description_fr text,
+ description_hu text, long_description_hu text,
+ description_it text, long_description_it text,
+ description_ja text, long_description_ja text,
+ description_ko text, long_description_ko text,
+ description_nl text, long_description_nl text,
+ description_pl text, long_description_pl text,
+ "description_pt_BR" text, "long_description_pt_BR" text,
+ description_ru text, long_description_ru text,
+ description_sk text, long_description_sk text,
+ description_sv text, long_description_sv text,
+ description_uk text, long_description_uk text,
+ "description_zh_CN" text, "long_description_zh_CN" text,
+ "description_zh_TW" text, "long_description_zh_TW" text
+ )"""
+_execute_udd_query(query)
# Sometimes the tasks file contains dependencies from virtual packages and we have to
# obtain the real packages which provide this dependency.
# First check whether there are such packages (only names)
-query = """PREPARE query_provides (text) AS
- SELECT package FROM packages WHERE provides = $1 GROUP BY package;"""
-curs.execute(query)
+query = """PREPARE query_provides (text[]) AS
+ SELECT DISTINCT package, provides FROM packages WHERE provides IS NOT NULL AND package = ANY($1) ;"""
+_execute_udd_query(query)
# Obtain more detailed information about packages that might provide a dependency
#query = """PREPARE query_provides_version_release (text) AS
# SELECT package, version, release FROM packages WHERE provides = $1
# GROUP BY version, package, release ORDER BY version DESC;"""
-#curs.execute(query)
-
-# Obtain the component a certain package might be in
-query = "PREPARE pkg_component (text) AS SELECT component FROM packages WHERE package = $1 GROUP BY component "
-curs.execute(query)
+#_execute_udd_query(query)
# Obtain the releases featuring a certain package, in case a package might show up in different components when
# considering different releases we apply a preference for main over contrib over non-free. If this is the case
# we mention only the releases of the selected component
query = "PREPARE pkg_releases (text, text) AS SELECT release FROM packages WHERE package = $1 AND component = $2 GROUP BY release "
-curs.execute(query)
-
-# Obtain available versions including the architectures where this version is available
-#query = """PREPARE pkg_versions_arch (text, text) AS
-# SELECT release, regexp_replace(version, '^[0-9]:', '') AS version, array_to_string(array_accum(architecture),',') AS architectures FROM
-# ( SELECT architecture, version,
-# release || CASE WHEN char_length(substring(distribution from '-.*')) > 0 THEN substring(distribution from '-.*') ELSE '' END AS release FROM packages
-# WHERE package = $1 AND component = $2
-# GROUP BY architecture, version, release, distribution
-# ORDER BY architecture
-# ) AS av
-# GROUP BY version, release ORDER BY version DESC;"""
-query = """PREPARE pkg_versions_arch (text) AS
- SELECT r as release, version, archs, component
- FROM versions_archs_component($1) AS (r text, version text, archs text, component text)
- -- you have to specify the column names because plain RECORD type is returned
- JOIN releases ON releases.release = r
- -- JOIN with release table to enable reasonable sorting
- WHERE r NOT LIKE '%-%'
- -- ignore releases like *-security etc.
- ORDER BY releases.sort ASC, version DESC;"""
-curs.execute(query)
+_execute_udd_query(query)
query = """PREPARE pkg_versions_stable_testing (text) AS
SELECT release, regexp_replace(regexp_replace(debversion, '-.*', ''), '[.+~]dfsg.*', '') AS version, debversion
FROM (SELECT r AS release, MAX(version) AS debversion
FROM versions_archs_component($1) AS (r text, version text, archs text, component text)
- WHERE r IN ('lenny', 'squeeze') GROUP BY r) AS zw;""" # Change here releases onace Squeeze is released
-curs.execute(query)
-
-# Obtain upstream version in case it is newer than Debian version
-query = """PREPARE pkg_dehs (text) AS
- SELECT DISTINCT d.source, unstable_upstream, unstable_parsed_version, unstable_status, experimental_parsed_version, experimental_status FROM dehs d
- JOIN packages p ON p.source = d.source
- WHERE p.package = $1 AND unstable_status = 'outdated' ;"""
-curs.execute(query)
-
-# Obtain Vcs Information from source table - use only latest version because this is
-query = """PREPARE src_vcs (text) AS
- SELECT vcs_type AS "vcs-type", vcs_url AS "vcs-url", vcs_browser AS "vcs-browser" FROM sources
- WHERE source = $1 AND (vcs_type IS NOT NULL OR vcs_url IS NOT NULL OR vcs_browser IS NOT NULL)
- ORDER BY version desc LIMIT 1"""
-curs.execute(query)
-
-# Obtain description in foreign language from DDTP project if available
-# For the moment just query for the highest available version of the description
-query = """PREPARE ddtp_description (text) AS
- SELECT language, description, long_description, version FROM DdtpLanguageMaxVersion($1)"""
-curs.execute(query)
-
-# Obtain popcon data:
-# inst : number of people who installed this package;
-# * vote : number of people who use this package regularly;
-# old : number of people who installed, but don't use this package regularly;
-# * recent : number of people who upgraded this package recently;
-# no-files : number of people whose entry didn't contain enough information (atime and ctime were 0).
-query = "PREPARE popcon (text) AS SELECT vote, recent FROM popcon WHERE package = $1"
-curs.execute(query)
-
-# Number of submissions
+ WHERE r IN ('lenny', 'squeeze') GROUP BY r) AS zw;""" # Change here releases once Squeeze is released
+_execute_udd_query(query)
+
+# Number of all submissions
query = "PREPARE popcon_submissions AS SELECT vote FROM popcon WHERE package = '_submissions'"
-curs.execute(query)
-
-# Debtags
-query = """PREPARE debtags (text) AS
- SELECT * FROM debtags WHERE
- package = $1 AND
- tag NOT LIKE 'implemented-in::%' AND
- tag NOT LIKE 'protocol::%' AND
- tag NOT LIKE '%::TODO' AND
- tag NOT LIKE '%not-yet-tagged%';"""
-curs.execute(query)
-
-query = """PREPARE query_screenshots (text) AS
- SELECT screenshot_url, large_image_url AS image, small_image_url AS icon, version
- FROM screenshots WHERE package = $1
- ORDER BY version DESC, image ;"""
-curs.execute(query)
-
-# This query might result in more packages than wanted, because when seeking for a
-# package 'foo' it also matches an enhances field of 'bar, xfoo-xx, foo-doc, bazz'
-# and thus we get a false positive. We have to preprocess the resultset by splitting
-# it into single packages and check again the whole word for matching. This is
-# implemented below in Python
-######################################################################################
-# ATTENTION: Call this "EXECUTE query_check_enhances('%"+pkg+"%')" #
-# I have no idea how to otherwise mask the '%' in the prepared statement. #
-######################################################################################
-query = """PREPARE query_check_enhances (text) AS
- SELECT DISTINCT package, enhances FROM packages WHERE enhances LIKE $1"""
-curs.execute(query)
-
-# Obtain e-mail address of latest uploader of source package
-query = """PREPARE query_get_latest_uploader (text) AS
- SELECT changed_by FROM upload_history WHERE source = $1 ORDER BY version DESC LIMIT 1;"""
-curs.execute(query)
+_execute_udd_query(query)
#########################################################################################
+
+def List2PgArray(list):
+ # turn a list of strings into the syntax for a PostgreSQL array:
+ # {"string1","string2",...,"stringN"}
+ if not list:
+ return '{}'
+ komma='{'
+ PgArray=''
+ for s in list:
+ PgArray=PgArray+komma+'"'+s+'"'
+ komma=','
+ return PgArray+'}'
+
+def List2PgSimilarArray(list):
+ # turn a list of strings surrounded by '%' into the syntax for a PostgreSQL array to enable LIKE conditions:
+ # {"%string1%","%string2%",...,"%stringN%"}
+ if not list:
+ return '{}'
+ komma='{'
+ PgSimArray=''
+ for s in list:
+ PgSimArray=PgSimArray+komma+'"%'+s+'%"'
+ komma=','
+ return PgSimArray+'}'
def ReadConfig(blendname=''):
# Try to read config file CONFDIR/<blendname>.conf
@@ -448,8 +464,11 @@
if os.path.isdir(tasksdir+'/.svn'):
logger.error("Trying old files in %s ..." % tasksdir)
else:
- logger.error("There are no old files in %s -> giving up" % tasksdir)
- exit(-1)
+ if os.listdir(tasksdir):
+ logger.warning("No .svn directory found in %s but trying those random files there as tasks files." % tasksdir)
+ else:
+ logger.error("There are no old files in %s -> giving up" % tasksdir)
+ exit(-1)
return data['datadir'] + '/tasks'
def RowDictionaries(cursor):
@@ -478,6 +497,8 @@
def BrowserFromVcsURL(vcs_type, vcs_url):
# Guess Vcs-Browser URL from VCS URL
+ if not vcs_type:
+ return HOMEPAGENONE
if vcs_type.lower().startswith('svn'):
ret_url = re.sub('^svn:', 'http:', vcs_url)
ret_url = re.sub('/svn/', '/wsvn/', ret_url)
@@ -563,233 +584,6 @@
self.remark = {} # Optional remark for a package
self.dep_strength = 0 # Type of Dependency (Depends, Recommends, Suggests, Experimental, New, Avoid, Ignore, WNPP
- def _QueryUDD4Package(self, source):
- # Query UDD for several properties of a package
- query = "EXECUTE pkg_component ('%s')" % self.pkg
- curs.execute(query)
- if curs.rowcount == 0:
- # the package does not exist in UDD table packages
- # verify whether we have some information in NEW queue
- query = "EXECUTE query_new ('%s')" % self.pkg
- curs.execute(query)
- if curs.rowcount == 0:
- return
-
- row = RowDictionaries(curs)[0]
- self.component = row['component']
- self.version = [{'release':'new', 'version': row['version'], 'archs':row['architecture']} , ]
- self.pkgstatus = 'new'
- self.properties['pkg-url'] = 'http://ftp-master.debian.org/new/%s_%s.html' % (row['source'], row['version'])
- for prop in PROPERTIES:
- self.properties[prop] = row[prop]
- self.desc['en']['short'] = MarkupString(to_unicode(row['description']), self.pkg, 'ShortDesc - New')
- self.desc['en']['long'] = Markup(render_longdesc(row['long_description'].splitlines()))
- (_name, _url) = email.Utils.parseaddr(row['maintainer'])
- self.responsible = '<a href="mailto:%s">%s</a>' % (_url, to_unicode(_name))
- logger.info("The package %s is not yet in Debian but it is just in the new queue. (Task %s)" % (self.pkg, self.taskname))
-
- return
- # = the package exists in UDD table packages
- else:
- # This might happen in cases where a package was in non-free and moved to main later or
- # something like this. If we find a certain package in more than one components we
- # prefer main over contrib over non-free.
- if curs.rowcount > 1:
- in_main = 0
- in_contrib = 0
- for component in curs.fetchall():
- if component[0] == 'main':
- self.component = 'main'
- in_main = 1
- break
- if component[0] == 'contrib':
- in_contrib = 1
- if in_main == 0:
- if in_contrib == 1:
- self.component = 'contrib'
- else:
- self.component = 'non-free'
- else:
- self.component = curs.fetchone()[0]
- # Debian Edu contains packages from main/debian-installer
- if self.component == 'main/debian-installer':
- self.component = 'main'
- self.properties['license'] = license_in_component[self.component]
-
- query = "EXECUTE pkg_releases ('%s', '%s')" % (self.pkg, self.component)
- curs.execute(query)
- has_official = 0
- for rel in curs.fetchall():
- self.releases.append(rel[0])
- if rel[0] != 'experimental':
- has_official = 1
-
- # query = "EXECUTE pkg_versions_arch ('%s', '%s')" % (self.pkg, self.component)
- query = "EXECUTE pkg_versions_arch ('%s')" % (self.pkg)
- curs.execute(query)
- if curs.rowcount > 0:
- for row in RowDictionaries(curs):
- if self.component.startswith(row['component']):
- self.version.append({'release':row['release'], 'version': row['version'], 'archs':row['archs']})
- else:
- self.version.append({'release':row['release'], 'version': row['version'] + ' (' + row['component'] + ')', 'archs':row['archs']})
-
- # Version in stable / testing for long table
- query = "EXECUTE pkg_versions_stable_testing ('%s')" % (self.pkg)
- curs.execute(query)
- if curs.rowcount > 0:
- prefix = 'Versions: '
- for row in RowDictionaries(curs):
- self.properties['stable_testing_version'].append((row['release'], row['debversion'], row['version'], prefix))
- prefix = ', '
-
- query = "EXECUTE pkg_dehs ('%s')" % (self.pkg)
- curs.execute(query)
- if curs.rowcount > 0:
- row = RowDictionaries(curs)[0]
- # it might be that the new upstream goes to experimental - this should be ignored here
- if row['experimental_status'] != 'uptodate':
- self.outdated['release'] = 'upstream'
- self.outdated['version'] = row['unstable_upstream']
- self.outdated['architectures'] = ''
-
- query = "EXECUTE popcon ('%s')" % (self.pkg)
- curs.execute(query)
- if curs.rowcount > 0:
- row = RowDictionaries(curs)[0]
- self.popcon['vote'] = row['vote']
- self.popcon['recent'] = row['recent']
-
- query = "EXECUTE debtags ('%s')" % (self.pkg)
- curs.execute(query)
- if curs.rowcount > 0:
- tagdict = {}
- taglist = []
- for row in RowDictionaries(curs):
- (tag,value) = row['tag'].split('::')
- if tagdict.has_key(tag):
- tagdict[tag] += ', ' + value
- else:
- tagdict[tag] = value
- taglist.append(tag)
- if taglist:
- taglist.sort()
- for tag in taglist:
- self.debtags.append({'tag':tag, 'value':tagdict[tag]})
-
- query = "EXECUTE query_screenshots ('%s')" % (self.pkg)
- curs.execute(query)
- if curs.rowcount > 0:
- rows = RowDictionaries(curs)
- self.icon = rows[0]['icon']
- self.image = rows[0]['image']
- self.screenshot_url = rows[0]['screenshot_url']
- # if a package has more than one screenshots provide a list of these
- if curs.rowcount > 1:
- for row in rows:
- self.screenshots.append({'version':row['version'], 'url':row['image']})
-
- if has_official == 1:
- if self.component == 'main':
- if self.dep_strength == 'Depends' or self.dep_strength == 'Recommends':
- self.pkgstatus = 'official_high'
- else:
- self.pkgstatus = 'official_low'
- else:
- self.pkgstatus = 'non-free'
- else:
- self.pkgstatus = 'experimental'
-
- # link to packages.debian.org search page to see overview about all
- # package versions in all releases
- self.properties['pkg-url'] = PKGURLMASK % self.pkg
-
- query = "EXECUTE query_pkg ('%s')" % self.pkg
- curs.execute(query)
-
- for row in RowDictionaries(curs):
- for prop in PROPERTIES:
- self.properties[prop] = row[prop]
- self.desc['en']['short'] = MarkupString(to_unicode(row['description']), self.pkg, 'ShortDesc')
- self.desc['en']['long'] = Markup(render_longdesc(to_unicode(row['long_description']).splitlines()))
- (_name, _url) = email.Utils.parseaddr(row['maintainer'])
- self.properties['maintainer'] = to_unicode(row['maintainer'])
- self.responsible = '<a href="mailto:%s">%s</a>' % (_url, to_unicode(_name))
-
- if self.properties.has_key('source'):
- query = "EXECUTE src_vcs ('%s')" % (self.properties['source'])
- curs.execute(query)
- if curs.rowcount > 0:
- # There is only one line returned by this query
- row = RowDictionaries(curs)[0]
- # If some information about Vcs is found in the database make sure it is ignored from tasks file
- self.vcs_found = 1
- for prop in row.keys():
- if row[prop]:
- self.properties[prop] = row[prop]
- if not self.properties.has_key('vcs-browser') or self.properties['vcs-browser'] == HOMEPAGENONE:
- try:
- self.properties['vcs-browser'] = BrowserFromVcsURL(self.properties['vcs-type'], self.properties['vcs-url'])
- except KeyError, err:
- logger.warning("Vcs Property missing in Database:", self.properties, err)
- if not self.properties.has_key('vcs-type') or not self.properties['vcs-type']:
- if self.properties.has_key('vcs-browser') and self.properties['vcs-browser'] != HOMEPAGENONE:
- self.properties['vcs-type'] = VcsTypeFromBrowserURL(self.properties['vcs-browser'])
- # We are only interested in source packages (for instance for Bugs page)
- if source == 1:
- self.pkg = self.properties['source']
- # Stop using source package in self.pkg because we need the source package to obtain latest uploaders and
- # and bugs should be rendered in the same job - so we need the differentiation anyway
- self.src = self.properties['source']
- else:
- logger.error("Failed to obtain source for package", self.pkg)
- return
-
- query = "EXECUTE query_get_latest_uploader ('%s')" % (self.src)
- curs.execute(query)
- try:
- changed = to_unicode(curs.fetchone()[0])
- except TypeError, err:
- changed = None
- logger.info("Query '%s' does not result in a valid changed entry (%s)" % (query, err))
- if changed:
- # self.properties['maintainer'] = to_unicode(self.properties['maintainer']) # .encode('utf-8')
- try:
- if not changed.startswith(self.properties['maintainer']):
- (_name, _url) = email.Utils.parseaddr(changed)
- changed = '<a href="mailto:%s">%s</a>' % (_url, _name)
- self.properties['changed_by'] = MarkupString(changed, self.pkg, 'changed_by')
- self.properties['last_uploader'] = to_unicode(changed)
- self.properties['last_uploader_simple'] = to_unicode('%s <%s>' % (_name, _url))
- except UnicodeDecodeError, err:
- logger.error("Failed to compare changed with maintainer - assume both are the same for package %s (%s)", self.pkg, err)
-
- if source == 0: # If we are querying for source packages to render BTS pages
- # tranlations are irrelevant - so only obtain ddtp translations
- # otherwise
- query = "EXECUTE ddtp_description ('%s')" % (self.pkg)
- curs.execute(query)
-
- for row in RowDictionaries(curs):
- lang = row['language']
- self.desc[lang] = {}
- self.desc[lang]['short'] = MarkupString(to_unicode(row['description']), self.pkg, 'ShortDesc - ' + lang)
- self.desc[lang]['long'] = Markup(render_longdesc(to_unicode(row['long_description']).splitlines()))
-
- query = "EXECUTE query_check_enhances ('%"+self.pkg+"%')"
- curs.execute(query)
-
- if curs.rowcount > 0:
- for row in RowDictionaries(curs):
- enhancelist = row['enhances'].split(', ')
- if self.pkg in enhancelist:
- if not row['package'] in self.properties['Enhances'].keys():
- self.properties['Enhances'][row['package']] = PKGURLMASK % row['package']
- # if self.properties['Enhances'] != {}:
- # print "The following packages are enhancing %s: " % self.pkg,
- # for enh in self.properties['Enhances'].keys():
- # print enh,
- # print
# sort these objects according to the package name
def __cmp__(self, other):
@@ -816,7 +610,17 @@
if self.desc:
ret += ", desc: " + str(self.desc)
for prop in self.properties.keys():
- ret += ", %s: %s" % (prop, str(self.properties[prop]))
+ try:
+ ret += ", %s: %s" % (prop, to_unicode(str(self.properties[prop])))
+ except UnicodeEncodeError:
+ ret += ", %s: <UnicodeEncodeError>" % (prop)
+ ret += ", popcon = %i (%i)" % (self.popcon['vote'], self.popcon['recent'])
+ ret += ", debtags = " + str(self.debtags)
+ if self.icon:
+ ret += ", screenshoticon = " + self.icon
+ if self.outdated:
+ ret += ", dehsstatus = " + str(self.outdated)
+
# if self.desc['en']:
# ret += ", desc['en']:" + str(self.desc['en'])
return ret
@@ -867,9 +671,6 @@
# If we want to subscribe ddpo we need the source package names.
# In this case set source=1
- if self.metapackagekeys == []:
- self._GetMetapackages()
-
# Obtain the prefix of the meta packages of the Blend using blends-dev tools blend_get_names
if os.access('/usr/share/blends-dev/blend-get-names', os.X_OK):
blend_get_names = '/usr/share/blends-dev/blend-get-names'
@@ -884,15 +685,42 @@
pipe = Popen(cmd, shell=True, stdout=PIPE).stdout
prefix = pipe.read().strip() + '-'
pipe.close()
+
+ metapackages = []
for task in self.metapackagekeys:
- td = TaskDependencies(self.blendname, task=task, tasksdir=self.tasksdir, prefix=prefix)
+ metapackages.append(prefix + task)
+
+ # Verify whether there are any translations of metapackage descriptions which on
+ # one hand is a sign that this Blend has uploaded metapackages at all and on the
+ # other hand gives a clue about whether it makes sense to query for description
+ # translations
+ query = "SELECT COUNT(*) FROM ddtp WHERE package = ANY ('%s')" % List2PgArray(metapackages)
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ hastranslations = curs.fetchone()[0]
+
+ metapkg_translations = {}
+ if hastranslations > 0:
+ query = "EXECUTE query_metapkg_trans('%s')" % List2PgArray(metapackages)
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ for row in RowDictionaries(curs):
+ metapkg_translations[row['package']] = row
+
+ for task in self.metapackagekeys:
+ td = TaskDependencies(self.blendname, task=task, tasksdir=self.tasksdir)
+ pkgname = prefix + task
+ translations = None
+ if metapkg_translations.has_key(pkgname):
+ translations = metapkg_translations[pkgname]
+ td.SetMetapackageInfo(pkgname, translations)
td.GetTaskDependencies(source)
self.tasks[task] = td
if source == 0:
# overall popcon submissions
query = "EXECUTE popcon_submissions"
- curs.execute(query)
+ _execute_udd_query(query)
if curs.rowcount > 0:
self.popconsubmit = curs.fetchone()[0]
self.LinkEnhances()
@@ -1080,7 +908,7 @@
class TaskDependencies:
# List of depencencies defined in one metapackage
- def __init__(self, blendname, task, tasksdir=None, prefix=''):
+ def __init__(self, blendname, task, tasksdir=None):
self.data = ReadConfig(blendname)
self.blendname = self.data['Blend']
@@ -1101,10 +929,6 @@
for pkgstat in pkgstatus:
self.dependencies[pkgstat] = []
- # Main information for a task
- self.metapkg = DependantPackage(self.blendname, self.task)
- self.metapkg.pkg = prefix + task
-
# If a Blend just bases on the meta package of an other Blend (this is the
# case in Debian Science which bases on med-bio for biology and gis-workstation
# for geography it makes no sense to build an own sentinel page but read
@@ -1115,6 +939,34 @@
#
# This is NOT YET implemented
self.metadepends = None
+
+ def SetMetapackageInfo(self, pkgname, ddtptranslations=None):
+ # Gather information (specifically description translations if exists) about metapackage itself
+ self.metapkg = DependantPackage(self.blendname, self.task)
+ self.metapkg.pkg = pkgname
+ if not ddtptranslations:
+ return
+ for lang in languages:
+ # we do not use the English package description but rather the description in the tasks file itself
+ # so ignore the English language here and moreover check whether a translation of the language in question exists
+ if lang != 'en' and \
+ ddtptranslations['description_'+lang]:
+ self.metapkg.desc[lang] = {}
+ try:
+ short = to_unicode(ddtptranslations['description_'+lang])
+ self.metapkg.desc[lang]['short'] = MarkupString(short, self.metapkg.pkg, 'taskShortDesc', lang)
+ except UnicodeEncodeError, err:
+ logger.error("===> UnicodeDecodeError in metapackage %s (lang='%s'): '%s'; ErrTxt: %s" % \
+ (self.metapkg.pkg, lang, ddtptranslations['description_'+lang], err))
+ short = to_unicode(ddtptranslations['description_'+lang],'latin1')
+ self.metapkg.desc[lang]['short'] = MarkupString(short, self.metapkg.pkg, 'taskShortDesc' + lang)
+
+ try:
+ self.metapkg.desc[lang]['long'] = Markup(render_longdesc(ddtptranslations['long_description_'+lang].splitlines()))
+ except UnicodeDecodeError, err:
+ logger.error("===> UnicodeDecodeError in metapackage long %s (lang='%s'): '%s'; ErrTxt: %s" % \
+ (self.metapkg.pkg, lang, ddtptranslations['long_description_'+lang], err))
+ self.metapkg.desc[lang]['long'] = 'UnicodeDecodeError'
def _AppendDependency2List(self, dep, source):
# Append dependency which was found in the tasks file if not Ignore / Avoid and
@@ -1142,6 +994,7 @@
global dep_strength_keys
f = file(self.taskfile)
+ found_description=False
for stanza in deb822.Sources.iter_paragraphs(f):
# Why and Responsible can be valid for more than one dependency
# Store them in strings and use them for all Dependent Package objects
@@ -1158,53 +1011,21 @@
self.metapkg.PrintedName = to_unicode(stanza['task'])
continue
if key == 'Description':
- (short, long) = SplitDescription(stanza['description'])
- # Markup strings to enable verbatim output of preformatted text
- self.metapkg.desc['en']['short'] = MarkupString(short.encode('utf-8'), self.metapkg.PrintedName, 'taskShortDesc')
- self.metapkg.desc['en']['long'] = MarkupString(long.encode('utf-8'), self.metapkg.PrintedName, 'taskLongDesc')
-
- # find maintainer and other metadata:
- query = "EXECUTE query_pkg ('%s')" % self.metapkg.pkg
- curs.execute(query)
- if curs.rowcount > 0:
- row = RowDictionaries(curs)[0]
- (_name, _url) = email.Utils.parseaddr(row['maintainer'])
- self.responsible = '<a href="mailto:%s">%s</a>' % (_url, to_unicode(_name))
-
- # if an English description was found chances are good that we are seeking for other languages
- # as well
- if source == 0: # If we are querying for source packages to render BTS pages
- # tranlations are irrelevant - so only obtain ddtp translations
- # otherwise
- query = "EXECUTE ddtp_description ('%s')" % (self.metapkg.pkg)
- curs.execute(query)
-
- for row in RowDictionaries(curs):
- lang = row['language']
- self.metapkg.desc[lang] = {}
- try:
- short = to_unicode(row['description'])
- self.metapkg.desc[lang]['short'] = MarkupString(to_unicode(short), \
- self.metapkg.PrintedName, 'taskShortDesc - ' + lang)
- except UnicodeEncodeError, err:
- logger.error("===> UnicodeDecodeError in metapackage %s (lang='%s'): '%s'; ErrTxt: %s" % \
- (self.metapkg.pkg, lang, row['description'], err))
- short = to_unicode(row['description'],'latin1')
- self.metapkg.desc[lang]['short'] = MarkupString(short, \
- self.metapkg.PrintedName, 'taskShortDesc - ' + lang)
-
- try:
- self.metapkg.desc[lang]['long'] = Markup(render_longdesc(to_unicode(row['long_description']).splitlines()))
- except UnicodeDecodeError, err:
- logger.error("===> UnicodeDecodeError in metapackage long %s (lang='%s'): '%s'; ErrTxt: %s" % \
- (self.metapkg.pkg, lang, row['long_description'], err))
- self.metapkg.desc[lang]['long'] = 'UnicodeDecodeError'
-
+ if found_description:
+ logger.error("Duplicate description entry in task %s; you probably want to use Pkg-Description field instead!" % self.task)
+ else:
+ (short, long) = SplitDescription(stanza['description'])
+ # Markup strings to enable verbatim output of preformatted text
+ self.metapkg.desc['en']['short'] = MarkupString(short.encode('utf-8'), self.metapkg.PrintedName, 'taskShortDesc')
+ self.metapkg.desc['en']['long'] = MarkupString(long.encode('utf-8'), self.metapkg.PrintedName, 'taskLongDesc')
+ found_description = True
continue
if key == 'Meta-Depends':
self.metadepends = stanza['meta-depends']
+ continue
if key == 'Meta-Suggests':
self.metadepends = stanza['meta-suggests']
+ continue
if key == 'Why':
why = stanza['why']
continue
@@ -1212,7 +1033,7 @@
responsible = stanza['responsible'].strip()
if not dep:
# Make sure there is really enough information to deal with provided by the package
- logger.error("Missing package information for field %s = %s" % (key, responsible))
+ logger.error("Missing package information for field %s = %s in task file %s" % (key, responsible, self.task))
continue
if dep.responsible != None:
# we are dealing with an official package that has a real maintainer who
@@ -1223,7 +1044,11 @@
continue
if responsible != '':
(_name, _url) = email.Utils.parseaddr(responsible)
- dep.responsible = '<a href="mailto:%s">%s</a>' % (_url, to_unicode(_name))
+ _name = to_unicode(_name)
+ try:
+ dep.responsible = '<a href="mailto:%s">%s</a>' % (_url, _name)
+ except UnicodeDecodeError, err:
+ logger.error("Unicode problem when decoding name of maintainer with mail address <%s> in task %s (%s)" % (_url, self.task, err))
continue
if key in dep_strength_keys:
@@ -1262,8 +1087,6 @@
dep.pkg = dep_in_line.lower()
logger.warning("Package names may not contain upper case letters, so %s is an invalid package name which is turned into %s" \
% (dep_in_line, dep.pkg))
-
- dep._QueryUDD4Package(source)
continue
@@ -1389,17 +1212,12 @@
% (key, stanza[key.lower()]))
elif key == 'WNPP':
if dep != None:
- # it makes sense to assume that a package where WNPP was recently closed
- # shows up in unstable first
- if dep.component != None:
- logger.info("WNPP for package %s just closed - extra information can be removed from task file %s." % (dep.pkg, dep.taskname))
- else:
- wnpp = stanza['wnpp'].strip()
- # in case somebody prepended a '#' sign before the bug number
- wnpp = re.sub('^#', '', wnpp)
- # if there is really a number given
- if re.compile("^\d+$").search(wnpp):
- dep.properties['wnpp'] = wnpp
+ wnpp = stanza['wnpp'].strip()
+ # in case somebody prepended a '#' sign before the bug number
+ wnpp = re.sub('^#', '', wnpp)
+ # if there is really a number given
+ if re.compile("^\d+$").search(wnpp):
+ dep.properties['wnpp'] = wnpp
else:
logger.error("Dep not initiated before WNPP %s -> something is wrong." \
% stanza['wnpp'])
@@ -1447,8 +1265,9 @@
if dep == None:
continue # ... with next stanza
# seek for certain field set in the tasks file to move the dependency into different
- # categories of development status of not yet included packages
- if dep.pkgstatus == 'unknown':
+ # categories of development status of not yet included packages provided that there
+ # is at least a package description given
+ if dep.pkgstatus == 'unknown' and dep.desc['en'] != {}:
flag = 0
# first check those packages where some work was just done
for status in ['pkgvcs', 'unofficial', 'wnpp', 'prospective']:
@@ -1461,11 +1280,308 @@
break
if flag == 1:
break
+
+ tmp_dep_list.append(dep)
+ # remarks which are common to several dependencies in a list have to be added to all of the dependencies
+ for dep in tmp_dep_list:
+ if remark != {}:
+ dep.remark = remark
+ if fields_obsolete != [] and dep.pkgstatus != 'new':
+ logger.info("Package %s is an official package and has information in UDD. The following extra information can be removed from tasks file %s: %s" % \
+ (dep.pkg, dep.taskname, str(fields_obsolete)))
+ self._AppendDependency2List(dep, source)
+
+ f.close()
+
+ alldepends=[]
+ for status in self.dependencies.keys():
+ for dep in self.dependencies[status]:
+ alldepends.append(dep.pkg)
+
+ if not alldepends:
+ logger.warning("No dependencies defined in taskfile %s" % self.task)
+ return
+ query = "EXECUTE query_pkgs ('%s', '%s')" % (List2PgArray(alldepends), List2PgSimilarArray(alldepends))
+ _execute_udd_query(query)
+ pkgs_in_pool = []
+ if curs.rowcount > 0:
+ for row in RowDictionaries(curs):
+ # seek for package name in list of packages mentioned in tasks file
+ found = False
+ for status in self.dependencies.keys():
+ for dep in self.dependencies[status]:
+ if dep.pkg == row['package']:
+ found = True
+ break
+ if found:
+ break
+ if not found:
+ # this should not happen ...
+ logger.info("The package %s was found in package pool but never mentioned in task %s." % (row['package'], self.task))
+ continue
+
+ # Now set the information for the package found in the database
+ # Debian Edu contains packages from main/debian-installer - that's why we use startswith here
+ if row['component'].startswith('main'):
+ dep.component = 'main'
+ if dep.dep_strength == 'Depends' or dep.dep_strength == 'Recommends':
+ dep.pkgstatus = 'official_high'
+ elif dep.dep_strength == 'Suggests':
+ dep.pkgstatus = 'official_low'
+ else:
+ dep.component = row['component']
+ # If a package is not found in main its status can be maximum non-free
+ dep.pkgstatus = 'non-free'
+ # if a package is released *only* in experimental decrease package status
+ if row['release'] == 'experimental':
+ dep.pkgstatus = 'experimental'
+
+ # move dependant package to different status list if needed because a status change was detected
+ if dep.pkgstatus != status:
+ self.dependencies[status].remove(dep)
+ self.dependencies[dep.pkgstatus].append(dep)
+
+ # Warn about remaining information of prospective package
+ if ( dep.desc['en'] and dep.desc['en']['short'] ) and \
+ not dep.debtags: # prevent informing about packages which are just duplicated because of a broken query
+ logger.info("WNPP for package %s just closed - extra information can be removed from task file %s." % (dep.pkg, dep.taskname))
+
+ dep.properties['license'] = license_in_component[dep.component]
+ for prop in PROPERTIES:
+ dep.properties[prop] = row[prop]
+ for prop in ('vcs-type', 'vcs-url'):
+ dep.properties[prop] = row[prop]
+ if row['vcs-browser']:
+ dep.properties['vcs-browser'] = row['vcs-browser']
+ else:
+ if dep.properties['vcs-browser'] == HOMEPAGENONE:
+ dep.properties['vcs-browser'] = BrowserFromVcsURL(dep.properties['vcs-type'], dep.properties['vcs-url'])
+
+ if row['enhanced']:
+ for pkg in row['enhanced']:
+ dep.properties['Enhances'][pkg] = PKGURLMASK % pkg
+
+ for i in range(len(row['releases'])):
+ dep.version.append({'release':row['releases'][i], 'version': row['versions'][i], 'archs':row['architectures'][i]})
+
+ dep.popcon['vote'] = row['vote']
+ dep.popcon['recent'] = row['recent']
+
+ # Debtags as sorted list of dict fields
+ if row['debtags']:
+ if dep.debtags: # there is no reasonable way that debtags was set before - so something is wrong here and a warning should be issued
+ logger.warning("Debtags for package '%s' was just set. A duplicated result from database query is suspected. Please check the result!" % dep.pkg)
+ tagdict = {}
+ taglist = []
+ for debtag in row['debtags']:
+ (tag,value) = debtag.split('::')
+ if tagdict.has_key(tag):
+ tagdict[tag] += ', ' + value
+ else:
+ tagdict[tag] = value
+ taglist.append(tag)
+ if taglist:
+ taglist.sort()
+ for tag in taglist:
+ dep.debtags.append({'tag':tag, 'value':tagdict[tag]})
+
+ # screenshots
+ if row['icon']:
+ dep.icon = row['icon'][0]
+ dep.image = row['image'][0]
+ dep.screenshot_url = 'http://screenshots.debian.net/package/' + dep.pkg
+ for i in range(1,len(row['image'])):
+ dep.screenshots.append({'version':row['screenshot_versions'][i], 'url':row['image'][i]})
+
+ # it might be that the new upstream goes to experimental - this should be ignored here
+ if row['experimental_status'] != 'uptodate' and row['unstable_parsed_version']:
+ dep.outdated['release'] = 'upstream'
+ dep.outdated['version'] = row['unstable_upstream']
+ dep.outdated['architectures'] = ''
+
+ if row['changed_by']:
+ try:
+ changed = to_unicode(row['changed_by'])
+ except TypeError, err:
+ changed = None
+ logger.warning("Encoding problem for last uploader of package '%s' in task %s (%s)" % (dep.pkg, dep.taskname, err))
+ if changed:
+ try:
+ (_name, _url) = email.Utils.parseaddr(changed)
+ changed = '<a href="mailto:%s">%s</a>' % (_url, _name)
+ dep.properties['changed_by'] = MarkupString(changed, dep.pkg, 'changed_by')
+ dep.properties['last_uploader'] = to_unicode(changed)
+ dep.properties['last_uploader_simple'] = to_unicode('%s <%s>' % (_name, _url))
+ except UnicodeDecodeError, err:
+ logger.error("Encoding problem for last uploader - assume same as maintainer for package %s (%s)", dep.pkg, err)
+
+ # link to packages.debian.org search page to see overview about all
+ # package versions in all releases
+ dep.properties['pkg-url'] = PKGURLMASK % dep.pkg
+
+ for l in languages:
+ if row['description_'+l]:
+ dep.desc[l] = {}
+ dep.desc[l]['short'] = MarkupString(to_unicode(row['description_'+l]), dep.pkg, 'ShortDesc')
+ if row['long_description_'+l]:
+ dep.desc[l]['long'] = Markup(render_longdesc(row['long_description_'+l].splitlines()))
+ (_name, _url) = email.Utils.parseaddr(row['maintainer'])
+ dep.properties['maintainer'] = to_unicode(row['maintainer'])
+ dep.responsible = '<a href="mailto:%s">%s</a>' % (_url, to_unicode(_name))
+
+ pkgs_in_pool.append(dep.pkg)
+ # DEBUG
+ # print dep
+
+ pkgs_not_in_pool = []
+ for status in self.dependencies.keys():
+ for dep in self.dependencies[status]:
+ if dep.pkg not in pkgs_in_pool:
+ pkgs_not_in_pool.append(dep.pkg)
+
+ # Gather information about packages in NEW
+ query = "EXECUTE query_new('%s')" % List2PgArray(pkgs_not_in_pool)
+ _execute_udd_query(query)
+ pkgs_in_new = []
+ if curs.rowcount > 0:
+ for row in RowDictionaries(curs):
+ pkgs_in_new.append(row['package'])
+ # seek for package name in list of packages mentioned in tasks file
+ found = False
+ for status in self.dependencies.keys():
+ for dep in self.dependencies[status]:
+ if dep.pkg == row['package']:
+ found = True
+ break
+ if found:
+ break
+ if not found:
+ # this should not happen ...
+ logger.info("The package %s was found in new but never mentioned in task %s." % (row['package'], self.task))
+ continue
+ # Check for korrekt status 'new'
+ if status != 'new':
+ self.dependencies[status].remove(dep)
+ self.dependencies['new'].append(dep)
+ dep.pkgstatus = 'new'
+ dep.component = row['component']
+ dep.version = [{'release':'new', 'version': row['version'], 'archs':row['architecture']} , ]
+ dep.properties['pkg-url'] = 'http://ftp-master.debian.org/new/%s_%s.html' % (row['source'], row['version'])
+ # Warn about remaining information of prospective package
+ if ( dep.desc['en'] and dep.desc['en']['short'] ) or dep.properties['homepage'] != HOMEPAGENONE:
+ logger.info("The package %s is not yet in Debian but it is just in the new queue. (Task %s)" % (dep.pkg, dep.taskname))
+ for prop in PROPERTIES:
+ dep.properties[prop] = row[prop]
+ dep.desc['en']['short'] = MarkupString(to_unicode(row['description_en']), dep.pkg, 'ShortDesc - New')
+ dep.desc['en']['long'] = Markup(render_longdesc(row['long_description_en'].splitlines()))
+ (_name, _url) = email.Utils.parseaddr(row['maintainer'])
+ dep.responsible = '<a href="mailto:%s">%s</a>' % (_url, to_unicode(_name))
+
+ # Verify whether there are virtual packages which are provided by some other packages in the list of dependencies
+ query = "EXECUTE query_provides('%s')" % List2PgArray(pkgs_in_pool)
+ _execute_udd_query(query)
+ pkgs_virtual = []
+ if curs.rowcount > 0:
+ virtual_pkgs = RowDictionaries(curs)
+ for status in self.dependencies.keys():
+ for dep in self.dependencies[status]:
+ if dep.pkg not in pkgs_in_pool and dep.pkg not in pkgs_in_new:
+ found = False
+ for vp in virtual_pkgs:
+ for pr in vp['provides'].split(','):
+ prs=pr.strip()
+ if dep.pkg == prs:
+ pkgs_virtual.append(prs)
+ logger.info("Virtual package %s is provided by package %s for task %s" % (dep.pkg, vp['package'], dep.taskname))
+ found = True
+ break
+ if found:
+ break
+
+ # Verify whether packages which are neither in pool, new nor virtual have sufficient information in task file
+ for status in self.dependencies.keys():
+ for dep in self.dependencies[status]:
+ if dep.pkg not in pkgs_in_pool and dep.pkg not in pkgs_in_new and dep.pkg not in pkgs_virtual and dep.pkgstatus == 'unknown':
+ if dep.properties['homepage'] == HOMEPAGENONE:
+ if dep.desc['en'] == {}:
+ logger.error("Package %s in task %s neither in pool nor new and is lacking homepage and description - ignored" % (dep.pkg, dep.taskname))
+ else:
+ logger.error("Package %s in task %s neither in pool nor new and has no homepage information - ignored (%s)" % (dep.pkg, dep.taskname, dep.pkgstatus))
+ else:
+ if dep.desc['en'] == {}:
+ logger.error("Package %s neither in pool nor new and has no description - ignored" % dep.pkg)
+
+ def _QueryUDD4Package(self, source):
+
+ query = "EXECUTE pkg_releases ('%s', '%s')" % (self.pkg, self.component)
+ _execute_udd_query(query)
+ has_official = 0
+ for rel in curs.fetchall():
+ self.releases.append(rel[0])
+ if rel[0] != 'experimental':
+ has_official = 1
+
+ # Version in stable / testing for long table
+ query = "EXECUTE pkg_versions_stable_testing ('%s')" % (self.pkg)
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ prefix = 'Versions: '
+ for row in RowDictionaries(curs):
+ self.properties['stable_testing_version'].append((row['release'], row['debversion'], row['version'], prefix))
+ prefix = ', '
+
+
+ if has_official == 1:
+ if self.component == 'main':
+ if self.dep_strength == 'Depends' or self.dep_strength == 'Recommends':
+ self.pkgstatus = 'official_high'
+ else:
+ self.pkgstatus = 'official_low'
+ else:
+ self.pkgstatus = 'non-free'
+ else:
+ self.pkgstatus = 'experimental'
+
+
+ if self.properties.has_key('source'):
+ query = "EXECUTE src_vcs ('%s')" % (self.properties['source'])
+ _execute_udd_query(query)
+ if curs.rowcount > 0:
+ # There is only one line returned by this query
+ row = RowDictionaries(curs)[0]
+ # If some information about Vcs is found in the database make sure it is ignored from tasks file
+ self.vcs_found = 1
+ for prop in row.keys():
+ if row[prop]:
+ self.properties[prop] = row[prop]
+ if not self.properties.has_key('vcs-browser') or self.properties['vcs-browser'] == HOMEPAGENONE:
+ try:
+ self.properties['vcs-browser'] = BrowserFromVcsURL(self.properties['vcs-type'], self.properties['vcs-url'])
+ except KeyError, err:
+ logger.warning("Vcs Property missing in Database:", self.properties, err)
+ if not self.properties.has_key('vcs-type') or not self.properties['vcs-type']:
+ if self.properties.has_key('vcs-browser') and self.properties['vcs-browser'] != HOMEPAGENONE:
+ self.properties['vcs-type'] = VcsTypeFromBrowserURL(self.properties['vcs-browser'])
+ # We are only interested in source packages (for instance for Bugs page)
+ if source == 1:
+ self.pkg = self.properties['source']
+ # Stop using source package in self.pkg because we need the source package to obtain latest uploaders and
+ # and bugs should be rendered in the same job - so we need the differentiation anyway
+ self.src = self.properties['source']
+ else:
+ logger.error("Failed to obtain source for package", self.pkg)
+ return
+
+ if source == 0: # If we are querying for source packages to render BTS pages
+ # tranlations are irrelevant - so only obtain ddtp translations
+ # otherwise
+
+######################
if flag == 0:
# If there was no such package found query UDD whether any package provides this name
# This is often the case for libraries with versions in the package name
query = "EXECUTE query_provides ('%s')" % (dep.pkg)
- curs.execute(query)
+ _execute_udd_query(query)
if curs.rowcount > 0:
has_expilicite = 0
VirtProvides = []
@@ -1478,7 +1594,7 @@
if has_expilicite == 1:
logger.error("Do not keep a record of virtual package %s which has explicite package dependencies" % dep.pkg)
# ATTENTION: THIS HAS TO BE CHANGED FOR blends-dev BY AN OPTIONAL parameter
- continue
+# continue
logger.error("Use real package %s instead of virtual package %s." % (VirtProvides[0], dep.pkg))
dep.pkg = VirtProvides[0]
@@ -1503,20 +1619,6 @@
else:
logger.warning("Dependency with unknown status: %s (Task %s)" % (dep.pkg, dep.taskname))
- tmp_dep_list.append(dep)
- # remarks which are common to several dependencies in a list have to be added to all of the dependencies
- for dep in tmp_dep_list:
- if remark != {}:
- dep.remark = remark
- if fields_obsolete != [] and dep.pkgstatus != 'new':
- logger.info("Package %s is an official package and has information in UDD. The following extra information can be removed from tasks file %s: %s" % \
- (dep.pkg, dep.taskname, str(fields_obsolete)))
- if dep.desc['en'] == {}:
- logger.error("Missing description for package %s in task %s. This package will be ignored completely." % (dep.pkg, dep.taskname))
- else:
- self._AppendDependency2List(dep, source)
-
- f.close()
for dependency in self.dependencies.keys():
self.dependencies[dependency].sort()
@@ -1595,10 +1697,10 @@
# For the moment just query for the highest available version of the description
query = """PREPARE bugs_query_source (text) AS
SELECT id, package, source, status, severity, done, title FROM bugs WHERE source = $1"""
-curs.execute(query)
+_execute_udd_query(query)
query = """PREPARE bugs_query_tags (int) AS SELECT tag FROM bugs_tags WHERE id = $1"""
-curs.execute(query)
+_execute_udd_query(query)
class BugEntry:
# Define a separate class for bug entries to be able to define a reasonably sorting mechanism
@@ -1614,7 +1716,7 @@
self.severity = bug['severity']
query = "EXECUTE bugs_query_tags (%i)" % self.bug
- curs.execute(query)
+ _execute_udd_query(query)
self.tags = ''
if curs.rowcount > 0:
@@ -1674,7 +1776,7 @@
bugs = None
query = "EXECUTE bugs_query_source ('%s')" % source
- curs.execute(query)
+ _execute_udd_query(query)
if curs.rowcount > 0:
for bug in RowDictionaries(curs):
Modified: blends/trunk/webtools/tasks.py
URL: http://svn.debian.org/wsvn/blends/blends/trunk/webtools/tasks.py?rev=2602&op=diff
==============================================================================
--- blends/trunk/webtools/tasks.py (original)
+++ blends/trunk/webtools/tasks.py Mon Jan 17 20:53:36 2011
@@ -18,33 +18,8 @@
from genshi.template.eval import UndefinedError
from blendstasktools import Tasks, GetDependencies2Use, pkgstatus, pkgstatus_sortedkeys
-from blendsunicode import to_unicode
-
-# The keys of the following dictionary are used to address the translation files of DDTP
-# The values contain dictionaries in turn with the following meaning:
-# 'short' = shortcut of language as extension of the output files
-# 'htaccess' = language definition in htaccess
-# 'title' = English name of the language used in title attribut of link
-# 'printed' = printed name for links
-language_dict = { 'en' : {'short':'en', 'htaccess':'en' , 'title':'English', 'printed':Markup('English')},
- 'cs' : {'short':'cs', 'htaccess':'cs' , 'title':'Czech', 'printed':Markup('česky')},
- 'da' : {'short':'da', 'htaccess':'da' , 'title':'Danish' , 'printed':Markup('dansk')},
- 'de' : {'short':'de', 'htaccess':'de' , 'title':'German', 'printed':Markup('Deutsch')},
- 'es' : {'short':'es', 'htaccess':'es' , 'title':'Spanish', 'printed':Markup('español')},
- 'fi' : {'short':'fi', 'htaccess':'fi' , 'title':'Finnish', 'printed':Markup('suomi')},
- 'fr' : {'short':'fr', 'htaccess':'fr' , 'title':'French', 'printed':Markup('français')},
- 'it' : {'short':'it', 'htaccess':'it' , 'title':'Italian', 'printed':Markup('Italiano')},
- 'ja' : {'short':'ja', 'htaccess':'ja' , 'title':'Japanese', 'printed':Markup('日本語 (Nihongo)')},
- 'ko' : {'short':'ko', 'htaccess':'ko' , 'title':'Korean', 'printed':Markup('한국어 (Hangul)')},
- 'nl' : {'short':'nl', 'htaccess':'nl' , 'title':'Dutch', 'printed':Markup('Nederlands')},
- 'pl' : {'short':'pl', 'htaccess':'pl' , 'title':'Polish', 'printed':Markup('polski')},
- 'pt_BR' : {'short':'pt', 'htaccess':'pt-BR', 'title':'Portuguese', 'printed':Markup('Português')},
- 'ru' : {'short':'ru', 'htaccess':'ru' , 'title':'Russian', 'printed':Markup('Русский (Russkij)')},
- 'zh_CN' : {'short':'zh_cn', 'htaccess':'zh-CN', 'title':'Chinese (China)', 'printed':Markup('中文(简)')},
- 'zh_TW' : {'short':'zh_tw', 'htaccess':'zh-TW', 'title':'Chinese (Taiwan)', 'printed':Markup('中文(正)')},
- }
-languages = language_dict.keys()
-languages.sort()
+from blendsunicode import to_unicode
+from blendslanguages import languages, language_dict
if len(argv) <= 1:
print >>stderr, "Usage: %s <Blend name>\n The <Blend name> needs a matching config file webconf/<Blend name>.conf"\
More information about the Blends-commit
mailing list