[Secure-testing-commits] r14625 - lib/python
Florian Weimer
fw at alioth.debian.org
Fri May 7 19:26:36 UTC 2010
Author: fw
Date: 2010-05-07 19:26:36 +0000 (Fri, 07 May 2010)
New Revision: 14625
Modified:
lib/python/parsers.py
Log:
lib/python/parsers.py: implement the inner annotation parsers
Modified: lib/python/parsers.py
===================================================================
--- lib/python/parsers.py 2010-05-07 18:51:15 UTC (rev 14624)
+++ lib/python/parsers.py 2010-05-07 19:26:36 UTC (rev 14625)
@@ -18,8 +18,10 @@
import operator
import re
+import debian_support
+import regexpcase
+import xcollections
import xpickle
-import debian_support
FORMAT = "1"
@@ -60,6 +62,135 @@
data[pkg_name] = pkg_version
return data
+def _sortedtuple(seq):
+ l = list(seq)
+ l.sort()
+ return tuple(l)
+
+Message = xcollections.namedtuple("Message", "file line level message")
+def addmessage(messages, file, line, level, msg):
+ if level not in ("error", "warning"):
+ raise ValueError("invalid message level: " + repr(level))
+ messages.append(Message(file, line, level, msg))
+
+FlagAnnotation = xcollections.namedtuple("FlagAnnotation", "line type")
+StringAnnotation = xcollections.namedtuple("StringAnnotation",
+ "line type description")
+XrefAnnotation = xcollections.namedtuple("XrefAnnotation", "line type bugs")
+PackageAnnotation = xcollections.namedtuple(
+ "PackageAnnotation",
+ "line type release package kind version description "
+ + "urgency debian_bugs bug_filed")
+
+def _annotationdispatcher():
+ # Parser for inner annotations, like (bug #1345; low)
+ urgencies=set("unimportant low medium high".split())
+ @regexpcase.rule('(bug filed|%s)' % '|'.join(urgencies))
+ def innerflag(groups, file, line, messages, flags, bugs):
+ f = groups[0]
+ if f in flags:
+ addmessage(messages, file, line, "error",
+ "duplicate flag: " + repr(f))
+ else:
+ flags.add(f)
+ @regexpcase.rule(r'bug #(\d+)')
+ def innerbug(groups, file, line, messages, flags, bugs):
+ no = int(groups[0])
+ if no in bugs:
+ messages.add(file, line, "error",
+ "duplicate bug number: " + groups[0])
+ else:
+ bugs.add(no)
+ def innerdefault(text, file, line, messages, flags, bugs):
+ addmessage(messages, file, line, "error",
+ "invalid inner annotation: " + repr(text))
+ innerdispatch = regexpcase.RegexpCase((innerflag, innerbug),
+ default=innerdefault)
+
+ def parseinner(file, line, messages, inner):
+ if not inner:
+ return (None, (), False)
+ flags = set()
+ bugs = set()
+ for innerann in inner.split(";"):
+ innerdispatch(innerann.strip(), file, line, messages, flags, bugs)
+
+ urgency = urgencies.intersection(flags)
+ if urgency:
+ if len(urgency) > 1:
+ addmessage(messages, file, line, "error",
+ "multiple urgencies: " + ", ".join(urgency))
+ else:
+ urgency = urgency.pop()
+ else:
+ urgency = None
+
+ bug_filed = "bug filed" in flags
+ if bugs and bug_filed:
+ addmessage(messages, file, line, "error",
+ "'bug filed' and bug numbers listed")
+ bug_filed = False
+
+ return (urgency, _sortedtuple(bugs), bug_filed)
+
+ # Parsers for indented annotations (NOT-FOR-US:, " - foo <unfixed>" etc.)
+
+ @regexpcase.rule(r'(?:\[([a-z]+)\]\s)?-\s([A-Za-z0-9:.+-]+)\s*'
+ + r'(?:\s([A-Za-z0-9:.+~-]+)\s*)?(?:\s\((.*)\))?')
+ def package_version(groups, file, line, messages, anns):
+ release, package, version, inner = groups
+ inner = parseinner(file, line, messages, inner)
+ if version is None:
+ kind = "unfixed"
+ else:
+ kind = "fixed"
+ anns.append(PackageAnnotation(
+ *((line, "package", release, package, kind, version, None)
+ + inner)))
+
+ pseudo_freetext = "no-dsa not-affected".split()
+ pseudo_struct = set("unfixed removed end-of-life itp undetermined".split())
+ @regexpcase.rule(r'(?:\[([a-z]+)\]\s)?-\s([A-Za-z0-9:.+-]+)'
+ + r'\s+<([a-z-]+)>\s*(?:\s\((.*)\))?')
+ def package_pseudo(groups, file, line, messages, anns):
+ release, package, version, inner = groups
+ if version in pseudo_freetext:
+ anns.append(PackageAnnotation(
+ line, "package", release, package, version, None, inner,
+ None, (), False))
+ elif version in pseudo_struct:
+ inner = parseinner(file, line, messages, inner)
+ if version == "itp" and not inner[1]:
+ addmessage(messages, file, line, "error",
+ "<itp> needs Debian bug reference")
+ anns.append(PackageAnnotation(
+ *((line, "package", release, package, version, None, None)
+ + inner)))
+ else:
+ addmessage(messages, file, line, "error",
+ "invalid pseudo-version: " + repr(version))
+
+ @regexpcase.rule(r'\{(.*)\}')
+ def xref(groups, file, line, messages, anns):
+ x = _sortedtuple(groups[0].strip().split())
+ if x:
+ anns.append(XrefAnnotation(line, "xref", x))
+ else:
+ addmessage(messages, file, line, "error", "empty cross-reference")
+
+ return regexpcase.RegexpCase(
+ ((r'(RESERVED|REJECTED)',
+ lambda groups, file, line, messages, anns:
+ anns.append(FlagAnnotation(line, groups[0]))),
+ (r'(NOT-FOR-US|NOTE|TODO):\s+(\S.*)',
+ lambda groups, file, line, messages, anns:
+ anns.append(StringAnnotation(line, *groups))),
+ package_version, package_pseudo, xref),
+ prefix=r"\s+", suffix=r"\s*",
+ default=lambda text, file, line, messages, anns:
+ addmessage(messages, file, line, "error", "invalid annotation"))
+_annotationdispatcher = _annotationdispatcher()
+
def _test():
o = binarypackages("../../data/packages/sid__main_i386_Packages")
assert type(o) == type(())
@@ -69,5 +200,84 @@
assert type(o) == type({})
assert "bash" in o
+ for (line, res, xmsgs) in [
+ (' - foo <unfixed>',
+ PackageAnnotation(17, "package", None, "foo", "unfixed", None,
+ None, None, (), False), ()),
+ (' - foo',
+ PackageAnnotation(17, "package", None, "foo", "unfixed", None,
+ None, None, (), False), ()),
+ (' [lenny] - foo <unfixed>',
+ PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None,
+ None, None, (), False), ()),
+ (' [lenny] - foo <undetermined> (bug #1234)',
+ PackageAnnotation(17, "package", "lenny", "foo", "undetermined",
+ None, None, None, (1234,), False), ()),
+ (' [lenny] - foo <itp> (bug #1234)',
+ PackageAnnotation(17, "package", "lenny", "foo", "itp", None,
+ None, None, (1234,), False), ()),
+ (' [lenny] - foo <itp>',
+ PackageAnnotation(17, "package", "lenny", "foo", "itp", None,
+ None, None, (), False),
+ (Message("CVE", 17, "error",
+ "<itp> needs Debian bug reference"),)),
+ (' [lenny] - foo 1.0',
+ PackageAnnotation(17, "package", "lenny", "foo", "fixed", "1.0" ,
+ None, None, (), False), ()),
+ (' [lenny] - foo <unfixed> (bug filed)',
+ PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None,
+ None, None, (), True), ()),
+ (' [lenny] - foo <unfixed> (bug filed; bug #1234)',
+ PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None,
+ None, None, (1234,), False),
+ (Message("CVE", 17, "error",
+ "'bug filed' and bug numbers listed"),)),
+ (' [lenny] - foo <unfixed> (low)',
+ PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None,
+ None, "low", (), False), ()),
+ (' [lenny] - foo <unfixed> (low; low)',
+ PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None,
+ None, "low", (), False),
+ (Message("CVE", 17, "error", "duplicate flag: 'low'"),)),
+ (' [lenny] - foo <unfixed> (bug #1234; garbled)',
+ PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None,
+ None, None, (1234,), False),
+ (Message("CVE", 17, "error",
+ "invalid inner annotation: 'garbled'"),)),
+ (' [lenny] - foo <no-dsa> (explanation goes here)',
+ PackageAnnotation(17, "package", "lenny", "foo", "no-dsa", None,
+ "explanation goes here", None, (), False), ()),
+ (' [lenny] - foo <not-affected> (explanation goes here)',
+ PackageAnnotation(17, "package", "lenny", "foo", "not-affected",
+ None,
+ "explanation goes here", None, (), False), ()),
+ ('\t{CVE-2009-1234 CVE-2009-1235}',
+ XrefAnnotation(17, "xref",
+ tuple("CVE-2009-1234 CVE-2009-1235".split())),
+ ()),
+ ('\t{}', None,
+ (Message("CVE", 17, "error", "empty cross-reference"),)),
+ (' NOT-FOR-US: Plan 9',
+ StringAnnotation(17, "NOT-FOR-US", "Plan 9"), ()),
+ (' TODO: to-do', StringAnnotation(17, "TODO", "to-do"), ()),
+ (' NOTE: note', StringAnnotation(17, "NOTE", "note"), ()),
+ (' RESERVED', FlagAnnotation(17, 'RESERVED'), ()),
+ (' REJECTED', FlagAnnotation(17, 'REJECTED'), ()),
+ (' garbled', None,
+ (Message("CVE", 17, "error", "invalid annotation"),)),
+ (' [lenny] - foo <garbled> (bug #1234)', None,
+ (Message("CVE", 17, "error",
+ "invalid pseudo-version: 'garbled'"),)),
+ ]:
+ anns = []
+ msgs = []
+ _annotationdispatcher(line, "CVE", 17, msgs, anns)
+ assert tuple(msgs) == xmsgs, repr(msgs)
+ if anns:
+ r = anns[0]
+ else:
+ r = None
+ assert r == res, repr(anns)
+
if __name__ == "__main__":
_test()
More information about the Secure-testing-commits
mailing list