[Git][security-tracker-team/security-tracker][master] 12 commits: Simplify DLAFile
Salvatore Bonaccorso
carnil at debian.org
Sun Jun 10 08:02:11 BST 2018
Salvatore Bonaccorso pushed to branch master at Debian Security Tracker / security-tracker
Commits:
0b664197 by Emilio Pozuelo Monfort at 2018-06-08T08:38:46+02:00
Simplify DLAFile
Subclass DSAFile rather than copying it.
- - - - -
375ba023 by Emilio Pozuelo Monfort at 2018-06-08T08:38:46+02:00
Merge DLAFile into DSAFile
The only difference is that the regular expressions look for DSA
or DLA, but we can just guess that based on the path.
- - - - -
0cb94dee by Emilio Pozuelo Monfort at 2018-06-08T09:26:45+02:00
Move source list to a config file
- - - - -
77190d32 by Emilio Pozuelo Monfort at 2018-06-08T10:04:06+02:00
Dynamically create announce queries
Based on the DSA-like files present in the config file.
- - - - -
ecbbab3e by Bastian Blank at 2018-06-08T10:04:21+02:00
Add support for CUSTOMER bugs and CVE extends
- - - - -
fafe4839 by Emilio Pozuelo Monfort at 2018-06-08T10:04:21+02:00
Simplify Extends support
BugExtend.writeDB() is pretty similar to BugBase's, so
update the latter to take extends into account when
necessary to avoid unneeded duplicated code.
- - - - -
bf1f037a by Emilio Pozuelo Monfort at 2018-06-08T10:04:21+02:00
gen-DSA: allow other gen-* links
- - - - -
b59cbe46 by Emilio Pozuelo Monfort at 2018-06-08T10:04:21+02:00
Rename CVECUSTOMERFile to CVEExtendFile
- - - - -
a0c20580 by Emilio Pozuelo Monfort at 2018-06-08T10:04:21+02:00
Document CVE extends support
- - - - -
240e8b95 by Salvatore Bonaccorso at 2018-06-10T08:59:50+02:00
Merge branch 'pochu/security-tracker-wip/extends'
- - - - -
5ba23a51 by Salvatore Bonaccorso at 2018-06-10T09:00:27+02:00
Unify name for bin/gen-MYSA script
- - - - -
cc4d0e94 by Salvatore Bonaccorso at 2018-06-10T09:01:30+02:00
Reformat data/config.json with jq
- - - - -
6 changed files:
- bin/check-syntax
- bin/gen-DSA
- data/config.json
- doc/security-team.d.o/security_tracker
- lib/python/bugs.py
- lib/python/security_db.py
Changes:
=====================================
bin/check-syntax
=====================================
--- a/bin/check-syntax
+++ b/bin/check-syntax
@@ -65,13 +65,10 @@ def parse_DSA(name):
def parse_DTSA(name):
do_parse(construct(bugs.DTSAFile, name))
-def parse_DLA(name):
- do_parse(construct(bugs.DLAFile, name))
-
file_types = {'CVE' : parse_CVE,
'DSA' : parse_DSA,
'DTSA' : parse_DTSA,
- 'DLA' : parse_DLA}
+ 'DLA' : parse_DSA}
if len(sys.argv) <> 3 or not file_types.has_key(sys.argv[1]):
l = file_types.keys()
=====================================
bin/gen-DSA
=====================================
--- a/bin/gen-DSA
+++ b/bin/gen-DSA
@@ -22,7 +22,7 @@ set -e
IDMODE=DSA
case "$(basename "$0")" in
- *gen-D[LS]A)
+ *gen-*)
IDMODE=${0#*gen-}
;;
esac
@@ -333,11 +333,7 @@ setvar DEBFULLNAME
setvar SPACEDDEBFULLNAME
setvar PACKAGE
setvar CVE "$CVE_LIST"
-if [ "$IDMODE" = DSA ]; then
- setvar DSAID "$DAID"
-else
- setvar DLAID "$DAID"
-fi
+setvar ${IDMODE}ID "$DAID"
setvar BUGNUM
setvar OLDOLDSTABLE
setvar OLDSTABLE
=====================================
data/config.json
=====================================
--- a/data/config.json
+++ b/data/config.json
@@ -1,69 +1,92 @@
{
- "repositories" : {
+ "repositories": {
"etch": "http://mirror.1und1.de/debian/dists/etch",
- "etch-security" :
- "http://security.debian.org/debian-security/dists/etch/updates",
- "etch-proposed-updates" :
- "http://mirror.1und1.de/debian/dists/etch-proposed-updates",
-
- "lenny": "http://mirror.1und1.de/debian/dists/lenny",
- "lenny-security" :
- "http://security.debian.org/debian-security/dists/lenny/updates",
- "lenny-proposed-updates" :
- "http://mirror.1und1.de/debian/dists/lenny-proposed-updates",
-
- "squeeze": "http://mirror.1und1.de/debian/dists/squeeze",
- "squeeze-security" :
- "http://security.debian.org/debian-security/dists/squeeze/updates",
- "squeeze-proposed-updates" :
- "http://mirror.1und1.de/debian/dists/squeeze-proposed-updates",
-
- "sid" : "http://mirror.1und1.de/debian/dists/sid"
+ "etch-security": "http://security.debian.org/debian-security/dists/etch/updates",
+ "etch-proposed-updates": "http://mirror.1und1.de/debian/dists/etch-proposed-updates",
+ "lenny": "http://mirror.1und1.de/debian/dists/lenny",
+ "lenny-security": "http://security.debian.org/debian-security/dists/lenny/updates",
+ "lenny-proposed-updates": "http://mirror.1und1.de/debian/dists/lenny-proposed-updates",
+ "squeeze": "http://mirror.1und1.de/debian/dists/squeeze",
+ "squeeze-security": "http://security.debian.org/debian-security/dists/squeeze/updates",
+ "squeeze-proposed-updates": "http://mirror.1und1.de/debian/dists/squeeze-proposed-updates",
+ "sid": "http://mirror.1und1.de/debian/dists/sid"
},
-
- "distributions" : {
- "potato" : {},
- "woody" : {},
- "sarge" : {},
- "etch" : {},
- "lenny" : {
- "members" : {
- "supported" : ["lenny", "lenny-security"],
- "optional" : ["lenny-proposed-updates"]
- },
+ "distributions": {
+ "potato": {},
+ "woody": {},
+ "sarge": {},
+ "etch": {},
+ "lenny": {
+ "members": {
+ "supported": [
+ "lenny",
+ "lenny-security"
+ ],
+ "optional": [
+ "lenny-proposed-updates"
+ ]
+ }
},
- "squeeze" : {
- "members" : {
- "supported" : ["squeeze", "squeeze-security"],
- "optional" : ["squeeze-proposed-updates"]
- },
+ "squeeze": {
+ "members": {
+ "supported": [
+ "squeeze",
+ "squeeze-security"
+ ],
+ "optional": [
+ "squeeze-proposed-updates"
+ ]
+ }
},
- "wheezy" : {
- "members" : {
- "supported" : ["wheezy", "wheezy-security"],
- "optional" : ["wheezy-proposed-updates"]
+ "wheezy": {
+ "members": {
+ "supported": [
+ "wheezy",
+ "wheezy-security"
+ ],
+ "optional": [
+ "wheezy-proposed-updates"
+ ]
},
- "release" : "oldstable"
+ "release": "oldstable"
},
- "jessie" : {
- "members" : {
- "supported" : ["jessie", "jessie-security"],
- "optional" : ["jessie-proposed-updates"]
+ "jessie": {
+ "members": {
+ "supported": [
+ "jessie",
+ "jessie-security"
+ ],
+ "optional": [
+ "jessie-proposed-updates"
+ ]
},
- "release" : "stable"
+ "release": "stable"
},
- "stretch" : {
- "members" : {
- "supported" : ["stretch", "stretch-security"],
- "optional" : ["stretch-proposed-updates"]
+ "stretch": {
+ "members": {
+ "supported": [
+ "stretch",
+ "stretch-security"
+ ],
+ "optional": [
+ "stretch-proposed-updates"
+ ]
},
- "release" : "testing"
+ "release": "testing"
},
- "sid" : {
- "members" : {
- "supported" : ["sid"]
+ "sid": {
+ "members": {
+ "supported": [
+ "sid"
+ ]
},
- "release" : "unstable"
+ "release": "unstable"
}
+ },
+ "sources": {
+ "/CVE/list": "CVEFile",
+ "/DSA/list": "DSAFile",
+ "/DTSA/list": "DTSAFile",
+ "/DLA/list": "DSAFile"
}
}
=====================================
doc/security-team.d.o/security_tracker
=====================================
--- a/doc/security-team.d.o/security_tracker
+++ b/doc/security-team.d.o/security_tracker
@@ -612,3 +612,23 @@ The following commands build the databases for stable and run a python local ser
make serve
The website is now available as `http://127.0.0.1:10605/tracker/`.
+
+Setting up an extended instance
+-------------------------------
+
+The security tracker supports extra sources of data, which can be used
+to override or extend the information in CVE/list, and to support your
+own announce lists. To do that, add a CVEExtendFile source to
+`data/config.json`. Entries in that file can add information to an
+existing CVE, e.g. to mark it as fixed or ignored, or to mark it as
+affecting additional source packages. For example:
+
+CVE-2018-11646
+ - webkitgtk <unfixed>
+CVE-2016-1000340
+ [wheezy] - bouncycastle <not-affected> (Vulnerable code introduced later)
+
+You can also add an announce list of type DSAFile to `data/config.json`,
+and then symlink `bin/gen-DSA` to e.g. `bin/gen-MYSA` and use that to
+create new advisories under your namespace. For that you will need to
+add a `data/mysa-needed.txt` file and `doc/MYSA.template`.
=====================================
lib/python/bugs.py
=====================================
--- a/lib/python/bugs.py
+++ b/lib/python/bugs.py
@@ -16,6 +16,7 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
import debian_support
+import os
import re
import types
import hashlib
@@ -200,6 +201,7 @@ class BugBase:
self.notes = []
self.xref = []
self.not_for_us = False
+ self.is_extend = False
def isFromCVE(self):
"""Returns True if the name has been officially assigned.
@@ -225,16 +227,18 @@ class BugBase:
not_for_us = 0
import apsw
- try:
- cursor.execute("""INSERT INTO bugs
- (name, cve_status, not_for_us, description, release_date,
- source_file, source_line)
- VALUES (?, ?, ?, ?, ?, ?, ?)""",
- (self.name, self.cveStatus(), not_for_us,
- self.description, self.date or '',
- self.source_file, self.source_line))
- except apsw.ConstraintError:
- raise ValueError, "bug name %s is not unique" % self.name
+
+ if not self.is_extend:
+ try:
+ cursor.execute("""INSERT INTO bugs
+ (name, cve_status, not_for_us, description, release_date,
+ source_file, source_line)
+ VALUES (?, ?, ?, ?, ?, ?, ?)""",
+ (self.name, self.cveStatus(), not_for_us,
+ self.description, self.date or '',
+ self.source_file, self.source_line))
+ except apsw.ConstraintError:
+ raise ValueError, "bug name %s is not unique" % self.name
for (typ, c) in self.comments:
cursor.execute("""INSERT INTO bugs_notes
@@ -257,7 +261,7 @@ class Bug(BugBase):
"""Class for bugs for which we have some data."""
def __init__(self, fname, lineno, date, name, description, comments, notes,
- xref, not_for_us=False):
+ xref, not_for_us=False, is_extend=False):
for n in notes:
assert isinstance(n, PackageNote) \
or isinstance(n, PackageNoteNoDSA)
@@ -268,6 +272,7 @@ class Bug(BugBase):
self.notes = notes
self.xref = xref
self.not_for_us = not_for_us
+ self.is_extend = is_extend
def mergeNotes(self):
"""Merge notes so that there is only one note for each
@@ -435,6 +440,7 @@ class FileBase(debian_support.PackageFile):
re_rejected = re.compile(r'^(?:NOTE:\s+rejected|REJECTED)\s*$')
re_note = re.compile(r'^NOTE:\s+(.*)$')
re_todo = re.compile(r'^TODO:\s+(.*)$')
+ is_extend = False
def __init__(self, name, fileObj=None):
debian_support.PackageFile.__init__(self, name, fileObj)
@@ -730,7 +736,8 @@ class FileBase(debian_support.PackageFile):
record_name = temp_bug_name(first_bug, description)
yield self.finishBug(Bug(self.file.name, first_lineno, date,
record_name, description,
- comments, notes=pkg_notes, xref=xref))
+ comments, notes=pkg_notes, xref=xref,
+ is_extend=self.is_extend))
def finishBug(self, bug):
"""Applies a transformation to the bug after it has been
@@ -773,6 +780,15 @@ class CVEFile(FileBase):
bug.mergeNotes()
return bug
+class CVEExtendFile(CVEFile):
+ # This is an extend file. The main CVEFile can have a 'CVE-2018-XXXX' (sic)
+ # identifier, which will get converted to TEMP-* automatically. However to
+ # refer to that one from here, we need to use the TEMP-* identifier, so we
+ # allow those in the regex
+ re_cve = re.compile(r'^(CVE-\d{4}-(?:\d{4,}|XXXX)|TEMP-\d+-\S+)\s+(.*?)\s*$')
+
+ is_extend = True
+
class DSAFile(FileBase):
"""A DSA file.
@@ -780,9 +796,13 @@ class DSAFile(FileBase):
reference point, and release dates.
"""
- re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] '
- + r'(DSA-\d+(?:-\d+)?)\s+'
- + r'(.*?)\s*$')
+ def __init__(self, name, fileObj=None):
+ FileBase.__init__(self, name, fileObj)
+
+ self.base = os.path.basename(os.path.dirname(self.name))
+ self.re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] '
+ + r'(' + self.base + '-\d+(?:-\d+)?)\s+'
+ + r'(.*?)\s*$')
month_names = {'Jan': 1,
'Feb': 2,
@@ -800,7 +820,7 @@ class DSAFile(FileBase):
def matchHeader(self, line):
match = self.re_dsa.match(line)
if not match:
- self.raiseSyntaxError("expected DSA record, got: %s" % `line`)
+ self.raiseSyntaxError("expected %s record, got: %s" % (self.base, `line`))
(record_name, description) = match.groups()
(day, month, year, name, desc) = match.groups()
try:
@@ -814,47 +834,7 @@ class DSAFile(FileBase):
bug.mergeNotes()
return bug
-class DLAFile(FileBase):
- """A DLA file.
- Similar to a CVE file, only that it contains DLAs as its main
- reference point, and release dates.
- """
-
- re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] '
- + r'(DLA-\d+(?:-\d+)?)\s+'
- + r'(.*?)\s*$')
-
- month_names = {'Jan': 1,
- 'Feb': 2,
- 'Mar': 3,
- 'Apr': 4,
- 'May': 5,
- 'Jun': 6,
- 'Jul': 7,
- 'Aug': 8,
- 'Sep': 9,
- 'Oct': 10,
- 'Nov': 11,
- 'Dec': 12}
-
- def matchHeader(self, line):
- match = self.re_dsa.match(line)
- if not match:
- self.raiseSyntaxError("expected DLA record, got: %s" % `line`)
- (record_name, description) = match.groups()
- (day, month, year, name, desc) = match.groups()
- try:
- month = self.month_names[month]
- except KeyError:
- self.raiseSyntaxError("invalid month name %s" % `month`)
- return ("%s-%02d-%s" % (year, month, day), name, desc)
-
- def finishBug(self, bug):
- # Merge identical package notes, for historical reasons.
- bug.mergeNotes()
- return bug
-
class DTSAFile(FileBase):
"""A DTSA file.
=====================================
lib/python/security_db.py
=====================================
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -856,6 +856,28 @@ class DB:
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
gen())
+ def getSources(self):
+ config = debian_support.getconfig()
+ sources = config["sources"]
+
+ return sources
+
+ def genDBAdvisoryString(self, field, dtsa=False):
+ sources = self.getSources()
+ advs = []
+
+ for path, cls in sources.iteritems():
+ name = path.split('/')[1]
+
+ if cls == 'DSAFile':
+ advs.append(name)
+
+ if cls == 'DTSAFile' and dtsa:
+ advs.append(name)
+
+ advs = ["{} LIKE '{}-%'".format(field, adv) for adv in advs]
+ return " OR ".join(advs)
+
def readBugs(self, cursor, path):
if self.verbose:
print "readBugs:"
@@ -913,15 +935,11 @@ class DB:
return True
source_removed_packages = '/packages/removed-packages'
- sources = ((bugs.CVEFile, '/CVE/list'),
- (bugs.DSAFile, '/DSA/list'),
- (bugs.DTSAFile, '/DTSA/list'),
- (bugs.DLAFile, '/DLA/list'),
- (None, source_removed_packages))
+ sources = self.getSources()
unchanged = True
- for (_, name) in sources:
- if has_changed(path + name):
+ for filename in sources.keys() + [source_removed_packages]:
+ if has_changed(path + filename):
unchanged = False
break
if unchanged:
@@ -940,9 +958,8 @@ class DB:
"""INSERT OR REPLACE INTO inodeprints (inodeprint, file)
VALUES (?, ?)""", (current_print, filename))
- for (cls, name) in sources:
- if cls is None:
- continue
+ for name, cls in sources.iteritems():
+ cls = getattr(bugs, cls)
read_one(cls(path + name))
if self.verbose:
@@ -967,9 +984,10 @@ class DB:
# Copy notes from DSA/DTSA/DLA to CVE.
old_source = ''
+ source_like = self.genDBAdvisoryString("source", dtsa=True)
for source, target in list(cursor.execute(
"""SELECT source, target FROM bugs_xref
- WHERE (source LIKE 'DTSA-%' OR source LIKE 'DSA-%' OR source LIKE 'DLA-%')
+ WHERE (""" + source_like + """)
AND target LIKE 'CVE-%'""")):
if source <> old_source:
source_bug = bugs.BugFromDB(cursor, source)
@@ -1848,11 +1866,12 @@ class DB:
return flag
def getDSAsForSourcePackage(self, cursor, package):
+ bugs_like = self.genDBAdvisoryString("bugs.name", dtsa=False)
for row in cursor.execute(
"""SELECT bugs.name, bugs.description
FROM bugs, package_notes as p
WHERE p.bug_name = bugs.name
- AND ( bugs.name LIKE 'DSA-%' OR bugs.name LIKE 'DLA-%')
+ AND ( """ + bugs_like + """ )
AND p.package = ?
ORDER BY bugs.release_date DESC""", (package,)):
yield DSAsForSourcePackage(*row)
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/compare/06f8fb550fc8e74e33d4d8e3c6ce928f42e6b83b...cc4d0e94483320ee378a326d5675bf1a766a224c
--
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/compare/06f8fb550fc8e74e33d4d8e3c6ce928f42e6b83b...cc4d0e94483320ee378a326d5675bf1a766a224c
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-security-tracker-commits/attachments/20180610/f2927cd8/attachment-0001.html>
More information about the debian-security-tracker-commits
mailing list