[Git][security-tracker-team/security-tracker][master] 3 commits: update-nvd: Fetch NVD JSON feeds instead of XML feeds
Salvatore Bonaccorso
carnil at debian.org
Sun Oct 20 21:31:19 BST 2019
Salvatore Bonaccorso pushed to branch master at Debian Security Tracker / security-tracker
Commits:
ce06805d by Salvatore Bonaccorso at 2019-10-20T19:17:11Z
update-nvd: Fetch NVD JSON feeds instead of XML feeds
As per October 16, 2019, NVD does not provide anymore the XML data
feeds which the security-tracker uses to fill in long description of a
CVE or determine the external severity.
In update-nvd target switch to fetch the json files from
https://nvd.nist.gov/feeds/json/cve/1.1/$feedname
instead.
Signed-off-by: Salvatore Bonaccorso <carnil at debian.org>
- - - - -
966aef09 by Salvatore Bonaccorso at 2019-10-20T20:27:01Z
Reimplement (incompletely) simplistic NVD parser to handle JSON feed
The reimplementation is focused on only the functionality actually
strictly required by the security-tracker. This includes fetching the
CVE id and corresponding description.
All of specific imapct metrics (severity, range, loss attributes) are
not implemented. Those will require a database schema version bump and
reimplementation as well for the security_db.
Closes: #942670
Signed-off-by: Salvatore Bonaccorso <carnil at debian.org>
- - - - -
07efc247 by Salvatore Bonaccorso at 2019-10-20T20:31:01Z
Merge branch 'fetch-nvd-json-feeds'
- - - - -
2 changed files:
- Makefile
- lib/python/nvd.py
Changes:
=====================================
Makefile
=====================================
@@ -112,14 +112,16 @@ update-lists:
# Since October 16, 2015 the XML data feeds are no longer available for
# download in an uncompressed format.
+# As per October 16, 2019, the XML data feeds were discontinued and NVD
+# only provides JSON feeds. Cf. https://bugs.debian.org/942670
update-nvd:
mkdir -p data/nvd
for x in $$(seq 2002 $$(date +%Y)) ; do \
- name=nvdcve-$$x.xml.gz; \
- wget -q -Odata/nvd/$$name https://nvd.nist.gov/download/$$name || true; \
+ name=nvdcve-1.1-$$x.json.gz; \
+ wget -q -Odata/nvd/$$name https://nvd.nist.gov/feeds/json/cve/1.1/$$name || true; \
gzip -f -d data/nvd/$$name || true; \
done
- bin/update-nvd data/nvd/nvdcve-*.xml
+ bin/update-nvd data/nvd/nvdcve-*.json
# Experimental code to compare the Debian and NVD CVE databases using
# CPE values as common key.
=====================================
lib/python/nvd.py
=====================================
@@ -1,82 +1,81 @@
# nvd.py -- simplistic NVD parser
# Copyright (C) 2005 Florian Weimer <fw at deneb.enyo.de>
-#
+# Copyright (C) 2019 Salvatore Bonaccorso <carnil at debian.org>
+#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
-#
+#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
-#
+#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-"""This module parses the XML files provided by the
+"""This module parses the JSON files provided by the
National Vulnerability Database (NVD) <https://nvd.nist.gov/>
"""
-import xml.sax
-import xml.sax.handler
+import json
-class _Parser(xml.sax.handler.ContentHandler):
+class _Parser:
"""Parser helper class."""
def __init__(self):
self.result = []
- self.start_dispatcher = {}
- for x in ('entry', 'local', 'network', 'local_network', 'user_init',
- 'avail', 'conf', 'int', 'sec_prot'):
- self.start_dispatcher[x] = getattr(self, 'TAG_' + x)
- self.path = []
-
- def _noop(*args):
- pass
-
- def startElement(self, name, attrs):
- self.path.append((name, attrs))
- self.start_dispatcher.get(name, self._noop)(name, attrs)
-
- def TAG_entry(self, name, attrs):
- self.name = attrs['name'].encode('utf-8')
- self.published = attrs['published'].encode('utf-8')
- self.severity = attrs.get('severity', u'').encode('utf-8')
- self.discovered = attrs.get('discovered', u'').encode('utf-8')
-
- self.cve_desc = ""
- self.range_local = self.range_remote = self.range_user_init = 0
-
- self.loss_avail = self.loss_conf = self.loss_int \
- = self.loss_sec_prot_user = self.loss_sec_prot_admin \
- = self.loss_sec_prot_other = 0
-
- def TAG_local(self, name, attrs):
- self.range_local = 1
- def TAG_network(self, name, attrs):
- self.range_remote = 1
- def TAG_local_network(self, name, attrs):
- self.range_remote = 1
- def TAG_user_init(self, name, attrs):
- self.range_user_init = 1
- def TAG_avail(self, name, attrs):
- self.loss_avail = 1
- def TAG_conf(self, name, attrs):
- self.loss_conf = 1
- def TAG_int(self, name, attrs):
- self.loss_int = 1
- def TAG_sec_prot(self, name, attrs):
- if 'user' in attrs:
- self.loss_sec_prot_user = 1
- if 'admin' in attrs:
- self.loss_sec_prot_admin = 1
- if 'other' in attrs:
- self.loss_sec_prot_other = 1
-
- def endElement(self, name):
- if name == 'entry':
+
+ def parse(self, file):
+ cve_data=json.load(file)
+
+ for entry in cve_data['CVE_Items']:
+ # get CVE ID name
+ if 'cve' not in entry:
+ raise ValueError("No CVE entry present in CVE_Items")
+ if 'CVE_data_meta' not in entry['cve']:
+ raise ValueError("No CVE metadata entry present")
+ if 'ID' not in entry['cve']['CVE_data_meta']:
+ raise VAlueError("No CVE ID present for entry")
+ self.name=entry['cve']['CVE_data_meta']['ID']
+
+ # get CVE description
+ self.cve_desc=""
+ try:
+ self.cve_desc=entry['cve']['description']['description_data'][0].get('value')
+ except KeyError:
+ pass
+
+ # get discovered date
+ # TODO: re-implement or change database schema
+ self.discovered=""
+
+ # get publication date
+ self.published=""
+ try:
+ self.published=entry.get('publishedDate')
+ except KeyError:
+ pass
+
+ # get severity
+ self.severity=""
+ try:
+ self.severity=entry['impact']['baseMetricV2'].get('severity')
+ except KeyError:
+ pass
+
+ # initalize defaults
+ self.range_local = self.range_remote = self.range_user_init = 0
+
+ self.loss_avail = self.loss_conf = self.loss_int \
+ = self.loss_sec_prot_user = self.loss_sec_prot_admin \
+ = self.loss_sec_prot_other = 0
+
+ # get range and loss values
+ # TODO: re-implement or change database schema
+
self.result.append((self.name,
self.cve_desc,
self.discovered,
@@ -91,12 +90,6 @@ class _Parser(xml.sax.handler.ContentHandler):
self.loss_sec_prot_user,
self.loss_sec_prot_admin,
self.loss_sec_prot_other))
- del self.path[-1]
-
- def characters(self, content):
- (name, attrs) = self.path[-1]
- if name == 'descript' and attrs['source'] == 'cve':
- self.cve_desc += content
def parse(file):
"""Parses the indicated file object. Returns a list of tuples,
@@ -116,14 +109,12 @@ def parse(file):
- security protection (admin) loss type flag
- security protection (other) loss type flag
"""
- parser = xml.sax.make_parser()
- parser.setFeature(xml.sax.handler.feature_namespaces, 0)
+
p = _Parser()
- parser.setContentHandler(p)
- parser.parse(file)
+ p.parse(file)
return p.result
-if __name__ == "__main__":
+if __name__ == '__main__':
import sys
for name in sys.argv[1:]:
parse(open(name))
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/compare/0affdcf340582a47fb0fe1e20668720321f50d40...07efc247c366a5b1e0150a709a467893891bde66
--
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/compare/0affdcf340582a47fb0fe1e20668720321f50d40...07efc247c366a5b1e0150a709a467893891bde66
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-security-tracker-commits/attachments/20191020/a5103eed/attachment-0001.html>
More information about the debian-security-tracker-commits
mailing list