[Git][security-tracker-team/security-tracker][master] 2 commits: update-xrefs: new script to update data/CVE/list Xrefs
Emilio Pozuelo Monfort (@pochu)
pochu at debian.org
Thu Apr 27 10:27:24 BST 2023
Emilio Pozuelo Monfort pushed to branch master at Debian Security Tracker / security-tracker
Commits:
3800e37a by Emilio Pozuelo Monfort at 2023-04-27T11:19:32+02:00
update-xrefs: new script to update data/CVE/list Xrefs
This partly replaces bin/updatelist.
- - - - -
a4c51f41 by Emilio Pozuelo Monfort at 2023-04-27T11:19:32+02:00
process-cve-records: new script to parse MITRE CVE 5.0 records
This replaces the other part of bin/updatelist, but using the
new CVE JSON 5.0 format.
Closes #17, #18.
- - - - -
2 changed files:
- + bin/process-cve-records
- + bin/update-xrefs
Changes:
=====================================
bin/process-cve-records
=====================================
@@ -0,0 +1,155 @@
+#!/usr/bin/python3
+#
+# Parse MITRE JSON 5.0 records and update data/CVE/list
+#
+# See https://github.com/CVEProject/cve-schema
+# and https://github.com/CVEProject/cvelistV5
+#
+# Copyright © 2023 Emilio Pozuelo Monfort <pochu at debian.org>
+
+import io
+import json
+import os
+import sys
+import zipfile
+
+import requests
+
+import setup_paths # noqa
+from sectracker import parsers
+
+CVE_ZIPFILE = 'https://github.com/CVEProject/cvelistV5/archive/refs/heads/main.zip'
+
+debug_enabled = False
+
+def debug(m):
+ if debug_enabled:
+ print(m)
+
+
+def get_annotation(annotations, ann_type):
+ for ann in annotations:
+ if isinstance(ann, ann_type):
+ return ann
+
+
+def is_published(record):
+ return record['cveMetadata']['state'] == 'PUBLISHED'
+
+
+def is_reserved(record):
+ return record['cveMetadata']['state'] == 'RESERVED'
+
+
+def is_rejected(record):
+ return record['cveMetadata']['state'] == 'REJECTED'
+
+
+def parse_record(record, cve):
+ # remove all flags, and add the current one if needed
+ ann = get_annotation(cve.annotations, parsers.FlagAnnotation)
+ if ann:
+ cve.annotations.remove(ann)
+
+ if is_published(record):
+ # no flag for published records
+ pass
+ elif is_reserved(record):
+ ann = parsers.FlagAnnotation(0, 'RESERVED')
+ cve.annotations.insert(0, ann)
+ elif is_rejected(record):
+ ann = parsers.FlagAnnotation(0, 'REJECTED')
+ cve.annotations.insert(0, ann)
+
+ if len(cve.header.description) == 0 \
+ and not is_rejected(record) and not is_reserved(record):
+ desc = [desc['value']
+ for desc in record['containers']['cna']['descriptions']
+ if desc['lang'].startswith('en')]
+ if desc:
+ desc = desc[0]
+ if desc and len(desc) > 70:
+ # for some reason descriptions contain new lines
+ desc = desc.replace('\n', ' ')
+ desc = desc[:70] + ' ...'
+ cve.header.description = f"({desc})"
+
+ if not is_reserved(record) and not is_rejected(record) \
+ and not get_annotation(cve.annotations, parsers.StringAnnotation):
+ ann = parsers.StringAnnotation(0, 'TODO', 'check')
+ cve.annotations.append(ann)
+
+
+def process_record_file(f):
+ global cve_dir
+ global cves
+
+ record = json.load(f)
+ cve_id = record['cveMetadata']['cveId']
+
+ try:
+ cve = cve_dir[cve_id]
+ except KeyError:
+ header = parsers.Header(0, cve_id, '')
+ cve = parsers.Bug('', header, list())
+ cves.insert(0, cve)
+ parse_record(record, cve)
+
+
+def process_record_filename(record_file):
+ with open(record_file) as f:
+ process_record_file(f)
+
+
+def process_record_dir(record_dir):
+ for year_dir in os.listdir(record_dir):
+ for record_file in os.listdir(year_dir):
+ debug("processing record " + record_file)
+ process_record_filename(record_file)
+ debug("record processed")
+
+
+def process_zip_file(zip_file):
+ z = zipfile.ZipFile(zip_file)
+ for fname in z.namelist():
+ if os.path.basename(fname).startswith('CVE-'):
+ f = z.open(fname)
+ debug("processing record " + fname)
+ process_record_file(f)
+ debug("record processed")
+
+
+def download_zip_file():
+ debug("downloading zip file...")
+ r = requests.get(CVE_ZIPFILE)
+ debug(f"downloaded, status {r.status_code}")
+ b = io.BytesIO(r.content)
+ process_zip_file(b)
+
+
+main_list = os.path.dirname(__file__) + '/../data/CVE/list'
+
+debug("reading cve file")
+cves = parsers.cvelist(main_list)
+debug("finished reading cve file")
+
+cve_dir = { cve.header.name: cve for cve in cves }
+
+if len(sys.argv) == 1:
+ # no argument, we download the CVE db
+ download_zip_file()
+elif sys.argv[1].endswith('.json'):
+ record_file = sys.argv[1]
+ debug("processing record " + record_file)
+ process_record_filename(record_file)
+ debug("record processed")
+elif sys.argv[1].endswith('.zip'):
+ zip_file = sys.argv[1]
+ process_zip_file(zip_file)
+else:
+ record_dir = sys.argv[1]
+ process_record_dir(record_dir)
+
+# write CVE file back
+with open(main_list, 'w') as f:
+ parsers.writecvelist(cves, f)
=====================================
bin/update-xrefs
=====================================
@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+#
+# Update xrefs in data/CVE/list
+#
+# Copyright © 2023 Emilio Pozuelo Monfort <pochu at debian.org>
+
+import os
+
+import setup_paths # noqa
+from sectracker import parsers
+
+def get_annotation(annotations, ann_type):
+ for ann in annotations:
+ if isinstance(ann, ann_type):
+ return ann
+
+
+def add_xref(cve, xref):
+ ann = get_annotation(cve.annotations, parsers.XrefAnnotation)
+ if not ann:
+ ann = parsers.XrefAnnotation(0, "xref", list())
+ # TODO: annotations order is important atm
+ flag_ann = get_annotation(cve.annotations, parsers.FlagAnnotation)
+ idx = cve.annotations.index(flag_ann) + 1 if flag_ann else 0
+ cve.annotations.insert(idx, ann)
+
+ if not xref in ann.bugs:
+ ann.bugs.append(xref)
+
+
+def parse_advfile(filename, parsemethod):
+ global cve_map
+
+ advs = parsemethod(filename)
+
+ for adv in advs:
+ for ann in adv.annotations:
+ if isinstance(ann, parsers.XrefAnnotation):
+ cves = ann.bugs
+ for cvename in cves:
+ if not cvename.startswith('CVE-'):
+ continue
+
+ cve = cve_map[cvename]
+ add_xref(cve, adv.header.name)
+
+
+def parse_dsafile(dsafile):
+ return parse_advfile(dsafile, parsers.dsalist)
+
+
+def parse_dtsafile(dtsafile):
+ return parse_advfile(dtsafile, parsers.dtsalist)
+
+
+def parse_dlafile(dlafile):
+ return parse_advfile(dlafile, parsers.dlalist)
+
+
+def remove_xrefs(cves):
+ for cve in cves:
+ #cve.annotations = [ann
+ # for ann in cve.annotations
+ # if not isinstance(ann, parsers.XrefAnnotation)]
+
+ ann = get_annotation(cve.annotations, parsers.XrefAnnotation)
+
+ if ann:
+ # we have CVE- cross-references, keep those and remove
+ # the rest, which we will re-add later if appropriate
+ ann.bugs = [bug for bug in ann.bugs if bug.startswith('CVE-')]
+ if len(ann.bugs) == 0:
+ cve.annotations.remove(ann)
+
+
+dsa_list = os.path.dirname(__file__) + '/../data/DSA/list'
+dtsa_list = os.path.dirname(__file__) + '/../data/DTSA/list'
+dla_list = os.path.dirname(__file__) + '/../data/DLA/list'
+main_list = os.path.dirname(__file__) + '/../data/CVE/list'
+
+cves = parsers.cvelist(main_list)
+cve_map = {cve.header.name: cve for cve in cves}
+
+# We remove the Xrefs, then re-parse the various advisory files and re-add
+# them
+remove_xrefs(cves)
+parse_dsafile(dsa_list)
+parse_dtsafile(dtsa_list)
+parse_dlafile(dla_list)
+
+# write the CVE file back
+with open(main_list, 'w') as f:
+ parsers.writecvelist(cves, f)
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/-/compare/da35af906a23f4df36a626e0890e03df0b3bbd86...a4c51f4130569e0e7309a180b78d6b022c5cb6ab
--
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/-/compare/da35af906a23f4df36a626e0890e03df0b3bbd86...a4c51f4130569e0e7309a180b78d6b022c5cb6ab
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-security-tracker-commits/attachments/20230427/a39bcfaa/attachment-0001.htm>
More information about the debian-security-tracker-commits
mailing list