[Git][security-tracker-team/security-tracker][master] 15 commits: check-new-issues: auto NFU based on a rules file
Emilio Pozuelo Monfort (@pochu)
pochu at debian.org
Sat Mar 1 10:19:35 GMT 2025
Emilio Pozuelo Monfort pushed to branch master at Debian Security Tracker / security-tracker
Commits:
9ad1c020 by Emilio Pozuelo Monfort at 2025-02-18T20:26:01+01:00
check-new-issues: auto NFU based on a rules file
Closes: #1073012
- - - - -
5a8bb831 by Emilio Pozuelo Monfort at 2025-02-18T20:34:25+01:00
Add initial NFU auto file
- - - - -
99673a79 by Emilio Pozuelo Monfort at 2025-02-20T16:09:10+01:00
check-new-issues: support complex NFU entries
- - - - -
22f834e7 by Emilio Pozuelo Monfort at 2025-02-20T16:09:10+01:00
nfu.yaml: add an entry using anyOf and allOf
- - - - -
4fc5a335 by Emilio Pozuelo Monfort at 2025-02-20T16:09:10+01:00
Move nfu.yaml to data/packages/
- - - - -
dceaadb4 by Emilio Pozuelo Monfort at 2025-02-21T12:29:49+01:00
check-new-issues: bump copyright
- - - - -
545d6cbc by Emilio Pozuelo Monfort at 2025-02-21T12:33:56+01:00
check-cna-nfu: new script to check issues by a CNA
This can be useful to determine rules for data/packages/nfu.yaml
to automatically mark some CVEs as NFU.
- - - - -
1d1b4c5e by Emilio Pozuelo Monfort at 2025-02-21T12:38:13+01:00
check-cna-nfu: report affected upstream products
- - - - -
3116cbe1 by Emilio Pozuelo Monfort at 2025-02-21T12:44:51+01:00
check-cna-nfu: also print debian source packages
- - - - -
cb65f9ee by Emilio Pozuelo Monfort at 2025-02-21T18:14:35+01:00
check-new-issues: support negating predicates
- - - - -
61df5656 by Emilio Pozuelo Monfort at 2025-02-21T18:15:17+01:00
nfu.yaml: add auto NFU for Adobe
- - - - -
f5394f95 by Emilio Pozuelo Monfort at 2025-02-21T18:39:39+01:00
check-new-issues: simplify return values
- - - - -
3540028c by Emilio Pozuelo Monfort at 2025-02-21T19:00:26+01:00
check-new-issues: support descriptions in auto NFU
The descriptions are expected to be regular expressions.
- - - - -
faec5e41 by Emilio Pozuelo Monfort at 2025-02-21T19:02:32+01:00
nfu.yaml: exercise descriptions
- - - - -
56f10758 by Emilio Pozuelo Monfort at 2025-03-01T10:19:29+00:00
Merge branch 'auto-nfu-v2' into 'master'
check-new-issues: auto NFU based on a rules file
See merge request security-tracker-team/security-tracker!204
- - - - -
3 changed files:
- + bin/check-cna-nfu
- bin/check-new-issues
- + data/packages/nfu.yaml
Changes:
=====================================
bin/check-cna-nfu
=====================================
@@ -0,0 +1,137 @@
+#!/usr/bin/python3
+#
+# Look for all CVEs issued by a CNA and report their NFU status in
+# the security tracker.
+#
+# Copyright © 2025 Emilio Pozuelo Monfort <pochu at debian.org>
+#
+# This file is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This file is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this file. If not, see <https://www.gnu.org/licenses/>.
+
+import argparse
+import collections
+import json
+import logging
+import os
+import sys
+import zipfile
+
+import requests
+
+import setup_paths # noqa
+from sectracker import parsers
+
+logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s')
+#logging.getLogger().setLevel("DEBUG")
+
+def debug(s):
+ if args.verbose:
+ print(s)
+
+def get_cve5(cve_id):
+ global cve5_zip
+
+ if cve_id not in cve5s:
+ return None
+
+ fname = cve5s[cve_id]
+
+ logging.info('loading file')
+ f = cve5_zip.open(fname)
+ logging.info('loading json')
+ return json.load(f)
+
+def read_cve5_file(f):
+ cve5s = {}
+
+ z = zipfile.ZipFile(cve5_file)
+ for fname in z.namelist():
+ if os.path.basename(fname).startswith('CVE-'):
+ debug("found record " + fname)
+ cve_id = os.path.basename(fname)[:-5]
+ cve5s[cve_id] = fname
+
+ return cve5s
+
+def parse_cves():
+ cvelist = parsers.cvelist(datafile)
+
+ # we want to use a dict to easily lookup and replace a cve, but we need
+ # to keep order from the list above for when we write the keys back.
+ cves = collections.OrderedDict()
+ for cve in cvelist:
+ name = cve.header.name
+ # skip temporary entries, we can't get CVE5 for them
+ if 'XXXX' in name:
+ continue
+
+ cves[name] = cve
+
+ return cves
+
+
+parser = argparse.ArgumentParser(description="check issues from a specific CNA")
+parser.add_argument('-c', '--cna', help='CNA short name')
+parser.add_argument('-D', '--no-download', action='store_true',
+ help='Skip downloading files')
+parser.add_argument('-v', '--verbose', action='store_true',
+ help='Verbose mode')
+
+args = parser.parse_args()
+
+cve5_file_url = 'https://github.com/CVEProject/cvelistV5/archive/refs/heads/main.zip'
+cve5_file = 'mitre.zip'
+datafile = "data/CVE/list"
+
+if not args.no_download:
+ debug("downloading files...")
+
+ r = requests.get(cve5_file_url)
+ with open(cve5_file, "wb") as f:
+ f.write(r.content)
+
+# used by read_cve5, used as a global so that we don't have to open the
+# file repeatedly, since we only read cve5s one by one on demand
+cve5_zip = zipfile.ZipFile(cve5_file)
+
+# We have CVE 5.0 JSON information coming from MITRE, we use cve5 for those
+# We also have CVE information coming from our data/CVE/list, we use cve there
+cves = parse_cves()
+cve5s = read_cve5_file(cve5_file)
+
+cves_from_cna = 0
+cves_from_cna_with_packages = 0
+
+for name, cve in cves.items():
+ if cve5 := get_cve5(name):
+ try:
+ if cve5['containers']['cna']['providerMetadata']['shortName'] == args.cna:
+ cves_from_cna += 1
+
+ # we don't check for NFU, as a CVE could be REJECTED or RESERVED.
+ # And it could be both RESERVED and still have a package assigned.
+ # So let's just check if it has a package, and flag those.
+ package_ann = [ann for ann in cve.annotations if isinstance(ann, parsers.PackageAnnotation)]
+ if package_ann:
+ cves_from_cna_with_packages += 1
+ products = [affected['product'] for affected in cve5['containers']['cna']['affected']]
+ products = ", ".join(products)
+ sources = [ann.package for ann in cve.annotations if isinstance(ann, parsers.PackageAnnotation) and not ann.release]
+ sources = ", ".join(sources)
+ print(f"{name}: upstream: {products}; debian: {sources}")
+ except KeyError:
+ # Some CVE5s don't have enough information, ignore them
+ pass
+
+print(f"Total CVEs from {args.cna}: {cves_from_cna}")
+print(f"Total CVEs from {args.cna} with packages assigned: {cves_from_cna_with_packages}")
=====================================
bin/check-new-issues
=====================================
@@ -5,7 +5,7 @@
#
# Based on a previous Perl script written by Stefan Fritsch and others.
#
-# Copyright © 2023 Emilio Pozuelo Monfort <pochu at debian.org>
+# Copyright © 2023-2025 Emilio Pozuelo Monfort <pochu at debian.org>
#
# This file is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -31,6 +31,7 @@ import subprocess
import sys
import tempfile
import textwrap
+import yaml
import zipfile
import requests
@@ -183,25 +184,49 @@ def parse_cves():
return cves
-def auto_nfu(name):
- debug(f'checking nfu for {name}')
- desc = get_cve5_description(name)
+def parse_not_for_us(auto_nfu_file):
+ with open(auto_nfu_file) as f:
+ return yaml.full_load(f)
+
+def nfu_entry_matches(nfu_entry, cve5):
+ if 'not' in nfu_entry:
+ subentry = nfu_entry['not']
+ return not nfu_entry_matches(subentry, cve5)
+ elif 'anyOf' in nfu_entry:
+ result = False
+ for subentry in nfu_entry['anyOf']:
+ result |= nfu_entry_matches(subentry, cve5)
+ return result
+ elif 'allOf' in nfu_entry:
+ result = True
+ for subentry in nfu_entry['allOf']:
+ result &= nfu_entry_matches(subentry, cve5)
+ return result
+
+ cna_name = cve5['containers']['cna']['providerMetadata']['shortName']
+ if 'cna' in nfu_entry and nfu_entry['cna'] == cna_name:
+ return True
+
+ products = [ affected['product'] for affected in cve5['containers']['cna']['affected'] ]
+ # only mark it if there's a single product, in case the CVE affects various
+ # projects and one of the others applies to us
+ if 'product' in nfu_entry and len(products) == 1 and nfu_entry['product'] in products:
+ return True
+
+ if 'description' in nfu_entry:
+ cve5_desc = get_cve5_description(cve5['cveMetadata']['cveId'])
+ if re.fullmatch(nfu_entry['description'], cve5_desc):
+ return True
- if not desc:
- return None
+ return False
- wordpress_re = re.compile(r".*in\s+the\s+(.+)\s+(plugin|theme)\s+(?:[\w\d.]+\s+)?(?:(?:(?:before|through)\s+)?[\w\d.]+\s+)?for\s+[Ww]ord[Pp]ress.*")
- m = wordpress_re.match(desc)
- if m:
- name, type = m.group(1, 2)
- return f"{name} {type} for WordPress"
+def auto_nfu(name, nfu_entries):
+ debug(f'checking nfu for {name}')
+ cve5 = get_cve5(name)
- nfu_re = re.compile(r".*\b(FS\s+.+?\s+Clone|Meinberg\s+LANTIME|Ecava\s+IntegraXor|Foxit\s+Reader|Cambium\s+Networks\s+.+?\s+firmware|Trend\s+Micro|(?:SAP|IBM|EMC|NetApp|Micro\sFocus).+?(?=tool|is|version|[\d(,])).*")
- m = nfu_re.match(desc)
- if m:
- name = m.group(1)
- name = name.strip()
- return name
+ for nfu_entry in nfu_entries:
+ if nfu_entry_matches(nfu_entry, cve5):
+ return nfu_entry['reason']
return None
@@ -363,6 +388,7 @@ removed_packages_file = "data/packages/removed-packages"
ignore_bug_file = "data/packages/ignored-debian-bug-packages"
wnppurl = "https://qa.debian.org/data/bts/wnpp_rm"
wnppfile = "../wnpp_rm"
+auto_nfu_file = "data/packages/nfu.yaml"
issue_re = re.compile(r'CVE-20(?:0[3-9]|[1-9][0-9])|TEMP')
auto_display_limit = 10
@@ -398,6 +424,8 @@ num_missing_bug = 0
wnpp = read_wnpp_file(wnppfile)
+nfu_entries = parse_not_for_us(auto_nfu_file)
+
# packages that should be ignored by -u/-U
ignore_missing_bugs = read_packages_file(removed_packages_file)
ignore_missing_bugs += read_packages_file(ignore_bug_file)
@@ -472,7 +500,7 @@ if args.list:
if args.auto:
# auto process
for todo in todos:
- if nfu_entry := auto_nfu(todo):
+ if nfu_entry := auto_nfu(todo, nfu_entries):
set_cve_nfu(todo, nfu_entry)
save_datafile(cves.values(), datafile)
@@ -599,7 +627,7 @@ def present_issue(name):
print_full_entry(name)
- if nfu_entry := auto_nfu(name):
+ if nfu_entry := auto_nfu(name, nfu_entries):
set_cve_nfu(name, nfu_entry)
print("New entry automatically set to NFU:")
entry = cves[name]
=====================================
data/packages/nfu.yaml
=====================================
@@ -0,0 +1,17 @@
+- reason: SAP
+ cna: sap
+- reason: WordPress plugin
+ cna: Wordfence
+- reason: IBM
+ allOf:
+ - cna: ibm
+ - anyOf:
+ - product: OpenPages with Watson
+ - product: Power Hardware Management Console
+- reason: Adobe
+ allOf:
+ - cna: adobe
+ - not:
+ product: XMP Toolkit
+- reason: Tenda
+ description: '.*\bTenda\b.*'
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/-/compare/0309da415d8482d07f26e7edee65cda95ba2a12d...56f10758d5b986d99402d3f6dc98deaf563e7940
--
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/-/compare/0309da415d8482d07f26e7edee65cda95ba2a12d...56f10758d5b986d99402d3f6dc98deaf563e7940
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-security-tracker-commits/attachments/20250301/7fd6a9fd/attachment-0001.htm>
More information about the debian-security-tracker-commits
mailing list