Skip to content

Commit a5877cc

Browse files
johnmhoranjohnmhoran
committed
Filter for name-affected.xml files, check CVE prefixes #1079
Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com>
1 parent c8c0f52 commit a5877cc

3 files changed

Lines changed: 229 additions & 40 deletions

File tree

‎vulnerabilities/importers/suse_oval.py‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818

1919

2020
class SuseOvalImporter(OvalImporter):
21-
2221
spdx_license_expression = "CC-BY-4.0"
2322
license_url = "https://ftp.suse.com/pub/projects/security/oval/LICENSE"
2423
base_url = "https://ftp.suse.com/pub/projects/security/oval/"
@@ -37,11 +36,34 @@ def _fetch(self):
3736
if node.get("href").endswith(".gz")
3837
]
3938

40-
for suse_file in suse_oval_files:
39+
for suse_file in filter(suse_oval_files):
4140
response = requests.get(suse_file)
4241

4342
extracted = gzip.decompress(response.content)
4443
yield (
4544
{"type": "rpm", "namespace": "opensuse"},
4645
ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))),
4746
)
47+
48+
49+
def filter(suse_oval_files):
50+
"""
51+
Filter to exclude "name.xml" when we also have "name-affected.xml", e.g.,
52+
"opensuse.leap.15.3.xml.gz" vs. "opensuse.leap.15.3-affected.xml.gz". See
53+
https://ftp.suse.com/pub/projects/security/oval/README: "name-affected.xml" includes
54+
"fixed security issues and the analyzed issues both affecting and NOT affecting SUSE" and
55+
"name.xml" includes "fixed security issues and the analyzed issues NOT affecting SUSE."
56+
"""
57+
affected_files = [
58+
affected_file for affected_file in suse_oval_files if "-affected" in affected_file
59+
]
60+
61+
trimmed_affected_files = [
62+
affected_file.replace("-affected", "") for affected_file in affected_files
63+
]
64+
65+
filtered_suse_oval_files = [
66+
gz_file for gz_file in suse_oval_files if gz_file not in trimmed_affected_files
67+
]
68+
69+
return filtered_suse_oval_files

‎vulnerabilities/oval_parser.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525

2626
class OvalParser:
2727
def __init__(self, translations: Dict, oval_document: ET.ElementTree):
28-
2928
self.translations = translations
3029
self.oval_document = OvalDocument(oval_document)
3130
self.all_definitions = self.oval_document.getDefinitions()
@@ -187,7 +186,22 @@ def get_vuln_id_from_definition(definition):
187186
for child in definition.element.iter():
188187
if child.get("ref_id") and child.get("source"):
189188
if child.get("source") == "CVE":
190-
cve_list.append(child.get("ref_id"))
189+
# TODO: I think this is here only for the print statement below, so need to revisit w/o print statement.
190+
# TODO: What if there's no "CVE" for the `split()` function? Use try/except?
191+
unwanted_prefix = child.get("ref_id").split("CVE")[0]
192+
if child.get("ref_id").startswith("CVE"):
193+
pass
194+
# else:
195+
# unwanted_prefix = child.get("ref_id").split("CVE")[0]
196+
# print(
197+
# "\n\nOh oh! This starts with '{}' and should be '{}' rather than '{}'".format(
198+
# unwanted_prefix,
199+
# child.get("ref_id").replace(unwanted_prefix, ""),
200+
# child.get("ref_id"),
201+
# )
202+
# )
203+
# cve_list.append(child.get("ref_id"))
204+
cve_list.append(child.get("ref_id").replace(unwanted_prefix, ""))
191205
# Debian OVAL files (no "ref_id") will get CVEs via this.
192206
if len(cve_list) == 0:
193207
cve_list.append(definition.getMetadata().getTitle())

‎vulnerabilities/tests/test_suse_oval.py‎

Lines changed: 189 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,17 @@
77
# See https://aboutcode.org for more information about nexB OSS projects.
88
#
99

10+
11+
import gzip
12+
import io
1013
import os
1114
import xml.etree.ElementTree as ET
1215

1316
from vulnerabilities.importers.suse_oval import SuseOvalImporter
17+
from vulnerabilities.importers.suse_oval import filter
1418
from vulnerabilities.oval_parser import OvalParser
1519
from vulnerabilities.tests import util_tests
1620

17-
TRACE = False
18-
1921
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
2022
TEST_DATA = os.path.join(BASE_DIR, "test_data/suse_oval")
2123

@@ -56,43 +58,194 @@ def test_suse_oval_importer_CVE_2008_5679():
5658
)
5759

5860

59-
# Explore parsing inspired by /vulnerablecode/vulnerabilities/tests/test_suse.py
60-
def test_suse_oval_parse_CVE_2008_5679():
61-
# xml_doc = ET.parse(os.path.join(TEST_DATA, "org.opensuse.CVE-2008-5679.xml"))
61+
def test_suse_oval_parse_leap_micro_5_3():
6262
xml_doc = ET.parse(os.path.join(TEST_DATA, "opensuse.leap.micro.5.3.xml"))
6363
translations = {"less than": "<", "equals": "=", "greater than or equal": ">="}
64-
6564
parsed_oval = OvalParser(translations, xml_doc)
66-
if TRACE:
67-
print("\n\ntype(parsed_oval) = {}\n".format(type(parsed_oval)))
6865

69-
print("parsed_oval.all_definitions = {}".format(parsed_oval.all_definitions))
70-
print("len(parsed_oval.all_definitions) = {}".format(len(parsed_oval.all_definitions)))
66+
# Get total number of definitions
67+
assert len(parsed_oval.all_definitions) == 104
7168

69+
# Get definition `id`: the `<definition>` element.
7270
definition_1 = parsed_oval.all_definitions[0]
73-
if TRACE:
74-
print("\ndefinition_1 = {}".format(definition_1))
75-
print("definition_1.getId() = {}\n".format(definition_1.getId()))
76-
77-
# if parsed_oval.all_definitions[1]:
78-
# definition_2 = parsed_oval.all_definitions[1]
79-
# print("definition_2 = {}".format(definition_2))
80-
# print("definition_2.getId() = {}".format(definition_2.getId()))
81-
82-
# For each definition, we can get tests for that definition
83-
# i.getId() for i in self.parsed_oval.get_tests_of_definition(self.definition_1)
84-
test_id_1 = {i.getId() for i in parsed_oval.get_tests_of_definition(definition_1)}
85-
if TRACE:
86-
print("\ntest_id_1 = {}\n".format(test_id_1))
87-
88-
try:
89-
definition_2 = parsed_oval.all_definitions[1]
90-
if TRACE:
91-
print("definition_2 = {}".format(definition_2))
92-
print("definition_2.getId() = {}".format(definition_2.getId()))
93-
94-
test_id_2 = {i.getId() for i in parsed_oval.get_tests_of_definition(definition_2)}
95-
if TRACE:
96-
print("\ntest_id_2 = {}\n".format(test_id_2))
97-
except IndexError:
98-
pass
71+
assert parsed_oval.all_definitions[0].getId() == "oval:org.opensuse.security:def:201918348"
72+
assert parsed_oval.all_definitions[1].getId() == "oval:org.opensuse.security:def:20192708"
73+
74+
# Get definition `test_ref`: the `<criterion>` element.
75+
definition_1_test_ids = {
76+
"oval:org.opensuse.security:tst:2009726610",
77+
"oval:org.opensuse.security:tst:2009726611",
78+
"oval:org.opensuse.security:tst:2009726612",
79+
}
80+
assert definition_1_test_ids == {
81+
i.getId() for i in parsed_oval.get_tests_of_definition(definition_1)
82+
}
83+
84+
# Get vuln_id from definition
85+
# TODO: Delete `Mitre` prefix
86+
vuln_id_1 = ["Mitre CVE-2019-18348"]
87+
assert vuln_id_1 == parsed_oval.get_vuln_id_from_definition(definition_1)
88+
89+
# Get total number of tests
90+
assert len(parsed_oval.oval_document.getTests()) == 3110
91+
92+
# Get test object and test state
93+
test_1 = parsed_oval.oval_document.getTests()[0]
94+
obj_t1, state_t1 = parsed_oval.get_object_state_of_test(test_1)
95+
assert obj_t1.getId() == "oval:org.opensuse.security:obj:2009030416"
96+
assert state_t1.getId() == "oval:org.opensuse.security:ste:2009169740"
97+
98+
# Get total number of packages: `rpminfo_object` elements
99+
assert len(parsed_oval.oval_document.getObjects()) == 336
100+
101+
# Get packages
102+
obj_t1 = parsed_oval.oval_document.getObjects()[0]
103+
obj_t2 = parsed_oval.oval_document.getObjects()[1]
104+
105+
pkg_set1 = set(parsed_oval.get_pkgs_from_obj(obj_t1))
106+
pkg_set2 = set(parsed_oval.get_pkgs_from_obj(obj_t2))
107+
108+
assert pkg_set1 == {"kernel-default"}
109+
assert pkg_set2 == {"kgraft-patch-3_12_38-44-default"}
110+
111+
# Get total number of versions: `rpminfo_state` elements
112+
assert len(parsed_oval.oval_document.getStates()) == 764
113+
114+
# Get versions
115+
state_1 = parsed_oval.oval_document.getStates()[0]
116+
state_2 = parsed_oval.oval_document.getStates()[1]
117+
118+
exp_range_1 = "=3.12.38-44.1"
119+
exp_range_2 = ">=5-2.1-0"
120+
121+
assert parsed_oval.get_version_range_from_state(state_1) == exp_range_1
122+
assert parsed_oval.get_version_range_from_state(state_2) == exp_range_2
123+
124+
# Get reference URLs: `ref_url` attribute from `reference` elements
125+
# We use the 2nd definition because the 1st has a lengthy list of references.
126+
definition_2 = parsed_oval.all_definitions[1]
127+
def2_urls = {
128+
"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-2708",
129+
"https://www.suse.com/security/cve/CVE-2019-2708",
130+
"https://lists.suse.com/pipermail/sle-security-updates/2022-November/013108.html",
131+
"https://lists.suse.com/pipermail/sle-security-updates/2022-November/013106.html",
132+
"https://lists.suse.com/pipermail/sle-security-updates/2022-November/013158.html",
133+
"https://www.suse.com/security/cve/CVE-2019-2708/",
134+
"https://bugzilla.suse.com/1174414",
135+
}
136+
137+
assert def2_urls == parsed_oval.get_urls_from_definition(definition_2)
138+
139+
140+
def test_compare_name_gz_vs_name_affected_gz():
141+
translations = {"less than": "<", "equals": "=", "greater than or equal": ">="}
142+
143+
# "name-affected.xml" example:
144+
145+
name_affected_gz = gzip.open(os.path.join(TEST_DATA, "opensuse.leap.15.3-affected.xml.gz"), "r")
146+
name_affected_xml = ET.parse(name_affected_gz)
147+
parsed_name_affected_xml = OvalParser(translations, name_affected_xml)
148+
149+
print("\n\nOVAL XML file = 'opensuse.leap.15.3-affected.xml.gz'\n")
150+
151+
# Get total number of definitions
152+
assert len(parsed_name_affected_xml.all_definitions) == 9138
153+
154+
print(
155+
"len(parsed_name_affected_xml.all_definitions) = {}\n".format(
156+
len(parsed_name_affected_xml.all_definitions)
157+
)
158+
)
159+
160+
assert (
161+
parsed_name_affected_xml.all_definitions[0].getId()
162+
== "oval:org.opensuse.security:def:20042771"
163+
)
164+
assert (
165+
parsed_name_affected_xml.all_definitions[1].getId()
166+
== "oval:org.opensuse.security:def:20054900"
167+
)
168+
169+
print(
170+
"parsed_name_affected_xml.all_definitions[0] = {}\n".format(
171+
parsed_name_affected_xml.all_definitions[0]
172+
)
173+
)
174+
175+
print(
176+
"parsed_name_affected_xml.get_vuln_id_from_definition(parsed_name_affected_xml.all_definitions[0]) = {}\n".format(
177+
parsed_name_affected_xml.get_vuln_id_from_definition(
178+
parsed_name_affected_xml.all_definitions[0]
179+
)
180+
)
181+
)
182+
183+
print(
184+
"parsed_name_affected_xml.get_vuln_id_from_definition(parsed_name_affected_xml.all_definitions[-1]) = {}\n".format(
185+
parsed_name_affected_xml.get_vuln_id_from_definition(
186+
parsed_name_affected_xml.all_definitions[-1]
187+
)
188+
)
189+
)
190+
191+
# Get definition `id`: the `<definition>` element.
192+
definition_1_name_affected_xml = parsed_name_affected_xml.all_definitions[0]
193+
194+
# TODO: How can we efficiently/simply test that name_xml is a subset of name_affected_xml?
195+
196+
# "name.xml" example:
197+
198+
name_gz = gzip.open(os.path.join(TEST_DATA, "opensuse.leap.15.3.xml.gz"), "r")
199+
name_xml = ET.parse(name_gz)
200+
parsed_name_xml = OvalParser(translations, name_xml)
201+
202+
# Get total number of definitions
203+
assert len(parsed_name_xml.all_definitions) == 9138
204+
205+
assert parsed_name_xml.all_definitions[0].getId() == "oval:org.opensuse.security:def:20042771"
206+
assert parsed_name_xml.all_definitions[1].getId() == "oval:org.opensuse.security:def:20054900"
207+
208+
# Get definition `id`: the `<definition>` element.
209+
definition_1_name_xml = parsed_name_xml.all_definitions[0]
210+
211+
# TODO: Repeating above TODO -- How can we efficiently/simply test that name_xml is a subset of name_affected_xml?
212+
213+
# =========================================================
214+
# Compare the 2 lists of definitions, confirm every item in `parsed_name_xml.all_definitions`
215+
# is also in `parsed_name_affected_xml.all_definitions`
216+
# =========================================================
217+
218+
219+
def test_filter_suse_gz_files():
220+
initial_suse_gz_files = [
221+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7-affected.xml.gz",
222+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7-patch.xml.gz",
223+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7.xml.gz",
224+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8-affected.xml.gz",
225+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8-patch.xml.gz",
226+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8.xml.gz",
227+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9-affected.xml.gz",
228+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9-patch.xml.gz",
229+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9.xml.gz",
230+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.6-affected.xml.gz",
231+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.6-patch.xml.gz",
232+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.6.xml.gz",
233+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.7-affected.xml.gz",
234+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.7-patch.xml.gz",
235+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.7.xml.gz",
236+
]
237+
238+
filtered_initial_suse_gz_files = [
239+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7-affected.xml.gz",
240+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7-patch.xml.gz",
241+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8-affected.xml.gz",
242+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8-patch.xml.gz",
243+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9-affected.xml.gz",
244+
"https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9-patch.xml.gz",
245+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.6-affected.xml.gz",
246+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.6-patch.xml.gz",
247+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.7-affected.xml.gz",
248+
"https://ftp.suse.com/pub/projects/security/oval/suse.storage.7-patch.xml.gz",
249+
]
250+
251+
assert filter(initial_suse_gz_files) == filtered_initial_suse_gz_files

0 commit comments

Comments
 (0)