Skip to content

Commit f0c5247

Browse files
johnmhoranjohnmhoran
committed
Refactor OVAL-relared code, fix failing tests #1079
Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com>
1 parent f8c0c83 commit f0c5247

12 files changed

Lines changed: 1701298 additions & 24019 deletions

‎vulnerabilities/importer.py‎

Lines changed: 2 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -421,83 +421,6 @@ def advisory_data(self) -> List[AdvisoryData]:
421421
)
422422
continue
423423

424-
# def get_data_from_xml_doc(
425-
# self, xml_doc: ET.ElementTree, pkg_metadata={}
426-
# ) -> Iterable[AdvisoryData]:
427-
# """
428-
# The orchestration method of the OvalDataSource. This method breaks an
429-
# OVAL xml ElementTree into a list of `Advisory`.
430-
431-
# Note: pkg_metadata is a mapping of Package URL data that MUST INCLUDE
432-
# "type" key.
433-
434-
# Example value of pkg_metadata:
435-
# {"type":"deb","qualifiers":{"distro":"buster"} }
436-
# """
437-
# oval_parsed_data = OvalParser(self.translations, xml_doc)
438-
# raw_data = oval_parsed_data.get_data()
439-
# oval_doc = oval_parsed_data.oval_document
440-
# timestamp = oval_doc.getGenerator().getTimestamp()
441-
442-
# print("\noval_parsed_data = {}\n".format(oval_parsed_data))
443-
# print("\nraw_data = {}\n".format(raw_data))
444-
445-
# # convert definition_data to Advisory objects
446-
# for definition_data in raw_data:
447-
# print("\ndefinition_data = {}\n".format(definition_data))
448-
# # These fields are definition level, i.e common for all elements
449-
# # connected/linked to an OvalDefinition
450-
451-
# # TODO: 2023-01-24 Tuesday 22:34:20. Is this where we'd loop through the list of CVEs/aliases?
452-
453-
# vuln_id = definition_data["vuln_id"]
454-
# description = definition_data["description"]
455-
456-
# severities = []
457-
# severity = definition_data.get("severity")
458-
# if severity:
459-
# severities.append(
460-
# VulnerabilitySeverity(system=severity_systems.GENERIC, value=severity)
461-
# )
462-
# references = [
463-
# Reference(url=url, severities=severities)
464-
# for url in definition_data["reference_urls"]
465-
# ]
466-
# affected_packages = []
467-
# print('\ndefinition_data["test_data"] = {}\n'.format(definition_data["test_data"]))
468-
# for test_data in definition_data["test_data"]:
469-
# print("\ntest_data['package_list'] = {}\n".format(test_data["package_list"]))
470-
# for package_name in test_data["package_list"]:
471-
# affected_version_range = test_data["version_ranges"]
472-
# vrc = RANGE_CLASS_BY_SCHEMES[pkg_metadata["type"]]
473-
# if affected_version_range:
474-
# try:
475-
# affected_version_range = vrc.from_native(affected_version_range)
476-
# except Exception as e:
477-
# logger.error(
478-
# f"Failed to parse version range {affected_version_range!r} "
479-
# f"for package {package_name!r}:\n{e}"
480-
# )
481-
# continue
482-
# if package_name:
483-
# affected_packages.append(
484-
# AffectedPackage(
485-
# package=self.create_purl(package_name, pkg_metadata),
486-
# affected_version_range=affected_version_range,
487-
# )
488-
# )
489-
# print("affected_packages = {}".format(affected_packages))
490-
# date_published = dateparser.parse(timestamp)
491-
# if not date_published.tzinfo:
492-
# date_published = date_published.replace(tzinfo=pytz.UTC)
493-
# yield AdvisoryData(
494-
# aliases=[vuln_id],
495-
# summary=description,
496-
# affected_packages=sorted(affected_packages),
497-
# references=sorted(references),
498-
# date_published=date_published,
499-
# )
500-
501424
def get_data_from_xml_doc(
502425
self, xml_doc: ET.ElementTree, pkg_metadata={}
503426
) -> Iterable[AdvisoryData]:
@@ -516,23 +439,15 @@ def get_data_from_xml_doc(
516439
oval_doc = oval_parsed_data.oval_document
517440
timestamp = oval_doc.getGenerator().getTimestamp()
518441

519-
print("\n== Run OvalImporter() get_data_from_xml_doc() ==\n")
520-
521-
# print("\noval_parsed_data = {}\n".format(oval_parsed_data))
522-
print("\n==> raw_data = {}\n".format(raw_data))
523-
524442
# convert definition_data to Advisory objects
525443
for definition_data in raw_data:
526-
print("\n==> definition_data = {}\n".format(definition_data))
527444
# These fields are definition level, i.e common for all elements
528445
# connected/linked to an OvalDefinition
529446

530447
# NOTE: This is where we loop through the list of CVEs/aliases.
531-
532448
vuln_id_list = definition_data["vuln_id"]
533449

534450
for vuln_id_item in vuln_id_list:
535-
# vuln_id = definition_data["vuln_id"]
536451
vuln_id = vuln_id_item
537452
description = definition_data["description"]
538453

@@ -547,13 +462,8 @@ def get_data_from_xml_doc(
547462
for url in definition_data["reference_urls"]
548463
]
549464
affected_packages = []
550-
print(
551-
'\n==> definition_data["test_data"] = {}\n'.format(definition_data["test_data"])
552-
)
465+
553466
for test_data in definition_data["test_data"]:
554-
print(
555-
"\n==> test_data['package_list'] = {}\n".format(test_data["package_list"])
556-
)
557467
for package_name in test_data["package_list"]:
558468
affected_version_range = test_data["version_ranges"]
559469
vrc = RANGE_CLASS_BY_SCHEMES[pkg_metadata["type"]]
@@ -573,7 +483,7 @@ def get_data_from_xml_doc(
573483
affected_version_range=affected_version_range,
574484
)
575485
)
576-
print("==> affected_packages = {}\n".format(affected_packages))
486+
577487
date_published = dateparser.parse(timestamp)
578488
if not date_published.tzinfo:
579489
date_published = date_published.replace(tzinfo=pytz.UTC)

‎vulnerabilities/oval_parser.py‎

Lines changed: 14 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -36,46 +36,29 @@ def get_data(self) -> List[Dict]:
3636
Return a list of OvalDefinition mappings.
3737
"""
3838
oval_data = []
39-
print("\nlen(self.all_definitions) = {}\n".format(len(self.all_definitions)))
4039
for definition in self.all_definitions:
41-
# print(definition)
42-
# print(list(definition))
43-
4440
matching_tests = self.get_tests_of_definition(definition)
4541
if not matching_tests:
4642
continue
4743
definition_data = {"test_data": []}
4844
# TODO:this could use some data cleaning
4945
definition_data["description"] = definition.getMetadata().getDescription() or ""
50-
5146
definition_data["vuln_id"] = self.get_vuln_id_from_definition(definition)
5247
definition_data["reference_urls"] = self.get_urls_from_definition(definition)
53-
5448
definition_data["severity"] = self.get_severity_from_definition(definition)
55-
print("\nlen(matching_tests) = {}\n".format(len(matching_tests)))
56-
# print("\nmatching_tests = {}\n".format(matching_tests))
49+
5750
for test in matching_tests:
58-
# print("\ntest = {}\n".format(test))
59-
# print("\ntest.element = {}\n".format(test.element))
6051
test_obj, test_state = self.get_object_state_of_test(test)
6152
if not test_obj or not test_state:
6253
continue
6354
test_data = {"package_list": []}
64-
# print("\ntest_obj = {}\n".format(test_obj))
6555
test_data["package_list"].extend(self.get_pkgs_from_obj(test_obj))
66-
print(
67-
"\nself.get_pkgs_from_obj(test_obj) = {}\n".format(
68-
self.get_pkgs_from_obj(test_obj)
69-
)
70-
)
7156
version_ranges = self.get_version_range_from_state(test_state)
7257
test_data["version_ranges"] = version_ranges
7358
definition_data["test_data"].append(test_data)
7459

7560
oval_data.append(definition_data)
7661

77-
# print('\ntest_data["package_list"] = {}\n'.format(test_data["package_list"]))
78-
7962
return oval_data
8063

8164
def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]:
@@ -86,25 +69,28 @@ def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]:
8669
criteria_refs = []
8770

8871
for child in definition.element.iter():
89-
9072
if "test_ref" in child.attrib:
9173
criteria_refs.append(child.get("test_ref"))
9274

9375
matching_tests = []
9476
for ref in criteria_refs:
9577
oval_test = self.oval_document.getElementByID(ref)
78+
# All matches will be `rpminfo_test` elements inside the `tests` element.
79+
# Test for len == 2 because this IDs a pair of nested `object` and `state` elements.
9680
if len(oval_test.element) == 2:
9781
_, state = self.get_object_state_of_test(oval_test)
9882
valid_test = True
9983
for child in state.element:
10084
if child.get("operation") not in self.translations:
10185
valid_test = False
102-
break
103-
if valid_test:
104-
matching_tests.append(self.oval_document.getElementByID(ref))
105-
print("\nThese are matching_tests: {}".format(matching_tests))
106-
for mt in matching_tests:
107-
print("mt = {}".format(mt.element))
86+
continue
87+
elif (
88+
child.get("operation") in self.translations
89+
# "debian_evr_string" is used in both Debian and Ubuntu test XML files; SUSE OVAL uses "evr_string".
90+
# See also https://github.com/OVALProject/Language/blob/master/docs/oval-common-schema.md
91+
and child.get("datatype") in ["evr_string", "debian_evr_string"]
92+
):
93+
matching_tests.append(self.oval_document.getElementByID(ref))
10894

10995
return list(set(matching_tests))
11096

@@ -126,6 +112,7 @@ def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]:
126112
pkg_list = []
127113

128114
for var in obj.element:
115+
# It appears that `var_ref` is used in Ubuntu OVAL but not Debian or SUSE.
129116
if var.get("var_ref"):
130117
var_elem = self.oval_document.getElementByID(var.get("var_ref"))
131118
comment = var_elem.element.get("comment")
@@ -195,26 +182,13 @@ def get_severity_from_definition(definition: OvalDefinition) -> Set[str]:
195182

196183
@staticmethod
197184
def get_vuln_id_from_definition(definition):
198-
# # SUSE and Ubuntu OVAL files will get cves via this loop
199-
# for child in definition.element.iter():
200-
# # if child.get("ref_id"):
201-
# # return child.get("ref_id")
202-
# # Must also check whether 'source' field exists and value is 'CVE'
203-
# # TODO: what if there are multiple elements that satisfy the condition?
204-
# # Add to list and report as separate AdvisoryData() objects?
205-
# if child.get("ref_id") and child.get("source"):
206-
# if child.get("source") == "CVE":
207-
# return child.get("ref_id")
208-
# # Debian OVAL files will get cves via this
209-
# return definition.getMetadata().getTitle()
210-
# ========================================================
185+
# SUSE and Ubuntu OVAL files will get CVEs via this loop.
211186
cve_list = []
212187
for child in definition.element.iter():
213188
if child.get("ref_id") and child.get("source"):
214189
if child.get("source") == "CVE":
215190
cve_list.append(child.get("ref_id"))
216-
217-
# Debian OVAL files will get cves via this
191+
# Debian OVAL files (no "ref_id") will get CVEs via this.
218192
if len(cve_list) == 0:
219193
cve_list.append(definition.getMetadata().getTitle())
220194

0 commit comments

Comments
 (0)