From 707f3db728133445a1df7a86bb0f2d64ed969991 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Tue, 19 May 2020 19:04:33 +0530 Subject: [PATCH 1/2] Add OVAL parser and tests for SUSE Signed-off-by: Shivam Sandbhor --- .travis.yml | 2 +- README.md | 2 +- vulnerabilities/scraper/oval_parser.py | 61 +++++- .../tests/test_data/suse_oval_data.xml | 163 ++++++++++++++++ vulnerabilities/tests/test_suse.py | 179 ++++++++++++++++++ 5 files changed, 402 insertions(+), 5 deletions(-) create mode 100644 vulnerabilities/tests/test_data/suse_oval_data.xml create mode 100644 vulnerabilities/tests/test_suse.py diff --git a/.travis.yml b/.travis.yml index 41fefae48..a126ab901 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: - SECRET_KEY="i1bn=oly)w*2yl-5yc&f!vvgt)p)fh3_2$r#spa!*sw36f5ov7" before_script: - - pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py --max-line-length=100 . + - pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py,test_suse.py --max-line-length=100 . - psql -c "CREATE DATABASE vulnerablecode;" -U postgres - ./manage.py migrate diff --git a/README.md b/README.md index 05874b3dc..79c818242 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ the code Django includes for this purpose: `SECRET_KEY=$(python -c "from django. ## Tests ``` -pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py --max-line-length=100 . +pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py,test_suse.py --max-line-length=100 . DJANGO_DEV=1 pytest ``` To skip tests which require internet connection: diff --git a/vulnerabilities/scraper/oval_parser.py b/vulnerabilities/scraper/oval_parser.py index 4335ba964..b8c4f89ee 100755 --- a/vulnerabilities/scraper/oval_parser.py +++ b/vulnerabilities/scraper/oval_parser.py @@ -1,8 +1,8 @@ +from typing import Optional from typing import List from typing import Dict from typing import Tuple from typing import Set -from typing import Optional import xml.etree.ElementTree as ET from dephell_specifier import RangeSpecifier @@ -20,7 +20,9 @@ def __init__(self, tree): self.id_to_test = {el.getId(): el for el in self.getTests()} self.id_to_object = {el.getId(): el for el in self.getObjects()} self.id_to_state = {el.getId(): el for el in self.getStates()} - self.id_to_variable = {el.getId(): el for el in self.getVariables()} + if self.getVariables(): + self.id_to_variable = { + el.getId(): el for el in self.getVariables()} def getElementByID(self, oval_id: str) -> Optional[OvalElement]: if not oval_id: @@ -70,7 +72,7 @@ def get_data(self) -> List[Dict]: continue definition_data = {'test_data': []} definition_data['description'] = definition.getMetadata( - ).getDescription() + ).getDescription() # this could use some data cleaning definition_data['vuln_id'] = self.get_vuln_id_from_definition( definition) definition_data['reference_urls'] = self.get_urls_from_definition( @@ -194,3 +196,56 @@ def get_urls_from_definition(definition: OvalDefinition) -> Set[str]: break return all_urls + + +class SUSEOvalParser(UbuntuOvalParser): + + def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]: + + criteria_refs = [] + + for child in definition.element.iter(): + + if 'test_ref' in child.attrib: + criteria_refs.append(child.get('test_ref')) + + matching_tests = [] + for ref in criteria_refs: + test = self.oval_document.getElementByID(ref) + _, state = self.get_object_state_of_test(test) + is_signature_test = False + is_suse_version_test = False + for child in state.element: + # This is to avoid signature tests, such as the one below + # b88b2fd43dbdc284 + if child.tag.endswith("signature_keyid"): + is_signature_test = True + break + # This is to avoid SUSE OS version tests, such as + # + # 15 + # + if child.get('operation') == "equals": + is_suse_version_test = True + break + if not is_suse_version_test and not is_signature_test: + matching_tests.append(test) + + return matching_tests + + @staticmethod + def get_urls_from_definition(definition: OvalDefinition) -> Set[str]: + + all_urls = set() + definition_metadata = definition.getMetadata().element + for child in definition_metadata: + if child.tag.endswith('reference'): + all_urls.add(child.get('ref_url')) + if child.tag.endswith('advisory'): + for grandchild in child: + if grandchild.get('href'): + all_urls.add(grandchild.get('href')) + break + + return all_urls diff --git a/vulnerabilities/tests/test_data/suse_oval_data.xml b/vulnerabilities/tests/test_data/suse_oval_data.xml new file mode 100644 index 000000000..ff8deac21 --- /dev/null +++ b/vulnerabilities/tests/test_data/suse_oval_data.xml @@ -0,0 +1,163 @@ + + + + Marcus Updateinfo to OVAL Converter + 5.5 + 2020-05-17T04:49:08 + + + + + CVE-2009-4112 + + openSUSE Leap 15.1 + + + + Cacti 0.8.7e and earlier allows remote authenticated administrators to gain privileges by modifying the "Data Input Method" for the "Linux - Get Memory Usage" setting to contain arbitrary commands. + + + Low + CVE-2009-4112 + SUSE bug 1122535 + SUSE bug 558664 + + + + + + + + + + + + + + + + + + + CVE-2011-2767 + + openSUSE Leap 15.1 + + + + mod_perl 2.0 through 2.0.10 allows attackers to execute arbitrary Perl code by placing it in a user-owned .htaccess file, because (contrary to the documentation) there is no configuration option that permits Perl code for the administrator's control of HTTP request processing without also permitting unprivileged users to run Perl code in the context of the user account that runs Apache HTTP Server processes. + + + Moderate + CVE-2011-2767 + SUSE bug 1156944 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + openSUSE-release + + + + cacti + + + + cacti-spine + + + + apache2-mod_perl + + + + apache2-mod_perl-devel + + + + + 15.1 + + + + 0:1.2.11-lp151.3.6 + + + + b88b2fd43dbdc284 + + + + 0:2.0.11-lp151.3.3 + + + + + \ No newline at end of file diff --git a/vulnerabilities/tests/test_suse.py b/vulnerabilities/tests/test_suse.py new file mode 100644 index 000000000..1d8fa478a --- /dev/null +++ b/vulnerabilities/tests/test_suse.py @@ -0,0 +1,179 @@ +import os +import unittest +import xml.etree.ElementTree as ET + +from dephell_specifier import RangeSpecifier + + +from vulnerabilities.scraper.oval_parser import SUSEOvalParser + + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/") + + +class TestUbuntuOvalParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + xml_doc = ET.parse(os.path.join(TEST_DATA, "suse_oval_data.xml")) + translator = {"less than": "<","equals":"=="} + # "equals" key is for just testing . In actual practice(check the last test for more info) + # all the elements which require "equals" are ignored(they are not useful) + cls.parsed_oval = SUSEOvalParser(translator, xml_doc) + + def setUp(self): + self.definition_1 = self.parsed_oval.all_definitions[0] + self.definition_2 = self.parsed_oval.all_definitions[1] + + def test_get_definitions(self): + + assert len(self.parsed_oval.all_definitions) == 2 + assert ( + self.parsed_oval.all_definitions[0].getId() + == "oval:org.opensuse.security:def:20094112" + ) + assert ( + self.parsed_oval.all_definitions[1].getId() + == "oval:org.opensuse.security:def:20112767" + ) + + def test_get_tests_of_definition(self): + + definition_1_test_ids = {"oval:org.opensuse.security:tst:2009281999", + "oval:org.opensuse.security:tst:2009282000", + } + definition_2_test_ids = {'oval:org.opensuse.security:tst:2009271113', + 'oval:org.opensuse.security:tst:2009271114', + } + + assert definition_1_test_ids == {i.getId() for i in self.parsed_oval.get_tests_of_definition(self.definition_1)} + + assert definition_2_test_ids == {i.getId() for i in self.parsed_oval.get_tests_of_definition(self.definition_2)} + + def test_get_vuln_id_from_definition(self): + + vuln_id_1 = "CVE-2009-4112" + vuln_id_2 = "CVE-2011-2767" + + assert vuln_id_1 == self.parsed_oval.get_vuln_id_from_definition( + self.definition_1 + ) + assert vuln_id_2 == self.parsed_oval.get_vuln_id_from_definition( + self.definition_2 + ) + + def test_get_object_state_of_test(self): + + # This method is inherited as it is from UbuntuOvalParser + # this test ensures that the method works with suse OVAL documents + + assert len(self.parsed_oval.oval_document.getTests()) == 9 + + test_1 = self.parsed_oval.oval_document.getTests()[0] + test_2 = self.parsed_oval.oval_document.getTests()[1] + + obj_t1, state_t1 = self.parsed_oval.get_object_state_of_test(test_1) + obj_t2, state_t2 = self.parsed_oval.get_object_state_of_test(test_2) + + assert state_t1.getId() == "oval:org.opensuse.security:ste:2009068342" + assert state_t2.getId() == "oval:org.opensuse.security:ste:2009072069" + + assert obj_t2.getId() == "oval:org.opensuse.security:obj:2009031297" + assert obj_t1.getId() == "oval:org.opensuse.security:obj:2009031246" + + def test_get_pkgs_from_obj(self): + + assert len(self.parsed_oval.oval_document.getObjects()) == 5 + + obj_t1 = self.parsed_oval.oval_document.getObjects()[0] + obj_t2 = self.parsed_oval.oval_document.getObjects()[1] + + pkg_set1 = set(self.parsed_oval.get_pkgs_from_obj(obj_t1)) + pkg_set2 = set(self.parsed_oval.get_pkgs_from_obj(obj_t2)) + + assert pkg_set1 == {'openSUSE-release'} + #In a full run we wont get pkg_set1 because we won't obtain + #it's object due to filters to avoid such tests in the first place + assert pkg_set2 == {'cacti'} + + + def test_get_versionsrngs_from_state(self): + + assert len(self.parsed_oval.oval_document.getStates()) == 4 + + state_1 = self.parsed_oval.oval_document.getStates()[0] + state_2 = self.parsed_oval.oval_document.getStates()[1] + + exp_range_1 = RangeSpecifier("==15.1") + exp_range_2 = RangeSpecifier("<0:1.2.11-lp151.3.6") + + assert self.parsed_oval.get_versionsrngs_from_state(state_1) == exp_range_1 + assert self.parsed_oval.get_versionsrngs_from_state(state_2) == exp_range_2 + + def test_get_urls_from_definition(self): + + def1_urls = {"http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2009-4112", + "https://www.suse.com/security/cve/CVE-2009-4112.html", + "https://bugzilla.suse.com/1122535", + "https://bugzilla.suse.com/558664" + } + + assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1) + + def2_urls = {"http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2011-2767", + "https://bugzilla.suse.com/1156944", + "https://www.suse.com/security/cve/CVE-2011-2767.html", + } + + assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2) + + def test_get_data(self): + + expected_data = [ + { + 'test_data': + [ + { + 'package_list': ['cacti'], + 'version_ranges': RangeSpecifier("<0:1.2.11-lp151.3.6") + } + , + { + 'package_list': ['cacti-spine'], + 'version_ranges': RangeSpecifier("<0:1.2.11-lp151.3.6") + } + ], + 'description':'\n Cacti 0.8.7e and earlier allows remote authenticated administrators to gain privileges by modifying the "Data Input Method" for the "Linux - Get Memory Usage" setting to contain arbitrary commands.\n ', + 'vuln_id': 'CVE-2009-4112', + 'reference_urls': { + 'https://bugzilla.suse.com/1122535', + 'https://bugzilla.suse.com/558664', + 'http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2009-4112', + 'https://www.suse.com/security/cve/CVE-2009-4112.html'} + }, + { 'test_data': + [ + { + 'package_list': ['apache2-mod_perl'], + 'version_ranges': RangeSpecifier("<0:2.0.11-lp151.3.3") + }, + { + 'package_list': ['apache2-mod_perl-devel'], + 'version_ranges': RangeSpecifier("<0:2.0.11-lp151.3.3")} + ], + 'description': "\n mod_perl 2.0 through 2.0.10 allows attackers to execute arbitrary Perl code by placing it in a user-owned .htaccess file, because (contrary to the documentation) there is no configuration option that permits Perl code for the administrator's control of HTTP request processing without also permitting unprivileged users to run Perl code in the context of the user account that runs Apache HTTP Server processes.\n ", + 'vuln_id': 'CVE-2011-2767', + 'reference_urls': { + 'https://bugzilla.suse.com/1156944', + 'https://www.suse.com/security/cve/CVE-2011-2767.html', + 'http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2011-2767' + } + } + + ] + + + + assert expected_data == self.parsed_oval.get_data() + + \ No newline at end of file From bcb0ff4610fb782b028c5162ba66b93ac05bea24 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Wed, 20 May 2020 11:44:54 +0530 Subject: [PATCH 2/2] Update lib_oval.py and remove PerformantOvalDocument from oval_parser.py Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/lib_oval.py | 39 +++++++++++++------------- vulnerabilities/scraper/oval_parser.py | 39 +------------------------- 2 files changed, 20 insertions(+), 58 deletions(-) diff --git a/vulnerabilities/scraper/lib_oval.py b/vulnerabilities/scraper/lib_oval.py index be8b36e12..18fc0e8d3 100644 --- a/vulnerabilities/scraper/lib_oval.py +++ b/vulnerabilities/scraper/lib_oval.py @@ -8,7 +8,6 @@ # * Neither the name of the Center for Internet Security, Inc. (CIS) nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER, CIS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER, CIS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """Library to simplify working with the OVAL XML structure @@ -160,8 +159,14 @@ def __init__(self, tree): gen.setSchemaVersion("5.10.1") root.append(gen.get_element()) return - + self.tree = tree + self.id_to_definition = { el.getId(): el for el in self.getDefinitions()} if self.getDefinitions() else {} + self.id_to_test = {el.getId(): el for el in self.getTests()} if self.getTests() else {} + self.id_to_object = {el.getId(): el for el in self.getObjects()} if self.getObjects() else {} + self.id_to_state = {el.getId(): el for el in self.getStates()} if self.getStates() else {} + self.id_to_variable = {el.getId(): el for el in self.getVariables()} if self.getVariables() else {} + def parseFromFile(self, filename): @@ -438,28 +443,17 @@ def getElementByID(self, ovalid): return None if oval_type == OvalDefinition.DEFINITION: - elist = self.getDefinitions() + return self.id_to_definition[ovalid] elif oval_type == OvalDefinition.TEST: - elist = self.getTests() + return self.id_to_test[ovalid] elif oval_type == OvalDefinition.OBJECT: - elist = self.getObjects() + return self.id_to_object[ovalid] elif oval_type == OvalDefinition.STATE: - elist = self.getStates() + return self.id_to_state[ovalid] elif oval_type == OvalDefinition.VARIABLE: - elist = self.getVariables() + return self.id_to_variable[ovalid] else: return None - - if not elist: - return None - - for element in elist: - defid = element.getId() - if defid and defid == ovalid: - return element - - - def addElement(self, element, replace=True): """ @@ -512,6 +506,7 @@ def addElement(self, element, replace=True): root.append(parent) parent.append(element.getElement()) + self.id_to_definition[ovalid] = element return True elif oval_type == OvalDefinition.TEST: @@ -521,6 +516,7 @@ def addElement(self, element, replace=True): root.append(parent) parent.append(element.getElement()) + self.id_to_test[ovalid] = element return True elif oval_type == OvalDefinition.OBJECT: @@ -530,6 +526,7 @@ def addElement(self, element, replace=True): root.append(parent) parent.append(element.getElement()) + self.id_to_object[ovalid] = element return True elif oval_type == OvalDefinition.STATE: @@ -539,6 +536,7 @@ def addElement(self, element, replace=True): root.append(parent) parent.append(element.getElement()) + self.id_to_state[ovalid] = element return True elif oval_type == OvalDefinition.VARIABLE: @@ -546,7 +544,8 @@ def addElement(self, element, replace=True): if parent is None: parent = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}variables") root.append(parent) - + + self.id_to_variable[ovalid] = element parent.append(element.getElement()) return True @@ -1383,4 +1382,4 @@ def __init__(self, element): def getType(self): - return OvalElement.VARIABLE + return OvalElement.VARIABLE \ No newline at end of file diff --git a/vulnerabilities/scraper/oval_parser.py b/vulnerabilities/scraper/oval_parser.py index 4335ba964..76fdac3ee 100755 --- a/vulnerabilities/scraper/oval_parser.py +++ b/vulnerabilities/scraper/oval_parser.py @@ -11,49 +11,12 @@ OvalDefinition, OvalDocument, OvalTest, OvalObject, OvalState, OvalElement) -class PerformantOvalDocument(OvalDocument): - - def __init__(self, tree): - super().__init__(tree) - self.id_to_definition = { - el.getId(): el for el in self.getDefinitions()} - self.id_to_test = {el.getId(): el for el in self.getTests()} - self.id_to_object = {el.getId(): el for el in self.getObjects()} - self.id_to_state = {el.getId(): el for el in self.getStates()} - self.id_to_variable = {el.getId(): el for el in self.getVariables()} - - def getElementByID(self, oval_id: str) -> Optional[OvalElement]: - if not oval_id: - return None - - root = self.getDocumentRoot() - if not root: - return None - try: - oval_type = OvalElement.getElementTypeFromOvalID(oval_id) - except Exception: - return None - - if oval_type == OvalDefinition.DEFINITION: - return self.id_to_definition[oval_id] - elif oval_type == OvalDefinition.TEST: - return self.id_to_test[oval_id] - elif oval_type == OvalDefinition.OBJECT: - return self.id_to_object[oval_id] - elif oval_type == OvalDefinition.STATE: - return self.id_to_state[oval_id] - elif oval_type == OvalDefinition.VARIABLE: - return self.id_to_variable[oval_id] - else: - return None - - class OvalExtractor: def __init__(self, translations: Dict, oval_document: ET.ElementTree): self.translations = translations - self.oval_document = PerformantOvalDocument(oval_document) + self.oval_document = OvalDocument(oval_document) self.all_definitions = self.oval_document.getDefinitions() self.all_tests = self.oval_document.getTests()