[WIP]Add OVAL parser for SUSE

.travis.yml

@@ -11,7 +11,7 @@ env:
   - SECRET_KEY="i1bn=oly)w*2yl-5yc&f!vvgt)p)fh3_2$r#spa!*sw36f5ov7"
 
 before_script:
-  - pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py --max-line-length=100 .
+  - pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py,test_suse.py --max-line-length=100 .
   - psql -c "CREATE DATABASE vulnerablecode;" -U postgres
   - ./manage.py migrate
 

README.md

@@ -53,7 +53,7 @@ the code Django includes for this purpose: `SECRET_KEY=$(python -c "from django.
 ## Tests
 
 ```
-pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py --max-line-length=100 .
+pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py,test_suse.py --max-line-length=100 .
 DJANGO_DEV=1 pytest 
 ```
 To skip tests which require internet connection:

vulnerabilities/scraper/lib_oval.py

@@ -8,7 +8,6 @@
 # * Neither the name of the Center for Internet Security, Inc. (CIS) nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
 
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER, CIS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER, CIS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 """Library to simplify working with the OVAL XML structure
 
 
@@ -160,8 +159,14 @@ def __init__(self, tree):
             gen.setSchemaVersion("5.10.1")
             root.append(gen.get_element())
             return        
-        
+
         self.tree = tree
+        self.id_to_definition = { el.getId(): el for el in self.getDefinitions()} if self.getDefinitions() else {}
+        self.id_to_test = {el.getId(): el for el in self.getTests()} if self.getTests() else {}
+        self.id_to_object = {el.getId(): el for el in self.getObjects()} if self.getObjects() else {}
+        self.id_to_state = {el.getId(): el for el in self.getStates()} if self.getStates() else {}
+        self.id_to_variable = {el.getId(): el for el in self.getVariables()} if self.getVariables() else {}
+
 
 
     def parseFromFile(self, filename):
@@ -438,28 +443,17 @@ def getElementByID(self, ovalid):
             return None
 
         if oval_type == OvalDefinition.DEFINITION:
-            elist = self.getDefinitions()
+            return self.id_to_definition[ovalid]
         elif oval_type == OvalDefinition.TEST:
-            elist = self.getTests()
+            return self.id_to_test[ovalid]
         elif oval_type == OvalDefinition.OBJECT:
-            elist = self.getObjects()
+            return self.id_to_object[ovalid]
         elif oval_type == OvalDefinition.STATE:
-            elist = self.getStates()
+            return self.id_to_state[ovalid]
         elif oval_type == OvalDefinition.VARIABLE:
-            elist = self.getVariables()
+            return self.id_to_variable[ovalid]
         else:
             return None
-
-        if not elist:
-            return None
-
-        for element in elist:
-            defid = element.getId()
-            if defid and defid == ovalid:
-                return element
-
-
-
 
     def addElement(self, element, replace=True):
         """
@@ -512,6 +506,7 @@ def addElement(self, element, replace=True):
                 root.append(parent)
 
             parent.append(element.getElement())
+            self.id_to_definition[ovalid] = element
             return True
 
         elif oval_type == OvalDefinition.TEST:
@@ -521,6 +516,7 @@ def addElement(self, element, replace=True):
                 root.append(parent)
 
             parent.append(element.getElement())
+            self.id_to_test[ovalid] = element
             return True
 
         elif oval_type == OvalDefinition.OBJECT:
@@ -530,6 +526,7 @@ def addElement(self, element, replace=True):
                 root.append(parent)
 
             parent.append(element.getElement())
+            self.id_to_object[ovalid] = element
             return True
 
         elif oval_type == OvalDefinition.STATE:
@@ -539,14 +536,16 @@ def addElement(self, element, replace=True):
                 root.append(parent)
 
             parent.append(element.getElement())
+            self.id_to_state[ovalid] = element
             return True
 
         elif oval_type == OvalDefinition.VARIABLE:
             parent = root.find("def:variables", OvalDocument.NS_DEFAULT)
             if parent is None:
                 parent = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}variables")
                 root.append(parent)
-
+
+            self.id_to_variable[ovalid] = element
             parent.append(element.getElement())
             return True
 
@@ -1383,4 +1382,4 @@ def __init__(self, element):
 
 
     def getType(self):
-        return OvalElement.VARIABLE
+        return OvalElement.VARIABLE

vulnerabilities/scraper/oval_parser.py

@@ -1,8 +1,8 @@
+from typing import Optional
 from typing import List
 from typing import Dict
 from typing import Tuple
 from typing import Set
-from typing import Optional
 import xml.etree.ElementTree as ET
 
 from dephell_specifier import RangeSpecifier
@@ -11,49 +11,12 @@
     OvalDefinition, OvalDocument, OvalTest, OvalObject, OvalState, OvalElement)
 
 
-class PerformantOvalDocument(OvalDocument):
-
-    def __init__(self, tree):
-        super().__init__(tree)
-        self.id_to_definition = {
-            el.getId(): el for el in self.getDefinitions()}
-        self.id_to_test = {el.getId(): el for el in self.getTests()}
-        self.id_to_object = {el.getId(): el for el in self.getObjects()}
-        self.id_to_state = {el.getId(): el for el in self.getStates()}
-        self.id_to_variable = {el.getId(): el for el in self.getVariables()}
-
-    def getElementByID(self, oval_id: str) -> Optional[OvalElement]:
-        if not oval_id:
-            return None
-
-        root = self.getDocumentRoot()
-        if not root:
-            return None
-        try:
-            oval_type = OvalElement.getElementTypeFromOvalID(oval_id)
-        except Exception:
-            return None
-
-        if oval_type == OvalDefinition.DEFINITION:
-            return self.id_to_definition[oval_id]
-        elif oval_type == OvalDefinition.TEST:
-            return self.id_to_test[oval_id]
-        elif oval_type == OvalDefinition.OBJECT:
-            return self.id_to_object[oval_id]
-        elif oval_type == OvalDefinition.STATE:
-            return self.id_to_state[oval_id]
-        elif oval_type == OvalDefinition.VARIABLE:
-            return self.id_to_variable[oval_id]
-        else:
-            return None
-
-
 class OvalExtractor:
 
     def __init__(self, translations: Dict, oval_document: ET.ElementTree):
 
         self.translations = translations
-        self.oval_document = PerformantOvalDocument(oval_document)
+        self.oval_document = OvalDocument(oval_document)
         self.all_definitions = self.oval_document.getDefinitions()
         self.all_tests = self.oval_document.getTests()
 
@@ -70,7 +33,7 @@ def get_data(self) -> List[Dict]:
                 continue
             definition_data = {'test_data': []}
             definition_data['description'] = definition.getMetadata(
-            ).getDescription()
+            ).getDescription()  # this could use some data cleaning
             definition_data['vuln_id'] = self.get_vuln_id_from_definition(
                 definition)
             definition_data['reference_urls'] = self.get_urls_from_definition(
@@ -194,3 +157,56 @@ def get_urls_from_definition(definition: OvalDefinition) -> Set[str]:
                 break
 
         return all_urls
+
+
+class SUSEOvalParser(UbuntuOvalParser):
+
+    def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]:
+
+        criteria_refs = []
+
+        for child in definition.element.iter():
+
+            if 'test_ref' in child.attrib:
+                criteria_refs.append(child.get('test_ref'))
+
+        matching_tests = []
+        for ref in criteria_refs:
+            test = self.oval_document.getElementByID(ref)
+            _, state = self.get_object_state_of_test(test)
+            is_signature_test = False
+            is_suse_version_test = False
+            for child in state.element:
+                # This is to avoid signature tests, such as the one below
+                # <signature_keyid operation="equals">b88b2fd43dbdc284</signature_keyid>
+                if child.tag.endswith("signature_keyid"):
+                    is_signature_test = True
+                    break
+            # This is to avoid SUSE OS version tests, such as
+            # <rpminfo_state id="oval:org.opensuse.security:ste:2009061809" version="1"
+            # xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+            #       <version operation="equals">15</version>
+            # </rpminfo_state>
+                if child.get('operation') == "equals":
+                    is_suse_version_test = True
+                    break
+            if not is_suse_version_test and not is_signature_test:
+                matching_tests.append(test)
+
+        return matching_tests
+
+    @staticmethod
+    def get_urls_from_definition(definition: OvalDefinition) -> Set[str]:
+
+        all_urls = set()
+        definition_metadata = definition.getMetadata().element
+        for child in definition_metadata:
+            if child.tag.endswith('reference'):
+                all_urls.add(child.get('ref_url'))
+            if child.tag.endswith('advisory'):
+                for grandchild in child:
+                    if grandchild.get('href'):
+                        all_urls.add(grandchild.get('href'))
+                break
+
+        return all_urls