From 5cbe40533529344b911c76106f7bfa609166dbdd Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 15 May 2020 19:55:27 +0530 Subject: [PATCH 1/9] Add oval parser Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/lib_oval.py | 1376 ++++++++++++++++++++++++ vulnerabilities/scraper/oval_parser.py | 145 +++ 2 files changed, 1521 insertions(+) create mode 100644 vulnerabilities/scraper/lib_oval.py create mode 100755 vulnerabilities/scraper/oval_parser.py diff --git a/vulnerabilities/scraper/lib_oval.py b/vulnerabilities/scraper/lib_oval.py new file mode 100644 index 000000000..0b01363d9 --- /dev/null +++ b/vulnerabilities/scraper/lib_oval.py @@ -0,0 +1,1376 @@ +#!/usr/bin/env/ python3 +"""Library to simplify working with the OVAL XML structure + + +Authors: Gunnar Engelbach + + + +Available classes: + - OvalDocument: operations at the OVAL document level, such as reading in an existing OVAL document from +file, creating a new one, finding or adding OVAL elements + - OvalElement: the base class for OVAL elements. Implements a few common methods inherited by the +subclasses for definition, test, state, object, and variable + - OvalDefinition: a type of OVAL element with certain attributes available. Additional classes used by the OvalDefinition class: + - OvalMetadata: the metadata associated with a definition, which includes the definition title and description. Metadata also contains: + - OvalAffected: The family and platforms affected by this definition + - OvalRepositoryInformation: Additional information added by the OVAL repository + - OvalTest: for working with OVAL test elements + - OvalObject: for working with OVAL object elements + - OvalState: for working with OVAL state elements + - OvalVariable: for working with OVAL variable elements + + + +Available exceptions: + - None at this time + + +:Usage: + +1. Create an OvalDocument: + + >>> tree = ElementTree() + >>> tree.parse("OvalTest.xml") + >>> document = OvalDocument(tree) + +2. Find an oval element within the loaded document: + + >>> element = document.getElementByID("oval:org.mitre.oval:def:22382") + >>> if element is not None: + >>> .... + +3. Read an XML file with a single OVAL Definition (error checking omitted for brevity): + + >>> tree = ElementTree() + >>> tree.parse('test-definition.xml') + >>> root = tree.getroot() + >>> definition = lib_oval.OvalDefinition(root) + +4. Change information in the definition from #3 and write the changes + + >>> meta = definition.getMetadata() + >>> repo = meta.getOvalRepositoryInformation() + >>> repo.setMinimumSchemaVersion("5.9") + >>> tree.write("outfilename.xml", UTF-8", True) + + + + + +TODO: + - Add exceptions that give more detail about why a value of None is sometimes returned + - Expand use of find() to allow for the possibility that the XML document is not using namespaces + - Lots of pydoc to be added + - Redo getter/setter for OvalRepository status elements. +""" + +import os, xml.etree +from xml.etree import ElementTree +from xml.etree.ElementTree import Element + +import datetime + + + +# __docformat__ = "Epytext en" + + + +class OvalDocument(object): + """ + For working with OVAL documents. That interaction will entail the use of the other classes. + Can be used to find certain elements within the document, update the document, and save the changes to a file + """ + + + # A time format to match what OVAL expects + TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" + + NS_DEFAULT = {"def": "http://oval.mitre.org/XMLSchema/oval-definitions-5"} + NS_OVAL = {"oval": "http://oval.mitre.org/XMLSchema/oval-common-5"} + NS_XSI = {"xsi": "http://www.w3.org/2001/XMLSchema-instance"} + +# xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5" +# xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5" +# xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" +# xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd +# http://oval.mitre.org/XMLSchema/oval-definitions-5#independent independent-definitions-schema.xsd +# http://oval.mitre.org/XMLSchema/oval-definitions-5#solaris solaris-definitions-schema.xsd +# http://oval.mitre.org/XMLSchema/oval-common-5 oval-common-schema.xsd +# http://oval.mitre.org/XMLSchema/oval-definitions-5#unix unix-definitions-schema.xsd">^M + + + @staticmethod + def indent(elem, level=0): + i = "\n" + level*" " + if len(elem): + if not elem.text or not elem.text.strip(): + elem.text = i + " " + if not elem.tail or not elem.tail.strip(): + elem.tail = i + for elem in elem: + OvalDocument.indent(elem, level+1) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + else: + if level and (not elem.tail or not elem.tail.strip()): + elem.tail = i + + + @staticmethod + def getOvalTimestamp(timestamp=None): + """Renders a datetime to a string formatted according to the OVAL specification. + if the timestamp argument is None (which it is by default) or is not of type datetime, + this function will return a string using the current datetime. + + @type timestamp: datetime + @param timestamp: A datetime to be formatted as an OVAL timestamp, or None to use the current time. + + @rtype: string + @return: a string formatted as per OVAL + """ + if timestamp is None or not isinstance(timestamp, datetime): + now = datetime.date.today() + return now.strftime(OvalDocument.TIME_FORMAT) + else: + return timestamp.strftime(OvalDocument.TIME_FORMAT) + + + + def __init__(self, tree): +# if not tree or not isinstance(tree, ElementTree): + if not tree: + root = Element("oval_definitions") + self.tree = ElementTree.ElementTree(root) + element = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}generator") + gen = OvalGenerator(element) + gen.setProduct("The CIS OVAL Repository") + gen.setTimestamp(None) + gen.setSchemaVersion("5.10.1") + root.append(gen.get_element()) + return + + self.tree = tree + + + def parseFromFile(self, filename): + """ + Load an OVAL document from a filename and parse that into an ElementTree + Returns False if the filename is empty or there is an error parsing the XML document + @type filename: string + @param filename: The path to the OVAL XML document to parse + + @rtype: boolean + @return: True on success, otherwise False + """ + try: + if not filename: + self.tree = None + return False + else: + self.tree = ElementTree.parse(filename) + return True + except Exception: + return False + + + def parseFromText(self, xmltext): + """ + Initializes the ElementTree by parsing the xmltext as XML + Returns False if the string could not be parsed as XML + + @rtype: boolean + @return: True on success, otherwise False + """ + try: + if not xmltext: + return False + else: + root = ElementTree.fromstring(xmltext) + self.tree = ElementTree(root) + return True + except Exception: + return False + + + def writeToFile(self, filename): + """ + Writes the internal representation of the XmlTree to the given file name + Returns False on error + + @rtype: boolean + @return: True on success, otherwise False + """ + try: + if not filename: + return False + if not self.tree: + return False + + ### TODO: Add all necessary namespaces + self.tree.write(filename, "UTF-8", True, OvalDocument.NS_DEFAULT, "xml") + + except Exception: + return False + + + def to_string(self): + + if not self.tree: + return None + + root = self.tree.getroot() + if root is None: + return "" + OvalDocument.indent(root) + return ElementTree.tostring(root, "UTF-8", "xml").decode("utf-8") + + + def getDocumentRoot(self): + """ + Returns the root element of the XML tree if one has been loaded. + Otherwise, returns None + + @rtype: Element + @return: The root Element of the OVAL document, or None + """ + if not self.tree: + return None + + return self.tree.getroot() + + + def getGenerator(self, create=False): + """ + Gets the generator for this OVAL document as an OvalGenerator object. + If the generator element does not exist, the default behavior is to + return none. However, setting the optional parameter to True will cause + a default generate element to be created, added to the document, and that will be returned. + A value of None may also be returned if this OvalDocument is empty + + @rtype: OvalGenerator + @return: An OvalGenerator object, or None if it does not exist and create was not set to True + """ + if not self.tree: + return None + + root = self.getDocumentRoot() + if not root: + return None + + gen_element = root.find("def:generator", OvalDocument.NS_DEFAULT) + + if gen_element is not None: + return OvalGenerator(gen_element) + + if not create: + return None + else: + element = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}generator") + gen = OvalGenerator(element) + gen.setProduct("The CIS OVAL Repository") + gen.setTimestamp(None) + gen.setSchemaVersion("5.10.1") + return gen + + + + + def getDefinitions(self): + """ + Returns a list of all definitions found in this OvalDocment where each item in the list is of type OvalDefinition + Returns None if no definitions could be found + + @rtype: List + @return: All definitions in the OVAL document or None if none were found + """ + root = self.getDocumentRoot() + if not root: + return None + + defroot = root.find("def:definitions", OvalDocument.NS_DEFAULT) + + if defroot is None: + return None + + element_list = list(defroot) + if not element_list: + return None + + return [OvalDefinition(element) for element in element_list] + + + + + def getTests(self): + """ + Returns a list of all tests in this OvalDocument where each list item is of type OvalTest + Returns None if no tests could be found + + @rtype: List + @return: All tests in the OVAL document or None if none were found + """ + root = self.getDocumentRoot() + if not root: + return None + + testroot = root.find("def:tests", OvalDocument.NS_DEFAULT) + + if testroot is None: + return None + + element_list = list(testroot) + if not element_list: + return None + + return [OvalTest(element) for element in element_list] + + + + def getObjects(self): + """ + Returns a list of all objects in this OvalDocument where each list item is of type OvalObject + Returns None if no objects could be found + + @rtype: List + @return: All objects in the OVAL document or None if none were found + """ + root = self.getDocumentRoot() + if not root: + return None + + objectroot = root.find("def:objects", OvalDocument.NS_DEFAULT) + + if objectroot is None: + return None + + element_list = list(objectroot) + if not element_list: + return None + + return [OvalObject(element) for element in element_list] + + + + def getStates(self): + """ + Returns a list of all states in this OvalDocument where each list item is of type OvalState + Returns None if no states could be found + + @rtype: List + @return: All states in the OVAL document or None if none were found + """ + root = self.getDocumentRoot() + if not root: + return None + + stateroot = root.find("def:states", OvalDocument.NS_DEFAULT) + + if stateroot is None: + return None + + element_list = list(stateroot) + if not element_list: + return None + + return [OvalState(element) for element in element_list] + + + + + def getVariables(self): + """ + Returns a list of all variables in this OvalDocument where each list item is of type OvalVariable + Returns None if no variables could be found + + @rtype: List + @return: All variables in the OVAL document or None if none were found + """ + root = self.getDocumentRoot() + if not root: + return None + + varroot = root.find("def:variables", OvalDocument.NS_DEFAULT) + + if varroot is None: + return None + + element_list = list(varroot) + if not element_list: + return None + + return [OvalVariable(element) for element in element_list] + + + + + def getElementByID(self, ovalid): + """ + Uses the ovalid argument to determine what type of element is being referenced and locate that element + in the OVAL ElementTree. + Returns an OvalElement of the appropriate class (OvalDefinition, OvalTest, ...) + or None if there is no ElementTree or if a matching item could not be found + + @rtype: OvalElement + @return: The located element as the appropriate OvalElement subclass, or None if no matching element was found. + """ + if not ovalid: + return None + + root = self.getDocumentRoot() + if not root: + return None + + try: + oval_type = OvalElement.getElementTypeFromOvalID(ovalid) + except Exception: + return None + + if oval_type == OvalDefinition.DEFINITION: + elist = self.getDefinitions() + elif oval_type == OvalDefinition.TEST: + elist = self.getTests() + elif oval_type == OvalDefinition.OBJECT: + elist = self.getObjects() + elif oval_type == OvalDefinition.STATE: + elist = self.getStates() + elif oval_type == OvalDefinition.VARIABLE: + elist = self.getVariables() + else: + return None + + if not elist: + return None + + for element in elist: + defid = element.getId() + if defid and defid == ovalid: + return element + + + + + def addElement(self, element, replace=True): + """ + Adds the element to the ElementTree for this OVAL document + The element argument must be of type OvalElement + This method uses the OVALID of the element to determine what type of element it is + and if an existing element with that OVALID already exists. + This method will also create the necessary structure (id est, adding , , etc) + if the ElementTree does not already contain it. + By default this method will replace an existing item with the same OVALID, but this behavior can + be overridden by changing the option second argument to a value of "False" + Returns True on success, otherwise False + + @rtype: boolean + @return: True if the element was added to the document, otherwise False + """ + if not element or element is None: + return False + if not self.tree or self.tree is None: + return False + + ovalid = element.getId() + if not ovalid: + return False + + root = self.tree.getroot() + if not root: + root = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}oval_definitions") + self.tree._setroot(root) + + # If replace has been set to False, then we want to exit with no changes + # when an element with this OVALID already appears in the document + if not replace: + existing = self.getElementByID(ovalid) + if existing: + return False; + + + try: + oval_type = OvalElement.getElementTypeFromOvalID(ovalid) + except Exception: + return False + + # Depending on the ID type, find the parent for it or create that parent if it doesn't exist + # Then append the current element + if oval_type == OvalDefinition.DEFINITION: + parent = root.find("def:definitions", OvalDocument.NS_DEFAULT) + if parent is None: + parent = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}definitions") + root.append(parent) + + parent.append(element.getElement()) + return True + + elif oval_type == OvalDefinition.TEST: + parent = root.find("def:tests", OvalDocument.NS_DEFAULT) + if parent is None: + parent = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}tests") + root.append(parent) + + parent.append(element.getElement()) + return True + + elif oval_type == OvalDefinition.OBJECT: + parent = root.find("def:objects", OvalDocument.NS_DEFAULT) + if parent is None: + parent = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}objects") + root.append(parent) + + parent.append(element.getElement()) + return True + + elif oval_type == OvalDefinition.STATE: + parent = root.find("def:states", OvalDocument.NS_DEFAULT) + if parent is None: + parent = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}states") + root.append(parent) + + parent.append(element.getElement()) + return True + + elif oval_type == OvalDefinition.VARIABLE: + parent = root.find("def:variables", OvalDocument.NS_DEFAULT) + if parent is None: + parent = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}variables") + root.append(parent) + + parent.append(element.getElement()) + return True + + else: + return False + +#--------------------- END OF OvalDocument class ---------------------------- + + + +class OvalGenerator(object): + """ + Contains information about this OvalDocument, such as the schema version, the product that produced it, and when it was produced + """ + + def __init__(self, element): + self.element = element + + + def getProduct(self): + """ + Gets the value of the product element + """ + if self.element is None: + return None + + +# child = self.element.find("{http://oval.mitre.org/XMLSchema/oval-common-5}product_name") + child = self.element.find("oval:product_name", OvalDocument.NS_OVAL) + if child is None: + return None + else: + return child.text + + + def get_element(self): + return self.element + + + def setProduct(self, product): + """ + Sets a value for the product element. If a product element does not already exist, one will be created + """ + if self.element is None: + return False + + if product is None: + return False + + child = self.element.find("oval:product_name", OvalDocument.NS_OVAL) + if child is not None: + child.text = product + else: + child = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}product_name") + child.text = product + self.element.append(child) + + + + def getSchemaVersion(self): + """ + Gets the value of the schema_version element + """ + if self.element is None: + return None + + child = self.element.find("oval:schema_version", OvalDocument.NS_OVAL) + if child is not None: + return child.text + else: + return None + + + def setSchemaVersion(self, version): + """ + Sets a value for the schema_version element. If that element does not exist, one will be created. + """ + if self.element is None: + return False + + if version is None: + return False + + child = self.element.find("oval:schema_version", OvalDocument.NS_OVAL) + if child is not None: + child.text = version + else: + child = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}schema_version") + child.text = version + self.element.append(child) + + + def getTimestamp(self): + """ + Gets the value of the timestamp element + """ + if self.element is None: + return None + + child = self.element.find("oval:timestamp", OvalDocument.NS_OVAL) + if child is not None: + return child.text + else: + return None + + + def setTimestamp(self, timestamp): + """ + Sets a value for the timestamp element. If that elememtn does not exist, one will be created. + If the timestamp argument is set to None, the timestamp will be set to the current time. + """ + if self.element is None: + return False + + if not timestamp or timestamp is None: + now = datetime.date.today() + timestamp = now.strftime(OvalDocument.TIME_FORMAT) + + child = self.element.find("oval:timestamp", OvalDocument.NS_OVAL) + if child is not None: + child.text = timestamp + else: + child = Element("{" + OvalDocument.NS_OVAL.get("oval") + "}timestamp") + child.text = timestamp + self.element.append(child) + + + def getExtra(self, name, namespace=None): + """ + Gets the value of the first child element of the generator where the tag name matches 'name' + If the namespace argument is not provided, it will be assumed that the child element does not have a namespace. + """ + if self.element is None: + return None + + if not name: + return None + + if namespace is not None: + child = self.element.find(name, namespace) + else: + child = self.element.find(name) + + if child is not None: + return child.text + else: + return None + + + def setExtra(self, name, value, namespace=None): + """ + Sets the value if the first child element with a tag name of 'name' to 'value'. If the namespace argument is None, + it will be assumed that the child element does not have a namespace + """ + if not self.element or not name or not value: + return None + + if namespace is not None: + child = self.element.find(name, namespace) + else: + child = self.element.find(name) + + if child is not None: + child.text = value + else: + if namespace is not None: + child = Element(name) + else: + child = Element(name, namespace) + child.text = value + self.element.append(child) + + + + +class OvalElement(object): + """ + The base class for the primary OVAL XML Elements. Contains a few basic operations common to all + OVAL Elements. + TODO: + """ + + DEFINITION = "definition" + TEST = "test" + OBJECT = "object" + STATE = "state" + VARIABLE = "variable" + + + def __init__(self, element): + self.element = element + + + + def getId(self): + """ + Returns the OVAL ID for this element, or None if + 1. This object was instantiated without an Element + 2. The underlying element does not have an "id" attribute + """ + if self.element is None: + return None + + return self.element.get("id") + + + def setId(self, ovalid): + """ + Sets the OVAL ID for this element + Returns False if there is no underlying Xml ELement for this object + """ + + if self.element is None: + return False + + if ovalid is not None: + self.element.set("id", ovalid) + + + def getVersion(self): + if self.element is None: + return None + + return self.element.get("version") + + + def setVersion(self, version): + if self.element is None: + return False + if not version: + return False + if not isinstance(version, int): + return False + + self.element.set("version", version) + return True + + + def incrementVersion(self): + version = self.getVersion() + if not version: + version = 1 + else: + if not isinstance(version, int): + version = 1 + else: + version = version + 1 + + self.setVersion(version) + + + + def getIndexSequence(self): + ovalid = self.getId() + if not ovalid or ovalid is None: + return 1000 + + # Get the numeric index from the end of the OVAL ID + position = ovalid.rfind(':') + if position < 0: + return 1000 + + try: + position = position + 1 + index = ovalid[position:] + + # Apply the modulus function to determine which bucket it belongs to + return int(int(index)/1000 + 1) * 1000 + # Or another way to do it: +# sequence = int(index) +# mod = sequence % 1000 +# return sequence - mod + 1000 + except Exception: + return 1000 + + + + def getFileName(self): + """ + Use my OVAL ID to create a base file name. That really just means replacing ':' with '_' + *NOTE* This does not include the path to the file. + """ + ovalid = self.getId() + if not ovalid or ovalid is None: + return None + + return ovalid.replace(':', '_') + ".xml" + + + + def getPredicate(self): + """ + The portion of the element name that precedes the "_" + So, for "password_test", the predicate would be "password" + """ + localname = self.getLocalName() + if not localname or localname is None: + return None + + return localname.rsplit('_',1)[0] + + + def getElement(self): + """ + Get the raw xml.etree.ElementTree.Element for this node. Can be used to directly manipulate the + XML in ways not currently supported by this library + """ + return self.element + + + + def getName(self): + """ + Get the tag name (XMl element name) of the underlying XML Element, which includes the schema URI + """ + if not self.element or self.element is None: + return None + + return self.element.tag + + + def getLocalName(self): + """ + Just the element name with the schema URI (if any) removed + """ + + if not self.element or self.element is None: + return None + + #Check if this node name is prefixed by a URI, in which case return every after the URI + if '}' in self.element.tag: + return str(self.element.tag).rsplit('}',1)[1] + + #If no namespace prefix, just return the node name + return self.element.tag + + + + def getNamespace(self): + """ + Returns the URI of the namespace or None if this node does not have a namepsace + """ + if not self.element or self.element is None: + return None + + tag = self.element.tag + + if not tag or tag is None: + return None + + # If the oval ID does not contain a namespace, then we can't determine the schema shortname + if not '}' in tag: + return None + + try: + position = tag.find('}') + if position < 0: + return None + + namespace = tag[:position] + return namespace[1:] + except Exception: + return None + + + def getSchemaShortName(self): + """ + """ + if not self.element or self.element is None: + return None + + tag = self.element.tag + + if not tag or tag is None: + return None + + # If the oval ID does not contain a namespace, then we can't determine the schema shortname + if not '}' in tag: + return None + + try: + schema = tag.rsplit('}', 1)[0] + if not '#' in schema: + return None + return schema.rsplit('#', 1)[1].strip() + except Exception: + return None + + + + + def writeToFile(self, path, with_xml_declaration=True): + """Writes this element as a single standalone XML file + @type path: string + @param path: The path to the file to write + @type with_xml_declaration: boolean + @param with_xml_declaration: True to include the XML declaration at the top of the file (the default), or False to exclude it + + @rtype: boolean + @return: True on success, otherwise False + """ + if not self or self is None: + return False; + if not self.element or self.element is None: + return False + if not path or path is None: + return False; + + + try: + namespace = self.getNamespace() + # Register this namespace with the parser as the default namespace + xml.etree.ElementTree.register_namespace('', namespace) + xml.etree.ElementTree.register_namespace('def', "http://oval.mitre.org/XMLSchema/oval-definitions-5") + xml.etree.ElementTree.register_namespace('oval', "http://oval.mitre.org/XMLSchema/oval-common-5") + xml.etree.ElementTree.register_namespace('xsi', "http://www.w3.org/2001/XMLSchema-instance") + + e = self.getElement() + # Fix up the element so it will print nicely + OvalDocument.indent(e) + # Create a new ElementTree with this element as the root + tree = ElementTree(e) + # And finally, write the full tree to a file + tree.write(path, "UTF-8", with_xml_declaration) + + return True + + except Exception: + return False; + + + @staticmethod + def fromStandaloneFile(path): + """For a file that contains a single OVAL XML element as the root element, instantiate the appropriate OvalElement sublcass for that element + @type path: string + @param path: the path to the file + + @rtype OvalElement + @return None on error, or an object of the appropriate OvalElement subclass + """ + + if not path or path is None: + return None + + if not os.path.exists(path): + return None + + try: + tree = ElementTree.ElementTree() + tree.parse(path) + root = tree.getroot() + return OvalElement.asOvalElement(root) + + except Exception: + return None + + + + + + @staticmethod + def getElementTypeFromOvalID(ovalid): + """ Gets element type from OVAL id. """ + + if not ovalid or ovalid is None: + raise ValueError("No OVAL ID given") + + segments = ovalid.split(':') + if len(segments) != 4: + raise ValueError('Invalid OVAL ID: {0}.'.format(ovalid)) + + code = segments[2] + if code == 'def': + return OvalElement.DEFINITION + elif code == 'tst': + return OvalElement.TEST + elif code == 'obj': + return OvalElement.OBJECT + elif code == 'ste': + return OvalElement.STATE + elif code == 'var': + return OvalElement.VARIABLE + else: + raise ValueError("Unknown OVAL object type '{0}' in {1}.".format(code, ovalid)) + + + + + @staticmethod + def asOvalElement(element): + """ + For an XML Element, determines if it fits one of the implemented OvalElement subclasses and, + if so, returns an instantiation of that class + """ + + if not element or element is None: + return None + + try: + ovalid = element.get("id") + if not ovalid or ovalid is None: + return None + + oval_type = OvalElement.getElementTypeFromOvalID(ovalid) + return OvalElement.create(oval_type, element) + except Exception: + return None + + + @staticmethod + def create(oval_type, element): + """ + Create an OvalElement of the proper OVAL element type + """ + if not oval_type: + return None + + + if oval_type == OvalDefinition.DEFINITION: + return OvalDefinition(element) + elif oval_type == OvalDefinition.TEST: + return OvalTest(element) + elif oval_type == OvalDefinition.OBJECT: + return OvalObject(element) + elif oval_type == OvalDefinition.STATE: + return OvalState(element) + elif oval_type == OvalDefinition.VARIABLE: + return OvalVariable(element) + else: + return None + + + + + +class OvalDefinition(OvalElement): + + + def __init__(self, element): + if element is not None: + #self.element = element.getElement() + self.element = element + else: + self.element = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}definition") + self.element.set("version", "1") + meta = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}metadata") + self.element.append(meta) + + + def getType(self): + return OvalElement.DEFINITION + + + + def getMetadata(self): + """ + Returns the metadata for this definition as an object of type OvalMetadata, or None if it that element does not exist + """ + if self.element is None: + return None + + metadata = self.element.find("def:metadata", OvalDocument.NS_DEFAULT) + if metadata is not None: + return OvalMetadata(metadata) + return None + + + + def getClass(self): + if self.element is None: + return None + + return self.element.get("class") + + def setClass(self, ovalclass): + if self.element is None: + return False + + if not ovalclass: + return False + + self.element.set("class", ovalclass) + return True + + + + def getReferencingIDs(self): + if self.element is None: + return None + + return self.xpath("//@*[name()='definition_ref' or name()='test_ref'") + + ''' + Collect a dictionary of the metadata's status changes + ''' + def get_last_status_change(self): + last_status_change = {} + + version = self.getVersion() + last_status_change["Version"] = version + + meta = self.getMetadata() + repo = meta.getOvalRepositoryInformation() + if repo: + status = repo.getStatus() + + last_status_change["Status"] = status + last_status_change["Submitted"] = repo.getSubmitted() + last_status_change["Modified"] = repo.getModified() + last_status_change["StatusChange"] = repo.getStatusChange() + else: + last_status_change["Status"] = None + last_status_change["Submitted"] = None + last_status_change["Modified"] = None + last_status_change["StatusChange"] = None + + return last_status_change + + def set_minimum_schema_version(self, min_schema_version): + meta = self.getMetadata() + repo = meta.getOvalRepositoryInformation() + + if repo: + repo.setMinimumSchemaVersion(min_schema_version) + + +class OvalMetadata(object): + + def __init__(self, element): + if element is not None: + self.element = element + else: + self.element = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}metadata") + + + def getTitle(self): + if self.element is None: + return None + + title_element = self.element.find("def:title", OvalDocument.NS_DEFAULT) + if title_element is not None: + return title_element.text + return None + + + + + def getDescription(self): + if self.element is None: + return None + + desc_element = self.element.find("def:description", OvalDocument.NS_DEFAULT) + if desc_element is not None: + return desc_element.text; + return None + + + def getAffected(self): + if self.element is None: + return None + + aff_element = self.element.find("def:affected", OvalDocument.NS_DEFAULT) + if aff_element is not None: + return OvalAffected(aff_element) + return None + + + def getOvalRepositoryInformation(self): + if self.element is None: + return None + + repo_element = self.element.find("def:oval_repository", OvalDocument.NS_DEFAULT) + if repo_element is not None: + return OvalRepositoryInformation(repo_element) + return None + + + +class OvalAffected(object): + + def __init__(self, element): + self.element = element + + + +class OvalRepositoryInformation(object): + + def __init__(self, element): + self.element = element + + + def getStatus(self): + if self.element is None: + return None + + status = self.element.find("def:status", OvalDocument.NS_DEFAULT) + + if status is None: + return None + + return status.text + + + def setStatus(self, status): + if self.element is None: + return + + if not status or status is None: + return + + element = self.element.find("def:status", OvalDocument.NS_DEFAULT) + + if element is None: + element = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}status") + self.element.append(element) + + element.text = status + + + def getMinimumSchemaVersion(self): + if self.element is None: + return None + + version = self.element.find("def:min_schema", OvalDocument.NS_DEFAULT) + + if version is None: + return None + + return version.text + + + def setMinimumSchemaVersion(self, version): + if self.element is None: + return + + if not version or version is None: + return + + child = self.element.find("def:min_schema_version", OvalDocument.NS_DEFAULT) + + if child is None: + child = Element("{" + OvalDocument.NS_DEFAULT.get("def") + "}min_schema_version") + self.element.append(child) + + child.text = version + + def getContributor(self, type): + results = {} + + if self.element is not None: + findstr = "def:dates/def:%s" % type + subs = self.element.findall(findstr, OvalDocument.NS_DEFAULT) + if subs is not None and len(subs) > 0: + sub = subs[(len(subs) - 1)] + results["Date"] = sub.get("date") + + contributors = [] + contribs = sub.findall("def:contributor", OvalDocument.NS_DEFAULT) + for c in contribs: + curr = {} + curr["Organization"] = c.get("organization") + curr["Contributor"] = c.text + contributors.append(curr) + + results["Contributors"] = contributors + + return results + + def getCreated(self): + return self.getContributor("created") + + def getSubmitted(self): + return self.getContributor("submitted") + + def getModified(self): + return self.getContributor("modified") + + def getStatusChange(self): + status_change = {} + + if self.element is not None: + scs = self.element.findall("def:dates/def:status_change", OvalDocument.NS_DEFAULT) + if scs is not None and len(scs) > 0: + sc = scs[(len(scs) - 1)] + status_change["Date"] = sc.get("date") + status_change["Status"] = sc.text + + return status_change + + #def add_status_change(self, status): + # if self.element is not None: + + + +class OvalTest(OvalElement): + + def __init__(self, element): + #self.element = element.getElement() + self.element = element + + + def getType(self): + return OvalElement.TEST + + +class OvalObject(OvalElement): + + def __init__(self, element): + #self.element = element.getElement() + self.element = element + + + def getType(self): + return OvalElement.OBJECT + + +class OvalState(OvalElement): + + def __init__(self, element): + #self.element = element.getElement() + self.element = element + + + + def getType(self): + return OvalElement.STATE + + +class OvalVariable(OvalElement): + + def __init__(self, element): + #self.element = element.getElement() + self.element = element + + + def getType(self): + return OvalElement.VARIABLE diff --git a/vulnerabilities/scraper/oval_parser.py b/vulnerabilities/scraper/oval_parser.py new file mode 100755 index 000000000..3036b4d95 --- /dev/null +++ b/vulnerabilities/scraper/oval_parser.py @@ -0,0 +1,145 @@ +import xml.etree.ElementTree as ET +from dephell_specifier import RangeSpecifier + +from vulnerabilities.scraper import lib_oval + + +class OvalExtractor: + + def __init__(self, translations, oval_document): + """ + translations : it is a dict + oval_document = it is an Etree parsed xml document + """ + self.translations = translations + self.oval_document = lib_oval.OvalDocument(oval_document) + self.all_definitions = self.oval_document.getDefinitions() + self.all_tests = self.oval_document.getTests() + + def get_data(self): + """ + Returns a list of dictionaries + """ + oval_data = [] + for definition in self.all_definitions: + + definition_data = {'test_data': []} + definition_data['description'] = definition.getMetadata( + ).getDescription() + definition_data['vuln_id'] = self.get_vuln_id_from_definition( + definition) + matching_tests = self.get_tests_of_definition(definition) + if not matching_tests: + continue + for test in matching_tests: + test_obj, test_state = self.get_object_state_of_test(test) + if not test_obj or not test_state: + continue + test_data = {'package_list': []} + test_data['package_list'].extend( + self.get_pkgs_from_obj(test_obj)) + test_data['version_ranges'] = self.get_versionsrngs_from_state( + test_state) + definition_data['test_data'].append(test_data) + oval_data.append(definition_data) + + return oval_data + + def get_tests_of_definition(self, definition): + """ + definition : type(definition) == OvalDefinition + returns a list of all valid tests of the passed definition + """ + pass + + def get_object_state_of_test(self, test): + """ + type(test) == OvalTest + returns a tuple of (OvalObject,OvalState) of an OvalTest + """ + pass + + def get_states_of_test(self, test): + """ + test : type(test) == OvalTest + returns a list of all related states of the passed test + """ + pass + + def get_pkgs_from_obj(self, obj): + """ + obj : type(obj) == OvalObject + returns a list of all related packages + """ + pass + + def get_versionsrngs_from_state(self, state): + """ + state : type(state) == OvalState + returns a list of all related version ranges + """ + pass + + @staticmethod + def get_vuln_id_from_definition(definition): + + for child in definition.element.iter(): + if child.get('ref_id'): + return child.get('ref_id') + + +class UbuntuOvalParser(OvalExtractor): + + def get_tests_of_definition(self, definition): + + criteria_refs = [] + + for child in definition.element.iter(): + + if 'test_ref' in child.attrib: + criteria_refs.append(child.get('test_ref')) + + # FIXME complexity of this can be reduced to O(1) by using a dictionary which maps + # oval_ids to their element. A simple imporvement could be instead of iterating over all + # tests, we could simply use OvalDocument.getElementById method + matching_tests = [] + for test in self.all_tests: + for ref in criteria_refs: + if test.getId() == ref and len(test.element) == 2: + matching_tests.append(test) + + return matching_tests + + def get_object_state_of_test(self, test): + + obj, state = list(test.element)[0].get( + 'object_ref'), list(test.element)[1].get('state_ref') + obj = self.oval_document.getElementByID(obj) + state = self.oval_document.getElementByID(state) + + return (obj, state) + + def get_pkgs_from_obj(self, obj): + + pkg_list = [] + + for var in obj.element: + if var.get('var_ref'): + var_elem = self.oval_document.getElementByID( + var.get('var_ref')) + for vals in var_elem.element: + pkg_list.append(vals.text) + else: + pkg_list.append(var.text) + + return pkg_list + + def get_versionsrngs_from_state(self, state): + + for var in state.element: + if var.get('operation'): + + operand = self.translations[var.get('operation')] + version = var.text + version_range = operand + version + return RangeSpecifier(version_range) From 30c813a09686281a641d03ed710b683ec4b75d2d Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 15 May 2020 19:57:10 +0530 Subject: [PATCH 2/9] Add test and a test oval file for the oval parser Signed-off-by: Shivam Sandbhor --- .../tests/test_data/ubuntu_oval_data.xml | 108 +++++++++++++++ vulnerabilities/tests/test_ubuntu.py | 128 ++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100644 vulnerabilities/tests/test_data/ubuntu_oval_data.xml create mode 100644 vulnerabilities/tests/test_ubuntu.py diff --git a/vulnerabilities/tests/test_data/ubuntu_oval_data.xml b/vulnerabilities/tests/test_data/ubuntu_oval_data.xml new file mode 100644 index 000000000..3a5faa6d9 --- /dev/null +++ b/vulnerabilities/tests/test_data/ubuntu_oval_data.xml @@ -0,0 +1,108 @@ + + + + Canonical CVE OVAL Generator + 1.1 + 5.11.1 + 2020-05-11T04:53:34 + + + + + + CVE-2016-8703 on Ubuntu 18.04 LTS (bionic) - medium. + Heap-based buffer overflow in the bm_readbody_bmp function in bitmap_io.c in potrace before 1.13 allows remote attackers to have unspecified impact via a crafted BMP image, a different vulnerability than CVE-2016-8698, CVE-2016-8699, CVE-2016-8700, CVE-2016-8701, and CVE-2016-8702. + + Ubuntu 18.04 LTS + + + + Medium + Copyright (C) 2017 Canonical Ltd. + 2017-01-31 22:59:00 UTC + Agostino Sarubbo + http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html + https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/ + + + + tyhicks> inkscape in xenial and earlier embeds libpotrace (LP: #1156664) mdeslaur> potrace in inkscape works on bitmaps already loaded, not mdeslaur> arbitrary images. Marking as not-affected for inkscape. + + + + + + + + + CVE-2016-8860 on Ubuntu 18.04 LTS (bionic) - medium. + Tor before 0.2.8.9 and 0.2.9.x before 0.2.9.4-alpha had internal functions that were entitled to expect that buf_t data had NUL termination, but the implementation of or/buffers.c did not ensure that NUL termination was present, which allows remote attackers to cause a denial of service (client, hidden service, relay, or authority crash) via crafted data. + + Ubuntu 18.04 LTS + + + + Medium + Copyright (C) 2017 Canonical Ltd. + 2017-01-04 20:59:00 UTC + http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html + https://trac.torproject.org/projects/tor/ticket/20384 + https://blog.torproject.org/blog/tor-0289-released-important-fixes + https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce + http://www.openwall.com/lists/oss-security/2016/10/18/11 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1.14-2 + + + 0.2.8.9-1ubuntu1 + + + + + + tor + tor-geoipdb + + + libpotrace0 + potrace + + + + \ No newline at end of file diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py new file mode 100644 index 000000000..10f0508fd --- /dev/null +++ b/vulnerabilities/tests/test_ubuntu.py @@ -0,0 +1,128 @@ +import os +import unittest +import xml.etree.ElementTree as ET + +from dephell_specifier import RangeSpecifier + + +from vulnerabilities.scraper.oval_parser import UbuntuOvalParser + + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/") + + +class TestUbuntuOvalParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + xml_doc = ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml")) + translator = {"less than": "<"} + cls.parsed_oval = UbuntuOvalParser(translator, xml_doc) + + def setUp(self): + self.definition_1 = self.parsed_oval.all_definitions[0] + self.definition_2 = self.parsed_oval.all_definitions[1] + + def test_get_definitions(self): + + assert len(self.parsed_oval.all_definitions) == 2 + assert ( + self.parsed_oval.all_definitions[0].getId() + == "oval:com.ubuntu.bionic:def:201687030000000" + ) + assert ( + self.parsed_oval.all_definitions[1].getId() + == "oval:com.ubuntu.bionic:def:201688600000000" + ) + + def test_get_tests_of_definition(self): + + definition_1_test_id = {"oval:com.ubuntu.bionic:tst:201686860000000"} + definition_2_test_id = {"oval:com.ubuntu.bionic:tst:201688600000000"} + + for test in self.parsed_oval.get_tests_of_definition(self.definition_1): + assert test.getId() in definition_1_test_id + + for test in self.parsed_oval.get_tests_of_definition(self.definition_2): + assert test.getId() in definition_2_test_id + + def test_get_vuln_id_from_definition(self): + + vuln_id_1 = "CVE-2016-8703" + vuln_id_2 = "CVE-2016-8860" + + assert vuln_id_1 == self.parsed_oval.get_vuln_id_from_definition( + self.definition_1 + ) + assert vuln_id_2 == self.parsed_oval.get_vuln_id_from_definition( + self.definition_2 + ) + + def test_get_object_state_of_test(self): + + assert len(self.parsed_oval.oval_document.getTests()) == 2 + + test_1 = self.parsed_oval.oval_document.getTests()[0] + test_2 = self.parsed_oval.oval_document.getTests()[1] + + obj_t1, state_t1 = self.parsed_oval.get_object_state_of_test(test_1) + obj_t2, state_t2 = self.parsed_oval.get_object_state_of_test(test_2) + + assert state_t2.getId() == "oval:com.ubuntu.bionic:ste:201688600000000" + assert state_t1.getId() == "oval:com.ubuntu.bionic:ste:201686860000000" + + assert obj_t2.getId() == "oval:com.ubuntu.bionic:obj:2017115650000000" + assert obj_t1.getId() == "oval:com.ubuntu.bionic:obj:201686860000000" + + def test_get_pkgs_from_obj(self): + + assert len(self.parsed_oval.oval_document.getObjects()) == 2 + + obj_t1 = self.parsed_oval.oval_document.getObjects()[0] + obj_t2 = self.parsed_oval.oval_document.getObjects()[1] + + pkg_set1 = set(self.parsed_oval.get_pkgs_from_obj(obj_t2)) + pkg_set2 = set(self.parsed_oval.get_pkgs_from_obj(obj_t1)) + + assert pkg_set1 == {"potrace", "libpotrace0"} + assert pkg_set2 == {"tor", "tor-geoipdb"} + + def test_get_versionsrngs_from_state(self): + + assert len(self.parsed_oval.oval_document.getStates()) == 2 + + state_1 = self.parsed_oval.oval_document.getStates()[0] + state_2 = self.parsed_oval.oval_document.getStates()[1] + + exp_range_1 = RangeSpecifier("<1.14-2") + exp_range_2 = RangeSpecifier("<0.2.8.9-1ubuntu1") + + assert self.parsed_oval.get_versionsrngs_from_state(state_1) == exp_range_1 + assert self.parsed_oval.get_versionsrngs_from_state(state_2) == exp_range_2 + + def test_get_data(self): + + expected_data = [ + { + "test_data": [ + { + "package_list": ["libpotrace0", "potrace"], + "version_ranges": RangeSpecifier("<1.14-2"), + } + ], + "description": "Heap-based buffer overflow in the bm_readbody_bmp function in bitmap_io.c in potrace before 1.13 allows remote attackers to have unspecified impact via a crafted BMP image, a different vulnerability than CVE-2016-8698, CVE-2016-8699, CVE-2016-8700, CVE-2016-8701, and CVE-2016-8702.", + "vuln_id": "CVE-2016-8703", + }, + { + "test_data": [ + { + "package_list": ["tor", "tor-geoipdb"], + "version_ranges": RangeSpecifier("<0.2.8.9-1ubuntu1"), + } + ], + "description": "Tor before 0.2.8.9 and 0.2.9.x before 0.2.9.4-alpha had internal functions that were entitled to expect that buf_t data had NUL termination, but the implementation of or/buffers.c did not ensure that NUL termination was present, which allows remote attackers to cause a denial of service (client, hidden service, relay, or authority crash) via crafted data.", + "vuln_id": "CVE-2016-8860", + }, + ] + + assert expected_data == self.parsed_oval.get_data() From fa531a652fc680a7c9a82688418d593d1db5f627 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 15 May 2020 19:58:32 +0530 Subject: [PATCH 3/9] Modify the style check to ignore the oval parser test and lib_oval.py Signed-off-by: Shivam Sandbhor --- .travis.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f7b3dc116..41fefae48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: - SECRET_KEY="i1bn=oly)w*2yl-5yc&f!vvgt)p)fh3_2$r#spa!*sw36f5ov7" before_script: - - pycodestyle --exclude=migrations,settings.py,lib --max-line-length=100 . + - pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py --max-line-length=100 . - psql -c "CREATE DATABASE vulnerablecode;" -U postgres - ./manage.py migrate diff --git a/README.md b/README.md index cd7184bf9..05874b3dc 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ the code Django includes for this purpose: `SECRET_KEY=$(python -c "from django. ## Tests ``` -pycodestyle --exclude=migrations,settings.py,venv --max-line-length=100 . +pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py --max-line-length=100 . DJANGO_DEV=1 pytest ``` To skip tests which require internet connection: From 1a0ba3d08441c734d545a8cf60969c247d8e9a0a Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sat, 16 May 2020 14:24:46 +0530 Subject: [PATCH 4/9] Added a method to obtain all urls nested within a definition Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/oval_parser.py | 60 +++++++++++++++----------- vulnerabilities/tests/test_ubuntu.py | 19 ++++++++ 2 files changed, 55 insertions(+), 24 deletions(-) diff --git a/vulnerabilities/scraper/oval_parser.py b/vulnerabilities/scraper/oval_parser.py index 3036b4d95..8d85363a8 100755 --- a/vulnerabilities/scraper/oval_parser.py +++ b/vulnerabilities/scraper/oval_parser.py @@ -1,22 +1,26 @@ import xml.etree.ElementTree as ET from dephell_specifier import RangeSpecifier -from vulnerabilities.scraper import lib_oval +from vulnerabilities.scraper.lib_oval import ( + OvalDefinition, OvalDocument, OvalTest, OvalObject, OvalState) +from typing import List +from typing import Dict +from typing import Tuple +from typing import Set class OvalExtractor: - def __init__(self, translations, oval_document): + def __init__(self, translations: Dict, oval_document: ET.ElementTree): """ - translations : it is a dict oval_document = it is an Etree parsed xml document """ self.translations = translations - self.oval_document = lib_oval.OvalDocument(oval_document) + self.oval_document = OvalDocument(oval_document) self.all_definitions = self.oval_document.getDefinitions() self.all_tests = self.oval_document.getTests() - def get_data(self): + def get_data(self) -> List[Dict]: """ Returns a list of dictionaries """ @@ -45,39 +49,32 @@ def get_data(self): return oval_data - def get_tests_of_definition(self, definition): + def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]: """ - definition : type(definition) == OvalDefinition returns a list of all valid tests of the passed definition """ pass - def get_object_state_of_test(self, test): + def get_object_state_of_test(self, test: OvalTest) -> Tuple[OvalObject, OvalState]: """ - type(test) == OvalTest returns a tuple of (OvalObject,OvalState) of an OvalTest """ pass - def get_states_of_test(self, test): + def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]: """ - test : type(test) == OvalTest - returns a list of all related states of the passed test + returns a list of all related packages """ pass - def get_pkgs_from_obj(self, obj): + def get_versionsrngs_from_state(self, state: OvalState) -> RangeSpecifier: """ - obj : type(obj) == OvalObject - returns a list of all related packages + returns a list of all related version ranges """ pass - def get_versionsrngs_from_state(self, state): - """ - state : type(state) == OvalState - returns a list of all related version ranges - """ + @staticmethod + def get_urls_from_definition(definition: OvalDefinition) -> Set[str]: pass @staticmethod @@ -90,7 +87,7 @@ def get_vuln_id_from_definition(definition): class UbuntuOvalParser(OvalExtractor): - def get_tests_of_definition(self, definition): + def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]: criteria_refs = [] @@ -110,7 +107,7 @@ def get_tests_of_definition(self, definition): return matching_tests - def get_object_state_of_test(self, test): + def get_object_state_of_test(self, test: OvalTest) -> Tuple[OvalObject, OvalState]: obj, state = list(test.element)[0].get( 'object_ref'), list(test.element)[1].get('state_ref') @@ -119,7 +116,7 @@ def get_object_state_of_test(self, test): return (obj, state) - def get_pkgs_from_obj(self, obj): + def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]: pkg_list = [] @@ -134,7 +131,7 @@ def get_pkgs_from_obj(self, obj): return pkg_list - def get_versionsrngs_from_state(self, state): + def get_versionsrngs_from_state(self, state: OvalState) -> RangeSpecifier: for var in state.element: if var.get('operation'): @@ -143,3 +140,18 @@ def get_versionsrngs_from_state(self, state): version = var.text version_range = operand + version return RangeSpecifier(version_range) + + @staticmethod + def get_urls_from_definition(definition: OvalDefinition) -> Set[str]: + all_urls = set() + definition_metadata = definition.getMetadata().element + for child in definition_metadata: + if child.tag.endswith('reference'): + all_urls.add(child.get('ref_url')) + if child.tag.endswith('advisory'): + for grandchild in child: + if grandchild.tag.endswith('ref'): + all_urls.add(grandchild.text) + break + + return all_urls diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 10f0508fd..4c1456dd8 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -126,3 +126,22 @@ def test_get_data(self): ] assert expected_data == self.parsed_oval.get_data() + + def test_get_urls_from_definition(self): + + def1_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html', + 'https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/', + 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703' + } + + assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1) + + def2_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html', + 'https://trac.torproject.org/projects/tor/ticket/20384', + 'https://blog.torproject.org/blog/tor-0289-released-important-fixes', + 'https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce', + 'http://www.openwall.com/lists/oss-security/2016/10/18/11', + 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860', + } + + assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2) \ No newline at end of file From 44a8843259c87b695a485a023d255f7e03a37e79 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sat, 16 May 2020 14:51:38 +0530 Subject: [PATCH 5/9] Capture URLs in get_data method and update tests and doc strings Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/oval_parser.py | 29 ++++++++------- vulnerabilities/tests/test_ubuntu.py | 49 +++++++++++++++++--------- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/vulnerabilities/scraper/oval_parser.py b/vulnerabilities/scraper/oval_parser.py index 8d85363a8..1ce1dbbaa 100755 --- a/vulnerabilities/scraper/oval_parser.py +++ b/vulnerabilities/scraper/oval_parser.py @@ -1,20 +1,19 @@ +from typing import List +from typing import Dict +from typing import Tuple +from typing import Set import xml.etree.ElementTree as ET + from dephell_specifier import RangeSpecifier from vulnerabilities.scraper.lib_oval import ( OvalDefinition, OvalDocument, OvalTest, OvalObject, OvalState) -from typing import List -from typing import Dict -from typing import Tuple -from typing import Set class OvalExtractor: def __init__(self, translations: Dict, oval_document: ET.ElementTree): - """ - oval_document = it is an Etree parsed xml document - """ + self.translations = translations self.oval_document = OvalDocument(oval_document) self.all_definitions = self.oval_document.getDefinitions() @@ -22,7 +21,8 @@ def __init__(self, translations: Dict, oval_document: ET.ElementTree): def get_data(self) -> List[Dict]: """ - Returns a list of dictionaries + This is the orchestration method, it returns a list of dictionaries, + where each dictionary represents data from an OvalDefinition """ oval_data = [] for definition in self.all_definitions: @@ -32,6 +32,9 @@ def get_data(self) -> List[Dict]: ).getDescription() definition_data['vuln_id'] = self.get_vuln_id_from_definition( definition) + definition_data['reference_urls'] = self.get_urls_from_definition( + definition + ) matching_tests = self.get_tests_of_definition(definition) if not matching_tests: continue @@ -51,7 +54,7 @@ def get_data(self) -> List[Dict]: def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]: """ - returns a list of all valid tests of the passed definition + returns a list of all valid tests of the passed OvalDefinition """ pass @@ -63,13 +66,15 @@ def get_object_state_of_test(self, test: OvalTest) -> Tuple[OvalObject, OvalStat def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]: """ - returns a list of all related packages + returns a list of all related packages nested within + an OvalObject """ pass def get_versionsrngs_from_state(self, state: OvalState) -> RangeSpecifier: """ - returns a list of all related version ranges + returns a list of all related version ranges within a + state """ pass @@ -97,7 +102,7 @@ def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]: criteria_refs.append(child.get('test_ref')) # FIXME complexity of this can be reduced to O(1) by using a dictionary which maps - # oval_ids to their element. A simple imporvement could be instead of iterating over all + # oval_ids to their element. A simple improvement could be instead of iterating over all # tests, we could simply use OvalDocument.getElementById method matching_tests = [] for test in self.all_tests: diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 4c1456dd8..60ec7630f 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -99,6 +99,25 @@ def test_get_versionsrngs_from_state(self): assert self.parsed_oval.get_versionsrngs_from_state(state_1) == exp_range_1 assert self.parsed_oval.get_versionsrngs_from_state(state_2) == exp_range_2 + + def test_get_urls_from_definition(self): + + def1_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html', + 'https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/', + 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703' + } + + assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1) + + def2_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html', + 'https://trac.torproject.org/projects/tor/ticket/20384', + 'https://blog.torproject.org/blog/tor-0289-released-important-fixes', + 'https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce', + 'http://www.openwall.com/lists/oss-security/2016/10/18/11', + 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860', + } + + assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2) def test_get_data(self): @@ -112,6 +131,12 @@ def test_get_data(self): ], "description": "Heap-based buffer overflow in the bm_readbody_bmp function in bitmap_io.c in potrace before 1.13 allows remote attackers to have unspecified impact via a crafted BMP image, a different vulnerability than CVE-2016-8698, CVE-2016-8699, CVE-2016-8700, CVE-2016-8701, and CVE-2016-8702.", "vuln_id": "CVE-2016-8703", + "reference_urls":{ + 'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html', + 'https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/', + 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703' + } + }, { "test_data": [ @@ -122,21 +147,8 @@ def test_get_data(self): ], "description": "Tor before 0.2.8.9 and 0.2.9.x before 0.2.9.4-alpha had internal functions that were entitled to expect that buf_t data had NUL termination, but the implementation of or/buffers.c did not ensure that NUL termination was present, which allows remote attackers to cause a denial of service (client, hidden service, relay, or authority crash) via crafted data.", "vuln_id": "CVE-2016-8860", - }, - ] - - assert expected_data == self.parsed_oval.get_data() - - def test_get_urls_from_definition(self): - - def1_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html', - 'https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/', - 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703' - } - - assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1) - - def2_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html', + "reference_urls":{ + 'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html', 'https://trac.torproject.org/projects/tor/ticket/20384', 'https://blog.torproject.org/blog/tor-0289-released-important-fixes', 'https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce', @@ -144,4 +156,9 @@ def test_get_urls_from_definition(self): 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860', } - assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2) \ No newline at end of file + }, + ] + + assert expected_data == self.parsed_oval.get_data() + + \ No newline at end of file From f4405dbcfd5709c2168d48a511eb2af323279b10 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sat, 16 May 2020 16:46:51 +0530 Subject: [PATCH 6/9] Add and use PerformantOvalDocument in the Oval parser to increase performance Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/oval_parser.py | 42 ++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/scraper/oval_parser.py b/vulnerabilities/scraper/oval_parser.py index 1ce1dbbaa..f8f42689c 100755 --- a/vulnerabilities/scraper/oval_parser.py +++ b/vulnerabilities/scraper/oval_parser.py @@ -2,12 +2,50 @@ from typing import Dict from typing import Tuple from typing import Set +from typing import Optional import xml.etree.ElementTree as ET from dephell_specifier import RangeSpecifier from vulnerabilities.scraper.lib_oval import ( - OvalDefinition, OvalDocument, OvalTest, OvalObject, OvalState) + OvalDefinition, OvalDocument, OvalTest, OvalObject, OvalState, OvalElement) + + +class PerformantOvalDocument(OvalDocument): + + def __init__(self, tree): + super().__init__(tree) + self.id_to_definition = { + el.getId(): el for el in self.getDefinitions()} + self.id_to_test = {el.getId(): el for el in self.getTests()} + self.id_to_object = {el.getId(): el for el in self.getObjects()} + self.id_to_state = {el.getId(): el for el in self.getStates()} + self.id_to_variable = {el.getId(): el for el in self.getVariables()} + + def getElementByID(self, oval_id: str) -> OvalElement: + if not oval_id: + return None + + root = self.getDocumentRoot() + if not root: + return None + try: + oval_type = OvalElement.getElementTypeFromOvalID(oval_id) + except Exception: + return None + + if oval_type == OvalDefinition.DEFINITION: + return self.id_to_definition[oval_id] + elif oval_type == OvalDefinition.TEST: + return self.id_to_test[oval_id] + elif oval_type == OvalDefinition.OBJECT: + return self.id_to_object[oval_id] + elif oval_type == OvalDefinition.STATE: + return self.id_to_state[oval_id] + elif oval_type == OvalDefinition.VARIABLE: + return self.id_to_variable[oval_id] + else: + return None class OvalExtractor: @@ -15,7 +53,7 @@ class OvalExtractor: def __init__(self, translations: Dict, oval_document: ET.ElementTree): self.translations = translations - self.oval_document = OvalDocument(oval_document) + self.oval_document = PerformantOvalDocument(oval_document) self.all_definitions = self.oval_document.getDefinitions() self.all_tests = self.oval_document.getTests() From 0f5de5587e25a25fbe040fe9f4ad4d340b33a036 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sat, 16 May 2020 17:21:34 +0530 Subject: [PATCH 7/9] Tweak get_tests_of_definition method Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/oval_parser.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/scraper/oval_parser.py b/vulnerabilities/scraper/oval_parser.py index f8f42689c..f7928741c 100755 --- a/vulnerabilities/scraper/oval_parser.py +++ b/vulnerabilities/scraper/oval_parser.py @@ -139,14 +139,10 @@ def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]: if 'test_ref' in child.attrib: criteria_refs.append(child.get('test_ref')) - # FIXME complexity of this can be reduced to O(1) by using a dictionary which maps - # oval_ids to their element. A simple improvement could be instead of iterating over all - # tests, we could simply use OvalDocument.getElementById method matching_tests = [] - for test in self.all_tests: - for ref in criteria_refs: - if test.getId() == ref and len(test.element) == 2: - matching_tests.append(test) + for ref in criteria_refs: + if len(self.oval_document.getElementByID(ref).element) == 2: + matching_tests.append(self.oval_document.getElementByID(ref)) return matching_tests From 88dcb9c9e7dabd252596982a311d5920283ad574 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Mon, 18 May 2020 20:58:11 +0530 Subject: [PATCH 8/9] Add license and terms of usage to lib_oval.py header Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/lib_oval.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vulnerabilities/scraper/lib_oval.py b/vulnerabilities/scraper/lib_oval.py index 0b01363d9..be8b36e12 100644 --- a/vulnerabilities/scraper/lib_oval.py +++ b/vulnerabilities/scraper/lib_oval.py @@ -1,4 +1,14 @@ #!/usr/bin/env/ python3 +# Copyright© 2010 United States Government. All Rights Reserved. + +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +# * Neither the name of the Center for Internet Security, Inc. (CIS) nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER, CIS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER, CIS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """Library to simplify working with the OVAL XML structure From 1490508414020f2e66a1aceaeefa5d5628753daa Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Mon, 18 May 2020 21:05:55 +0530 Subject: [PATCH 9/9] Replace OvalElement with Optional[OvalElement] as type annotatation in oval_parser.py's getElementById method, also improve OvalExtractor's get_data method by checking for mathing_tests earlier Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/oval_parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/scraper/oval_parser.py b/vulnerabilities/scraper/oval_parser.py index f7928741c..4335ba964 100755 --- a/vulnerabilities/scraper/oval_parser.py +++ b/vulnerabilities/scraper/oval_parser.py @@ -22,7 +22,7 @@ def __init__(self, tree): self.id_to_state = {el.getId(): el for el in self.getStates()} self.id_to_variable = {el.getId(): el for el in self.getVariables()} - def getElementByID(self, oval_id: str) -> OvalElement: + def getElementByID(self, oval_id: str) -> Optional[OvalElement]: if not oval_id: return None @@ -65,6 +65,9 @@ def get_data(self) -> List[Dict]: oval_data = [] for definition in self.all_definitions: + matching_tests = self.get_tests_of_definition(definition) + if not matching_tests: + continue definition_data = {'test_data': []} definition_data['description'] = definition.getMetadata( ).getDescription() @@ -73,9 +76,6 @@ def get_data(self) -> List[Dict]: definition_data['reference_urls'] = self.get_urls_from_definition( definition ) - matching_tests = self.get_tests_of_definition(definition) - if not matching_tests: - continue for test in matching_tests: test_obj, test_state = self.get_object_state_of_test(test) if not test_obj or not test_state: