Merge pull request #393 from Hritik14/collect_mozilla

Collect Mozilla

Author: Hritik14

SOURCES.rst

@@ -47,3 +47,5 @@
 +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
 |suse_scores     | https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml                                |vulnerability severity scores by SUSE               |
 +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
+|mozilla         | https://github.com/mozilla/foundation-security-advisories                                            |mozilla                                             |
++----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+

pytest.ini

@@ -30,5 +30,6 @@ addopts =
 	--ignore=vulnerabilities/importers/suse_backports.py
 	--ignore=vulnerabilities/importers/suse_scores.py
 	--ignore=vulnerabilities/importers/ubuntu_usn.py
+	--ignore=vulnerabilities/importers/mozilla.py
 	--ignore=vulnerabilities/management/commands/create_cpe_to_purl_map.py
 	--ignore=vulnerabilities/lib_oval.py

requirements.txt

@@ -17,3 +17,5 @@ lxml>=4.6.4
 gunicorn>=20.1.0
 django-environ==0.4.5
 defusedxml==0.7.1
+
+Markdown==3.3.4

vulnerabilities/helpers.py

@@ -72,6 +72,41 @@ def fetch_yaml(url):
 create_etag = MagicMock()
 
 
+def split_markdown_front_matter(lines: str) -> Tuple[str, str]:
+    """
+    This function splits lines into markdown front matter and the markdown body
+    and returns list of lines for both
+
+    for example :
+        lines =
+        ---
+        title: ISTIO-SECURITY-2019-001
+        description: Incorrect access control.
+        cves: [CVE-2019-12243]
+        ---
+        # Markdown starts here
+
+    split_markdown_front_matter(lines) would return
+    ['title: ISTIO-SECURITY-2019-001','description: Incorrect access control.'
+    ,'cves: [CVE-2019-12243]'],
+    ["# Markdown starts here"]
+    """
+
+    fmlines = []
+    mdlines = []
+    splitter = mdlines
+
+    for index, line in enumerate(lines.split("\n")):
+        if index == 0 and line.strip().startswith("---"):
+            splitter = fmlines
+        elif line.strip().startswith("---"):
+            splitter = mdlines
+        else:
+            splitter.append(line)
+
+    return "\n".join(fmlines), "\n".join(mdlines)
+
+
 def contains_alpha(string):
     """
     Return True if the input 'string' contains any alphabet

vulnerabilities/importers/mozilla.py

@@ -0,0 +1,180 @@
+import re
+from typing import List
+from typing import Set
+
+import yaml
+from bs4 import BeautifulSoup
+from markdown import markdown
+from packageurl import PackageURL
+
+from vulnerabilities.importer import Advisory
+from vulnerabilities.importer import GitImporter
+from vulnerabilities.importer import Reference
+from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.helpers import is_cve
+from vulnerabilities.helpers import split_markdown_front_matter
+from vulnerabilities.severity_systems import SCORING_SYSTEMS
+
+REPOSITORY = "mozilla/foundation-security-advisories"
+MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$")
+
+
+class MozillaImporter(GitImporter):
+    def __enter__(self):
+        super(MozillaImporter, self).__enter__()
+
+        if not getattr(self, "_added_files", None):
+            self._added_files, self._updated_files = self.file_changes(
+                recursive=True, subdir="announce"
+            )
+
+    def updated_advisories(self) -> Set[Advisory]:
+        files = self._updated_files.union(self._added_files)
+        files = [
+            f for f in files if f.endswith(".md") or f.endswith(".yml")
+        ]  # skip irrelevant files
+
+        advisories = []
+        for path in files:
+            advisories.extend(to_advisories(path))
+
+        return self.batch_advisories(advisories)
+
+
+def to_advisories(path: str) -> List[Advisory]:
+    """
+    Convert a file to corresponding advisories.
+    This calls proper method to handle yml/md files.
+    """
+    mfsa_id = mfsa_id_from_filename(path)
+    if not mfsa_id:
+        return []
+
+    with open(path) as lines:
+        if path.endswith(".md"):
+            return get_advisories_from_md(mfsa_id, lines)
+        if path.endswith(".yml"):
+            return get_advisories_from_yml(mfsa_id, lines)
+
+    return []
+
+
+def get_advisories_from_yml(mfsa_id, lines) -> List[Advisory]:
+    advisories = []
+    data = yaml.safe_load(lines)
+    data["mfsa_id"] = mfsa_id
+
+    fixed_package_urls = get_package_urls(data.get("fixed_in"))
+    references = get_yml_references(data)
+
+    if not data.get("advisories"):
+        return []
+
+    for cve, advisory in data["advisories"].items():
+        # These may contain HTML tags
+        summary = BeautifulSoup(advisory.get("description", ""), features="lxml").get_text()
+
+        advisories.append(
+            Advisory(
+                summary=summary,
+                vulnerability_id=cve if is_cve(cve) else "",
+                impacted_package_urls=[],
+                resolved_package_urls=fixed_package_urls,
+                references=references,
+            )
+        )
+
+    return advisories
+
+
+def get_advisories_from_md(mfsa_id, lines) -> List[Advisory]:
+    yamltext, mdtext = split_markdown_front_matter(lines.read())
+    data = yaml.safe_load(yamltext)
+    data["mfsa_id"] = mfsa_id
+
+    fixed_package_urls = get_package_urls(data.get("fixed_in"))
+    references = get_yml_references(data)
+    cves = re.findall(r"CVE-\d+-\d+", yamltext + mdtext, re.IGNORECASE)
+    for cve in cves:
+        references.append(
+            Reference(
+                reference_id=cve,
+                url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}",
+            )
+        )
+
+    description = html_get_p_under_h3(markdown(mdtext), "description")
+
+    return [
+        Advisory(
+            summary=description,
+            vulnerability_id="",
+            impacted_package_urls=[],
+            resolved_package_urls=fixed_package_urls,
+            references=references,
+        )
+    ]
+
+
+def html_get_p_under_h3(html, h3: str):
+    soup = BeautifulSoup(html, features="lxml")
+    h3tag = soup.find("h3", text=lambda txt: txt.lower() == h3)
+    p = ""
+    if h3tag:
+        for tag in h3tag.next_siblings:
+            if tag.name:
+                if tag.name != "p":
+                    break
+                p += tag.get_text()
+    return p
+
+
+def mfsa_id_from_filename(filename):
+    match = MFSA_FILENAME_RE.search(filename)
+    if match:
+        return "mfsa" + match.group(1)
+
+    return None
+
+
+def get_package_urls(pkgs: List[str]) -> List[PackageURL]:
+    package_urls = [
+        PackageURL(
+            type="mozilla",
+            # pkg is of the form "Firefox ESR 1.21" or "Thunderbird 2.21"
+            name=pkg.rsplit(None, 1)[0],
+            version=pkg.rsplit(None, 1)[1],
+        )
+        for pkg in pkgs
+        if pkg
+    ]
+    return package_urls
+
+
+def get_yml_references(data: any) -> List[Reference]:
+    """
+    Returns a list of references
+    Currently only considers the given mfsa as a reference
+    """
+    # FIXME: Needs improvement
+    # Should we add 'bugs' section in references too?
+    # Should we add 'impact'/severity of CVE in references too?
+    # If yes, then fix alpine_linux importer as well
+    # Otherwise, do we need severity field for adversary as well?
+
+    severities = ["critical", "high", "medium", "low", "none"]
+    severity = "none"
+    if data.get("impact"):
+        impact = data.get("impact").lower()
+        for s in severities:
+            if s in impact:
+                severity = s
+                break
+
+    return [
+        Reference(
+            reference_id=data["mfsa_id"],
+            url="https://www.mozilla.org/en-US/security/advisories/{}".format(data["mfsa_id"]),
+            severities=[VulnerabilitySeverity(scoring_systems["generic_textual"], severity)],
+        )
+    ]

vulnerabilities/tests/conftest.py

@@ -74,4 +74,5 @@ def no_rmtree(monkeypatch):
     "test_importer_yielder.py",
     "test_upstream.py",
     "test_istio.py",
+    "test_mozilla.py",
 ]

vulnerabilities/tests/test_data/mozilla.zip

@@ -0,0 +1,97 @@
+import os
+import shutil
+import tempfile
+import zipfile
+from unittest.mock import patch
+
+from django.test import TestCase
+
+from vulnerabilities import models
+from vulnerabilities.import_runner import ImportRunner
+from vulnerabilities.importers.npm import categorize_versions
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_DATA = os.path.join(BASE_DIR, "test_data/")
+
+
+@patch("vulnerabilities.importers.MozillaImporter._update_from_remote")
+class MozillaImportTest(TestCase):
+
+    tempdir = None
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.tempdir = tempfile.mkdtemp()
+        zip_path = os.path.join(TEST_DATA, "mozilla.zip")
+
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(cls.tempdir)
+
+        cls.importer = models.Importer.objects.create(
+            name="mozilla_unittests",
+            license="",
+            last_run=None,
+            data_source="MozillaImporter",
+            data_source_cfg={
+                "repository_url": "https://example.git",
+                "working_directory": os.path.join(cls.tempdir, "mozilla_test"),
+                "create_working_directory": False,
+                "remove_working_directory": False,
+            },
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        # Make sure no requests for unexpected package names have been made during the tests.
+        shutil.rmtree(cls.tempdir)
+
+    def test_import(self, _):
+        runner = ImportRunner(self.importer, 100)
+
+        # Remove if we don't need set_api in MozillaImporter
+        # with patch("vulnerabilities.importers.MozillaImporter.versions", new=MOCK_VERSION_API):
+        #     with patch("vulnerabilities.importers.MozillaImporter.set_api"):
+        #         runner.run()
+        runner.run()
+
+        assert models.Vulnerability.objects.count() == 9
+        assert models.VulnerabilityReference.objects.count() == 10
+        assert models.VulnerabilitySeverity.objects.count() == 9
+        assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=False).count() == 16
+
+        assert models.Package.objects.count() == 12
+
+        self.assert_for_package("Firefox ESR", "mfsa2021-06", "78.7.1")
+        self.assert_for_package("Firefox ESR", "mfsa2021-04", "78.7", "CVE-2021-23953")
+        self.assert_for_package("Firefox for Android", "mfsa2021-01", "84.1.3", "CVE-2020-16044")
+        self.assert_for_package("Thunderbird", "mfsa2014-30", "24.4")
+        self.assert_for_package("Thunderbird", "mfsa2014-30", "24.4")
+        self.assert_for_package("Mozilla Suite", "mfsa2005-29", "1.7.6")
+
+    def assert_for_package(
+        self,
+        package_name,
+        mfsa_id,
+        resolved_version,
+        vulnerability_id=None,
+        impacted_version=None,
+    ):
+        vuln = None
+
+        pkg = models.Package.objects.get(name=package_name, version=resolved_version)
+        vuln = pkg.vulnerabilities.first()
+
+        if vulnerability_id:
+            assert vuln.vulnerability_id == vulnerability_id
+
+        ref_url = f"https://www.mozilla.org/en-US/security/advisories/{mfsa_id}"
+        assert models.VulnerabilityReference.objects.get(url=ref_url, vulnerability=vuln)
+
+        assert models.PackageRelatedVulnerability.objects.filter(
+            package=pkg, vulnerability=vuln, is_vulnerable=False
+        )
+
+
+def test_categorize_versions_ranges():
+    # Populate if impacted version is filled
+    pass