Migrate mozilla importer

CHANGELOG.rst

@@ -2,6 +2,12 @@ Release notes
 =============
 
 
+Next release
+----------------
+
+- We re-enabled support for the mozilla vulnerabilities advisories importer.
+
+
 Version v31.1.1
 ---------------
 

vulnerabilities/importers/__init__.py

@@ -14,6 +14,7 @@
 from vulnerabilities.importers import debian_oval
 from vulnerabilities.importers import github
 from vulnerabilities.importers import gitlab
+from vulnerabilities.importers import mozilla
 from vulnerabilities.importers import nginx
 from vulnerabilities.importers import npm
 from vulnerabilities.importers import nvd
@@ -43,6 +44,7 @@
     npm.NpmImporter,
     retiredotnet.RetireDotnetImporter,
     apache_httpd.ApacheHTTPDImporter,
+    mozilla.MozillaImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/mozilla.py

@@ -7,73 +7,74 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import logging
 import re
+from pathlib import Path
+from typing import Iterable
 from typing import List
-from typing import Set
 
 import yaml
 from bs4 import BeautifulSoup
 from markdown import markdown
 from packageurl import PackageURL
+from univers.versions import SemverVersion
 
 from vulnerabilities import severity_systems
 from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import GitImporter
+from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.utils import is_cve
 from vulnerabilities.utils import split_markdown_front_matter
 
-REPOSITORY = "mozilla/foundation-security-advisories"
 MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$")
+logger = logging.getLogger(__name__)
 
 
-class MozillaImporter(GitImporter):
-    def __enter__(self):
-        super(MozillaImporter, self).__enter__()
+class MozillaImporter(Importer):
+    spdx_license_expression = "MPL-2.0"
+    license_url = "https://github.com/mozilla/foundation-security-advisories/blob/master/LICENSE"
+    repo_url = "git+https://github.com/mozilla/foundation-security-advisories/"
 
-        if not getattr(self, "_added_files", None):
-            self._added_files, self._updated_files = self.file_changes(
-                recursive=True, subdir="announce"
-            )
-
-    def updated_advisories(self) -> Set[AdvisoryData]:
-        files = self._updated_files.union(self._added_files)
-        files = [
-            f for f in files if f.endswith(".md") or f.endswith(".yml")
-        ]  # skip irrelevant files
+    def advisory_data(self) -> Iterable[AdvisoryData]:
+        try:
+            self.clone(self.repo_url)
+            path = Path(self.vcs_response.dest_dir)
 
-        advisories = []
-        for path in files:
-            advisories.extend(to_advisories(path))
+            vuln = path / "announce"
+            paths = list(vuln.glob("**/*.yml")) + list(vuln.glob("**/*.md"))
+            for file_path in paths:
+                yield from to_advisories(file_path)
+        finally:
+            if self.vcs_response:
+                self.vcs_response.delete()
 
-        return self.batch_advisories(advisories)
 
-
-def to_advisories(path: str) -> List[AdvisoryData]:
+def to_advisories(path: Path) -> List[AdvisoryData]:
     """
     Convert a file to corresponding advisories.
     This calls proper method to handle yml/md files.
     """
+    path = str(path)
     mfsa_id = mfsa_id_from_filename(path)
     if not mfsa_id:
         return []
 
     with open(path) as lines:
         if path.endswith(".md"):
-            return get_advisories_from_md(mfsa_id, lines)
+            yield from get_advisories_from_md(mfsa_id, lines)
         if path.endswith(".yml"):
-            return get_advisories_from_yml(mfsa_id, lines)
+            yield from get_advisories_from_yml(mfsa_id, lines)
 
     return []
 
 
 def get_advisories_from_yml(mfsa_id, lines) -> List[AdvisoryData]:
-    advisories = []
     data = yaml.safe_load(lines)
     data["mfsa_id"] = mfsa_id
 
-    fixed_package_urls = get_package_urls(data.get("fixed_in"))
+    affected_packages = get_affected_packages(data.get("fixed_in") or [])
     references = get_yml_references(data)
 
     if not data.get("advisories"):
@@ -82,47 +83,35 @@ def get_advisories_from_yml(mfsa_id, lines) -> List[AdvisoryData]:
     for cve, advisory in data["advisories"].items():
         # These may contain HTML tags
         summary = BeautifulSoup(advisory.get("description", ""), features="lxml").get_text()
-
-        advisories.append(
-            AdvisoryData(
+        if is_cve(cve):
+            yield AdvisoryData(
                 summary=summary,
-                vulnerability_id=cve if is_cve(cve) else "",
-                impacted_package_urls=[],
-                resolved_package_urls=fixed_package_urls,
+                aliases=[cve],
                 references=references,
+                affected_packages=list(affected_packages),
             )
-        )
-
-    return advisories
 
 
 def get_advisories_from_md(mfsa_id, lines) -> List[AdvisoryData]:
     yamltext, mdtext = split_markdown_front_matter(lines.read())
     data = yaml.safe_load(yamltext)
     data["mfsa_id"] = mfsa_id
 
-    fixed_package_urls = get_package_urls(data.get("fixed_in"))
+    affected_packages = get_affected_packages(data.get("fixed_in") or [])
     references = get_yml_references(data)
     cves = re.findall(r"CVE-\d+-\d+", yamltext + mdtext, re.IGNORECASE)
+    description = html_get_p_under_h3(markdown(mdtext), "description")
     for cve in cves:
-        references.append(
-            Reference(
-                reference_id=cve,
-                url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}",
-            )
+        cve_ref = Reference(
+            reference_id=cve,
+            url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}",
         )
-
-    description = html_get_p_under_h3(markdown(mdtext), "description")
-
-    return [
-        AdvisoryData(
+        yield AdvisoryData(
             summary=description,
-            vulnerability_id="",
-            impacted_package_urls=[],
-            resolved_package_urls=fixed_package_urls,
-            references=references,
+            aliases=[cve],
+            affected_packages=list(affected_packages),
+            references=references + [cve_ref],
         )
-    ]
 
 
 def html_get_p_under_h3(html, h3: str):
@@ -146,18 +135,28 @@ def mfsa_id_from_filename(filename):
     return None
 
 
-def get_package_urls(pkgs: List[str]) -> List[PackageURL]:
-    package_urls = [
-        PackageURL(
-            type="mozilla",
+def get_affected_packages(pkgs: List[str]) -> List[PackageURL]:
+    for pkg in pkgs:
+        if not pkg:
+            continue
             # pkg is of the form "Firefox ESR 1.21" or "Thunderbird 2.21"
-            name=pkg.rsplit(None, 1)[0],
-            version=pkg.rsplit(None, 1)[1],
-        )
-        for pkg in pkgs
-        if pkg
-    ]
-    return package_urls
+        name, _, version = pkg.rpartition(" ")
+        if version and name:
+            try:
+                # count no of "." in version
+                # if 3, then it is not a valid semver version
+                if version.count(".") == 3:
+                    continue
+                fixed_version = SemverVersion(version)
+                yield AffectedPackage(
+                    package=PackageURL(
+                        type="mozilla",
+                        name=name,
+                    ),
+                    fixed_version=fixed_version,
+                )
+            except Exception:
+                logger.exception(f"Error parsing version {version} for {name}")
 
 
 def get_yml_references(data: any) -> List[Reference]:

vulnerabilities/tests/conftest.py

@@ -32,7 +32,6 @@ def no_rmtree(monkeypatch):
     "test_gentoo.py",
     "test_istio.py",
     "test_models.py",
-    "test_mozilla.py",
     "test_msr2019.py",
     "test_package_managers.py",
     "test_ruby.py",

vulnerabilities/tests/test_data/mozilla/expected-md.json

@@ -0,0 +1,41 @@
+[
+  {
+    "aliases": [
+      "CVE-2006-0294"
+    ],
+    "summary": "Dynamically changing the style of an element from position:relative\nto position:static can cause Gecko to operate on freed memory.\nIt may be possible to exploit this in order to run arbitrary\ncode.This flaw was introduced during Firefox 1.5 and SeaMonkey 1.0\ndevelopment and does not affect Firefox 1.0 or the Mozilla Suite 1.7Thunderbird 1.5 could be vulnerable if JavaScript is\nenabled in mail. This is not the default setting and we strongly\ndiscourage users from turning on JavaScript in mail. Thunderbird\nis not vulnerable in its default configuration.Update (13 April 2006)\nThis flaw has been fixed in Thunderbird 1.5.0.2",
+    "affected_packages": [
+      {
+        "package": {
+          "type": "mozilla",
+          "namespace": null,
+          "name": "SeaMonkey",
+          "version": null,
+          "qualifiers": null,
+          "subpath": null
+        },
+        "affected_version_range": null,
+        "fixed_version": "1.0.0"
+      }
+    ],
+    "references": [
+      {
+        "reference_id": "mfsa2006-02",
+        "url": "https://www.mozilla.org/en-US/security/advisories/mfsa2006-02",
+        "severities": [
+          {
+            "system": "generic_textual",
+            "value": "none",
+            "scoring_elements": ""
+          }
+        ]
+      },
+      {
+        "reference_id": "CVE-2006-0294",
+        "url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2006-0294",
+        "severities": []
+      }
+    ],
+    "date_published": null
+  }
+]