Add CWE support for gitlab and redhat

Signed-off-by: ziadhany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/gitlab.py

@@ -9,14 +9,17 @@
 
 import logging
 import traceback
+from datetime import datetime
 from pathlib import Path
 from typing import Iterable
 from typing import List
+from typing import Mapping
 from typing import Optional
 
 import pytz
 import saneyaml
 from dateutil import parser as dateparser
+from django.db.models.query import QuerySet
 from packageurl import PackageURL
 from univers.version_range import RANGE_CLASS_BY_SCHEMES
 from univers.version_range import VersionRange
@@ -27,11 +30,23 @@
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
+from vulnerabilities.importer import UnMergeablePackageError
+from vulnerabilities.improver import Improver
+from vulnerabilities.improver import Inference
+from vulnerabilities.models import Advisory
+from vulnerabilities.package_managers import VERSION_API_CLASSES_BY_PACKAGE_TYPE
+from vulnerabilities.package_managers import GoproxyVersionAPI
+from vulnerabilities.package_managers import VersionAPI
+from vulnerabilities.package_managers import get_api_package_name
+from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
 from vulnerabilities.utils import build_description
+from vulnerabilities.utils import get_affected_packages_by_patched_package
+from vulnerabilities.utils import get_cwe_id
+from vulnerabilities.utils import nearest_patched_package
+from vulnerabilities.utils import resolve_version_range
 
 logger = logging.getLogger(__name__)
 
-
 PURL_TYPE_BY_GITLAB_SCHEME = {
     # "conan": "conan",
     "gem": "gem",
@@ -44,7 +59,6 @@
     "pypi": "pypi",
 }
 
-
 GITLAB_SCHEME_BY_PURL_TYPE = {v: k for k, v in PURL_TYPE_BY_GITLAB_SCHEME.items()}
 
 
@@ -188,6 +202,10 @@ def parse_gitlab_advisory(file):
     summary = build_description(gitlab_advisory.get("title"), gitlab_advisory.get("description"))
     urls = gitlab_advisory.get("urls")
     references = [Reference.from_url(u) for u in urls]
+
+    cwe_ids = gitlab_advisory.get("cwe_ids") or []
+    cwe_list = list(map(get_cwe_id, cwe_ids))
+
     date_published = dateparser.parse(gitlab_advisory.get("pubdate"))
     date_published = date_published.replace(tzinfo=pytz.UTC)
     package_slug = gitlab_advisory.get("package_slug")
@@ -253,4 +271,102 @@ def parse_gitlab_advisory(file):
         references=references,
         date_published=date_published,
         affected_packages=affected_packages,
+        weaknesses=cwe_list,
     )
+
+
+class GitLabBasicImprover(Improver):
+    """
+    Get the nearest fixed_version and then resolve the version range with the help of all valid versions.
+    Generate inference between all the affected packages and the fixed_version that fixes all those affected packages.
+
+    In case of gitlab advisory data we get a list of fixed_versions and a affected_version_range.
+    Since we can not determine which package fixes which range.
+    """
+
+    def __init__(self) -> None:
+        self.versions_fetcher_by_purl: Mapping[str, VersionAPI] = {}
+
+    @property
+    def interesting_advisories(self) -> QuerySet:
+        return Advisory.objects.filter(created_by=GitLabAPIImporter.qualified_name)
+
+    def get_package_versions(
+        self, package_url: PackageURL, until: Optional[datetime] = None
+    ) -> List[str]:
+        """
+        Return a list of `valid_versions` for the `package_url`
+        """
+        api_name = get_api_package_name(purl=package_url)
+        if not api_name:
+            logger.error(f"Could not get versions for {package_url!r}")
+            return []
+        versions_fetcher = self.versions_fetcher_by_purl.get(package_url)
+        if not versions_fetcher:
+            versions_fetcher: VersionAPI = VERSION_API_CLASSES_BY_PACKAGE_TYPE[package_url.type]
+            self.versions_fetcher_by_purl[package_url] = versions_fetcher()
+
+        versions_fetcher = self.versions_fetcher_by_purl[package_url]
+
+        self.versions_fetcher_by_purl[package_url] = versions_fetcher
+        return versions_fetcher.get_until(package_name=api_name, until=until).valid_versions
+
+    def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
+        """
+        Yield Inferences for the given advisory data
+        """
+        if not advisory_data.affected_packages:
+            return iter([])
+        try:
+            purl, affected_version_ranges, _ = AffectedPackage.merge(
+                advisory_data.affected_packages
+            )
+        except UnMergeablePackageError:
+            logger.error(f"Cannot merge with different purls {advisory_data.affected_packages!r}")
+            return iter([])
+
+        pkg_type = purl.type
+        pkg_namespace = purl.namespace
+        pkg_name = purl.name
+        if purl.type == "golang":
+            # Problem with the Golang and Go that they provide full path
+            # FIXME: We need to get the PURL subpath for Go module
+            versions_fetcher = self.versions_fetcher_by_purl.get(purl)
+            if not versions_fetcher:
+                versions_fetcher = GoproxyVersionAPI()
+                self.versions_fetcher_by_purl[purl] = versions_fetcher
+            pkg_name = versions_fetcher.module_name_by_package_name.get(pkg_name, pkg_name)
+
+        valid_versions = self.get_package_versions(
+            package_url=purl, until=advisory_data.date_published
+        )
+        for affected_version_range in affected_version_ranges:
+            aff_vers, unaff_vers = resolve_version_range(
+                affected_version_range=affected_version_range,
+                package_versions=valid_versions,
+                ignorable_versions=[],
+            )
+            affected_purls = [
+                PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version)
+                for version in aff_vers
+            ]
+
+            unaffected_purls = [
+                PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version)
+                for version in unaff_vers
+            ]
+
+            affected_packages: List[LegacyAffectedPackage] = nearest_patched_package(
+                vulnerable_packages=affected_purls, resolved_packages=unaffected_purls
+            )
+
+            for (
+                fixed_package,
+                affected_packages,
+            ) in get_affected_packages_by_patched_package(affected_packages).items():
+                yield Inference.from_advisory_data(
+                    advisory_data,  # We are getting all valid versions to get this inference
+                    confidence=100,
+                    affected_purls=affected_packages,
+                    fixed_purl=fixed_package,
+                )

vulnerabilities/importers/redhat.py

@@ -8,6 +8,7 @@
 #
 
 import logging
+import re
 from typing import Dict
 from typing import Iterable
 from typing import List
@@ -23,6 +24,7 @@
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.rpm_utils import rpm_to_purl
+from vulnerabilities.utils import get_cwe_id
 from vulnerabilities.utils import get_item
 from vulnerabilities.utils import requests_with_5xx_retry
 
@@ -61,7 +63,6 @@ def get_data_from_url(url):
 
 
 class RedhatImporter(Importer):
-
     spdx_license_expression = "CC-BY-4.0"
     license_url = "https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0/html/red_hat_security_data_api/legal-notice"
 
@@ -164,6 +165,11 @@ def to_advisory(advisory_data):
                 scoring_elements=cvssv3_vector,
             )
         )
+    cwe_list = []
+    # cwe_string : CWE-409","CWE-121->CWE-787","(CWE-401|CWE-404)","(CWE-190|CWE-911)->CWE-416"
+    cwe_string = advisory_data.get("CWE")
+    if cwe_string:
+        cwe_list = list(map(get_cwe_id, re.findall("CWE-[0-9]+", cwe_string)))
 
     aliases = []
     alias = advisory_data.get("CVE")
@@ -177,4 +183,5 @@ def to_advisory(advisory_data):
         summary=advisory_data.get("bugzilla_description") or "",
         affected_packages=affected_packages,
         references=references,
+        weaknesses=cwe_list,
     )

vulnerabilities/improvers/default.py

@@ -54,6 +54,7 @@ def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
                         affected_purls=affected_purls,
                         fixed_purl=None,
                         references=advisory_data.references,
+                        weaknesses=advisory_data.weaknesses,
                     )
                 else:
                     for fixed_purl in fixed_purls or []:
@@ -64,6 +65,7 @@ def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
                             affected_purls=affected_purls,
                             fixed_purl=fixed_purl,
                             references=advisory_data.references,
+                            weaknesses=advisory_data.weaknesses,
                         )
 
         else:

vulnerabilities/tests/test_data/gitlab/composer-expected.json

@@ -25,5 +25,5 @@
     }
   ],
   "date_published": "2018-03-15T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035,937]
 }

vulnerabilities/tests/test_data/gitlab/composer-improver-expected.json

@@ -22,6 +22,6 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937]
   }
 ]

vulnerabilities/tests/test_data/gitlab/golang-expected.json

@@ -31,5 +31,5 @@
     }
   ],
   "date_published": "2021-05-20T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035,937]
 }

vulnerabilities/tests/test_data/gitlab/golang-improver-expected.json

@@ -37,7 +37,7 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937]
   },
   {
     "vulnerability_id": null,
@@ -68,6 +68,6 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937]
   }
 ]

vulnerabilities/tests/test_data/gitlab/maven-expected.json

@@ -46,5 +46,5 @@
     }
   ],
   "date_published": "2021-11-15T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035,937,94]
 }

vulnerabilities/tests/test_data/gitlab/maven-improver-expected.json

@@ -100,7 +100,7 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937,94]
   },
   {
     "vulnerability_id": null,
@@ -146,6 +146,6 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937,94]
   }
 ]

vulnerabilities/tests/test_data/gitlab/npm-expected.json

@@ -36,5 +36,5 @@
     }
   ],
   "date_published": "2020-06-05T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035,937]
 }