Refactor github api importer

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

Author: sbs2001

vulnerabilities/importers/github.py

@@ -38,6 +38,7 @@
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
 from vulnerabilities.data_source import DataSourceConfiguration
+from vulnerabilities.data_source import VulnerabilityReferenceUnit
 
 
 # set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET'}
@@ -109,7 +110,8 @@ def fetch(self) -> Mapping[str, List[Mapping]]:
             while True:
 
                 query_json = {"query": query % (ecosystem, end_cursor_exp)}
-                resp = requests.post(self.config.endpoint, headers=headers, json=query_json).json()
+                resp = requests.post(self.config.endpoint,
+                                     headers=headers, json=query_json).json()
                 print(resp)
                 if resp.get("message") == "Bad credentials":
                     raise GitHubTokenError("Invalid GitHub token")
@@ -168,40 +170,50 @@ def process_response(self) -> List[Advisory]:
                     )
 
                     affected_purls = {
-                        PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
+                        PackageURL(name=pkg_name, namespace=ns,
+                                   version=version, type=pkg_type)
                         for version in aff_vers
                     }
 
                     unaffected_purls = {
-                        PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
+                        PackageURL(name=pkg_name, namespace=ns,
+                                   version=version, type=pkg_type)
                         for version in unaff_vers
                     }
 
                     cve_ids = set()
-                    ref_ids = set()
+                    vuln_references = []
                     vuln_desc = adv["node"]["advisory"]["summary"]
 
                     for vuln in adv["node"]["advisory"]["identifiers"]:
                         if vuln["type"] == "CVE":
                             cve_ids.add(vuln["value"])
-                        else:
-                            ref_ids.add(vuln["value"])
+
+                        elif vuln["type"] == "GHSA":
+                            ghsa = vuln['value']
+                            vuln_references.append(VulnerabilityReferenceUnit(
+                                reference_id=ghsa,
+                                url="https://github.com/advisories/{}".format(
+                                    ghsa)
+                            ))
+
                     for cve_id in cve_ids:
                         adv_list.append(
                             Advisory(
                                 cve_id=cve_id,
                                 summary=vuln_desc,
                                 impacted_package_urls=affected_purls,
                                 resolved_package_urls=unaffected_purls,
-                                reference_ids=ref_ids,
+                                vuln_references=vuln_references,
                             )
                         )
         return adv_list
 
     @staticmethod
     def categorize_versions(version_range: str, all_versions: Set[str]) -> Tuple[Set[str], Set[str]]:  # nopep8
         version_range = RangeSpecifier(version_range)
-        affected_versions = {version for version in all_versions if version in version_range}
+        affected_versions = {
+            version for version in all_versions if version in version_range}
         return (affected_versions, all_versions - affected_versions)
 
 

vulnerabilities/tests/test_github.py

@@ -32,13 +32,14 @@
 from requests.models import Response
 from packageurl import PackageURL
 
+from vulnerabilities.data_source import Advisory
+from vulnerabilities.data_source import VulnerabilityReferenceUnit
 from vulnerabilities.importers.github import GitHubAPIDataSource
 from vulnerabilities.importers.github import MavenVersionAPI
 from vulnerabilities.importers.github import ComposerVersionAPI
 from vulnerabilities.importers.github import NugetVersionAPI
 from vulnerabilities.importers.github import GitHubTokenError
 from vulnerabilities.importers.github import query
-from vulnerabilities.data_source import Advisory
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 TEST_DATA = os.path.join(BASE_DIR, 'test_data')
@@ -186,8 +187,9 @@ def test_process_response(self):
                         subpath=None,
                     ),
                 },
-                reference_urls=[],
-                reference_ids={'GHSA-qcxh-w3j9-58qr'},
+                vuln_references=[VulnerabilityReferenceUnit(
+                    url='https://github.com/advisories/GHSA-qcxh-w3j9-58qr',
+                    reference_id='GHSA-qcxh-w3j9-58qr')],
                 cve_id='CVE-2019-0199',
             ),
             Advisory(
@@ -212,8 +214,9 @@ def test_process_response(self):
                         subpath=None,
                     )
                 },
-                reference_urls=[],
-                reference_ids={'GHSA-qcxh-w3j9-58qr'},
+                vuln_references=[VulnerabilityReferenceUnit(
+                    url='https://github.com/advisories/GHSA-qcxh-w3j9-58qr',
+                    reference_id='GHSA-qcxh-w3j9-58qr')],
                 cve_id='CVE-2019-0199',
             ),
             Advisory(
@@ -237,8 +240,9 @@ def test_process_response(self):
                         subpath=None,
                     ),
                 },
-                reference_urls=[],
-                reference_ids={'GHSA-c9hw-wf7x-jp9j'},
+                vuln_references=[VulnerabilityReferenceUnit(
+                    url='https://github.com/advisories/GHSA-c9hw-wf7x-jp9j',
+                    reference_id='GHSA-c9hw-wf7x-jp9j')],
                 cve_id='CVE-2020-1938',
             ),
             Advisory(
@@ -262,8 +266,9 @@ def test_process_response(self):
                         subpath=None,
                     ),
                 },
-                reference_urls=[],
-                reference_ids={'GHSA-c9hw-wf7x-jp9j'},
+                vuln_references=[VulnerabilityReferenceUnit(
+                    url='https://github.com/advisories/GHSA-c9hw-wf7x-jp9j',
+                    reference_id='GHSA-c9hw-wf7x-jp9j')],
                 cve_id='CVE-2020-1938',
             ),
             Advisory(
@@ -288,8 +293,9 @@ def test_process_response(self):
                         subpath=None,
                     )
                 },
-                reference_urls=[],
-                reference_ids={'GHSA-c9hw-wf7x-jp9j'},
+                vuln_references=[VulnerabilityReferenceUnit(
+                    url='https://github.com/advisories/GHSA-c9hw-wf7x-jp9j',
+                    reference_id='GHSA-c9hw-wf7x-jp9j')],
                 cve_id='CVE-2020-1938',
             ),
         ]
@@ -433,7 +439,8 @@ def test_artifact_url(self):
 
     def test_extract_versions(self):
         expected_versions = {'1.2.2', '1.2.3', '1.3.0'}
-        assert expected_versions == self.version_api.extract_versions(self.response)
+        assert expected_versions == self.version_api.extract_versions(
+            self.response)
 
     def test_load_to_api(self):