Bulk rename DataSource -> Importer

The name DataSource and Importer were majorly used interchangeably
although they represent the same concept.
Also, the importers now require a mandatory ``spdx_license_expression``
to run

Signed-off-by: Hritik Vijay <hritikxx8@gmail.com>

Author: Hritik14

vulnerabilities/fixtures/github.json

@@ -96,7 +96,7 @@
             "name": "github",
             "license": "",
             "last_run": "2021-03-06T09:09:01.523Z",
-            "data_source": "GitHubAPIDataSource",
+            "data_source": "GitHubAPIImporter",
             "data_source_cfg": {
                 "endpoint": "https://api.github.com/graphql",
                 "ecosystems": [
@@ -115,4 +115,4 @@
             "reference": 136
         }
     }
-]
+]

vulnerabilities/fixtures/openssl.json

@@ -40538,7 +40538,7 @@
             "name": "rust",
             "license": "cc0-1.0",
             "last_run": "2020-09-14T12:47:17.451Z",
-            "data_source": "RustDataSource",
+            "data_source": "RustImporter",
             "data_source_cfg": {
                 "branch": null,
                 "repository_url": "https://github.com/RustSec/advisory-db",
@@ -40555,7 +40555,7 @@
             "name": "alpine",
             "license": "",
             "last_run": null,
-            "data_source": "AlpineDataSource",
+            "data_source": "AlpineImporter",
             "data_source_cfg": {
                 "branch": null,
                 "repository_url": "https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb"
@@ -40569,7 +40569,7 @@
             "name": "archlinux",
             "license": "mit",
             "last_run": null,
-            "data_source": "ArchlinuxDataSource",
+            "data_source": "ArchlinuxImporter",
             "data_source_cfg": {
                 "archlinux_tracker_url": "https://security.archlinux.org/json"
             }
@@ -40582,7 +40582,7 @@
             "name": "debian",
             "license": "mit",
             "last_run": null,
-            "data_source": "DebianDataSource",
+            "data_source": "DebianImporter",
             "data_source_cfg": {
                 "debian_tracker_url": "https://security-tracker.debian.org/tracker/data/json"
             }
@@ -40595,7 +40595,7 @@
             "name": "safetydb",
             "license": "cc-by-nc-4.0",
             "last_run": null,
-            "data_source": "SafetyDbDataSource",
+            "data_source": "SafetyDbImporter",
             "data_source_cfg": {
                 "url": "https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json",
                 "etags": {}
@@ -40609,7 +40609,7 @@
             "name": "npm",
             "license": "mit",
             "last_run": null,
-            "data_source": "NpmDataSource",
+            "data_source": "NpmImporter",
             "data_source_cfg": {
                 "repository_url": "https://github.com/nodejs/security-wg.git"
             }
@@ -40622,7 +40622,7 @@
             "name": "ruby",
             "license": "",
             "last_run": null,
-            "data_source": "RubyDataSource",
+            "data_source": "RubyImporter",
             "data_source_cfg": {
                 "repository_url": "https://github.com/rubysec/ruby-advisory-db.git"
             }
@@ -40635,7 +40635,7 @@
             "name": "ubuntu",
             "license": "gpl-2.0",
             "last_run": null,
-            "data_source": "UbuntuDataSource",
+            "data_source": "UbuntuImporter",
             "data_source_cfg": {
                 "etags": {},
                 "releases": [
@@ -40655,7 +40655,7 @@
             "name": "retiredotnet",
             "license": "mit",
             "last_run": null,
-            "data_source": "RetireDotnetDataSource",
+            "data_source": "RetireDotnetImporter",
             "data_source_cfg": {
                 "repository_url": "https://github.com/RetireNet/Packages.git"
             }
@@ -40668,7 +40668,7 @@
             "name": "suse_backports",
             "license": "",
             "last_run": null,
-            "data_source": "SUSEBackportsDataSource",
+            "data_source": "SUSEBackportsImporter",
             "data_source_cfg": {
                 "url": "http://ftp.suse.com/pub/projects/security/yaml/",
                 "etags": {}
@@ -40682,7 +40682,7 @@
             "name": "debian_oval",
             "license": "",
             "last_run": null,
-            "data_source": "DebianOvalDataSource",
+            "data_source": "DebianOvalImporter",
             "data_source_cfg": {
                 "etags": {},
                 "releases": [
@@ -40701,7 +40701,7 @@
             "name": "redhat",
             "license": "cc-by-4.0",
             "last_run": null,
-            "data_source": "RedhatDataSource",
+            "data_source": "RedhatImporter",
             "data_source_cfg": {}
         }
     },
@@ -40712,7 +40712,7 @@
             "name": "gentoo",
             "license": "",
             "last_run": null,
-            "data_source": "GentooDataSource",
+            "data_source": "GentooImporter",
             "data_source_cfg": {
                 "repository_url": "https://anongit.gentoo.org/git/data/glsa.git"
             }
@@ -40725,7 +40725,7 @@
             "name": "openssl",
             "license": "",
             "last_run": "2020-09-14T12:52:58.762Z",
-            "data_source": "OpenSSLDataSource",
+            "data_source": "OpenSSLImporter",
             "data_source_cfg": {
                 "etags": {
                     "https://www.openssl.org/news/vulnerabilities.xml": "\"513bd-5aee0a1c716f0-gzip\""
@@ -40740,7 +40740,7 @@
             "name": "ubuntu_usn",
             "license": "gpl-2.0",
             "last_run": null,
-            "data_source": "UbuntuUSNDataSource",
+            "data_source": "UbuntuUSNImporter",
             "data_source_cfg": {
                 "etags": {},
                 "db_url": "https://usn.ubuntu.com/usn-db/database-all.json.bz2"
@@ -40754,7 +40754,7 @@
             "name": "github",
             "license": "",
             "last_run": null,
-            "data_source": "GitHubAPIDataSource",
+            "data_source": "GitHubAPIImporter",
             "data_source_cfg": {
                 "endpoint": "https://api.github.com/graphql",
                 "ecosystems": [
@@ -40765,4 +40765,4 @@
             }
         }
     }
-]
+]

vulnerabilities/import_runner.py

@@ -31,8 +31,8 @@
 
 from vulnerabilities import models
 from vulnerabilities.models import Advisory
-from vulnerabilities.data_source import AdvisoryData
-from vulnerabilities.data_source import DataSource
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import Importer
 
 logger = logging.getLogger(__name__)
 
@@ -48,7 +48,7 @@ class ImportRunner:
         - No valid data from the data source must be skipped or truncated.
     """
 
-    def __init__(self, importer: DataSource):
+    def __init__(self, importer: Importer):
         self.importer = importer
 
     def run(self) -> None:
@@ -60,9 +60,7 @@ def run(self) -> None:
         logger.info(f"Starting import for {importer_name}")
         advisory_datas = importer_class().advisory_data()
         count = process_advisories(advisory_datas=advisory_datas, importer_name=importer_name)
-        logger.info(
-            f"Finished import for {importer_name}. Imported {count} advisories."
-        )
+        logger.info(f"Finished import for {importer_name}. Imported {count} advisories.")
 
 
 def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: str) -> List:

vulnerabilities/data_source.py vulnerabilities/importer.pyvulnerabilities/data_source.py renamed to vulnerabilities/importer.py

@@ -212,22 +212,22 @@ def __post_init__(self):
             logger.warn(f"AdvisoryData with no tzinfo: {self!r}")
 
 
-class NoLicenseWarning(Warning):
+class NoLicenseError(Exception):
     pass
 
 
-class DataSource:
+class Importer:
     """
-    A DataSource collects data from various upstreams and returns corresponding
+    An Importer collects data from various upstreams and returns corresponding
     AdvisoryData objects in its advisory_data method.
     Subclass this class to implement an importer
     """
 
-    license = ""
+    spdx_license_expression = ""
 
     def __init__(self):
-        if not self.license:
-            warnings.warn(f"Running importer {self!r} without a license", NoLicenseWarning)
+        if not self.spdx_license_expression:
+            raise Exception(f"Cannot run importer {self!r} without a license")
 
     @classmethod
     @property
@@ -245,23 +245,8 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
         raise NotImplementedError
 
 
-# TODO: Adopt the same design as that for DataSource
-class DataSourceConfiguration:
-    pass
-
-
-@dataclasses.dataclass
-class GitDataSourceConfiguration(DataSourceConfiguration):
-    repository_url: str
-    branch: Optional[str] = None
-    create_working_directory: bool = True
-    remove_working_directory: bool = True
-    working_directory: Optional[str] = None
-
-
-class GitDataSource(DataSource):
-    CONFIG_CLASS = GitDataSourceConfiguration
-
+# TODO: Needs rewrite
+class GitImporter(Importer):
     def validate_configuration(self) -> None:
 
         if not self.config.create_working_directory and self.config.working_directory is None:
@@ -449,7 +434,8 @@ def _include_file(
     return match
 
 
-class OvalDataSource(DataSource):
+# TODO: Needs rewrite
+class OvalImporter(Importer):
     """
     All data sources which collect data from OVAL files must inherit from this
     `OvalDataSource` class. Subclasses must implement the methods `_fetch` and `set_api`.

vulnerabilities/importers/__init__.py

@@ -21,6 +21,6 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 from vulnerabilities.importers import nginx
 
-IMPORTERS = [nginx.NginxDataSource]
+IMPORTERS = [nginx.NginxImporter]
 
 IMPORTER_REGISTRY = {x.qualified_name: x for x in IMPORTERS}

vulnerabilities/importers/alpine_linux.py

@@ -29,15 +29,15 @@
 import saneyaml
 from bs4 import BeautifulSoup
 
-from vulnerabilities.data_source import Advisory
-from vulnerabilities.data_source import DataSource
-from vulnerabilities.data_source import Reference
+from vulnerabilities.importer import Advisory
+from vulnerabilities.importer import Importer
+from vulnerabilities.importer import Reference
 from vulnerabilities.helpers import is_cve
 
 BASE_URL = "https://secdb.alpinelinux.org/"
 
 
-class AlpineDataSource(DataSource):
+class AlpineImporter(Importer):
     @staticmethod
     def fetch_advisory_links():
         index_page = BeautifulSoup(requests.get(BASE_URL).content, features="lxml")

vulnerabilities/importers/apache_httpd.py

@@ -30,24 +30,17 @@
 from univers.versions import SemverVersion
 from univers.version_specifier import VersionSpecifier
 
-from vulnerabilities.data_source import Advisory
-from vulnerabilities.data_source import DataSource
-from vulnerabilities.data_source import DataSourceConfiguration
-from vulnerabilities.data_source import Reference
-from vulnerabilities.data_source import VulnerabilitySeverity
+from vulnerabilities.importer import Advisory
+from vulnerabilities.importer import Importer
+from vulnerabilities.importer import Reference
+from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.package_managers import GitHubTagsAPI
 from vulnerabilities.severity_systems import scoring_systems
 from vulnerabilities.helpers import nearest_patched_package
 
 
-@dataclasses.dataclass
-class ApacheHTTPDDataSourceConfiguration(DataSourceConfiguration):
-    etags: dict
+class ApacheHTTPDImporter(Importer):
 
-
-class ApacheHTTPDDataSource(DataSource):
-
-    CONFIG_CLASS = ApacheHTTPDDataSourceConfiguration
     base_url = "https://httpd.apache.org/security/json/"
 
     def set_api(self):

vulnerabilities/importers/apache_kafka.py

@@ -28,17 +28,17 @@
 from univers.versions import MavenVersion
 from univers.version_specifier import VersionSpecifier
 
-from vulnerabilities.data_source import Advisory
-from vulnerabilities.data_source import DataSource
-from vulnerabilities.data_source import Reference
+from vulnerabilities.importer import Advisory
+from vulnerabilities.importer import Importer
+from vulnerabilities.importer import Reference
 from vulnerabilities.package_managers import GitHubTagsAPI
 from vulnerabilities.helpers import nearest_patched_package
 
 GH_PAGE_URL = "https://raw.githubusercontent.com/apache/kafka-site/asf-site/cve-list.html"
 ASF_PAGE_URL = "https://kafka.apache.org/cve-list"
 
 
-class ApacheKafkaDataSource(DataSource):
+class ApacheKafkaImporter(Importer):
     @staticmethod
     def fetch_advisory_page():
         page = requests.get(GH_PAGE_URL)

vulnerabilities/importers/apache_tomcat.py

@@ -32,23 +32,16 @@
 from univers.versions import SemverVersion
 from packageurl import PackageURL
 
-from vulnerabilities.data_source import Advisory
-from vulnerabilities.data_source import DataSource
-from vulnerabilities.data_source import DataSourceConfiguration
-from vulnerabilities.data_source import Reference
+from vulnerabilities.importer import Advisory
+from vulnerabilities.importer import Importer
+from vulnerabilities.importer import Reference
 from vulnerabilities.helpers import create_etag
 from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.package_managers import MavenVersionAPI
 
 
-@dataclasses.dataclass
-class ApacheTomcatDataSourceConfiguration(DataSourceConfiguration):
-    etags: dict
+class ApacheTomcatImporter(Importer):
 
-
-class ApacheTomcatDataSource(DataSource):
-
-    CONFIG_CLASS = ApacheTomcatDataSourceConfiguration
     base_url = "https://tomcat.apache.org/security-{}"
 
     def __init__(self, *args, **kwargs):