diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index c5e87ae88..666c7233d 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -31,6 +31,7 @@ from vulnerabilities.importers import retiredotnet from vulnerabilities.importers import suse_scores from vulnerabilities.importers import ubuntu +from vulnerabilities.importers import ubuntu_usn from vulnerabilities.importers import xen IMPORTERS_REGISTRY = [ @@ -59,6 +60,7 @@ elixir_security.ElixirSecurityImporter, apache_tomcat.ApacheTomcatImporter, xen.XenImporter, + ubuntu_usn.UbuntuUSNImporter, ] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index f78b5a93a..bb6dbc1dc 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -19,8 +19,48 @@ class UbuntuImporter(OvalImporter): - spdx_license_expression = "GPL" - license_url = "https://ubuntu.com/legal/terms" + spdx_license_expression = "LicenseRef-scancode-other-permissive" + notice = """ + From: Seth Arnold + Date: Wed, Jan 25, 2023 at 2:02 AM + Subject: Re: [ubuntu-hardened] Usage of Ubuntu Security Data in VulnerableCode + To: Tushar Goel + Cc: , Philippe Ombredanne , jmhoran@nexb.com + + + On Wed, Jan 11, 2023 at 06:27:38PM +0530, Tushar Goel wrote: + > We would like to integrate the Ubuntu usn data[1][2] and + > Ubuntu security data (OVAL format)[3] in vulnerablecode[4] + > which is a FOSS db of FOSS vulnerability data. We were not + > able to know under which license this security data comes. + > We would be grateful to have your acknowledgement over usage of + > the ubuntu security data in vulnerablecode and have + > some kind of licensing declaration from your side. + + Hello Tushar, we do not have an explicit license on this data. + + We share our data with the intention that others will use it. Please + feel free to use it for the general furtherance of security. + + Much of the data that's contained within our databases is sourced from + third parties, who also shared their data with the intention that others + will use it. I'm not sure what it would look like to try to put a license + on data that is crowd-sourced from thousands of contributors. (If you were + to start such a project today, it'd probably be one of the first things to + formalize. But when CVE was started two decades ago, the primary goal was + sharing knowledge and simplifying the vulnerability remediation process, + and licensing the data was, as far as I can remember, not considered. + Sharing was the goal.) + + I will ask that vulnerablecode 'be nice' to our infrastructure that + hosts the databases -- some automated uses of our infrastructure by + vulnerability scanner tools has lead to significant load and engineering + effort. In general, please prefer a small handful of systems updating + mirrors roughly twice a day rather than thousands of hosts pulling + data hourly. + + Thanks + """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py index e67a3662e..586514128 100644 --- a/vulnerabilities/importers/ubuntu_usn.py +++ b/vulnerabilities/importers/ubuntu_usn.py @@ -15,55 +15,81 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference -from vulnerabilities.utils import create_etag from vulnerabilities.utils import is_cve class UbuntuUSNImporter(Importer): - def updated_advisories(self): - advisories = [] - if create_etag(data_src=self, url=self.config.db_url, etag_key="etag"): - advisories.extend(self.to_advisories(fetch(self.config.db_url))) - - return self.batch_advisories(advisories) - - def create_etag(self, url): - etag = requests.head(url).headers.get("etag") - if not etag: - return True - - elif url in self.config.etags: - if self.config.etags[url] == etag: - return False - - self.config.etags[url] = etag - return True + db_url = "https://usn.ubuntu.com/usn-db/database-all.json.bz2" + spdx_license_expression = "LicenseRef-scancode-other-permissive" + notice = """ + From: Seth Arnold + Date: Wed, Jan 25, 2023 at 2:02 AM + Subject: Re: [ubuntu-hardened] Usage of Ubuntu Security Data in VulnerableCode + To: Tushar Goel + Cc: , Philippe Ombredanne , jmhoran@nexb.com + + + On Wed, Jan 11, 2023 at 06:27:38PM +0530, Tushar Goel wrote: + > We would like to integrate the Ubuntu usn data[1][2] and + > Ubuntu security data (OVAL format)[3] in vulnerablecode[4] + > which is a FOSS db of FOSS vulnerability data. We were not + > able to know under which license this security data comes. + > We would be grateful to have your acknowledgement over usage of + > the ubuntu security data in vulnerablecode and have + > some kind of licensing declaration from your side. + + Hello Tushar, we do not have an explicit license on this data. + + We share our data with the intention that others will use it. Please + feel free to use it for the general furtherance of security. + + Much of the data that's contained within our databases is sourced from + third parties, who also shared their data with the intention that others + will use it. I'm not sure what it would look like to try to put a license + on data that is crowd-sourced from thousands of contributors. (If you were + to start such a project today, it'd probably be one of the first things to + formalize. But when CVE was started two decades ago, the primary goal was + sharing knowledge and simplifying the vulnerability remediation process, + and licensing the data was, as far as I can remember, not considered. + Sharing was the goal.) + + I will ask that vulnerablecode 'be nice' to our infrastructure that + hosts the databases -- some automated uses of our infrastructure by + vulnerability scanner tools has lead to significant load and engineering + effort. In general, please prefer a small handful of systems updating + mirrors roughly twice a day rather than thousands of hosts pulling + data hourly. + + Thanks + """ + + def advisory_data(self): + usn_db = fetch(self.db_url) + yield from self.to_advisories(usn_db=usn_db) @staticmethod def to_advisories(usn_db): - advisories = [] for usn in usn_db: - reference = get_usn_references(usn_db[usn]["id"]) - for cve in usn_db[usn].get("cves", [""]): + usn_data = usn_db[usn] + references = get_usn_references(usn_data.get("id")) + for cve in usn_data.get("cves", []): # The db sometimes contains entries like # {'cves': ['python-pgsql vulnerabilities', 'CVE-2006-2313', 'CVE-2006-2314']} # This `if` filters entries like 'python-pgsql vulnerabilities' if not is_cve(cve): - cve = "" + continue - advisories.append( - AdvisoryData( - vulnerability_id=cve, - summary="", - references=[reference], - ) + yield AdvisoryData( + aliases=[cve], + summary="", + references=references, ) - return advisories - def get_usn_references(usn_id): - return Reference(reference_id="USN-" + usn_id, url="https://usn.ubuntu.com/{}/".format(usn_id)) + if not usn_id: + return [] + return [Reference(reference_id=f"USN-{usn_id}", url=f"https://usn.ubuntu.com/{usn_id}/")] def fetch(url): diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index b2bb96038..d1d304429 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -33,6 +33,5 @@ def no_rmtree(monkeypatch): "test_rust.py", "test_suse_backports.py", "test_suse.py", - "test_ubuntu_usn.py", "test_upstream.py", ] diff --git a/vulnerabilities/tests/test_data/ubuntu_usn_db/ubuntu-usn-expected.json b/vulnerabilities/tests/test_data/ubuntu_usn_db/ubuntu-usn-expected.json new file mode 100644 index 000000000..1173a0473 --- /dev/null +++ b/vulnerabilities/tests/test_data/ubuntu_usn_db/ubuntu-usn-expected.json @@ -0,0 +1,34 @@ +[ + { + "aliases": [ + "CVE-2009-0698" + ], + "summary": "", + "affected_packages": [], + "references": [ + { + "reference_id": "USN-763-1", + "url": "https://usn.ubuntu.com/763-1/", + "severities": [] + } + ], + "date_published": null, + "weaknesses": [] + }, + { + "aliases": [ + "CVE-2009-1274" + ], + "summary": "", + "affected_packages": [], + "references": [ + { + "reference_id": "USN-763-1", + "url": "https://usn.ubuntu.com/763-1/", + "severities": [] + } + ], + "date_published": null, + "weaknesses": [] + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_ubuntu_usn.py b/vulnerabilities/tests/test_ubuntu_usn.py index 2032a4ac3..cac58e47a 100644 --- a/vulnerabilities/tests/test_ubuntu_usn.py +++ b/vulnerabilities/tests/test_ubuntu_usn.py @@ -17,71 +17,21 @@ from packageurl import PackageURL -import vulnerabilities.importers.ubuntu_usn as ubuntu_usn from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference +from vulnerabilities.importers.ubuntu_usn import UbuntuUSNImporter +from vulnerabilities.tests import util_tests BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/", "ubuntu_usn_db", "database-all.json.bz2") - - -class TestUbuntuUSNImporter(TestCase): - @classmethod - def setUpClass(cls): - data_src_cfg = {"etags": {}, "db_url": "http://exampledb.com"} - cls.data_src = ubuntu_usn.UbuntuUSNImporter(batch_size=1, config=data_src_cfg) - with open(TEST_DATA, "rb") as f: - cls.raw_data = f.read() - cls.db = json.loads(bz2.decompress(cls.raw_data)) - - def test_get_usn_references(self): - - eg_usn = "435-1" - expected_references = Reference( - reference_id="USN-435-1", url="https://usn.ubuntu.com/435-1/" - ) - - found_references = ubuntu_usn.get_usn_references(eg_usn) - assert found_references == expected_references - - def test_fetch(self): - - mock_response = MagicMock() - mock_response.content = self.raw_data - with patch("vulnerabilities.importers.ubuntu_usn.requests.get", return_value=mock_response): - assert ubuntu_usn.fetch("www.db.com") == self.db - - def test_to_advisories(self): - - expected_advisories = [ - Advisory( - summary="", - references=[ - Reference(url="https://usn.ubuntu.com/763-1/", reference_id="USN-763-1") - ], - vulnerability_id="CVE-2009-0698", - ), - Advisory( - summary="", - references=[ - Reference(url="https://usn.ubuntu.com/763-1/", reference_id="USN-763-1") - ], - vulnerability_id="CVE-2009-1274", - ), - ] - found_advisories = self.data_src.to_advisories(self.db) - - found_advisories = list(map(Advisory.normalized, found_advisories)) - expected_advisories = list(map(Advisory.normalized, expected_advisories)) - assert sorted(found_advisories) == sorted(expected_advisories) - - def test_create_etag(self): - assert self.data_src.config.etags == {} - - mock_response = MagicMock() - mock_response.headers = {"etag": "2131151243&2191"} - - with patch("vulnerabilities.importers.ubuntu.requests.head", return_value=mock_response): - assert self.data_src.create_etag("https://example.org") - assert self.data_src.config.etags == {"https://example.org": "2131151243&2191"} - assert not self.data_src.create_etag("https://example.org") +TEST_DIR = os.path.join(BASE_DIR, "test_data", "ubuntu_usn_db") + + +def test_ubuntu_usn(): + database = os.path.join(TEST_DIR, "database-all.json.bz2") + with open(database, "rb") as f: + raw_data = f.read() + db = json.loads(bz2.decompress(raw_data)) + advisories = UbuntuUSNImporter().to_advisories(db) + expected_file = os.path.join(TEST_DIR, f"ubuntu-usn-expected.json") + result = [data.to_dict() for data in list(advisories)] + util_tests.check_results_against_json(result, expected_file)