From e8c5bf15775315660838d9da9094243a6c092d7b Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Fri, 29 Jan 2021 18:52:23 +0530 Subject: [PATCH 1/4] Add istio importer and tests Signed-off-by: Tushar912 --- vulnerabilities/importer_yielder.py | 16 ++ vulnerabilities/importers/__init__.py | 2 + vulnerabilities/importers/istio.py | 173 ++++++++++++++++++ .../tests/test_data/istio/test_file.md | 15 ++ vulnerabilities/tests/test_istio.py | 120 ++++++++++++ 5 files changed, 326 insertions(+) create mode 100644 vulnerabilities/importers/istio.py create mode 100644 vulnerabilities/tests/test_data/istio/test_file.md create mode 100644 vulnerabilities/tests/test_istio.py diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 5abc69e25..1b2631359 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -228,6 +228,13 @@ "data_source": "ApacheTomcatDataSource", "data_source_cfg": {"etags": {}}, }, + { + "name": "apache_tomcat", + "license": "", + "last_run": None, + "data_source": "ApacheTomcatDataSource", + "data_source_cfg": {"etags": {}}, + }, { "name": "apache_kafka", "license": "", @@ -235,6 +242,15 @@ "data_source": "ApacheKafkaDataSource", "data_source_cfg": {}, }, + { + 'name': 'istio', + 'license': '', + 'last_run': None, + 'data_source': 'IstioDataSource', + 'data_source_cfg': { + 'repository_url': 'https://github.com/istio/istio.io' + }, + }, ] diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 72c92de02..9f468632a 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -48,3 +48,5 @@ from vulnerabilities.importers.ubuntu import UbuntuDataSource from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource +from vulnerabilities.importers.apache_kafka import ApacheKafkaDataSource +from vulnerabilities.importers.istio import IstioDataSource diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py new file mode 100644 index 000000000..ce3244d09 --- /dev/null +++ b/vulnerabilities/importers/istio.py @@ -0,0 +1,173 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import asyncio +from typing import List, Set + +from dephell_specifier import RangeSpecifier +from packageurl import PackageURL + +from vulnerabilities.data_source import Advisory, GitDataSource, Reference +from vulnerabilities.package_managers import GitHubTagsAPI + + +class IstioDataSource(GitDataSource): + def __enter__(self): + super(IstioDataSource, self).__enter__() + + if not getattr(self, "_added_files", None): + self._added_files, self._updated_files = self.file_changes( + recursive=True, file_ext="md", subdir="./content/en/news/security" + ) + self.version_api = GitHubTagsAPI() + self.set_api() + + def set_api(self): + asyncio.run(self.version_api.load_api(["istio/istio"])) + + def updated_advisories(self) -> Set[Advisory]: + files = self._updated_files + advisories = [] + for f in files: + processed_data = self.process_file(f) + if processed_data: + advisories.extend(processed_data) + return self.batch_advisories(advisories) + + def added_advisories(self) -> Set[Advisory]: + files = self._added_files + advisories = [] + for f in files: + processed_data = self.process_file(f) + if processed_data: + advisories.extend(processed_data) + return self.batch_advisories(advisories) + + def get_versions_for_pkg_from_range_list(self, version_range_list): + # Takes a list of version ranges(affected) of a package + # as parameter and returns a tuple of safe package versions and + # vulnerable package versions + + safe_pkg_versions = [] + vuln_pkg_versions = [] + all_version_list = self.version_api.get("istio/istio") + if not version_range_list: + return all_version_list, [] + version_ranges = {RangeSpecifier(r) for r in version_range_list} + for version in all_version_list: + if any([version in v for v in version_ranges]): + vuln_pkg_versions.append(version) + + safe_pkg_versions = set(all_version_list) - set(vuln_pkg_versions) + return safe_pkg_versions, vuln_pkg_versions + + def get_data_from_md(self, file): + data = {} + for line in file: + line = line.strip() + line = line.split() + if len(line) > 0 and line is not None: + + start = line[0] + + if start == "title:": + data["title"] = " ".join(line[1:]) + elif start == "description:": + data["description"] = " ".join(line[1:]) + elif start == "cves:": + data["cves"] = " ".join(line[1:]) + data["cves"] = data["cves"].replace("[", "") + data["cves"] = data["cves"].replace("]", "") + data["cves"] = data["cves"].split(",") + + elif start == "releases:": + data["releases"] = " ".join(line[1:]) + data["releases"] = data["releases"].replace("[", "") + data["releases"] = data["releases"].replace("]", "") + data["releases"] = data["releases"].replace('"', "") + data["releases"] = data["releases"].split(",") + releases = [] + if data.get("releases"): + for release in data["releases"]: + release = release.strip() + release = release.split(" ") + if len(release) > 2: + lbound = ">=" + release[0] + ubound = "<=" + release[2] + releases.append(lbound + "," + ubound) + data["releases"] = releases + + return data + + def process_file(self, path): + + advisories = [] + + with open(path) as f: + data = {} + + data = self.get_data_from_md(f) + + if not data.get("cves"): + data["cves"] = [""] + + for cve_id in data["cves"]: + + if not cve_id.startswith("CVE"): + continue + + safe_pkg_versions = [] + vuln_pkg_versions = [] + + if not data.get("releases"): + data["releases"] = [] + + ( + safe_pkg_versions, + vuln_pkg_versions, + ) = self.get_versions_for_pkg_from_range_list(data["releases"]) + + safe_purls = [] + vuln_purls = [] + + cve_id = cve_id + + safe_purls = { + PackageURL(name="istio", type="golang", version=version) + for version in safe_pkg_versions + } + + vuln_purls = { + PackageURL(name="istio", type="golang", version=version) + for version in vuln_pkg_versions + } + + advisories.append( + Advisory( + summary=data["description"], + impacted_package_urls=vuln_purls, + resolved_package_urls=safe_purls, + cve_id=cve_id, + ) + ) + + return advisories diff --git a/vulnerabilities/tests/test_data/istio/test_file.md b/vulnerabilities/tests/test_data/istio/test_file.md new file mode 100644 index 000000000..f58b20d9b --- /dev/null +++ b/vulnerabilities/tests/test_data/istio/test_file.md @@ -0,0 +1,15 @@ +--- +title: ISTIO-SECURITY-2019-001 +subtitle: Security Bulletin +description: Incorrect access control. +cves: [CVE-2019-12243] +cvss: "8.9" +vector: "CVSS:3.0/AV:A/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:N/E:H/RL:O/RC:C" +releases: ["1.1 to 1.1.15", "1.2 to 1.2.6", "1.3 to 1.3.1"] +publishdate: 2019-05-28 +keywords: [CVE] +skip_seealso: true +aliases: + - /blog/2019/cve-2019-12243 + - /news/2019/cve-2019-12243 +--- diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py new file mode 100644 index 000000000..4a22c76ed --- /dev/null +++ b/vulnerabilities/tests/test_istio.py @@ -0,0 +1,120 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import os +from collections import OrderedDict +from unittest import TestCase + +from packageurl import PackageURL + +from vulnerabilities.data_source import Advisory, Reference +from vulnerabilities.importers.istio import IstioDataSource +from vulnerabilities.package_managers import GitHubTagsAPI + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class TestIstioDataSource(TestCase): + @classmethod + def setUpClass(cls): + data_source_cfg = { + "repository_url": "", + } + cls.data_src = IstioDataSource(1, config=data_source_cfg) + cls.data_src.version_api = GitHubTagsAPI( + { + "istio/istio": [ + "1.1.0-rc.0", + "1.1.0-rc.1", + "1.1.0-rc.2", + "1.1.0-rc.3", + "1.1.0-rc.4", + "1.1.0-rc.5", + "1.1.0-rc.6", + "1.1.0-snapshot.2", + "1.1.0-snapshot.3", + ] + } + ) + + def test_process_file(self): + + path = os.path.join(BASE_DIR, "test_data/istio/test_file.md") + expected_data = [ + Advisory( + summary=("Incorrect access control."), + impacted_package_urls={ + PackageURL( + type="golang", + name="istio", + version="1.1.0-snapshot.2", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-snapshot.3", + ), + }, + resolved_package_urls={ + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.2", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.4", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.3", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.0", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.5", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.1", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.6", + ), + }, + cve_id="CVE-2019-12243", + ) + ] + + found_data = self.data_src.process_file(path) + + assert expected_data == found_data From 05b7ba65d9929eb5d5987d1173911cd41a2173ca Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Tue, 9 Feb 2021 18:33:24 +0530 Subject: [PATCH 2/4] Use yaml_lines of file to extract data. Signed-off-by: Tushar912 --- vulnerabilities/importers/istio.py | 123 +++++++++++++---------------- 1 file changed, 57 insertions(+), 66 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index ce3244d09..27c4b269e 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -22,6 +22,7 @@ import asyncio from typing import List, Set +import yaml from dephell_specifier import RangeSpecifier from packageurl import PackageURL @@ -80,31 +81,27 @@ def get_versions_for_pkg_from_range_list(self, version_range_list): safe_pkg_versions = set(all_version_list) - set(vuln_pkg_versions) return safe_pkg_versions, vuln_pkg_versions - def get_data_from_md(self, file): - data = {} - for line in file: + def get_data_from_yaml_lines(self, yaml_lines): + + return yaml.safe_load("\n".join(yaml_lines)) + + def get_yaml_lines(self, lines): + + for line in lines: line = line.strip() - line = line.split() - if len(line) > 0 and line is not None: - - start = line[0] - - if start == "title:": - data["title"] = " ".join(line[1:]) - elif start == "description:": - data["description"] = " ".join(line[1:]) - elif start == "cves:": - data["cves"] = " ".join(line[1:]) - data["cves"] = data["cves"].replace("[", "") - data["cves"] = data["cves"].replace("]", "") - data["cves"] = data["cves"].split(",") - - elif start == "releases:": - data["releases"] = " ".join(line[1:]) - data["releases"] = data["releases"].replace("[", "") - data["releases"] = data["releases"].replace("]", "") - data["releases"] = data["releases"].replace('"', "") - data["releases"] = data["releases"].split(",") + if line.startswith("---"): + continue + elif line.endswith("---"): + break + else: + yield line + + def process_file(self, path): + + advisories = [] + + data = self.get_data_from_md(path) + releases = [] if data.get("releases"): for release in data["releases"]: @@ -114,60 +111,54 @@ def get_data_from_md(self, file): lbound = ">=" + release[0] ubound = "<=" + release[2] releases.append(lbound + "," + ubound) - data["releases"] = releases - - return data - def process_file(self, path): - - advisories = [] - - with open(path) as f: - data = {} - - data = self.get_data_from_md(f) + data["releases"] = releases - if not data.get("cves"): - data["cves"] = [""] + if not data.get("cves"): + data["cves"] = [""] - for cve_id in data["cves"]: + for cve_id in data["cves"]: - if not cve_id.startswith("CVE"): - continue + if not cve_id.startswith("CVE"): + continue - safe_pkg_versions = [] - vuln_pkg_versions = [] + safe_pkg_versions = [] + vuln_pkg_versions = [] - if not data.get("releases"): - data["releases"] = [] + if not data.get("releases"): + data["releases"] = [] - ( - safe_pkg_versions, - vuln_pkg_versions, - ) = self.get_versions_for_pkg_from_range_list(data["releases"]) + safe_pkg_versions, vuln_pkg_versions = self.get_versions_for_pkg_from_range_list( + data["releases"] + ) - safe_purls = [] - vuln_purls = [] + safe_purls = [] + vuln_purls = [] - cve_id = cve_id + cve_id = cve_id - safe_purls = { - PackageURL(name="istio", type="golang", version=version) - for version in safe_pkg_versions - } + safe_purls = { + PackageURL(name="istio", type="golang", version=version) + for version in safe_pkg_versions + } - vuln_purls = { - PackageURL(name="istio", type="golang", version=version) - for version in vuln_pkg_versions - } + vuln_purls = { + PackageURL(name="istio", type="golang", version=version) + for version in vuln_pkg_versions + } - advisories.append( - Advisory( - summary=data["description"], - impacted_package_urls=vuln_purls, - resolved_package_urls=safe_purls, - cve_id=cve_id, - ) + advisories.append( + Advisory( + summary=data["description"], + impacted_package_urls=vuln_purls, + resolved_package_urls=safe_purls, + cve_id=cve_id, ) + ) return advisories + + def get_data_from_md(self, path): + with open(path) as f: + yaml_lines = self.get_yaml_lines(f) + return self.get_data_from_yaml_lines(yaml_lines) From 43e4adde7e60913dfb04712997ff8b7013c2cccd Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Wed, 10 Feb 2021 18:28:10 +0530 Subject: [PATCH 3/4] Add docstrings and add test for get_data_from_md. Also add github as package type Signed-off-by: Tushar912 --- vulnerabilities/importers/istio.py | 71 +++++++++++++++---- .../tests/test_data/istio/test_file.md | 5 -- vulnerabilities/tests/test_istio.py | 62 ++++++++++++++++ 3 files changed, 118 insertions(+), 20 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 27c4b269e..e81396b96 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -22,12 +22,14 @@ import asyncio from typing import List, Set -import yaml +import yaml from dephell_specifier import RangeSpecifier from packageurl import PackageURL -from vulnerabilities.data_source import Advisory, GitDataSource, Reference +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import GitDataSource +from vulnerabilities.data_source import Reference from vulnerabilities.package_managers import GitHubTagsAPI @@ -64,28 +66,52 @@ def added_advisories(self) -> Set[Advisory]: return self.batch_advisories(advisories) def get_versions_for_pkg_from_range_list(self, version_range_list): - # Takes a list of version ranges(affected) of a package - # as parameter and returns a tuple of safe package versions and - # vulnerable package versions + """Takes a list of version ranges(affected) of a package + as parameter and returns a tuple of safe package versions and + vulnerable package versions""" safe_pkg_versions = [] vuln_pkg_versions = [] - all_version_list = self.version_api.get("istio/istio") + all_version = self.version_api.get("istio/istio") if not version_range_list: - return all_version_list, [] + return all_version, [] version_ranges = {RangeSpecifier(r) for r in version_range_list} - for version in all_version_list: + for version in all_version: if any([version in v for v in version_ranges]): vuln_pkg_versions.append(version) - safe_pkg_versions = set(all_version_list) - set(vuln_pkg_versions) + safe_pkg_versions = set(all_version) - set(vuln_pkg_versions) return safe_pkg_versions, vuln_pkg_versions def get_data_from_yaml_lines(self, yaml_lines): + """Return a mapping of data from a iterable of yaml_lines + for example : + ['title: ISTIO-SECURITY-2019-001', + 'description: Incorrect access control.','cves: [CVE-2019-12243]'] + + would give {'title':'ISTIO-SECURITY-2019-001', + 'description': 'Incorrect access control.', + 'cves': '[CVE-2019-12243]'} + """ return yaml.safe_load("\n".join(yaml_lines)) def get_yaml_lines(self, lines): + """The istio advisory file contains lines similar to yaml format . + This function extracts those lines and return an iterable of lines + + for example : + lines = + --- + title: ISTIO-SECURITY-2019-001 + description: Incorrect access control. + cves: [CVE-2019-12243] + --- + + get_yaml_lines(lines) would return + ['title: ISTIO-SECURITY-2019-001','description: Incorrect access control.' + ,'cves: [CVE-2019-12243]'] + """ for line in lines: line = line.strip() @@ -129,24 +155,35 @@ def process_file(self, path): data["releases"] = [] safe_pkg_versions, vuln_pkg_versions = self.get_versions_for_pkg_from_range_list( - data["releases"] - ) + data["releases"]) safe_purls = [] vuln_purls = [] cve_id = cve_id - safe_purls = { - PackageURL(name="istio", type="golang", version=version) + safe_purls_golang = { + PackageURL(type="golang", name="istio", version=version) for version in safe_pkg_versions } - vuln_purls = { - PackageURL(name="istio", type="golang", version=version) + safe_purls_github = { + PackageURL(type="github", name="istio", version=version) + for version in safe_pkg_versions + } + safe_purls = safe_purls_github | safe_purls_golang + + vuln_purls_golang = { + PackageURL(type="golang", name="istio", version=version) for version in vuln_pkg_versions } + vuln_purls_github = { + PackageURL(type="github", name="istio", version=version) + for version in vuln_pkg_versions + } + vuln_purls = vuln_purls_github | vuln_purls_golang + advisories.append( Advisory( summary=data["description"], @@ -159,6 +196,10 @@ def process_file(self, path): return advisories def get_data_from_md(self, path): + """Return a mapping of vulnerability data from istio . The data is + in the form of yaml_lines inside a .md file. + """ + with open(path) as f: yaml_lines = self.get_yaml_lines(f) return self.get_data_from_yaml_lines(yaml_lines) diff --git a/vulnerabilities/tests/test_data/istio/test_file.md b/vulnerabilities/tests/test_data/istio/test_file.md index f58b20d9b..019d79c19 100644 --- a/vulnerabilities/tests/test_data/istio/test_file.md +++ b/vulnerabilities/tests/test_data/istio/test_file.md @@ -7,9 +7,4 @@ cvss: "8.9" vector: "CVSS:3.0/AV:A/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:N/E:H/RL:O/RC:C" releases: ["1.1 to 1.1.15", "1.2 to 1.2.6", "1.3 to 1.3.1"] publishdate: 2019-05-28 -keywords: [CVE] -skip_seealso: true -aliases: - - /blog/2019/cve-2019-12243 - - /news/2019/cve-2019-12243 --- diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index 4a22c76ed..dafa6f0ca 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -20,6 +20,7 @@ # VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. +import datetime import os from collections import OrderedDict from unittest import TestCase @@ -56,6 +57,22 @@ def setUpClass(cls): } ) + def test_get_data_from_md(self): + path = os.path.join(BASE_DIR, "test_data/istio/test_file.md") + actual_data = self.data_src.get_data_from_md(path) + expected_data = { + "title": "ISTIO-SECURITY-2019-001", + "subtitle": "Security Bulletin", + "description": "Incorrect access control.", + "cves": ["CVE-2019-12243"], + "cvss": "8.9", + "vector": "CVSS:3.0/AV:A/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:N/E:H/RL:O/RC:C", + "releases": ["1.1 to 1.1.15", "1.2 to 1.2.6", "1.3 to 1.3.1"], + "publishdate": datetime.date(2019, 5, 28), + } + + assert expected_data == actual_data + def test_process_file(self): path = os.path.join(BASE_DIR, "test_data/istio/test_file.md") @@ -73,6 +90,16 @@ def test_process_file(self): name="istio", version="1.1.0-snapshot.3", ), + PackageURL( + type="github", + name="istio", + version="1.1.0-snapshot.2", + ), + PackageURL( + type="github", + name="istio", + version="1.1.0-snapshot.3", + ), }, resolved_package_urls={ PackageURL( @@ -110,6 +137,41 @@ def test_process_file(self): name="istio", version="1.1.0-rc.6", ), + PackageURL( + type="github", + name="istio", + version="1.1.0-rc.2", + ), + PackageURL( + type="github", + name="istio", + version="1.1.0-rc.4", + ), + PackageURL( + type="github", + name="istio", + version="1.1.0-rc.3", + ), + PackageURL( + type="github", + name="istio", + version="1.1.0-rc.0", + ), + PackageURL( + type="github", + name="istio", + version="1.1.0-rc.5", + ), + PackageURL( + type="github", + name="istio", + version="1.1.0-rc.1", + ), + PackageURL( + type="github", + name="istio", + version="1.1.0-rc.6", + ), }, cve_id="CVE-2019-12243", ) From 60f6c012445f3eeb03717aa783e9a8448b219a44 Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Tue, 16 Feb 2021 14:03:57 +0530 Subject: [PATCH 4/4] Remove added_advisories method and assign cve to empty string if its N/A Signed-off-by: Tushar912 --- vulnerabilities/importer_yielder.py | 37 +++------- vulnerabilities/importers/__init__.py | 1 - vulnerabilities/importers/istio.py | 70 +++++++++---------- .../tests/test_data/istio/test_file.md | 46 ++++++++++++ vulnerabilities/tests/test_istio.py | 2 +- 5 files changed, 88 insertions(+), 68 deletions(-) diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 1b2631359..ebcd51079 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -102,10 +102,7 @@ "license": "", "last_run": None, "data_source": "SUSEBackportsDataSource", - "data_source_cfg": { - "url": "http://ftp.suse.com/pub/projects/security/yaml/", - "etags": {}, - }, + "data_source_cfg": {"url": "http://ftp.suse.com/pub/projects/security/yaml/", "etags": {}}, }, { "name": "suse_scores", @@ -119,10 +116,7 @@ "license": "", "last_run": None, "data_source": "DebianOvalDataSource", - "data_source_cfg": { - "etags": {}, - "releases": ["wheezy", "stretch", "jessie", "buster"], - }, + "data_source_cfg": {"etags": {}, "releases": ["wheezy", "stretch", "jessie", "buster"]}, }, { "name": "redhat", @@ -136,9 +130,7 @@ "license": "", "last_run": None, "data_source": "NVDDataSource", - "data_source_cfg": { - "etags": {}, - }, + "data_source_cfg": {"etags": {}}, }, { "name": "gentoo", @@ -228,13 +220,6 @@ "data_source": "ApacheTomcatDataSource", "data_source_cfg": {"etags": {}}, }, - { - "name": "apache_tomcat", - "license": "", - "last_run": None, - "data_source": "ApacheTomcatDataSource", - "data_source_cfg": {"etags": {}}, - }, { "name": "apache_kafka", "license": "", @@ -243,13 +228,11 @@ "data_source_cfg": {}, }, { - 'name': 'istio', - 'license': '', - 'last_run': None, - 'data_source': 'IstioDataSource', - 'data_source_cfg': { - 'repository_url': 'https://github.com/istio/istio.io' - }, + "name": "istio", + "license": "apache-2.0", + "last_run": None, + "data_source": "IstioDataSource", + "data_source_cfg": {"repository_url": "https://github.com/istio/istio.io"}, }, ] @@ -258,9 +241,7 @@ def load_importers(): for importer in IMPORTER_REGISTRY: imp, created = Importer.objects.get_or_create( - name=importer["name"], - data_source=importer["data_source"], - license=importer["license"], + name=importer["name"], data_source=importer["data_source"], license=importer["license"] ) if created: diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 9f468632a..e7bbf70af 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -48,5 +48,4 @@ from vulnerabilities.importers.ubuntu import UbuntuDataSource from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource -from vulnerabilities.importers.apache_kafka import ApacheKafkaDataSource from vulnerabilities.importers.istio import IstioDataSource diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index e81396b96..fa0f1f903 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -21,15 +21,14 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import asyncio +import re from typing import List, Set import yaml + from dephell_specifier import RangeSpecifier from packageurl import PackageURL - -from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import GitDataSource -from vulnerabilities.data_source import Reference +from vulnerabilities.data_source import Advisory, GitDataSource, Reference from vulnerabilities.package_managers import GitHubTagsAPI @@ -56,26 +55,14 @@ def updated_advisories(self) -> Set[Advisory]: advisories.extend(processed_data) return self.batch_advisories(advisories) - def added_advisories(self) -> Set[Advisory]: - files = self._added_files - advisories = [] - for f in files: - processed_data = self.process_file(f) - if processed_data: - advisories.extend(processed_data) - return self.batch_advisories(advisories) - - def get_versions_for_pkg_from_range_list(self, version_range_list): + def get_pkg_versions_from_ranges(self, version_range_list): """Takes a list of version ranges(affected) of a package as parameter and returns a tuple of safe package versions and vulnerable package versions""" - + all_version = self.version_api.get("istio/istio") safe_pkg_versions = [] vuln_pkg_versions = [] - all_version = self.version_api.get("istio/istio") - if not version_range_list: - return all_version, [] - version_ranges = {RangeSpecifier(r) for r in version_range_list} + version_ranges = [RangeSpecifier(r) for r in version_range_list] for version in all_version: if any([version in v for v in version_ranges]): vuln_pkg_versions.append(version) @@ -113,9 +100,9 @@ def get_yaml_lines(self, lines): ,'cves: [CVE-2019-12243]'] """ - for line in lines: + for index, line in enumerate(lines): line = line.strip() - if line.startswith("---"): + if line.startswith("---") and index == 0: continue elif line.endswith("---"): break @@ -131,14 +118,23 @@ def process_file(self, path): releases = [] if data.get("releases"): for release in data["releases"]: - release = release.strip() - release = release.split(" ") - if len(release) > 2: + # If it is of form "All versions prior to x" + if "All releases" in release: + release = release.strip() + release = release.split(" ") + releases.append("<" + release[4]) + # If it is of form "a to b" + elif "to" in release: + release = release.strip() + release = release.split(" ") lbound = ">=" + release[0] ubound = "<=" + release[2] releases.append(lbound + "," + ubound) + # If it is a single release + elif is_release(release): + releases.append(release) - data["releases"] = releases + data["release_ranges"] = releases if not data.get("cves"): data["cves"] = [""] @@ -146,21 +142,17 @@ def process_file(self, path): for cve_id in data["cves"]: if not cve_id.startswith("CVE"): - continue + cve_id = "" safe_pkg_versions = [] vuln_pkg_versions = [] - if not data.get("releases"): - data["releases"] = [] + if not data.get("release_ranges"): + data["release_ranges"] = [] - safe_pkg_versions, vuln_pkg_versions = self.get_versions_for_pkg_from_range_list( - data["releases"]) - - safe_purls = [] - vuln_purls = [] - - cve_id = cve_id + safe_pkg_versions, vuln_pkg_versions = self.get_pkg_versions_from_ranges( + data["release_ranges"] + ) safe_purls_golang = { PackageURL(type="golang", name="istio", version=version) @@ -171,7 +163,7 @@ def process_file(self, path): PackageURL(type="github", name="istio", version=version) for version in safe_pkg_versions } - safe_purls = safe_purls_github | safe_purls_golang + safe_purls = safe_purls_github.union(safe_purls_golang) vuln_purls_golang = { PackageURL(type="golang", name="istio", version=version) @@ -182,14 +174,14 @@ def process_file(self, path): PackageURL(type="github", name="istio", version=version) for version in vuln_pkg_versions } - vuln_purls = vuln_purls_github | vuln_purls_golang + vuln_purls = vuln_purls_github.union(vuln_purls_golang) advisories.append( Advisory( summary=data["description"], impacted_package_urls=vuln_purls, resolved_package_urls=safe_purls, - cve_id=cve_id, + vulnerability_id=cve_id, ) ) @@ -203,3 +195,5 @@ def get_data_from_md(self, path): with open(path) as f: yaml_lines = self.get_yaml_lines(f) return self.get_data_from_yaml_lines(yaml_lines) + + is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match diff --git a/vulnerabilities/tests/test_data/istio/test_file.md b/vulnerabilities/tests/test_data/istio/test_file.md index 019d79c19..530cb0311 100644 --- a/vulnerabilities/tests/test_data/istio/test_file.md +++ b/vulnerabilities/tests/test_data/istio/test_file.md @@ -7,4 +7,50 @@ cvss: "8.9" vector: "CVSS:3.0/AV:A/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:N/E:H/RL:O/RC:C" releases: ["1.1 to 1.1.15", "1.2 to 1.2.6", "1.3 to 1.3.1"] publishdate: 2019-05-28 + --- + +{{< security_bulletin >}} + +During review of the [Istio 1.1.7](/news/releases/1.1.x/announcing-1.1.7) release notes, we realized that [issue 13868](https://github.com/istio/istio/issues/13868), +which is fixed in the release, actually represents a security vulnerability. + +Initially we thought the bug was impacting the [TCP Authorization](/about/feature-stages/#security-and-policy-enforcement) feature advertised +as alpha stability, which would not have required invoking this security advisory process, but we later realized that the +[Deny Checker](https://istio.io/v1.6/docs/reference/config/policy-and-telemetry/adapters/denier/) and +[List Checker](https://istio.io/v1.6/docs/reference/config/policy-and-telemetry/adapters/list/) feature were affected and those are considered stable features. +We are revisiting our processes to flag vulnerabilities that are initially reported as bugs instead of through the +[private disclosure process](/about/security-vulnerabilities/). + +We tracked the bug to a code change introduced in Istio 1.1 and affecting all releases up to 1.1.6. + +## Impact and detection + +Since Istio 1.1, In the default Istio installation profile, policy enforcement is disabled by default. + +You can check the status of policy enforcement for your mesh with the following command: + +{{< text bash >}} +$ kubectl -n istio-system get cm istio -o jsonpath="{@.data.mesh}" | grep disablePolicyChecks +disablePolicyChecks: true +{{< /text >}} + +You are not impacted by this vulnerability if `disablePolicyChecks` is set to true. + +You are impacted by the vulnerability issue if the following conditions are all true: + +* You are running one of the affected Istio releases. +* `disablePolicyChecks` is set to false (follow the steps mentioned above to check) +* Your workload is NOT using HTTP, HTTP/2, or gRPC protocols +* A mixer adapter (e.g., Deny Checker, List Checker) is used to provide authorization for your backend TCP service. + +## Mitigation + +* Users of Istio 1.0.x are not affected. +* For Istio 1.1.x deployments: update to [Istio 1.1.7](/news/releases/1.1.x/announcing-1.1.7) or later. + +## Credit + +The Istio team would like to thank `Haim Helman` for the original bug report. + +{{< boilerplate "security-vulnerability" >}} \ No newline at end of file diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index dafa6f0ca..b907afe0f 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -173,7 +173,7 @@ def test_process_file(self): version="1.1.0-rc.6", ), }, - cve_id="CVE-2019-12243", + vulnerability_id="CVE-2019-12243", ) ]