From 38ec05ff262b73cf9e4d6d8fa95fb8ff3ed97c40 Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Thu, 3 Dec 2020 19:52:28 +0530 Subject: [PATCH 1/8] add elixir security importer and test Signed-off-by: Tushar912 add elixir security to init.py Signed-off-by: Tushar912 add test for elixir security Signed-off-by: Tushar912 fixed code style Signed-off-by: Tushar912 --- vulnerabilities/importer_yielder.py | 9 ++ vulnerabilities/importers/__init__.py | 1 + vulnerabilities/importers/elixir_security.py | 128 ++++++++++++++++++ .../test_data/elixir_security/test_file.yml | 12 ++ vulnerabilities/tests/test_elixir_security.py | 85 ++++++++++++ 5 files changed, 235 insertions(+) create mode 100644 vulnerabilities/importers/elixir_security.py create mode 100644 vulnerabilities/tests/test_data/elixir_security/test_file.yml create mode 100644 vulnerabilities/tests/test_elixir_security.py diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index d3567abc5..6f6d481bc 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -226,6 +226,15 @@ 'data_source': 'PostgreSQLDataSource', 'data_source_cfg': {}, }, + { + 'name': 'elixir_security', + 'license': '', + 'last_run': None, + 'data_source': 'ElixirSecurityDataSource', + 'data_source_cfg': { + 'repository_url': 'https://github.com/dependabot/elixir-security-advisories' + }, + }, ] diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index ddeb0afe8..12a610a72 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -43,3 +43,4 @@ from vulnerabilities.importers.kaybee import KaybeeDataSource from vulnerabilities.importers.nginx import NginxDataSource from vulnerabilities.importers.postgresql import PostgreSQLDataSource +from vulnerabilities.importers.elixir_security import ElixirSecurityDataSource diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py new file mode 100644 index 000000000..f603fd0c9 --- /dev/null +++ b/vulnerabilities/importers/elixir_security.py @@ -0,0 +1,128 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import yaml +import re +import json +import requests +from typing import Set +from typing import List + +from packageurl import PackageURL + +from vulnerabilities.data_source import GitDataSource +from vulnerabilities.data_source import GitDataSourceConfiguration +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import Reference + + +class ElixirSecurityDataSource(GitDataSource): + def __enter__(self): + super(ElixirSecurityDataSource, self).__enter__() + + if not getattr(self, "_added_files", None): + self._added_files, self._updated_files = self.file_changes( + recursive=True, file_ext="yml", subdir="./packages" + ) + + def updated_advisories(self) -> Set[Advisory]: + files = self._updated_files + advisories = [] + for f in files: + processed_data = self.process_file(f) + if processed_data: + advisories.append(processed_data) + return self.batch_advisories(advisories) + + def added_advisories(self) -> Set[Advisory]: + files = self._added_files + advisories = [] + for f in files: + processed_data = self.process_file(f) + if processed_data: + advisories.append(processed_data) + return self.batch_advisories(advisories) + + @staticmethod + def generate_all_versions_list(pkg_name): + resp = requests.get(f"https://hex.pm/api/packages/{pkg_name}") + resp = resp.content + json_resp = json.loads(resp) + versions_list = [] + for release in json_resp["releases"]: + versions_list.append(release["version"]) + return versions_list + + def get_pkg_from_range(self, versions_list, pkg_name): + pkg_versions = [] + all_versions_list = self.generate_all_versions_list(pkg_name) + if versions_list is None: + return + for version in versions_list: + if re.match("^>=", version): + index = all_versions_list.index(version[3:]) + pkg_versions = pkg_versions + all_versions_list[0: index + 1] + elif re.match("^>", version): + index = all_versions_list.index(version[2:]) + pkg_versions = pkg_versions + all_versions_list[0:index] + elif re.match("^<", version): + index = all_versions_list.index(version[2:]) + pkg_versions = pkg_versions + all_versions_list[index + 1: -1] + else: + pkg_versions.append(version[3:]) + return pkg_versions + + def process_file(self, path): + with open(path) as f: + yaml_file = yaml.safe_load(f) + pkg_name = yaml_file["package"] + safe_pkg_versions = [] + if yaml_file.get("unaffected_versions"): + safe_pkg_versions = self.get_pkg_from_range( + yaml_file["patched_versions"] + yaml_file["unaffected_versions"], + pkg_name, + ) + else: + safe_pkg_versions = self.get_pkg_from_range( + yaml_file["patched_versions"], pkg_name + ) + cve_id = yaml_file["cve"] + safe_purls = [] + if safe_pkg_versions is not None: + safe_purls = { + PackageURL(name=pkg_name, type="hex", version=version) + for version in safe_pkg_versions + } + + vuln_reference = [ + Reference( + url=yaml_file["link"], + ) + ] + + return Advisory( + summary=yaml_file["description"], + impacted_package_urls=[], + resolved_package_urls=safe_purls, + cve_id=cve_id, + vuln_references=vuln_reference, + ) diff --git a/vulnerabilities/tests/test_data/elixir_security/test_file.yml b/vulnerabilities/tests/test_data/elixir_security/test_file.yml new file mode 100644 index 000000000..519cc60f9 --- /dev/null +++ b/vulnerabilities/tests/test_data/elixir_security/test_file.yml @@ -0,0 +1,12 @@ +--- +id: 2aae6e3a-24a3-4d5f-86ff-b964eaf7c6d1 +package: coherence +disclosure_date: 2017-08-02 +cve: 2018-20301 +link: https://github.com/smpallen99/coherence/issues/270 +title: | + Permissive parameters and privilege escalation +description: | + The Coherence library has "Mass Assignment"-like vulnerabilities. +patched_versions: + - ">= 0.5.2" \ No newline at end of file diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py new file mode 100644 index 000000000..e4125968e --- /dev/null +++ b/vulnerabilities/tests/test_elixir_security.py @@ -0,0 +1,85 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import os +from unittest import TestCase +from collections import OrderedDict + +from vulnerabilities.data_source import Reference +from packageurl import PackageURL + +from vulnerabilities.importers.elixir_security import ElixirSecurityDataSource +from vulnerabilities.data_source import Advisory + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class TestElixirSecurityDataSource(TestCase): + @classmethod + def setUpClass(cls): + data_source_cfg = { + "repository_url": "https://test.net", + } + cls.data_src = ElixirSecurityDataSource(1, config=data_source_cfg) + + def test_generate_all_versions_list(self): + package = "coherence" + actual_list = self.data_src.generate_all_versions_list(package) + expected_list = [ + "0.5.2", + "0.5.1", + "0.5.0", + "0.4.0", + "0.3.1", + "0.3.0", + "0.2.0", + "0.1.3", + "0.1.2", + "0.1.1", + "0.1.0", + ] + assert actual_list == expected_list + + def test_process_file(self): + + path = os.path.join(BASE_DIR, "test_data/elixir_security/test_file.yml") + expected_data = Advisory( + summary=( + 'The Coherence library has "Mass Assignment"-like vulnerabilities.\n' + ), + impacted_package_urls=[], + resolved_package_urls={ + PackageURL( + type="hex", + name="coherence", + version="0.5.2", + ), + }, + vuln_references=[ + Reference(url="https://github.com/smpallen99/coherence/issues/270") + ], + cve_id="2018-20301", + ) + + found_data = self.data_src.process_file(path) + + assert expected_data == found_data From 70ac4cd6fdc8070cfbe93ae2045a0ff1467ea332 Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Wed, 9 Dec 2020 15:21:19 +0530 Subject: [PATCH 2/8] use dephell_specifier for version ranges and sort imports Signed-off-by: Tushar912 --- vulnerabilities/importers/__init__.py | 28 +++++++-------- vulnerabilities/importers/elixir_security.py | 34 +++++++------------ vulnerabilities/tests/test_elixir_security.py | 6 ++-- 3 files changed, 30 insertions(+), 38 deletions(-) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 12a610a72..249caa48e 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -22,25 +22,25 @@ from vulnerabilities.importers.alpine_linux import AlpineDataSource +from vulnerabilities.importers.apache_httpd import ApacheHTTPDDataSource from vulnerabilities.importers.archlinux import ArchlinuxDataSource from vulnerabilities.importers.debian import DebianDataSource -from vulnerabilities.importers.npm import NpmDataSource -from vulnerabilities.importers.rust import RustDataSource -from vulnerabilities.importers.safety_db import SafetyDbDataSource -from vulnerabilities.importers.ruby import RubyDataSource -from vulnerabilities.importers.ubuntu import UbuntuDataSource -from vulnerabilities.importers.retiredotnet import RetireDotnetDataSource -from vulnerabilities.importers.suse_backports import SUSEBackportsDataSource from vulnerabilities.importers.debian_oval import DebianOvalDataSource -from vulnerabilities.importers.redhat import RedhatDataSource +from vulnerabilities.importers.elixir_security import ElixirSecurityDataSource from vulnerabilities.importers.gentoo import GentooDataSource -from vulnerabilities.importers.openssl import OpenSSLDataSource -from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource from vulnerabilities.importers.github import GitHubAPIDataSource -from vulnerabilities.importers.nvd import NVDDataSource -from vulnerabilities.importers.project_kb_msr2019 import ProjectKBMSRDataSource -from vulnerabilities.importers.apache_httpd import ApacheHTTPDDataSource from vulnerabilities.importers.kaybee import KaybeeDataSource from vulnerabilities.importers.nginx import NginxDataSource +from vulnerabilities.importers.npm import NpmDataSource +from vulnerabilities.importers.nvd import NVDDataSource +from vulnerabilities.importers.openssl import OpenSSLDataSource from vulnerabilities.importers.postgresql import PostgreSQLDataSource -from vulnerabilities.importers.elixir_security import ElixirSecurityDataSource +from vulnerabilities.importers.project_kb_msr2019 import ProjectKBMSRDataSource +from vulnerabilities.importers.redhat import RedhatDataSource +from vulnerabilities.importers.retiredotnet import RetireDotnetDataSource +from vulnerabilities.importers.ruby import RubyDataSource +from vulnerabilities.importers.rust import RustDataSource +from vulnerabilities.importers.safety_db import SafetyDbDataSource +from vulnerabilities.importers.suse_backports import SUSEBackportsDataSource +from vulnerabilities.importers.ubuntu import UbuntuDataSource +from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index f603fd0c9..425e66fa1 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# Copyright (c) nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/vulnerablecode/ # The VulnerableCode software is licensed under the Apache License version 2.0. # Data generated with VulnerableCode require an acknowledgment. @@ -26,7 +26,7 @@ import requests from typing import Set from typing import List - +from dephell_specifier import RangeSpecifier from packageurl import PackageURL from vulnerabilities.data_source import GitDataSource @@ -63,32 +63,24 @@ def added_advisories(self) -> Set[Advisory]: return self.batch_advisories(advisories) @staticmethod - def generate_all_versions_list(pkg_name): + def generate_all_version_list(pkg_name): resp = requests.get(f"https://hex.pm/api/packages/{pkg_name}") resp = resp.content json_resp = json.loads(resp) - versions_list = [] + version_list = [] for release in json_resp["releases"]: - versions_list.append(release["version"]) - return versions_list + version_list.append(release["version"]) + return version_list - def get_pkg_from_range(self, versions_list, pkg_name): + def get_pkg_from_range(self, version_list, pkg_name): pkg_versions = [] - all_versions_list = self.generate_all_versions_list(pkg_name) - if versions_list is None: + all_version_list = self.generate_all_version_list(pkg_name) + if version_list is None: return - for version in versions_list: - if re.match("^>=", version): - index = all_versions_list.index(version[3:]) - pkg_versions = pkg_versions + all_versions_list[0: index + 1] - elif re.match("^>", version): - index = all_versions_list.index(version[2:]) - pkg_versions = pkg_versions + all_versions_list[0:index] - elif re.match("^<", version): - index = all_versions_list.index(version[2:]) - pkg_versions = pkg_versions + all_versions_list[index + 1: -1] - else: - pkg_versions.append(version[3:]) + version_ranges = {RangeSpecifier(r) for r in version_list} + for version in all_version_list: + if any([version in v for v in version_ranges]): + pkg_versions.append(version) return pkg_versions def process_file(self, path): diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py index e4125968e..e1bcc95cc 100644 --- a/vulnerabilities/tests/test_elixir_security.py +++ b/vulnerabilities/tests/test_elixir_security.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# Copyright (c) nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/vulnerablecode/ # The VulnerableCode software is licensed under the Apache License version 2.0. # Data generated with VulnerableCode require an acknowledgment. @@ -41,9 +41,9 @@ def setUpClass(cls): } cls.data_src = ElixirSecurityDataSource(1, config=data_source_cfg) - def test_generate_all_versions_list(self): + def test_generate_all_version_list(self): package = "coherence" - actual_list = self.data_src.generate_all_versions_list(package) + actual_list = self.data_src.generate_all_version_list(package) expected_list = [ "0.5.2", "0.5.1", From f809a89972e17c2eb65a4297991fc3310fdb1c82 Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Sat, 12 Dec 2020 13:13:48 +0530 Subject: [PATCH 3/8] added HexVersionAPI and mock test it Signed-off-by: Tushar912 --- vulnerabilities/importers/elixir_security.py | 45 ++++++++++++------ vulnerabilities/package_managers.py | 21 +++++++++ vulnerabilities/tests/test_elixir_security.py | 46 ++++++++++++++++--- 3 files changed, 91 insertions(+), 21 deletions(-) diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 425e66fa1..b6015e530 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -20,12 +20,14 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. +import asyncio import yaml -import re import json import requests +import re from typing import Set from typing import List + from dephell_specifier import RangeSpecifier from packageurl import PackageURL @@ -33,7 +35,7 @@ from vulnerabilities.data_source import GitDataSourceConfiguration from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import Reference - +from vulnerabilities.package_managers import HexVersionAPI class ElixirSecurityDataSource(GitDataSource): def __enter__(self): @@ -43,6 +45,11 @@ def __enter__(self): self._added_files, self._updated_files = self.file_changes( recursive=True, file_ext="yml", subdir="./packages" ) + self.pkg_manager_api = HexVersionAPI() + self.set_api(self.collect_packages()) + + def set_api(self, packages): + asyncio.run(self.pkg_manager_api.load_api(packages)) def updated_advisories(self) -> Set[Advisory]: files = self._updated_files @@ -62,17 +69,25 @@ def added_advisories(self) -> Set[Advisory]: advisories.append(processed_data) return self.batch_advisories(advisories) - @staticmethod - def generate_all_version_list(pkg_name): - resp = requests.get(f"https://hex.pm/api/packages/{pkg_name}") - resp = resp.content - json_resp = json.loads(resp) - version_list = [] - for release in json_resp["releases"]: - version_list.append(release["version"]) + def collect_packages(self): + packages = set() + files = self._updated_files.union(self._added_files) + for f in files: + with open(f) as file: + data = yaml.safe_load(file) + if data.get("package"): + packages.add(data["package"]) + + return packages + + def generate_all_version_list(self,pkg_name): + if not getattr(self, 'pkg_manager_api', None): + self.pkg_manager_api = HexVersionAPI() + version_list = self.pkg_manager_api.get( + pkg_name) return version_list - def get_pkg_from_range(self, version_list, pkg_name): + def get_versions_from_range(self, version_list, pkg_name): pkg_versions = [] all_version_list = self.generate_all_version_list(pkg_name) if version_list is None: @@ -89,15 +104,16 @@ def process_file(self, path): pkg_name = yaml_file["package"] safe_pkg_versions = [] if yaml_file.get("unaffected_versions"): - safe_pkg_versions = self.get_pkg_from_range( + safe_pkg_versions = self.get_versions_from_range( yaml_file["patched_versions"] + yaml_file["unaffected_versions"], pkg_name, ) else: - safe_pkg_versions = self.get_pkg_from_range( + safe_pkg_versions = self.get_versions_from_range( yaml_file["patched_versions"], pkg_name ) - cve_id = yaml_file["cve"] + + cve_id = "CVE-"+str(yaml_file["cve"]) safe_purls = [] if safe_pkg_versions is not None: safe_purls = { @@ -107,6 +123,7 @@ def process_file(self, path): vuln_reference = [ Reference( + reference_id=yaml_file["id"], url=yaml_file["link"], ) ] diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index 220bfb4d5..896d27f3f 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -321,3 +321,24 @@ async def fetch(self, owner_repo: str, session) -> None: resp = await resp.json() print(resp) self.cache[owner_repo] = [release["ref"].split("/")[-1] for release in resp] + +class HexVersionAPI(VersionAPI): + async def load_api(self, pkg_set): + async with ClientSession(raise_for_status=True) as session: + await asyncio.gather( + *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] + ) + + async def fetch(self, pkg, session): + url = f"https://hex.pm/api/packages/{pkg}" + versions = set() + try: + response = await session.request(method="GET", url=url) + response = await response.json() + for release in response["releases"]: + versions.add(release["version"]) + except (ClientResponseError, JSONDecodeError): + pass + + self.cache[pkg] = versions + \ No newline at end of file diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py index e1bcc95cc..b1b14693d 100644 --- a/vulnerabilities/tests/test_elixir_security.py +++ b/vulnerabilities/tests/test_elixir_security.py @@ -22,13 +22,16 @@ import os from unittest import TestCase +from unittest.mock import patch from collections import OrderedDict from vulnerabilities.data_source import Reference from packageurl import PackageURL -from vulnerabilities.importers.elixir_security import ElixirSecurityDataSource from vulnerabilities.data_source import Advisory +from vulnerabilities.importers.elixir_security import ElixirSecurityDataSource +from vulnerabilities.package_managers import HexVersionAPI + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -37,11 +40,26 @@ class TestElixirSecurityDataSource(TestCase): @classmethod def setUpClass(cls): data_source_cfg = { - "repository_url": "https://test.net", + "repository_url": 'https://github.com/dependabot/elixir-security-advisories', } cls.data_src = ElixirSecurityDataSource(1, config=data_source_cfg) + cls.data_src.pkg_manager_api = HexVersionAPI() - def test_generate_all_version_list(self): + @patch('vulnerabilities.package_managers.HexVersionAPI.get', + return_value=[ + "0.5.2", + "0.5.1", + "0.5.0", + "0.4.0", + "0.3.1", + "0.3.0", + "0.2.0", + "0.1.3", + "0.1.2", + "0.1.1", + "0.1.0", + ]) + def test_generate_all_version_list(self,mock_write): package = "coherence" actual_list = self.data_src.generate_all_version_list(package) expected_list = [ @@ -58,8 +76,21 @@ def test_generate_all_version_list(self): "0.1.0", ] assert actual_list == expected_list - - def test_process_file(self): + @patch('vulnerabilities.package_managers.HexVersionAPI.get', + return_value=[ + "0.5.2", + "0.5.1", + "0.5.0", + "0.4.0", + "0.3.1", + "0.3.0", + "0.2.0", + "0.1.3", + "0.1.2", + "0.1.1", + "0.1.0", + ]) + def test_process_file(self,mock_write): path = os.path.join(BASE_DIR, "test_data/elixir_security/test_file.yml") expected_data = Advisory( @@ -75,9 +106,10 @@ def test_process_file(self): ), }, vuln_references=[ - Reference(url="https://github.com/smpallen99/coherence/issues/270") + Reference(reference_id='2aae6e3a-24a3-4d5f-86ff-b964eaf7c6d1', + url="https://github.com/smpallen99/coherence/issues/270") ], - cve_id="2018-20301", + cve_id="CVE-2018-20301", ) found_data = self.data_src.process_file(path) From 922e34f5ebb73e3189bcf23cde18ae9601619d46 Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Sat, 12 Dec 2020 13:25:59 +0530 Subject: [PATCH 4/8] fixed code style Signed-off-by: Tushar912 --- vulnerabilities/importers/elixir_security.py | 10 ++++++---- vulnerabilities/package_managers.py | 2 +- vulnerabilities/tests/test_elixir_security.py | 11 ++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index b6015e530..ea1e11bd9 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -37,6 +37,7 @@ from vulnerabilities.data_source import Reference from vulnerabilities.package_managers import HexVersionAPI + class ElixirSecurityDataSource(GitDataSource): def __enter__(self): super(ElixirSecurityDataSource, self).__enter__() @@ -79,8 +80,8 @@ def collect_packages(self): packages.add(data["package"]) return packages - - def generate_all_version_list(self,pkg_name): + + def generate_all_version_list(self, pkg_name): if not getattr(self, 'pkg_manager_api', None): self.pkg_manager_api = HexVersionAPI() version_list = self.pkg_manager_api.get( @@ -105,14 +106,15 @@ def process_file(self, path): safe_pkg_versions = [] if yaml_file.get("unaffected_versions"): safe_pkg_versions = self.get_versions_from_range( - yaml_file["patched_versions"] + yaml_file["unaffected_versions"], + yaml_file["patched_versions"] + + yaml_file["unaffected_versions"], pkg_name, ) else: safe_pkg_versions = self.get_versions_from_range( yaml_file["patched_versions"], pkg_name ) - + cve_id = "CVE-"+str(yaml_file["cve"]) safe_purls = [] if safe_pkg_versions is not None: diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index 896d27f3f..020025a07 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -322,6 +322,7 @@ async def fetch(self, owner_repo: str, session) -> None: print(resp) self.cache[owner_repo] = [release["ref"].split("/")[-1] for release in resp] + class HexVersionAPI(VersionAPI): async def load_api(self, pkg_set): async with ClientSession(raise_for_status=True) as session: @@ -341,4 +342,3 @@ async def fetch(self, pkg, session): pass self.cache[pkg] = versions - \ No newline at end of file diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py index b1b14693d..0d74e4426 100644 --- a/vulnerabilities/tests/test_elixir_security.py +++ b/vulnerabilities/tests/test_elixir_security.py @@ -58,8 +58,8 @@ def setUpClass(cls): "0.1.2", "0.1.1", "0.1.0", - ]) - def test_generate_all_version_list(self,mock_write): + ]) + def test_generate_all_version_list(self, mock_write): package = "coherence" actual_list = self.data_src.generate_all_version_list(package) expected_list = [ @@ -76,6 +76,7 @@ def test_generate_all_version_list(self,mock_write): "0.1.0", ] assert actual_list == expected_list + @patch('vulnerabilities.package_managers.HexVersionAPI.get', return_value=[ "0.5.2", @@ -89,8 +90,8 @@ def test_generate_all_version_list(self,mock_write): "0.1.2", "0.1.1", "0.1.0", - ]) - def test_process_file(self,mock_write): + ]) + def test_process_file(self, mock_write): path = os.path.join(BASE_DIR, "test_data/elixir_security/test_file.yml") expected_data = Advisory( @@ -107,7 +108,7 @@ def test_process_file(self,mock_write): }, vuln_references=[ Reference(reference_id='2aae6e3a-24a3-4d5f-86ff-b964eaf7c6d1', - url="https://github.com/smpallen99/coherence/issues/270") + url="https://github.com/smpallen99/coherence/issues/270") ], cve_id="CVE-2018-20301", ) From ee6dd622cfd007dd44b60c1806653fed9d4f30f7 Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Fri, 18 Dec 2020 21:15:27 +0530 Subject: [PATCH 5/8] initialize hexapi with cache and sort imports Signed-off-by: Tushar912 --- vulnerabilities/importers/elixir_security.py | 108 +++++++++--------- vulnerabilities/tests/test_elixir_security.py | 51 ++------- 2 files changed, 62 insertions(+), 97 deletions(-) diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index ea1e11bd9..1292349d6 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -17,17 +17,13 @@ # OR CONDITIONS OF ANY KIND, either express or implied. No content created from # VulnerableCode should be considered or used as legal advice. Consult an Attorney # for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. import asyncio -import yaml -import json -import requests -import re -from typing import Set -from typing import List +from typing import List, Set +import yaml from dephell_specifier import RangeSpecifier from packageurl import PackageURL @@ -74,23 +70,18 @@ def collect_packages(self): packages = set() files = self._updated_files.union(self._added_files) for f in files: - with open(f) as file: - data = yaml.safe_load(file) - if data.get("package"): - packages.add(data["package"]) + data = load_yaml(f) + if data.get("package"): + packages.add(data["package"]) return packages - def generate_all_version_list(self, pkg_name): + def get_versions_from_range(self, version_list, pkg_name): + pkg_versions = [] if not getattr(self, 'pkg_manager_api', None): self.pkg_manager_api = HexVersionAPI() - version_list = self.pkg_manager_api.get( + all_version_list = self.pkg_manager_api.get( pkg_name) - return version_list - - def get_versions_from_range(self, version_list, pkg_name): - pkg_versions = [] - all_version_list = self.generate_all_version_list(pkg_name) if version_list is None: return version_ranges = {RangeSpecifier(r) for r in version_list} @@ -100,40 +91,49 @@ def get_versions_from_range(self, version_list, pkg_name): return pkg_versions def process_file(self, path): - with open(path) as f: - yaml_file = yaml.safe_load(f) - pkg_name = yaml_file["package"] - safe_pkg_versions = [] - if yaml_file.get("unaffected_versions"): - safe_pkg_versions = self.get_versions_from_range( - yaml_file["patched_versions"] + - yaml_file["unaffected_versions"], - pkg_name, - ) - else: - safe_pkg_versions = self.get_versions_from_range( - yaml_file["patched_versions"], pkg_name - ) - - cve_id = "CVE-"+str(yaml_file["cve"]) - safe_purls = [] - if safe_pkg_versions is not None: - safe_purls = { - PackageURL(name=pkg_name, type="hex", version=version) - for version in safe_pkg_versions - } - - vuln_reference = [ - Reference( - reference_id=yaml_file["id"], - url=yaml_file["link"], - ) - ] - - return Advisory( - summary=yaml_file["description"], - impacted_package_urls=[], - resolved_package_urls=safe_purls, - cve_id=cve_id, - vuln_references=vuln_reference, + yaml_file = load_yaml(path) + pkg_name = yaml_file["package"] + safe_pkg_versions = [] + if yaml_file.get("unaffected_versions"): + safe_pkg_versions = self.get_versions_from_range( + yaml_file["patched_versions"] + + yaml_file["unaffected_versions"], + pkg_name, + ) + else: + safe_pkg_versions = self.get_versions_from_range( + yaml_file["patched_versions"], pkg_name + ) + if yaml_file.get('cve'): + cve_id = "CVE-" + yaml_file["cve"] + else: + cve_id = "" + + safe_purls = [] + if safe_pkg_versions is not None: + safe_purls = { + PackageURL(name=pkg_name, type="hex", version=version) + for version in safe_pkg_versions + } + + vuln_reference = [ + Reference( + reference_id=yaml_file["id"], + ), + Reference( + url=yaml_file["link"], ) + ] + + return Advisory( + summary=yaml_file["description"], + impacted_package_urls=[], + resolved_package_urls=safe_purls, + cve_id=cve_id, + vuln_references=vuln_reference, + ) + + +def load_yaml(path): + with open(path) as f: + return yaml.safe_load(f) diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py index 0d74e4426..ede63c50a 100644 --- a/vulnerabilities/tests/test_elixir_security.py +++ b/vulnerabilities/tests/test_elixir_security.py @@ -17,22 +17,20 @@ # OR CONDITIONS OF ANY KIND, either express or implied. No content created from # VulnerableCode should be considered or used as legal advice. Consult an Attorney # for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. import os -from unittest import TestCase -from unittest.mock import patch from collections import OrderedDict +from unittest import TestCase -from vulnerabilities.data_source import Reference from packageurl import PackageURL from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import Reference from vulnerabilities.importers.elixir_security import ElixirSecurityDataSource from vulnerabilities.package_managers import HexVersionAPI - BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -43,26 +41,7 @@ def setUpClass(cls): "repository_url": 'https://github.com/dependabot/elixir-security-advisories', } cls.data_src = ElixirSecurityDataSource(1, config=data_source_cfg) - cls.data_src.pkg_manager_api = HexVersionAPI() - - @patch('vulnerabilities.package_managers.HexVersionAPI.get', - return_value=[ - "0.5.2", - "0.5.1", - "0.5.0", - "0.4.0", - "0.3.1", - "0.3.0", - "0.2.0", - "0.1.3", - "0.1.2", - "0.1.1", - "0.1.0", - ]) - def test_generate_all_version_list(self, mock_write): - package = "coherence" - actual_list = self.data_src.generate_all_version_list(package) - expected_list = [ + cls.data_src.pkg_manager_api = HexVersionAPI({'coherence': [ "0.5.2", "0.5.1", "0.5.0", @@ -74,24 +53,9 @@ def test_generate_all_version_list(self, mock_write): "0.1.2", "0.1.1", "0.1.0", - ] - assert actual_list == expected_list + ]}) - @patch('vulnerabilities.package_managers.HexVersionAPI.get', - return_value=[ - "0.5.2", - "0.5.1", - "0.5.0", - "0.4.0", - "0.3.1", - "0.3.0", - "0.2.0", - "0.1.3", - "0.1.2", - "0.1.1", - "0.1.0", - ]) - def test_process_file(self, mock_write): + def test_process_file(self): path = os.path.join(BASE_DIR, "test_data/elixir_security/test_file.yml") expected_data = Advisory( @@ -108,7 +72,8 @@ def test_process_file(self, mock_write): }, vuln_references=[ Reference(reference_id='2aae6e3a-24a3-4d5f-86ff-b964eaf7c6d1', - url="https://github.com/smpallen99/coherence/issues/270") + ), + Reference(url='https://github.com/smpallen99/coherence/issues/270') ], cve_id="CVE-2018-20301", ) From 6e62cdf89c0d135e2bda0b6e2e8d177996278ed0 Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Mon, 21 Dec 2020 18:34:32 +0530 Subject: [PATCH 6/8] add license and categorize left versions as vuln Signed-off-by: Tushar912 --- vulnerabilities/importer_yielder.py | 2 +- vulnerabilities/importers/elixir_security.py | 62 ++++++------ vulnerabilities/tests/test_elixir_security.py | 96 +++++++++++++++---- 3 files changed, 109 insertions(+), 51 deletions(-) diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 8189efae4..9400979b3 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -228,7 +228,7 @@ }, { 'name': 'elixir_security', - 'license': '', + 'license': 'cc0-1.0', 'last_run': None, 'data_source': 'ElixirSecurityDataSource', 'data_source_cfg': { diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 18297d84a..671ebf4b9 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -78,58 +78,62 @@ def collect_packages(self): return packages def get_versions_from_range(self, version_list, pkg_name): - pkg_versions = [] - if not getattr(self, 'pkg_manager_api', None): - self.pkg_manager_api = HexVersionAPI() - all_version_list = self.pkg_manager_api.get( - pkg_name) - if version_list is None: - return + safe_pkg_versions = [] + vuln_pkg_versions = [] + all_version_list = self.pkg_manager_api.get(pkg_name) + if not version_list: + return [], all_version_list version_ranges = {RangeSpecifier(r) for r in version_list} for version in all_version_list: if any([version in v for v in version_ranges]): - pkg_versions.append(version) - return pkg_versions + safe_pkg_versions.append(version) + + vuln_pkg_versions = set(all_version_list) - set(safe_pkg_versions) + return safe_pkg_versions, vuln_pkg_versions def process_file(self, path): yaml_file = load_yaml(path) pkg_name = yaml_file["package"] safe_pkg_versions = [] - if yaml_file.get("unaffected_versions"): - safe_pkg_versions = self.get_versions_from_range( - yaml_file["patched_versions"] + - yaml_file["unaffected_versions"], - pkg_name, - ) - else: - safe_pkg_versions = self.get_versions_from_range( - yaml_file["patched_versions"], pkg_name - ) - if yaml_file.get('cve'): + vuln_pkg_versions = [] + if not yaml_file.get("patched_versions"): + yaml_file["patched_versions"] = [] + if not yaml_file.get("unaffected_versions"): + yaml_file["unaffected_versions"] = [] + safe_pkg_versions, vuln_pkg_versions = self.get_versions_from_range( + yaml_file.get("patched_versions", []) + yaml_file.get("unaffected_versions", []), + pkg_name, + ) + + if yaml_file.get("cve"): cve_id = "CVE-" + yaml_file["cve"] else: cve_id = "" safe_purls = [] - if safe_pkg_versions is not None: - safe_purls = { - PackageURL(name=pkg_name, type="hex", version=version) - for version in safe_pkg_versions - } + vuln_purls = [] + + safe_purls = { + PackageURL(name=pkg_name, type="hex", version=version) for version in safe_pkg_versions + } + + vuln_purls = { + PackageURL(name=pkg_name, type="hex", version=version) for version in vuln_pkg_versions + } - vuln_reference = [ + vuln_references = [ Reference( reference_id=yaml_file["id"], ), Reference( url=yaml_file["link"], - ) + ), ] return Advisory( summary=yaml_file["description"], - impacted_package_urls=[], + impacted_package_urls=vuln_purls, resolved_package_urls=safe_purls, cve_id=cve_id, - vuln_references=vuln_reference, + vuln_references=vuln_references, ) diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py index ede63c50a..717189c98 100644 --- a/vulnerabilities/tests/test_elixir_security.py +++ b/vulnerabilities/tests/test_elixir_security.py @@ -38,31 +38,84 @@ class TestElixirSecurityDataSource(TestCase): @classmethod def setUpClass(cls): data_source_cfg = { - "repository_url": 'https://github.com/dependabot/elixir-security-advisories', + "repository_url": "https://github.com/dependabot/elixir-security-advisories", } cls.data_src = ElixirSecurityDataSource(1, config=data_source_cfg) - cls.data_src.pkg_manager_api = HexVersionAPI({'coherence': [ - "0.5.2", - "0.5.1", - "0.5.0", - "0.4.0", - "0.3.1", - "0.3.0", - "0.2.0", - "0.1.3", - "0.1.2", - "0.1.1", - "0.1.0", - ]}) + cls.data_src.pkg_manager_api = HexVersionAPI( + { + "coherence": [ + "0.5.2", + "0.5.1", + "0.5.0", + "0.4.0", + "0.3.1", + "0.3.0", + "0.2.0", + "0.1.3", + "0.1.2", + "0.1.1", + "0.1.0", + ] + } + ) def test_process_file(self): path = os.path.join(BASE_DIR, "test_data/elixir_security/test_file.yml") expected_data = Advisory( - summary=( - 'The Coherence library has "Mass Assignment"-like vulnerabilities.\n' - ), - impacted_package_urls=[], + summary=('The Coherence library has "Mass Assignment"-like vulnerabilities.\n'), + impacted_package_urls={ + PackageURL( + type="hex", + name="coherence", + version="0.5.1", + ), + PackageURL( + type="hex", + name="coherence", + version="0.5.0", + ), + PackageURL( + type="hex", + name="coherence", + version="0.4.0", + ), + PackageURL( + type="hex", + name="coherence", + version="0.3.1", + ), + PackageURL( + type="hex", + name="coherence", + version="0.3.0", + ), + PackageURL( + type="hex", + name="coherence", + version="0.2.0", + ), + PackageURL( + type="hex", + name="coherence", + version="0.1.3", + ), + PackageURL( + type="hex", + name="coherence", + version="0.1.2", + ), + PackageURL( + type="hex", + name="coherence", + version="0.1.1", + ), + PackageURL( + type="hex", + name="coherence", + version="0.1.0", + ), + }, resolved_package_urls={ PackageURL( type="hex", @@ -71,9 +124,10 @@ def test_process_file(self): ), }, vuln_references=[ - Reference(reference_id='2aae6e3a-24a3-4d5f-86ff-b964eaf7c6d1', - ), - Reference(url='https://github.com/smpallen99/coherence/issues/270') + Reference( + reference_id="2aae6e3a-24a3-4d5f-86ff-b964eaf7c6d1", + ), + Reference(url="https://github.com/smpallen99/coherence/issues/270"), ], cve_id="CVE-2018-20301", ) From fd25df38d56d7520901058c2c8b1bd9d37c793d1 Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Wed, 23 Dec 2020 02:49:54 +0530 Subject: [PATCH 7/8] change name of function get_version_from range and add comments in it Signed-off-by: Tushar912 --- vulnerabilities/importers/elixir_security.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 671ebf4b9..8504a306a 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -77,13 +77,17 @@ def collect_packages(self): return packages - def get_versions_from_range(self, version_list, pkg_name): + def get_versions_for_pkg_from_range_list(self, version_range_list, pkg_name): + # Takes a list of version ranges(pathced and unaffected) of a package + # as parameter and returns a tuple of safe package versions and + # vulnerable package versions + safe_pkg_versions = [] vuln_pkg_versions = [] all_version_list = self.pkg_manager_api.get(pkg_name) - if not version_list: + if not version_range_list: return [], all_version_list - version_ranges = {RangeSpecifier(r) for r in version_list} + version_ranges = {RangeSpecifier(r) for r in version_range_list} for version in all_version_list: if any([version in v for v in version_ranges]): safe_pkg_versions.append(version) @@ -98,10 +102,12 @@ def process_file(self, path): vuln_pkg_versions = [] if not yaml_file.get("patched_versions"): yaml_file["patched_versions"] = [] + if not yaml_file.get("unaffected_versions"): yaml_file["unaffected_versions"] = [] - safe_pkg_versions, vuln_pkg_versions = self.get_versions_from_range( - yaml_file.get("patched_versions", []) + yaml_file.get("unaffected_versions", []), + + safe_pkg_versions, vuln_pkg_versions = self.get_versions_for_pkg_from_range_list( + yaml_file.get("patched_versions") + yaml_file.get("unaffected_versions"), pkg_name, ) From 0fa1bf4175d47fcd136052de3fc4de72f4e8479c Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Fri, 25 Dec 2020 18:17:56 +0530 Subject: [PATCH 8/8] remove unused import and add name to authors and add importer to sources Signed-off-by: Tushar912 remove single quote at end of url Signed-off-by: Tushar912 --- AUTHORS.rst | 3 ++- SOURCES.rst | 2 ++ vulnerabilities/importers/elixir_security.py | 3 +-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index a7fe33387..039459f93 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -10,4 +10,5 @@ The following organizations or individuals have contributed to this repo: - Ayush Lohani @lohani2280 - Islam Elhakmi @EslamHiko - Edoardo Lanzini @elanzini -- Navonil Das @NavonilDas \ No newline at end of file +- Navonil Das @NavonilDas +- Tushar Upadhyay @tushar912 \ No newline at end of file diff --git a/SOURCES.rst b/SOURCES.rst index 02b2ff6fd..4bba04be2 100644 --- a/SOURCES.rst +++ b/SOURCES.rst @@ -43,3 +43,5 @@ +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ |postgresql | https://www.postgresql.org/support/security/ |postgresql | +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ +|elixir_security | https://github.com/dependabot/elixir-security-advisories |hex packages | ++----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 8504a306a..9293b0ded 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -28,7 +28,6 @@ from packageurl import PackageURL from vulnerabilities.data_source import GitDataSource -from vulnerabilities.data_source import GitDataSourceConfiguration from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import Reference from vulnerabilities.package_managers import HexVersionAPI @@ -107,7 +106,7 @@ def process_file(self, path): yaml_file["unaffected_versions"] = [] safe_pkg_versions, vuln_pkg_versions = self.get_versions_for_pkg_from_range_list( - yaml_file.get("patched_versions") + yaml_file.get("unaffected_versions"), + yaml_file["patched_versions"] + yaml_file["unaffected_versions"], pkg_name, )