diff --git a/requirements.txt b/requirements.txt index 621436e00..1d079bd0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,6 @@ pytest-mock==1.13.0 pytz==2019.3 PyYAML==5.3 saneyaml==0.4 -semantic-version==2.8.2 six==1.13.0 soupsieve==1.9.5 sqlparse==0.3.0 diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py index 831252c37..7e8368876 100644 --- a/vulnerabilities/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -174,6 +174,7 @@ def npm_dump(extract_data): for version in data['affected_versions']: package_affected = Package.objects.create( name=package_name, + type='npm', version=version, ) ImpactedPackage.objects.create( diff --git a/vulnerabilities/scraper/npm.py b/vulnerabilities/scraper/npm.py index cd698c4e1..133132aec 100644 --- a/vulnerabilities/scraper/npm.py +++ b/vulnerabilities/scraper/npm.py @@ -22,38 +22,30 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import json -import re -import semantic_version +from dephell_specifier import RangeSpecifier from urllib.request import urlopen +from urllib.error import HTTPError + NPM_URL = 'https://registry.npmjs.org{}' PAGE = '/-/npm/v1/security/advisories?page=0' -def remove_spaces(x): - """ - Remove Multiple Space, spaces after relational operator - and remove v charecter in front of version string (ex v1.2.3) - """ - x = re.sub(r' +', ' ', x) - x = re.sub(r'< +', '<', x) - x = re.sub(r'> +', '>', x) - x = re.sub(r'<= +', '<=', x) - x = re.sub(r'>= +', '>=', x) - x = re.sub(r'>=[vV]', '>=', x) - x = re.sub(r'<=[vV]', '<=', x) - x = re.sub(r'>[vV]', '>', x) - x = re.sub(r'<[vV]', '<', x) - return x - - def get_all_versions(package_name): """ Returns all versions available for a module """ + package_name = package_name.strip() package_url = NPM_URL.format(f'/{package_name}') - with urlopen(package_url) as response: - data = json.load(response) + try: + with urlopen(package_url) as response: + data = json.load(response) + except HTTPError as e: + if e.code == 404: + return [] + else: + raise + # NPM registry has no data regarding this package, we skip these return [v for v in data.get('versions', {})] @@ -62,20 +54,21 @@ def extract_versions(package_name, aff_version_range, fixed_version_range): Seperate list of affected versions and unaffected versions from all versions using the ranges specified. """ - aff_spec = semantic_version.NpmSpec(remove_spaces(aff_version_range)) - fix_spec = semantic_version.NpmSpec(remove_spaces(fixed_version_range)) + aff_spec = RangeSpecifier(aff_version_range) + fix_spec = RangeSpecifier(fixed_version_range) all_ver = get_all_versions(package_name) + if not all_ver: + # NPM registry has no data regarding this package, we skip these + return ([], []) aff_ver = [] fix_ver = [] # Unaffected version is that version which is in the fixed_version_range # or which is absent in the aff_version_range for ver in all_ver: - cur_version = semantic_version.Version(ver) - if cur_version in fix_spec or cur_version not in aff_spec: + if ver in fix_spec or ver not in aff_spec: fix_ver.append(ver) else: aff_ver.append(ver) - return (aff_ver, fix_ver) @@ -96,6 +89,9 @@ def extract_data(JSON): obj.get('vulnerable_versions', ''), obj.get('patched_versions', '') ) + if not affected_versions and not fixed_versions: + continue + # NPM registry has no data regarding this package finally we skip these package_vulnerabilities.append({ 'package_name': package_name, @@ -113,12 +109,19 @@ def scrape_vulnerabilities(): """ Extract JSON From NPM registry """ - nextpage = PAGE package_vulnerabilities = [] + nextpage = PAGE while nextpage: - cururl = NPM_URL.format(nextpage) - response = json.load(urlopen(cururl)) - package_vulnerabilities.extend(extract_data(response)) - next_page = response.get('urls', {}).get('next') + try: + cururl = NPM_URL.format(nextpage) + response = json.load(urlopen(cururl)) + package_vulnerabilities.extend(extract_data(response)) + nextpage = response.get('urls', {}).get('next') + + except HTTPError as error: + if error.code == 404: + break + else: + raise return package_vulnerabilities diff --git a/vulnerabilities/tests/test_npm.py b/vulnerabilities/tests/test_npm.py index 2eb2644cc..8fdc1a057 100644 --- a/vulnerabilities/tests/test_npm.py +++ b/vulnerabilities/tests/test_npm.py @@ -27,20 +27,11 @@ from vulnerabilities.scraper.npm import extract_data from vulnerabilities.scraper.npm import get_all_versions -from vulnerabilities.scraper.npm import remove_spaces BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data/') -def test_remove_space(): - res = remove_spaces(">= 1.2.1 || <= 2.1.1") - assert res == '>=1.2.1 || <=2.1.1' - - res = remove_spaces(">= v1.2.1 || <= V2.1.1") - assert res == '>=1.2.1 || <=2.1.1' - - def test_get_all_versions(): x = get_all_versions('electron') expected = ['0.1.2', '2.0.0', '3.0.0',