Skip to content

Commit 6fe0d62

Browse files
TG1999TG1999
committed
Migrate istio importer
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent 92d1bf0 commit 6fe0d62

2 files changed

Lines changed: 82 additions & 119 deletions

File tree

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from vulnerabilities.importers import debian_oval
1414
from vulnerabilities.importers import github
1515
from vulnerabilities.importers import gitlab
16+
from vulnerabilities.importers import istio
1617
from vulnerabilities.importers import nginx
1718
from vulnerabilities.importers import npm
1819
from vulnerabilities.importers import nvd
@@ -41,6 +42,7 @@
4142
debian_oval.DebianOvalImporter,
4243
npm.NpmImporter,
4344
retiredotnet.RetireDotnetImporter,
45+
istio.IstioImporter,
4446
]
4547

4648
IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

‎vulnerabilities/importers/istio.py‎

Lines changed: 80 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -6,165 +6,126 @@
66
# See https://github.com/nexB/vulnerablecode for support or download.
77
# See https://aboutcode.org for more information about nexB OSS projects.
88
#
9-
import asyncio
109
import re
10+
from pathlib import Path
1111
from typing import Set
1212

1313
import pytz
1414
import saneyaml
1515
from dateutil import parser
1616
from packageurl import PackageURL
17-
from univers.version_range import VersionRange
17+
from univers.version_range import GitHubVersionRange
18+
from univers.version_range import GolangVersionRange
1819
from univers.versions import SemverVersion
1920

2021
from vulnerabilities.importer import AdvisoryData
21-
from vulnerabilities.importer import GitImporter
22+
from vulnerabilities.importer import AffectedPackage
23+
from vulnerabilities.importer import Importer
2224
from vulnerabilities.importer import Reference
23-
from vulnerabilities.package_managers import GitHubTagsAPI
24-
from vulnerabilities.utils import nearest_patched_package
2525
from vulnerabilities.utils import split_markdown_front_matter
2626

2727
is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match
2828

2929

30-
class IstioImporter(GitImporter):
31-
def __enter__(self):
32-
super(IstioImporter, self).__enter__()
30+
class IstioImporter(Importer):
31+
spdx_license_expression = "Apache-2.0"
32+
license_url = "https://github.com/istio/istio.io/blob/master/LICENSE"
3333

34-
if not getattr(self, "_added_files", None):
35-
self._added_files, self._updated_files = self.file_changes(
36-
recursive=True, file_ext="md", subdir="./content/en/news/security"
37-
)
38-
self.version_api = GitHubTagsAPI()
39-
self.set_api()
40-
41-
def set_api(self):
42-
asyncio.run(self.version_api.load_api(["istio/istio"]))
43-
44-
def updated_advisories(self) -> Set[AdvisoryData]:
45-
files = self._added_files.union(self._updated_files)
46-
advisories = []
47-
for f in files:
34+
def advisory_data(self) -> Set[AdvisoryData]:
35+
self.clone(repo_url="git+https://github.com/istio/istio.io/")
36+
path = Path(self.vcs_response.dest_dir)
37+
vuln = path / "content/en/news/security/"
38+
for f in vuln.glob("**/*.md"):
4839
# Istio website has files with name starting with underscore, these contain metadata
4940
# required for rendering the website. We're not interested in these.
5041
# See also https://github.com/nexB/vulnerablecode/issues/563
42+
f = str(f)
5143
if f.endswith("_index.md"):
5244
continue
53-
processed_data = self.process_file(f)
54-
if processed_data:
55-
advisories.extend(processed_data)
56-
return self.batch_advisories(advisories)
57-
58-
def get_pkg_versions_from_ranges(self, version_range_list, release_date):
59-
"""Takes a list of version ranges(affected) of a package
60-
as parameter and returns a tuple of safe package versions and
61-
vulnerable package versions"""
62-
all_version = self.version_api.get("istio/istio", release_date).valid_versions
63-
safe_pkg_versions = []
64-
vuln_pkg_versions = []
65-
version_ranges = [
66-
VersionRange.from_scheme_version_spec_string("semver", r) for r in version_range_list
67-
]
68-
for version in all_version:
69-
version_obj = SemverVersion(version)
70-
if any([version_obj in v for v in version_ranges]):
71-
vuln_pkg_versions.append(version)
72-
73-
safe_pkg_versions = set(all_version) - set(vuln_pkg_versions)
74-
return safe_pkg_versions, vuln_pkg_versions
45+
yield from self.process_file(f)
7546

7647
def process_file(self, path):
7748

78-
advisories = []
79-
8049
data = self.get_data_from_md(path)
8150
release_date = parser.parse(data["publishdate"]).replace(tzinfo=pytz.UTC)
8251

83-
releases = []
84-
if data.get("releases"):
85-
for release in data["releases"]:
86-
# If it is of form "All releases prior to x"
87-
if "All releases prior" in release:
88-
release = release.strip()
89-
release = release.split(" ")
90-
releases.append("<" + release[4])
91-
92-
# Eg. 'All releases 1.5 and later'
93-
elif "All releases" in release and "and later" in release:
94-
release = release.split()[2].strip()
95-
releases.append(f">={release}")
96-
97-
elif "to" in release:
98-
release = release.strip()
99-
release = release.split(" ")
100-
lbound = ">=" + release[0]
101-
ubound = "<=" + release[2]
102-
releases.append(lbound + "," + ubound)
103-
# If it is a single release
104-
elif is_release(release):
105-
releases.append(release)
106-
107-
data["release_ranges"] = releases
108-
109-
if not data.get("cves"):
110-
data["cves"] = [""]
111-
112-
for cve_id in data["cves"]:
52+
constraints = []
53+
54+
for release in data.get("releases") or []:
55+
# If it is of form "All releases prior to x"
56+
if "All releases prior" in release:
57+
release = release.strip()
58+
release = release.split(" ")
59+
constraints.append(
60+
VersionConstraint(version=SemverVersion(release[4]), comparator="<")
61+
)
62+
63+
# Eg. 'All releases 1.5 and later'
64+
elif "All releases" in release and "and later" in release:
65+
release = release.split()[2].strip()
66+
constraints.append(
67+
VersionConstraint(version=SemverVersion(release), comparator=">")
68+
)
69+
70+
elif "to" in release:
71+
release = release.strip()
72+
release = release.split(" ")
73+
constraints.append(
74+
VersionConstraint(version=SemverVersion(release[0]), comparator=">=")
75+
)
76+
constraints.append(
77+
VersionConstraint(version=SemverVersion(release[2]), comparator="<=")
78+
)
79+
80+
# If it is a single release
81+
elif is_release(release):
82+
constraints.append(
83+
VersionConstraint(version=SemverVersion(release), comparator="=")
84+
)
85+
86+
for cve_id in data.get("cves") or []:
11387

11488
if not cve_id.startswith("CVE"):
11589
cve_id = ""
11690

117-
safe_pkg_versions = []
118-
vuln_pkg_versions = []
119-
12091
if not data.get("release_ranges"):
12192
data["release_ranges"] = []
12293

123-
safe_pkg_versions, vuln_pkg_versions = self.get_pkg_versions_from_ranges(
124-
data["release_ranges"], release_date
125-
)
126-
12794
affected_packages = []
12895

129-
safe_purls_golang = [
130-
PackageURL(type="golang", name="istio", version=version)
131-
for version in safe_pkg_versions
132-
]
133-
134-
vuln_purls_golang = [
135-
PackageURL(type="golang", name="istio", version=version)
136-
for version in vuln_pkg_versions
137-
]
138-
139-
affected_packages.extend(nearest_patched_package(vuln_purls_golang, safe_purls_golang))
140-
141-
safe_purls_github = [
142-
PackageURL(type="github", name="istio", version=version)
143-
for version in safe_pkg_versions
144-
]
145-
146-
vuln_purls_github = [
147-
PackageURL(type="github", name="istio", version=version)
148-
for version in vuln_pkg_versions
149-
]
150-
151-
affected_packages.extend(nearest_patched_package(vuln_purls_github, safe_purls_github))
152-
153-
advisories.append(
154-
AdvisoryData(
155-
vulnerability_id=cve_id,
156-
summary=data["description"],
157-
affected_packages=affected_packages,
158-
references=[
159-
Reference(
160-
reference_id=data["title"],
161-
url=f"https://istio.io/latest/news/security/{data['title']}/",
162-
)
163-
],
96+
affected_packages.append(
97+
AffectedPackage(
98+
package=PackageURL(type="golang", name="istio"),
99+
affected_version_range=GolangVersionRange(constraints=constraints),
100+
)
101+
)
102+
103+
affected_packages.append(
104+
AffectedPackage(
105+
package=PackageURL(type="github", name="istio"),
106+
affected_version_range=GitHubVersionRange(constraints=constraints),
164107
)
165108
)
166109

167-
return advisories
110+
title = data.get("title") or ""
111+
references = []
112+
if title:
113+
references.append(
114+
Reference(
115+
reference_id=title,
116+
url=f"https://istio.io/latest/news/security/{title}/",
117+
)
118+
)
119+
120+
summary = data.get("description") or ""
121+
122+
yield AdvisoryData(
123+
aliases=[cve_id],
124+
summary=summary,
125+
affected_packages=affected_packages,
126+
references=references,
127+
date_published=release_date,
128+
)
168129

169130
def get_data_from_md(self, path):
170131
"""Return a mapping of vulnerability data extracted from an advisory."""

0 commit comments

Comments
 (0)