Skip to content

Commit f520627

Browse files
keshav-spacekeshav-space
committed
add GitLab DataSource
Signed-off-by: Keshav Priyadarshi <git@keshav.space>
1 parent 78dd5ae commit f520627

3 files changed

Lines changed: 321 additions & 1 deletion

File tree

‎vulntotal/datasources/__init__.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
2323

2424

25-
DATASOURCE_REGISTRY = []
25+
from vulntotal.datasources import gitlab
26+
27+
DATASOURCE_REGISTRY = [
28+
gitlab.GitlabDataSource,
29+
]
2630

2731
DATASOURCE_REGISTRY = {x.__module__.split(".")[-1]: x for x in DATASOURCE_REGISTRY}

‎vulntotal/datasources/gitlab.py‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
4+
# The VulnTotal software is licensed under the Apache License version 2.0.
5+
# Data generated with VulnTotal require an acknowledgment.
6+
#
7+
# You may not use this software except in compliance with the License.
8+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
9+
# Unless required by applicable law or agreed to in writing, software distributed
10+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
# specific language governing permissions and limitations under the License.
13+
#
14+
# When you publish or redistribute any data created with VulnTotal or any VulnTotal
15+
# derivative work, you must accompany this data with the following acknowledgment:
16+
#
17+
# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
18+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
19+
# VulnTotal should be considered or used as legal advice. Consult an Attorney
20+
# for any legal advice.
21+
# VulnTotal is a free software tool from nexB Inc. and others.
22+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
23+
24+
25+
import json
26+
import logging
27+
import os
28+
import shutil
29+
import tarfile
30+
from pathlib import Path
31+
from typing import Iterable
32+
33+
import requests
34+
import saneyaml
35+
from fetchcode import fetch
36+
from packageurl import PackageURL
37+
38+
from vulntotal.validator import DataSource
39+
from vulntotal.validator import VendorData
40+
from vulntotal.vulntotal_utils import gitlab_constraints_satisfied
41+
42+
logger = logging.getLogger(__name__)
43+
44+
45+
class GitlabDataSource(DataSource):
46+
spdx_license_expression = "TODO"
47+
license_url = "TODO"
48+
49+
def datasource_advisory(self, purl) -> Iterable[VendorData]:
50+
package_slug = get_package_slug(purl)
51+
location = download_subtree(package_slug, speculative_execution=True)
52+
if not location:
53+
clear_download(location)
54+
path = self.supported_ecosystem()[purl.type]
55+
casesensitive_package_slug = get_casesensitive_slug(path, package_slug)
56+
location = download_subtree(casesensitive_package_slug)
57+
if location:
58+
interesting_advisories = parse_interesting_advisories(location, purl.version, delete_download=True)
59+
return interesting_advisories
60+
clear_download(location)
61+
62+
@classmethod
63+
def supported_ecosystem(cls):
64+
return {
65+
"composer": "packagist",
66+
"conan": "conan",
67+
"gem": "gem",
68+
"golang": "go",
69+
"maven": "maven",
70+
"npm": "npm",
71+
"nuget": "nuget",
72+
"pypi": "pypi",
73+
}
74+
75+
76+
def get_package_slug(purl):
77+
supported_ecosystem = GitlabDataSource.supported_ecosystem()
78+
79+
if purl.type not in supported_ecosystem:
80+
return
81+
82+
ecosystem = supported_ecosystem[purl.type]
83+
package_name = purl.name
84+
85+
if purl.type in ("maven", "composer", "golang"):
86+
package_name = f"{purl.namespace}/{purl.name}"
87+
88+
return f"{ecosystem}/{package_name}"
89+
90+
91+
def download_subtree(package_slug: str, speculative_execution=False):
92+
url = f"https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/archive/master/gemnasium-db-master.tar.gz?path={package_slug}"
93+
response = fetch(url)
94+
if os.path.getsize(response.location) > 0:
95+
extracted_location = Path(response.location).parent.joinpath(
96+
"temp_vulntotal_gitlab_datasource"
97+
)
98+
with tarfile.open(response.location, "r") as file_obj:
99+
file_obj.extractall(extracted_location)
100+
os.remove(response.location)
101+
return extracted_location
102+
if not speculative_execution:
103+
logger.error(f"{package_slug} doesn't exist")
104+
os.remove(response.location)
105+
106+
107+
def clear_download(location):
108+
if location:
109+
shutil.rmtree(location)
110+
111+
112+
def get_casesensitive_slug(path, package_slug):
113+
payload = [
114+
{
115+
"operationName": "getPaginatedTree",
116+
"variables": {
117+
"projectPath": "gitlab-org/security-products/gemnasium-db",
118+
"ref": "master",
119+
"path": path,
120+
"nextPageCursor": "",
121+
"pageSize": 100,
122+
},
123+
"query": """
124+
fragment TreeEntry on Entry {
125+
flatPath
126+
}
127+
query getPaginatedTree($projectPath: ID!, $path: String, $ref: String!, $nextPageCursor: String) {
128+
project(fullPath: $projectPath) {
129+
repository {
130+
paginatedTree(path: $path, ref: $ref, after: $nextPageCursor) {
131+
pageInfo {
132+
endCursor
133+
startCursor
134+
hasNextPage
135+
}
136+
nodes {
137+
trees {
138+
nodes {
139+
...TreeEntry
140+
}
141+
}
142+
}
143+
}
144+
}
145+
}
146+
} """,
147+
}
148+
]
149+
url = "https://gitlab.com/api/graphql"
150+
hasnext = True
151+
152+
while hasnext:
153+
response = requests.post(url, json=payload).json()
154+
paginated_tree = response[0]["data"]["project"]["repository"]["paginatedTree"]
155+
156+
for slug in paginated_tree["nodes"][0]["trees"]["nodes"]:
157+
if slug["flatPath"].lower() == package_slug.lower():
158+
return slug["flatPath"]
159+
160+
# If the namespace/subfolder contains multiple packages, then progressive transverse through folders tree
161+
if package_slug.lower().startswith(slug["flatPath"].lower()):
162+
return get_gitlab_style_slug(slug["flatPath"], package_slug)
163+
164+
payload[0]["variables"]["nextPageCursor"] = paginated_tree["pageInfo"]["endCursor"]
165+
hasnext = paginated_tree["pageInfo"]["hasNextPage"]
166+
167+
168+
def parse_interesting_advisories(location, version, delete_download=False) -> Iterable[VendorData]:
169+
path = Path(location)
170+
glob = "**/*.yml"
171+
files = (p for p in path.glob(glob) if p.is_file())
172+
for file in files:
173+
with open(file) as f:
174+
gitlab_advisory = saneyaml.load(f)
175+
if gitlab_constraints_satisfied(gitlab_advisory["affected_range"], version):
176+
yield VendorData(
177+
aliases=sorted(gitlab_advisory["identifiers"]),
178+
affected_versions=[gitlab_advisory["affected_range"]],
179+
fixed_versions=sorted(gitlab_advisory["fixed_versions"]),
180+
)
181+
if delete_download:
182+
clear_download(location)

‎vulntotal/vulntotal_utils.py‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
4+
# The VulnTotal software is licensed under the Apache License version 2.0.
5+
# Data generated with VulnTotal require an acknowledgment.
6+
#
7+
# You may not use this software except in compliance with the License.
8+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
9+
# Unless required by applicable law or agreed to in writing, software distributed
10+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
# specific language governing permissions and limitations under the License.
13+
#
14+
# When you publish or redistribute any data created with VulnTotal or any VulnTotal
15+
# derivative work, you must accompany this data with the following acknowledgment:
16+
#
17+
# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
18+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
19+
# VulnTotal should be considered or used as legal advice. Consult an Attorney
20+
# for any legal advice.
21+
# VulnTotal is a free software tool from nexB Inc. and others.
22+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
23+
24+
import operator
25+
26+
27+
class GenericVersion:
28+
def __init__(self, version):
29+
self.value = version.replace(" ", "").lstrip("v")
30+
31+
self.decomposed = tuple(
32+
[int(com) if com.isnumeric() else com for com in self.value.split(".")]
33+
)
34+
35+
def __str__(self):
36+
return str(self.value)
37+
38+
def __eq__(self, other):
39+
if not isinstance(other, self.__class__):
40+
return NotImplemented
41+
return self.value.__eq__(other.value)
42+
43+
def __lt__(self, other):
44+
if not isinstance(other, self.__class__):
45+
return NotImplemented
46+
for i, j in zip(self.decomposed, other.decomposed):
47+
if not isinstance(i, type(j)):
48+
continue
49+
if i.__gt__(j):
50+
return False
51+
return True
52+
53+
def __le__(self, other):
54+
if not isinstance(other, self.__class__):
55+
return NotImplemented
56+
return self.__lt__(other) or self.__eq__(other)
57+
58+
59+
def compare(version, package_comparator, package_version):
60+
operator_comparator = {
61+
"<": operator.lt,
62+
">": operator.gt,
63+
"=": operator.eq,
64+
"<=": operator.le,
65+
">=": operator.ge,
66+
"==": operator.eq,
67+
"!=": operator.ne,
68+
")": operator.lt,
69+
"]": operator.le,
70+
"(": operator.gt,
71+
"[": operator.ge,
72+
}
73+
compare = operator_comparator[package_comparator]
74+
return compare(version, package_version)
75+
76+
77+
def parse_constraint(constraint):
78+
if constraint.startswith(("<=", ">=", "==", "!=")):
79+
return constraint[:2], constraint[2:]
80+
81+
if constraint.startswith(("<", ">", "=", "[", "]", "(", ")")):
82+
return constraint[0], constraint[1:]
83+
84+
if constraint.endswith(("[", "]", "(", ")")):
85+
return constraint[-1], constraint[:-1]
86+
87+
88+
def github_constraints_satisfied(github_constrain, version):
89+
gh_constraints = github_constrain.strip().replace(" ", "")
90+
constraints = gh_constraints.split(",")
91+
for constraint in constraints:
92+
gh_comparator, gh_version = parse_constraint(constraint)
93+
if not gh_version:
94+
continue
95+
# TODO: Replace the GenericVersion with ecosystem specific from univers
96+
if not compare(GenericVersion(version), gh_comparator, GenericVersion(gh_version)):
97+
return False
98+
return True
99+
100+
101+
def snky_constraints_satisfied(snyk_constrain, version):
102+
snyk_constraints = snyk_constrain.strip().replace(" ", "")
103+
constraints = snyk_constraints.split(",")
104+
for constraint in constraints:
105+
snyk_comparator, snyk_version = parse_constraint(constraint)
106+
if not snyk_version:
107+
continue
108+
# TODO: Replace the GenericVersion with ecosystem specific from univers or maybe not if snyk is normalizing versions to semver
109+
if not compare(GenericVersion(version), snyk_comparator, GenericVersion(snyk_version)):
110+
return False
111+
return True
112+
113+
114+
def gitlab_constraints_satisfied(gitlab_constrain, version):
115+
gitlab_constraints = gitlab_constrain.strip()
116+
constraints = gitlab_constraints.split("||")
117+
118+
for constraint in constraints:
119+
is_constraint_satisfied = True
120+
121+
for subcontraint in constraint.strip().split(" "):
122+
123+
gitlab_comparator, gitlab_version = parse_constraint(subcontraint.strip())
124+
if not gitlab_version:
125+
continue
126+
# TODO: Replace the GenericVersion with ecosystem specific from univers
127+
if not compare(
128+
GenericVersion(version), gitlab_comparator, GenericVersion(gitlab_version)
129+
):
130+
is_constraint_satisfied = False
131+
break
132+
133+
if is_constraint_satisfied:
134+
return True

0 commit comments

Comments
 (0)