Skip to content

Commit bc44a84

Browse files
ziadhanyziadhany
committed
Add api test for epss
Keep the EPSS score separate from the severity score range Add epss tab Add epss to severity scoring Add published_at date to the Vulnerability score model. Add EPSS importer Add EPSS model Add EPSS UI Add EPSS to api Fix api test Signed-off-by: ziadhany <ziadhany2016@gmail.com>
1 parent 433d3a6 commit bc44a84

11 files changed

Lines changed: 256 additions & 8 deletions

File tree

‎vulnerabilities/api.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,14 @@
3737
class VulnerabilitySeveritySerializer(serializers.ModelSerializer):
3838
class Meta:
3939
model = VulnerabilitySeverity
40-
fields = ["value", "scoring_system", "scoring_elements"]
40+
fields = ["value", "scoring_system", "scoring_elements", "published_at"]
41+
42+
def to_representation(self, instance):
43+
data = super().to_representation(instance)
44+
published_at = data.get("published_at", None)
45+
if not published_at:
46+
data.pop("published_at")
47+
return data
4148

4249

4350
class VulnerabilityReferenceSerializer(serializers.ModelSerializer):

‎vulnerabilities/import_runner.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver
189189
defaults={
190190
"value": str(severity.value),
191191
"scoring_elements": str(severity.scoring_elements),
192+
"published_at": str(severity.published_at),
192193
},
193194
)
194195
if updated:

‎vulnerabilities/importer.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,17 @@ class VulnerabilitySeverity:
5252
system: ScoringSystem
5353
value: str
5454
scoring_elements: str = ""
55+
published_at: Optional[datetime.datetime] = None
5556

5657
def to_dict(self):
58+
published_at_dict = (
59+
{"published_at": self.published_at.isoformat()} if self.published_at else {}
60+
)
5761
return {
5862
"system": self.system.identifier,
5963
"value": self.value,
6064
"scoring_elements": self.scoring_elements,
65+
**published_at_dict,
6166
}
6267

6368
@classmethod
@@ -70,6 +75,7 @@ def from_dict(cls, severity: dict):
7075
system=SCORING_SYSTEMS[severity["system"]],
7176
value=severity["value"],
7277
scoring_elements=severity.get("scoring_elements", ""),
78+
published_at=severity.get("published_at"),
7379
)
7480

7581

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from vulnerabilities.importers import debian
1616
from vulnerabilities.importers import debian_oval
1717
from vulnerabilities.importers import elixir_security
18+
from vulnerabilities.importers import epss
1819
from vulnerabilities.importers import fireeye
1920
from vulnerabilities.importers import gentoo
2021
from vulnerabilities.importers import github
@@ -71,6 +72,7 @@
7172
oss_fuzz.OSSFuzzImporter,
7273
ruby.RubyImporter,
7374
github_osv.GithubOSVImporter,
75+
epss.EPSSImporter,
7476
]
7577

7678
IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

‎vulnerabilities/importers/epss.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
import csv
10+
import gzip
11+
import logging
12+
import urllib.request
13+
from datetime import datetime
14+
from typing import Iterable
15+
16+
from vulnerabilities import severity_systems
17+
from vulnerabilities.importer import AdvisoryData
18+
from vulnerabilities.importer import Importer
19+
from vulnerabilities.importer import Reference
20+
from vulnerabilities.importer import VulnerabilitySeverity
21+
22+
logger = logging.getLogger(__name__)
23+
24+
25+
class EPSSImporter(Importer):
26+
"""Exploit Prediction Scoring System (EPSS) Importer"""
27+
28+
advisory_url = "https://epss.cyentia.com/epss_scores-current.csv.gz"
29+
spdx_license_expression = "unknown"
30+
importer_name = "EPSS Importer"
31+
32+
def advisory_data(self) -> Iterable[AdvisoryData]:
33+
response = urllib.request.urlopen(self.advisory_url)
34+
with gzip.open(response, "rb") as f:
35+
lines = [l.decode("utf-8") for l in f.readlines()]
36+
37+
epss_reader = csv.reader(lines)
38+
model_version, score_date = next(
39+
epss_reader
40+
) # score_date='score_date:2024-05-19T00:00:00+0000'
41+
published_at = datetime.strptime(score_date[11::], "%Y-%m-%dT%H:%M:%S%z")
42+
43+
next(epss_reader) # skip the header row
44+
for epss_row in epss_reader:
45+
cve, score, percentile = epss_row
46+
47+
if not cve or not score or not percentile:
48+
logger.error(f"Invalid epss row: {epss_row}")
49+
continue
50+
51+
severity = VulnerabilitySeverity(
52+
system=severity_systems.EPSS,
53+
value=score,
54+
scoring_elements=percentile,
55+
published_at=published_at,
56+
)
57+
58+
references = Reference(
59+
url=f"https://api.first.org/data/v1/epss?cve={cve}",
60+
severities=[severity],
61+
)
62+
63+
yield AdvisoryData(
64+
aliases=[cve],
65+
references=[references],
66+
url=self.advisory_url,
67+
)

‎vulnerabilities/migrations/0057_vulnerabilityseverity_published_at_and_more.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Generated by Django 4.1.13 on 2024-05-22 21:51
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
("vulnerabilities", "0056_alter_packagechangelog_software_version_and_more"),
10+
]
11+
12+
operations = [
13+
migrations.AddField(
14+
model_name="vulnerabilityseverity",
15+
name="published_at",
16+
field=models.DateTimeField(
17+
blank=True,
18+
help_text="UTC Date of publication of the vulnerability severity",
19+
null=True,
20+
),
21+
),
22+
migrations.AlterField(
23+
model_name="vulnerabilityseverity",
24+
name="scoring_system",
25+
field=models.CharField(
26+
choices=[
27+
("cvssv2", "CVSSv2 Base Score"),
28+
("cvssv3", "CVSSv3 Base Score"),
29+
("cvssv3.1", "CVSSv3.1 Base Score"),
30+
("rhbs", "RedHat Bugzilla severity"),
31+
("rhas", "RedHat Aggregate severity"),
32+
("archlinux", "Archlinux Vulnerability Group Severity"),
33+
("cvssv3.1_qr", "CVSSv3.1 Qualitative Severity Rating"),
34+
("generic_textual", "Generic textual severity rating"),
35+
("apache_httpd", "Apache Httpd Severity"),
36+
("apache_tomcat", "Apache Tomcat Severity"),
37+
("epss", "Exploit Prediction Scoring System"),
38+
],
39+
help_text="Identifier for the scoring system used. Available choices are: cvssv2: CVSSv2 Base Score,\ncvssv3: CVSSv3 Base Score,\ncvssv3.1: CVSSv3.1 Base Score,\nrhbs: RedHat Bugzilla severity,\nrhas: RedHat Aggregate severity,\narchlinux: Archlinux Vulnerability Group Severity,\ncvssv3.1_qr: CVSSv3.1 Qualitative Severity Rating,\ngeneric_textual: Generic textual severity rating,\napache_httpd: Apache Httpd Severity,\napache_tomcat: Apache Tomcat Severity,\nepss: Exploit Prediction Scoring System ",
40+
max_length=50,
41+
),
42+
),
43+
]

‎vulnerabilities/models.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -920,6 +920,10 @@ class VulnerabilitySeverity(models.Model):
920920
"For example a CVSS vector string as used to compute a CVSS score.",
921921
)
922922

923+
published_at = models.DateTimeField(
924+
blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity"
925+
)
926+
923927
class Meta:
924928
unique_together = ["reference", "scoring_system", "value"]
925929
ordering = ["reference", "scoring_system", "value"]
@@ -1121,7 +1125,6 @@ class Meta:
11211125

11221126

11231127
class ChangeLog(models.Model):
1124-
11251128
action_time = models.DateTimeField(
11261129
# check if dates are actually UTC
11271130
default=timezone.now,
@@ -1261,7 +1264,6 @@ def log_action(self, package, action_type, actor_name, source_url, related_vulne
12611264

12621265

12631266
class PackageChangeLog(ChangeLog):
1264-
12651267
AFFECTED_BY = 1
12661268
FIXING = 2
12671269

‎vulnerabilities/severity_systems.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,19 @@ def get(self, scoring_elements: str) -> dict:
157157
"Low",
158158
]
159159

160+
161+
@dataclasses.dataclass(order=True)
162+
class EPSSScoringSystem(ScoringSystem):
163+
def compute(self, scoring_elements: str):
164+
return NotImplementedError
165+
166+
167+
EPSS = EPSSScoringSystem(
168+
identifier="epss",
169+
name="Exploit Prediction Scoring System",
170+
url="https://www.first.org/epss/",
171+
)
172+
160173
SCORING_SYSTEMS = {
161174
system.identifier: system
162175
for system in (
@@ -170,5 +183,6 @@ def get(self, scoring_elements: str) -> dict:
170183
GENERIC,
171184
APACHE_HTTPD,
172185
APACHE_TOMCAT,
186+
EPSS,
173187
)
174188
}

‎vulnerabilities/templates/vulnerability_details.html‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,15 @@
6060
</span>
6161
</a>
6262
</li>
63+
64+
<li data-tab="epss">
65+
<a>
66+
<span>
67+
EPSS
68+
</span>
69+
</a>
70+
</li>
71+
6372
<li data-tab="history">
6473
<a>
6574
<span>
@@ -374,6 +383,52 @@
374383
</tr>
375384
{% endfor %}
376385
</div>
386+
387+
{% for severity in severities %}
388+
{% if severity.scoring_system == 'epss' %}
389+
<div class="tab-div content" data-content="epss">
390+
<div class="has-text-weight-bold tab-nested-div ml-1 mb-1 mt-1">
391+
Exploit Prediction Scoring System
392+
</div>
393+
<table class="table vcio-table width-100-pct mt-2">
394+
<tbody>
395+
<tr>
396+
<td class="two-col-left">
397+
<span class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left"
398+
data-tooltip="the percentile of the current score, the proportion of all scored vulnerabilities with the same or a lower EPSS score">
399+
Percentile:
400+
</span>
401+
</td>
402+
<td class="two-col-right">{{ severity.scoring_elements }}</td>
403+
</tr>
404+
405+
<tr>
406+
<td class="two-col-left">
407+
<span class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left"
408+
data-tooltip="the EPSS score representing the probability [0-1] of exploitation in the wild in the next 30 days (following score publication)">
409+
EPSS score:
410+
</span>
411+
</td>
412+
<td class="two-col-right">{{ severity.value }}</td>
413+
</tr>
414+
415+
<tr>
416+
<td class="two-col-left">
417+
<span
418+
class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left"
419+
data-tooltip="When was the first time we fetched epss">
420+
Published at:
421+
</span>
422+
</td>
423+
<td class="two-col-right">{{ severity.published_at }}</td>
424+
</tr>
425+
426+
</tbody>
427+
</table>
428+
</div>
429+
{% endif %}
430+
{% endfor %}
431+
377432
<div class="tab-div content" data-content="history">
378433
<table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth">
379434
<thead>

‎vulnerabilities/tests/test_api.py‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@
2828
from vulnerabilities.models import Vulnerability
2929
from vulnerabilities.models import VulnerabilityReference
3030
from vulnerabilities.models import VulnerabilityRelatedReference
31+
from vulnerabilities.models import VulnerabilitySeverity
3132
from vulnerabilities.models import Weakness
33+
from vulnerabilities.severity_systems import EPSS
3234

3335
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
3436
TEST_DATA = os.path.join(BASE_DIR, "test_data")
@@ -199,6 +201,23 @@ def setUp(self):
199201
PackageRelatedVulnerability.objects.create(
200202
package=pkg, vulnerability=self.vulnerability, fix=True
201203
)
204+
205+
self.reference1 = VulnerabilityReference.objects.create(
206+
reference_id="",
207+
url="https://.com",
208+
)
209+
210+
VulnerabilitySeverity.objects.create(
211+
reference=self.reference1,
212+
scoring_system=EPSS.identifier,
213+
scoring_elements=".0016",
214+
value="0.526",
215+
)
216+
217+
VulnerabilityRelatedReference.objects.create(
218+
reference=self.reference1, vulnerability=self.vulnerability
219+
)
220+
202221
self.weaknesses = Weakness.objects.create(cwe_id=119)
203222
self.weaknesses.vulnerabilities.add(self.vulnerability)
204223
self.invalid_weaknesses = Weakness.objects.create(
@@ -242,7 +261,20 @@ def test_api_with_single_vulnerability(self):
242261
},
243262
],
244263
"affected_packages": [],
245-
"references": [],
264+
"references": [
265+
{
266+
"reference_url": "https://.com",
267+
"reference_id": "",
268+
"scores": [
269+
{
270+
"value": "0.526",
271+
"scoring_system": "epss",
272+
"scoring_elements": ".0016",
273+
}
274+
],
275+
"url": "https://.com",
276+
}
277+
],
246278
"weaknesses": [
247279
{
248280
"cwe_id": 119,
@@ -272,7 +304,20 @@ def test_api_with_single_vulnerability_with_filters(self):
272304
},
273305
],
274306
"affected_packages": [],
275-
"references": [],
307+
"references": [
308+
{
309+
"reference_url": "https://.com",
310+
"reference_id": "",
311+
"scores": [
312+
{
313+
"value": "0.526",
314+
"scoring_system": "epss",
315+
"scoring_elements": ".0016",
316+
}
317+
],
318+
"url": "https://.com",
319+
}
320+
],
276321
"weaknesses": [
277322
{
278323
"cwe_id": 119,

0 commit comments

Comments
 (0)