Skip to content

Commit 67fc00e

Browse files
authored
Merge pull request #525 from Hritik14/importer-refactor
Separate import and improve operations Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
2 parents ed21316 + 0e74bea commit 67fc00e

25 files changed

Lines changed: 1583 additions & 944 deletions

.gitignore

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,3 +125,12 @@ Pipfile
125125

126126
# VSCode
127127
.vscode
128+
129+
# Various junk and temp files
130+
.DS_Store
131+
*~
132+
.*.sw[po]
133+
.build
134+
.ve
135+
*.bak
136+
/.cache/

pytest.ini

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,34 @@
11
[pytest]
22
DJANGO_SETTINGS_MODULE = vulnerablecode.settings
33
markers =
4-
webtest
4+
webtest
5+
addopts =
6+
--doctest-modules
7+
# Ignore the following doctests until these files are migrated to
8+
# import-improve structure
9+
--ignore=vulnerabilities/importers/alpine_linux.py
10+
--ignore=vulnerabilities/importers/apache_httpd.py
11+
--ignore=vulnerabilities/importers/apache_kafka.py
12+
--ignore=vulnerabilities/importers/apache_tomcat.py
13+
--ignore=vulnerabilities/importers/archlinux.py
14+
--ignore=vulnerabilities/importers/debian.py
15+
--ignore=vulnerabilities/importers/elixir_security.py
16+
--ignore=vulnerabilities/importers/gentoo.py
17+
--ignore=vulnerabilities/importers/github.py
18+
--ignore=vulnerabilities/importers/istio.py
19+
--ignore=vulnerabilities/importers/kaybee.py
20+
--ignore=vulnerabilities/importers/npm.py
21+
--ignore=vulnerabilities/importers/nvd.py
22+
--ignore=vulnerabilities/importers/openssl.py
23+
--ignore=vulnerabilities/importers/postgresql.py
24+
--ignore=vulnerabilities/importers/project_kb_msr2019.py
25+
--ignore=vulnerabilities/importers/redhat.py
26+
--ignore=vulnerabilities/importers/retiredotnet.py
27+
--ignore=vulnerabilities/importers/ruby.py
28+
--ignore=vulnerabilities/importers/rust.py
29+
--ignore=vulnerabilities/importers/safety_db.py
30+
--ignore=vulnerabilities/importers/suse_backports.py
31+
--ignore=vulnerabilities/importers/suse_scores.py
32+
--ignore=vulnerabilities/importers/ubuntu_usn.py
33+
--ignore=vulnerabilities/management/commands/create_cpe_to_purl_map.py
34+
--ignore=vulnerabilities/lib_oval.py

requirements.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@ django-widget-tweaks>=1.4.8
88
packageurl-python>=0.9.4
99
binaryornot>=0.4.4
1010
GitPython>=3.1.17
11-
univers>=21.4.16.6
11+
univers>=30.0.0
1212
saneyaml>=0.5.2
1313
beautifulsoup4>=4.9.3
1414
python-dateutil>=2.8.1
1515
toml>=0.10.2
16-
lxml>=4.6.3
16+
lxml>=4.6.4
1717
gunicorn>=20.1.0
1818
django-environ==0.4.5
1919
defusedxml==0.7.1

vulnerabilities/data_inference.py

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import dataclasses
2+
import logging
3+
from typing import List
4+
from typing import Optional
5+
from uuid import uuid4
6+
7+
from packageurl import PackageURL
8+
from django.db.models.query import QuerySet
9+
10+
from vulnerabilities.data_source import Reference
11+
from vulnerabilities.data_source import AdvisoryData
12+
13+
logger = logging.getLogger(__name__)
14+
15+
MAX_CONFIDENCE = 100
16+
17+
18+
@dataclasses.dataclass(order=True)
19+
class Inference:
20+
"""
21+
This data class expresses the contract between data improvers and the improve runner.
22+
23+
Only inferences with highest confidence for one vulnerability <-> package
24+
relationship is to be inserted into the database
25+
"""
26+
27+
vulnerability_id: str = None
28+
aliases: List[str] = dataclasses.field(default_factory=list)
29+
confidence: int = MAX_CONFIDENCE
30+
summary: Optional[str] = None
31+
affected_purls: List[PackageURL] = dataclasses.field(default_factory=list)
32+
fixed_purl: PackageURL = dataclasses.field(default_factory=list)
33+
references: List[Reference] = dataclasses.field(default_factory=list)
34+
35+
def __post_init__(self):
36+
if self.confidence > MAX_CONFIDENCE or self.confidence < 0:
37+
raise ValueError
38+
39+
assert (
40+
self.vulnerability_id
41+
or self.aliases
42+
or self.summary
43+
or self.affected_purls
44+
or self.fixed_purl
45+
or self.references
46+
)
47+
48+
versionless_purls = []
49+
for purl in self.affected_purls + [self.fixed_purl]:
50+
if not purl.version:
51+
versionless_purls.append(purl)
52+
53+
assert (
54+
not versionless_purls
55+
), f"Version-less purls are not supported in an Inference: {versionless_purls}"
56+
57+
@classmethod
58+
def from_advisory_data(cls, advisory_data, confidence, affected_purls, fixed_purl):
59+
"""
60+
Return an Inference object while keeping the same values as of advisory_data
61+
for vulnerability_id, summary and references
62+
"""
63+
return cls(
64+
aliases=advisory_data.aliases,
65+
confidence=confidence,
66+
summary=advisory_data.summary,
67+
affected_purls=affected_purls,
68+
fixed_purl=fixed_purl,
69+
references=advisory_data.references,
70+
)
71+
72+
73+
class Improver:
74+
"""
75+
Improvers are responsible to improve the already imported data by a datasource.
76+
Inferences regarding the data could be generated based on multiple factors.
77+
"""
78+
79+
@property
80+
def interesting_advisories(self) -> QuerySet:
81+
"""
82+
Return QuerySet for the advisories this improver is interested in
83+
"""
84+
raise NotImplementedError
85+
86+
def get_inferences(self, advisory_data: AdvisoryData) -> List[Inference]:
87+
"""
88+
Generate and return Inferences for the given advisory data
89+
"""
90+
raise NotImplementedError
91+
92+
@classmethod
93+
def qualified_name(cls):
94+
"""
95+
Fully qualified name prefixed with the module name of the improver
96+
used in logging.
97+
"""
98+
return f"{cls.__module__}.{cls.__qualname__}"

0 commit comments

Comments
 (0)