Skip to content

Commit 6e9dde0

Browse files
haikoscholhaikoschol
authored
Merge pull request #194 from sbs2001/suse_backport_importer
Add SUSE backport data source and it's tests
2 parents 826de10 + 2a1cf66 commit 6e9dde0

5 files changed

Lines changed: 313 additions & 0 deletions

File tree

‎vulnerabilities/importers/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,4 @@
3030
from vulnerabilities.importers.ruby import RubyDataSource
3131
from vulnerabilities.importers.ubuntu import UbuntuDataSource
3232
from vulnerabilities.importers.retiredotnet import RetireDotnetDataSource
33+
from vulnerabilities.importers.suse_backports import SUSEBackportsDataSource

‎vulnerabilities/importers/suse_backports.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
23+
import yaml
24+
import dataclasses
25+
26+
import requests
27+
from packageurl import PackageURL
28+
from bs4 import BeautifulSoup
29+
30+
from vulnerabilities.data_source import Advisory
31+
from vulnerabilities.data_source import DataSource
32+
from vulnerabilities.data_source import DataSourceConfiguration
33+
34+
35+
@dataclasses.dataclass
36+
class SUSEBackportsConfiguration(DataSourceConfiguration):
37+
url: str
38+
etags: dict
39+
40+
41+
class SUSEBackportsDataSource(DataSource):
42+
43+
CONFIG_CLASS = SUSEBackportsConfiguration
44+
45+
@staticmethod
46+
def get_all_urls_of_backports(url):
47+
r = requests.get(url)
48+
soup = BeautifulSoup(r.content, 'lxml')
49+
for a_tag in soup.find_all('a', href=True):
50+
if a_tag['href'].endswith('.yaml') and a_tag['href'].startswith('backports'):
51+
yield url + a_tag['href']
52+
53+
def updated_advisories(self):
54+
advisories = []
55+
all_urls = self.get_all_urls_of_backports(self.config.url)
56+
for url in all_urls:
57+
if not self.create_etag(url):
58+
continue
59+
advisories.extend(self.process_file(self._fetch_yaml(url)))
60+
return self.batch_advisories(advisories)
61+
62+
def create_etag(self, url):
63+
etag = requests.head(url).headers.get('ETag')
64+
if not etag:
65+
# Kind of inaccurate to return True since etag is
66+
# not created
67+
return True
68+
elif url in self.config.etags:
69+
if self.config.etags[url] == etag:
70+
return False
71+
self.config.etags[url] = etag
72+
return True
73+
74+
def _fetch_yaml(self, url):
75+
76+
try:
77+
resp = requests.get(url)
78+
resp.raise_for_status()
79+
return yaml.safe_load(resp.content)
80+
81+
except requests.HTTPError:
82+
return {}
83+
84+
@staticmethod
85+
def process_file(yaml_file):
86+
advisories = []
87+
try:
88+
for pkg in yaml_file[0]['packages']:
89+
for version in yaml_file[0]['packages'][pkg]['fixed']:
90+
for vuln in yaml_file[0]['packages'][pkg]['fixed'][version]:
91+
# yaml_file specific data can be added
92+
purl = [PackageURL(
93+
name=pkg, type="rpm", version=version, namespace='opensuse')]
94+
advisories.append(
95+
Advisory(cve_id=vuln,
96+
resolved_package_urls=purl,
97+
summary='',
98+
impacted_package_urls=[])
99+
)
100+
except TypeError:
101+
# could've used pass
102+
return advisories
103+
104+
return advisories

‎vulnerabilities/migrations/0011_susebackports_importer.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
23+
from django.db import migrations
24+
25+
26+
def add_suse_backports_importer(apps, _):
27+
Importer = apps.get_model('vulnerabilities', 'Importer')
28+
29+
Importer.objects.create(
30+
name='suse_backports',
31+
license='',
32+
last_run=None,
33+
data_source='SUSEBackportsDataSource',
34+
data_source_cfg={
35+
'url':'http://ftp.suse.com/pub/projects/security/yaml/',
36+
'etags':{},
37+
},
38+
)
39+
40+
41+
def remove_suse_backports_importer(apps, _):
42+
Importer = apps.get_model('vulnerabilities', 'Importer')
43+
qs = Importer.objects.filter(name='suse_backports')
44+
if qs:
45+
qs[0].delete()
46+
47+
48+
class Migration(migrations.Migration):
49+
50+
dependencies = [
51+
('vulnerabilities', '0010_retiredotnet_importer'),
52+
]
53+
54+
operations = [
55+
migrations.RunPython(add_suse_backports_importer, remove_suse_backports_importer),
56+
]

‎vulnerabilities/tests/test_data/suse_backports/backports-sle11-sp0.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
- name: SLES
3+
version: 11
4+
packages:
5+
MozillaFirefox:
6+
fixed:
7+
3.0.10-1.1.1:
8+
- CVE-2009-1313
9+
MozillaFirefox-branding-SLED:
10+
fixed:
11+
3.5-1.1.5:
12+
- CVE-2009-1313
13+
MozillaFirefox-translations:
14+
fixed:
15+
3.0.10-1.1.1:
16+
- CVE-2009-1313
17+
NetworkManager:
18+
fixed:
19+
0.7.0.r4359-15.9.2:
20+
- CVE-2009-0365
21+
- CVE-2009-0578

‎vulnerabilities/tests/test_suse_backports.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
23+
from collections import OrderedDict
24+
import os
25+
from unittest import TestCase
26+
import yaml
27+
28+
from packageurl import PackageURL
29+
30+
from vulnerabilities.importers.suse_backports import SUSEBackportsDataSource
31+
from vulnerabilities.data_source import Advisory
32+
33+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
34+
35+
36+
def yaml_loader():
37+
path = os.path.join(BASE_DIR, "test_data/suse_backports/")
38+
yaml_files = {}
39+
for file in os.listdir(path):
40+
with open(os.path.join(path, file)) as f:
41+
yaml_files[file] = yaml.safe_load(f)
42+
return yaml_files
43+
44+
45+
class TestSUSEBackportsDataSource(TestCase):
46+
47+
@classmethod
48+
def setUpClass(cls):
49+
data_source_cfg = {
50+
'url': 'https://endpoint.com',
51+
'etags': {}}
52+
cls.data_src = SUSEBackportsDataSource(1, config=data_source_cfg)
53+
54+
def test_process_file(self):
55+
parsed_yamls = yaml_loader()
56+
expected_data = [
57+
Advisory(
58+
summary='',
59+
impacted_package_urls=[],
60+
resolved_package_urls=[
61+
PackageURL(
62+
type='rpm',
63+
namespace='opensuse',
64+
name='MozillaFirefox',
65+
version='3.0.10-1.1.1',
66+
qualifiers=OrderedDict(),
67+
subpath=None)],
68+
reference_urls=[],
69+
reference_ids=[],
70+
cve_id='CVE-2009-1313'),
71+
Advisory(
72+
summary='',
73+
impacted_package_urls=[],
74+
resolved_package_urls=[
75+
PackageURL(
76+
type='rpm',
77+
namespace='opensuse',
78+
name='MozillaFirefox-branding-SLED',
79+
version='3.5-1.1.5',
80+
qualifiers=OrderedDict(),
81+
subpath=None)],
82+
reference_urls=[],
83+
reference_ids=[],
84+
cve_id='CVE-2009-1313'),
85+
Advisory(
86+
summary='',
87+
impacted_package_urls=[],
88+
resolved_package_urls=[
89+
PackageURL(
90+
type='rpm',
91+
namespace='opensuse',
92+
name='MozillaFirefox-translations',
93+
version='3.0.10-1.1.1',
94+
qualifiers=OrderedDict(),
95+
subpath=None)],
96+
reference_urls=[],
97+
reference_ids=[],
98+
cve_id='CVE-2009-1313'),
99+
Advisory(
100+
summary='',
101+
impacted_package_urls=[],
102+
resolved_package_urls=[
103+
PackageURL(
104+
type='rpm',
105+
namespace='opensuse',
106+
name='NetworkManager',
107+
version='0.7.0.r4359-15.9.2',
108+
qualifiers=OrderedDict(),
109+
subpath=None)],
110+
reference_urls=[],
111+
reference_ids=[],
112+
cve_id='CVE-2009-0365'),
113+
Advisory(
114+
summary='',
115+
impacted_package_urls=[],
116+
resolved_package_urls=[
117+
PackageURL(
118+
type='rpm',
119+
namespace='opensuse',
120+
name='NetworkManager',
121+
version='0.7.0.r4359-15.9.2',
122+
qualifiers=OrderedDict(),
123+
subpath=None)],
124+
reference_urls=[],
125+
reference_ids=[],
126+
cve_id='CVE-2009-0578'),
127+
]
128+
129+
found_data = self.data_src.process_file(
130+
parsed_yamls['backports-sle11-sp0.yaml'])
131+
assert expected_data == found_data

0 commit comments

Comments
 (0)