Skip to content

Commit 478b7e8

Browse files
pombredannepombredanne
authored
Merge pull request #221 from sbs2001/remove_migrations
Remove migrations and add importer yielder module
2 parents 72abb0d + b5927b3 commit 478b7e8

19 files changed

Lines changed: 272 additions & 820 deletions

‎vulnerabilities/importer_yielder.py‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
# Copyright (c) nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an 'AS IS' BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
23+
from vulnerabilities.models import Importer
24+
25+
IMPORTER_REGISTRY = [
26+
{
27+
'name': 'rust',
28+
'license': 'https://creativecommons.org/publicdomain/zero/1.0/',
29+
'last_run': None,
30+
'data_source': 'RustDataSource',
31+
'data_source_cfg': {
32+
'branch': None,
33+
'repository_url': 'https://github.com/RustSec/advisory-db',
34+
},
35+
},
36+
{
37+
'name': 'alpine',
38+
'license': '',
39+
'last_run': None,
40+
'data_source': 'AlpineDataSource',
41+
'data_source_cfg': {
42+
'branch': None,
43+
'repository_url': 'https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb',
44+
},
45+
},
46+
{
47+
'name': 'archlinux',
48+
'license': '',
49+
'last_run': None,
50+
'data_source': 'ArchlinuxDataSource',
51+
'data_source_cfg': {
52+
'archlinux_tracker_url': 'https://security.archlinux.org/json'
53+
},
54+
},
55+
{
56+
'name': 'debian',
57+
'license': 'MIT',
58+
'last_run': None,
59+
'data_source': 'DebianDataSource',
60+
'data_source_cfg': {
61+
'debian_tracker_url': 'https://security-tracker.debian.org/tracker/data/json'
62+
},
63+
},
64+
{
65+
'name': 'safetydb',
66+
'license': '',
67+
'last_run': None,
68+
'data_source': 'SafetyDbDataSource',
69+
'data_source_cfg': {
70+
'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json' # nopep8
71+
},
72+
},
73+
{
74+
'name': 'npm',
75+
'license': '',
76+
'last_run': None,
77+
'data_source': 'NpmDataSource',
78+
'data_source_cfg': {},
79+
},
80+
{
81+
'name': 'ruby',
82+
'license': '',
83+
'last_run': None,
84+
'data_source': 'RubyDataSource',
85+
'data_source_cfg': {
86+
'repository_url': 'https://github.com/rubysec/ruby-advisory-db.git'
87+
},
88+
},
89+
{
90+
'name': 'ubuntu',
91+
'license': '',
92+
'last_run': None,
93+
'data_source': 'UbuntuDataSource',
94+
'data_source_cfg': {
95+
'etags': {},
96+
'releases': ['bionic', 'trusty', 'focal', 'eoan', 'xenial'],
97+
},
98+
},
99+
{
100+
'name': 'retiredotnet',
101+
'license': 'MIT',
102+
'last_run': None,
103+
'data_source': 'RetireDotnetDataSource',
104+
'data_source_cfg': {
105+
'repository_url': 'https://github.com/RetireNet/Packages.git'
106+
},
107+
},
108+
{
109+
'name': 'suse_backports',
110+
'license': '',
111+
'last_run': None,
112+
'data_source': 'SUSEBackportsDataSource',
113+
'data_source_cfg': {
114+
'url': 'http://ftp.suse.com/pub/projects/security/yaml/',
115+
'etags': {},
116+
},
117+
},
118+
{
119+
'name': 'debian_oval',
120+
'license': '',
121+
'last_run': None,
122+
'data_source': 'DebianOvalDataSource',
123+
'data_source_cfg': {
124+
'etags': {},
125+
'releases': ['wheezy', 'stretch', 'jessie', 'buster'],
126+
},
127+
},
128+
{
129+
'name': 'redhat',
130+
'license': '',
131+
'last_run': None,
132+
'data_source': 'RedhatDataSource',
133+
'data_source_cfg': {},
134+
},
135+
{
136+
'name': 'gentoo',
137+
'license': '',
138+
'last_run': None,
139+
'data_source': 'GentooDataSource',
140+
'data_source_cfg': {
141+
'repository_url': 'https://anongit.gentoo.org/git/data/glsa.git'
142+
},
143+
},
144+
{
145+
'name': 'openssl',
146+
'license': '',
147+
'last_run': None,
148+
'data_source': 'OpenSSLDataSource',
149+
'data_source_cfg': {},
150+
},
151+
{
152+
'name': 'ubuntu_usn',
153+
'license': '',
154+
'last_run': None,
155+
'data_source': 'UbuntuUSNDataSource',
156+
'data_source_cfg': {
157+
'etags': {},
158+
'db_url': 'https://usn.ubuntu.com/usn-db/database-all.json.bz2'
159+
},
160+
}
161+
162+
]
163+
164+
165+
def load_importers():
166+
167+
for importer in IMPORTER_REGISTRY:
168+
imp, created = Importer.objects.get_or_create(
169+
name=importer['name'],
170+
data_source=importer['data_source'],
171+
license=importer['license'],
172+
)
173+
174+
if created:
175+
# Sets the dynamic fields equal to the default values
176+
imp.data_source_cfg = importer['data_source_cfg']
177+
imp.last_run = importer['last_run']
178+
imp.save()

‎vulnerabilities/management/commands/import.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
33
# http://nexb.com and https://github.com/nexB/vulnerablecode/
44
# The VulnerableCode software is licensed under the Apache License version 2.0.
55
# Data generated with VulnerableCode require an acknowledgment.
@@ -28,6 +28,7 @@
2828

2929
from vulnerabilities.models import Importer
3030
from vulnerabilities.import_runner import ImportRunner
31+
from vulnerabilities.importer_yielder import load_importers
3132

3233

3334
class Command(BaseCommand):
@@ -55,6 +56,8 @@ def add_arguments(self, parser):
5556
'--batch_size', help='The batch size to be used for bulk inserting data')
5657

5758
def handle(self, *args, **options):
59+
# load_importers() seeds the DB with Importers
60+
load_importers()
5861
if options['list']:
5962
self.list_sources()
6063
return
@@ -75,15 +78,13 @@ def handle(self, *args, **options):
7578

7679
def list_sources(self):
7780
importers = Importer.objects.all()
78-
7981
self.stdout.write(
8082
'Vulnerability data can be imported from the following sources:')
8183
self.stdout.write(', '.join([i.name for i in importers]))
8284

8385
def import_data(self, names, cutoff_date):
8486
importers = []
8587
unknown_importers = set()
86-
8788
# make sure all arguments are valid before running any importers
8889
for name in names:
8990
try:

‎vulnerabilities/migrations/0002_rust_importer.py‎

Lines changed: 0 additions & 55 deletions
This file was deleted.

‎vulnerabilities/migrations/0003_alpine_importer.py‎

Lines changed: 0 additions & 55 deletions
This file was deleted.

‎vulnerabilities/migrations/0004_archlinux_importer.py‎

Lines changed: 0 additions & 55 deletions
This file was deleted.

0 commit comments

Comments
 (0)