diff --git a/.travis.yml b/.travis.yml index 5beb4259b..cdf937200 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,15 @@ language: python python: 3.6 +services: + - postgresql + install: - pip install -r requirements.txt -env: - - DJANGO_DEV=1 - before_script: - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . + - psql -c "CREATE DATABASE vulnerablecode;" -U postgres - ./manage.py migrate script: diff --git a/README.md b/README.md index 0b23264a7..51a5e6509 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,33 @@ Clone the source code: git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode ``` +System requirements + +- Get Python 3.6+ installed first and pip (pip is included in the + Python.org downloads since Python 2.7.9) + +- Install PostgreSQL 9 of later. (11.2 preferred) + On Debian distros use: `sudo apt-get install postgresql` + +- Install extra utilities if needed: `sudo apt-get install wget build-essential redis-server` + + +Configure a local test database + +- Create a local test `vulnerablecode` database user. Use `vulnerablecode` as password when prompted + (otherwise use any password and update your settings locally). + `sudo -u postgres createuser --no-createrole --no-superuser --login --inherit --createdb --pwprompt vulnerablecode` + +- Create a local test `vulnerablecode` database. + `createdb --encoding=utf-8 --owner=vulnerablecode --user=vulnerablecode --password --host=localhost --port=5432 vulnerablecode` + Activate a virtualenv, install dependencies, and run the database migrations: ``` -python3.6 -m venv . +python3 -m venv . source bin/activate pip install -r requirements.txt -DJANGO_DEV=1 >> .env -DJANGO_DEV=1 ./manage.py migrate +python manage.py migrate ``` Tests @@ -32,14 +51,14 @@ python3.6 -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/te For Django based tests ``` -DJANGO_DEV=1 ./manage.py test vulnerabilities/tests +python manage.py test vulnerabilities/tests ``` Scrape and save to the database ------------------------------- ``` -DJANGO_DEV=1 ./manage.py shell +python manage.py shell ``` ``` @@ -61,7 +80,7 @@ API Start the webserver ``` -DJANGO_DEV=1 ./manage.py runserver +python manage.py runserver ``` In your browser access: diff --git a/requirements.txt b/requirements.txt index f4679f73a..145ec7b6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ setuptools==36.5.0 -beautifulsoup4==4.6.0 -lxml==4.0.0 -django==1.11.6 -djangorestframework==3.7.0 -django-filter==1.0.4 +beautifulsoup4==4.7.1 +lxml==4.3.3 +django==2.2 +djangorestframework==3.9.2 +django-filter==2.1.0 # Tests pytest==3.2.3 diff --git a/vulnerabilities/migrations/0003_auto_20190406_0950.py b/vulnerabilities/migrations/0003_auto_20190406_0950.py new file mode 100644 index 000000000..8cb2c8455 --- /dev/null +++ b/vulnerabilities/migrations/0003_auto_20190406_0950.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2 on 2019-04-06 09:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0002_package_vulnerabilities'), + ] + + operations = [ + migrations.AlterField( + model_name='vulnerability', + name='cvss', + field=models.FloatField(help_text='CVSS Score', max_length=100, null=True), + ), + migrations.AlterField( + model_name='vulnerability', + name='summary', + field=models.CharField(blank=True, help_text='Summary of the vulnerability', max_length=100), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 97b0e6dd1..d74302230 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -29,8 +29,8 @@ class Vulnerability(models.Model): A software vulnerability with minimal information. Identifiers are stored as VulnerabilityReference. """ - summary = models.CharField(max_length=50, help_text='Summary of the vulnerability', blank=True) - cvss = models.FloatField(max_length=50, help_text='CVSS Score', null=True) + summary = models.CharField(max_length=100, help_text='Summary of the vulnerability', blank=True) + cvss = models.FloatField(max_length=100, help_text='CVSS Score', null=True) def __str__(self): return self.summary @@ -42,10 +42,14 @@ class VulnerabilityReference(models.Model): vulnerability data on such as a CVE ID and its web page at the NVD, a bug id and similar references. """ - vulnerability = models.ForeignKey('Vulnerability') - source = models.CharField(max_length=50, help_text='Source(s) name eg:NVD', blank=True) - reference_id = models.CharField(max_length=50, help_text='Reference ID, eg:CVE-ID', blank=True) - url = models.URLField(max_length=1024, help_text='URL of Vulnerability data', blank=True) + vulnerability = models.ForeignKey( + Vulnerability, on_delete=models.CASCADE) + source = models.CharField( + max_length=50, help_text='Source(s) name eg:NVD', blank=True) + reference_id = models.CharField( + max_length=50, help_text='Reference ID, eg:CVE-ID', blank=True) + url = models.URLField( + max_length=1024, help_text='URL of Vulnerability data', blank=True) class Meta: unique_together = ('vulnerability', 'source', 'reference_id', 'url') @@ -54,12 +58,26 @@ def __str__(self): return self.source +class Package(models.Model): + """ + A software package with minimal identifying information. + Other identifiers are stored as PackageReference. + """ + platform = models.CharField(max_length=50, help_text='Package platform eg:maven', blank=True) + name = models.CharField(max_length=50, help_text='Package name', blank=True) + version = models.CharField(max_length=50, help_text='Package version', blank=True) + vulnerabilities = models.ManyToManyField(to='Vulnerability', through='ImpactedPackage') + + def __str__(self): + return self.name + + class ImpactedPackage(models.Model): """ Relates a vulnerability to package(s) impacted by it. """ - vulnerability = models.ForeignKey('Vulnerability') - package = models.ForeignKey('Package') + vulnerability = models.ForeignKey(Vulnerability, on_delete=models.CASCADE) + package = models.ForeignKey(Package, on_delete=models.CASCADE) class Meta: unique_together = ('vulnerability', 'package') @@ -70,22 +88,8 @@ class ResolvedPackage(models.Model): Relates a vulnerability to package(s) that contain a fix or resolution of this vulnerability. """ - vulnerability = models.ForeignKey('Vulnerability') - package = models.ForeignKey('Package') - - -class Package(models.Model): - """ - A software package with minimal identifying information. - Other identifiers are stored as PackageReference. - """ - platform = models.CharField(max_length=50, help_text='Package platform eg:maven', blank=True) - name = models.CharField(max_length=50, help_text='Package name', blank=True) - version = models.CharField(max_length=50, help_text='Package version', blank=True) - vulnerabilities = models.ManyToManyField(to='Vulnerability', through='ImpactedPackage') - - def __str__(self): - return self.name + vulnerability = models.ForeignKey(Vulnerability, on_delete=models.CASCADE) + package = models.ForeignKey(Package, on_delete=models.CASCADE) class PackageReference(models.Model): @@ -93,7 +97,7 @@ class PackageReference(models.Model): One or more identifiers and references for a software package in a package repository, such as a Debian, Maven or NPM repository. """ - package = models.ForeignKey('Package') + package = models.ForeignKey(Package, on_delete=models.CASCADE) repository = models.CharField( max_length=50, help_text='Repository URL eg:http://central.maven.org', diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index a350be9c1..970744e75 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -72,6 +72,7 @@ 'django.template.context_processors.debug', 'django.template.context_processors.static', 'django.template.context_processors.request', + 'django.contrib.messages.context_processors.messages' ], }, }, @@ -86,9 +87,18 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'vulnerablecode', + 'USER': 'vulnerablecode', + 'PASSWORD': 'vulnerablecode', + 'HOST': 'localhost', + 'PORT': '5432', } } +if 'TRAVIS' in os.environ: + DATABASES['default']['USER'] = 'postgres' + DATABASES['default']['PASSWORD'] = '' + # Update database configuration with $DATABASE_URL. import dj_database_url db_from_env = dj_database_url.config(conn_max_age=500)