diff --git a/.travis.yml b/.travis.yml index cdf937200..de9364aef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,9 @@ services: install: - pip install -r requirements.txt +env: + - SECRET_KEY="i1bn=oly)w*2yl-5yc&f!vvgt)p)fh3_2$r#spa!*sw36f5ov7" + before_script: - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . - psql -c "CREATE DATABASE vulnerablecode;" -U postgres diff --git a/README.md b/README.md index 733fda04d..fde92ef3e 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ [![Build Status](https://travis-ci.org/nexB/vulnerablecode.svg?branch=develop)](https://travis-ci.org/nexB/vulnerablecode) -Setup ------ -VulnerableCode requires Python 3.6+, get the latest version at https://www.python.org/ +## Setup Clone the source code: @@ -12,53 +10,62 @@ Clone the source code: git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode ``` -System requirements +### System requirements -- Get Python 3.6+ installed first and pip (pip is included in the - Python.org downloads since Python 2.7.9) +- Python 3.6+ -- Install PostgreSQL 9 of later. (11.2 preferred) - On Debian distros use: `sudo apt-get install postgresql` +- PostgreSQL 9+ or [Docker](https://hub.docker.com/search/?type=edition&offering=community) -- Install extra utilities if needed: `sudo apt-get install wget build-essential redis-server` +- Compiler toolchain and development files for Python and PostgreSQL +On Debian-based distros, these can be installed with `sudo apt install python3-venv python3-dev postgresql libpq-dev build-essential`. Leave out `postgresql` if you want to run it in Docker. -Configure a local test database +### Database configuration -- Create a local test `vulnerablecode` database user. Use `vulnerablecode` as password when prompted - (otherwise use any password and update your settings locally). +Either run PostgreSQL in Docker: +`docker run --name pg-vulnerablecode -e POSTGRES_USER=vulnerablecode -e POSTGRES_PASSWORD=vulnerablecode -e POSTGRES_DB=vulnerablecode -p 5432:5432 postgres` + +Or without: + +- Create a user named `vulnerablecode`. Use `vulnerablecode` as password when prompted: `sudo -u postgres createuser --no-createrole --no-superuser --login --inherit --createdb --pwprompt vulnerablecode` -- Create a local test `vulnerablecode` database. +- Create a databased named `vulnerablecode`: `createdb --encoding=utf-8 --owner=vulnerablecode --user=vulnerablecode --password --host=localhost --port=5432 vulnerablecode` +### Application dependencies + Activate a virtualenv, install dependencies, and run the database migrations: ``` -python3 -m venv . -source bin/activate +python3 -m venv venv +source venv/bin/activate pip install -r requirements.txt -python manage.py migrate +DJANGO_DEV=1 python manage.py migrate ``` -Tests ------ +The environment variable `DJANGO_DEV` is used to load settings suitable for development, defined in `vulnerablecode/dev.py`. If you don't want to type +it every time use `export DJANGO_DEV=1` instead. + +When not running in development mode, an environment variable named `SECRET_KEY` needs to be set. The recommended way to generate this key is to use +the code Django includes for this purpose: `SECRET_KEY=$(python -c "from django.core.management import utils; print(utils.get_random_secret_key())")`. + +## Tests ``` -pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . -python3.6 -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py +pycodestyle --exclude=migrations,settings.py,venv,tests --max-line-length=100 . +python -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py ``` For Django based tests ``` -python manage.py test vulnerabilities/tests +DJANGO_DEV=1 python manage.py test vulnerabilities/tests ``` -Scrape and save to the database -------------------------------- +## Data import ``` -python manage.py shell +DJANGO_DEV=1 python manage.py shell ``` ``` @@ -78,22 +85,22 @@ archlinux_vulnerabilities = archlinux.scrape_vulnerabilities() archlinux_dump(archlinux_vulnerabilities) ``` -API ----- +## API + Start the webserver ``` -python manage.py runserver +DJANGO_DEV=1 python manage.py runserver ``` In your browser access: + ``` http://127.0.0.1:8000/api/ http://127.0.0.1:8000/api/packages/?name= ``` -Deployment on Heroku --------------------- +## Deployment on Heroku See https://devcenter.heroku.com/articles/django-app-configuration#creating-a-new-django-project https://devcenter.heroku.com/articles/deploying-python#how-to-keep-build-artifacts-out-of-git @@ -108,10 +115,12 @@ https://devcenter.heroku.com/articles/deploying-python#how-to-keep-build-artifac 5. Create Heroku app: `heroku create` -6. Deploy: `git push heroku :master` +6. Generate a secret key and pass it as an environment variable: `heroku config:set SECRET_KEY=$(python -c "from django.core.management import utils; print(utils.get_random_secret_key())")` + +7. Deploy: `git push heroku :master` -7. Migrate the database: `heroku run python manage.py migrate` +8. Migrate the database: `heroku run python manage.py migrate` -8. Load the data referring to chapter "Scrape and save to the database" above. +9. Load the data referring to chapter "Data import" above. -9. To check the logs: `heroku logs --tail` +10. To check the logs: `heroku logs --tail` diff --git a/vulnerablecode/dev.py b/vulnerablecode/dev.py index 15f8a2874..f957c4340 100644 --- a/vulnerablecode/dev.py +++ b/vulnerablecode/dev.py @@ -1,14 +1,7 @@ import os -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - DEBUG = True ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1'] -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} +SECRET_KEY = 'hhismo6l@1uj)nr6@o7$b2u68w5*_o^liji+=uzq=954-f$8_1' diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 970744e75..220925f52 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -12,6 +12,8 @@ import os +DEV_MODE = os.environ.get('DJANGO_DEV', False) + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -19,7 +21,8 @@ # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'hhismo6l@1uj)nr6@o7$b2u68w5*_o^liji+=uzq=954-f$8_1' +if not DEV_MODE: + SECRET_KEY = os.environ['SECRET_KEY'] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False @@ -28,7 +31,6 @@ '.herokuapp.com', ] - # Application definition INSTALLED_APPS = [ @@ -159,5 +161,5 @@ # Set `DJANGO_DEV=1` in env to enable dev mode -if os.environ.get('DJANGO_DEV', False): +if DEV_MODE: from vulnerablecode.dev import *