From ec1508599547e82977572d2d1d7ac1eec816ddb8 Mon Sep 17 00:00:00 2001 From: Haiko Schol Date: Fri, 13 Sep 2019 17:15:44 +0200 Subject: [PATCH 1/2] Improve setup instructions in README - Fix instructions for Debian-based distros - Add instructions for running Postgres in Docker - Set DJANGO_DEV=1 in runserver call to have localhost in - ALLOWED_HOSTS closes #47 Signed-off-by: Haiko Schol --- README.md | 55 ++++++++++++++++++++++--------------------- vulnerablecode/dev.py | 9 ------- 2 files changed, 28 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 733fda04d..251012964 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ [![Build Status](https://travis-ci.org/nexB/vulnerablecode.svg?branch=develop)](https://travis-ci.org/nexB/vulnerablecode) -Setup ------ -VulnerableCode requires Python 3.6+, get the latest version at https://www.python.org/ +## Setup Clone the source code: @@ -12,41 +10,45 @@ Clone the source code: git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode ``` -System requirements +### System requirements -- Get Python 3.6+ installed first and pip (pip is included in the - Python.org downloads since Python 2.7.9) +- Python 3.6+ -- Install PostgreSQL 9 of later. (11.2 preferred) - On Debian distros use: `sudo apt-get install postgresql` +- PostgreSQL 9+ or [Docker](https://hub.docker.com/search/?type=edition&offering=community) -- Install extra utilities if needed: `sudo apt-get install wget build-essential redis-server` +- Compiler toolchain and development files for Python and PostgreSQL +On Debian-based distros, these can be installed with `sudo apt install python3-venv python3-dev postgresql libpq-dev build-essential`. Leave out `postgresql` if you want to run it in Docker. -Configure a local test database +### Database configuration -- Create a local test `vulnerablecode` database user. Use `vulnerablecode` as password when prompted - (otherwise use any password and update your settings locally). +Either run PostgreSQL in Docker: +`docker run --name pg-vulnerablecode -e POSTGRES_USER=vulnerablecode -e POSTGRES_PASSWORD=vulnerablecode -e POSTGRES_DB=vulnerablecode -p 5432:5432 postgres` + +Or without: + +- Create a user named `vulnerablecode`. Use `vulnerablecode` as password when prompted: `sudo -u postgres createuser --no-createrole --no-superuser --login --inherit --createdb --pwprompt vulnerablecode` -- Create a local test `vulnerablecode` database. +- Create a databased named `vulnerablecode`: `createdb --encoding=utf-8 --owner=vulnerablecode --user=vulnerablecode --password --host=localhost --port=5432 vulnerablecode` +### Application dependencies + Activate a virtualenv, install dependencies, and run the database migrations: ``` -python3 -m venv . -source bin/activate +python3 -m venv venv +source venv/bin/activate pip install -r requirements.txt python manage.py migrate ``` -Tests ------ +## Tests ``` -pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . -python3.6 -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py +pycodestyle --exclude=migrations,settings.py,venv,tests --max-line-length=100 . +python -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py ``` For Django based tests @@ -54,8 +56,7 @@ For Django based tests python manage.py test vulnerabilities/tests ``` -Scrape and save to the database -------------------------------- +## Data import ``` python manage.py shell @@ -78,22 +79,22 @@ archlinux_vulnerabilities = archlinux.scrape_vulnerabilities() archlinux_dump(archlinux_vulnerabilities) ``` -API ----- +## API + Start the webserver ``` -python manage.py runserver +DJANGO_DEV=1 python manage.py runserver ``` In your browser access: + ``` http://127.0.0.1:8000/api/ http://127.0.0.1:8000/api/packages/?name= ``` -Deployment on Heroku --------------------- +## Deployment on Heroku See https://devcenter.heroku.com/articles/django-app-configuration#creating-a-new-django-project https://devcenter.heroku.com/articles/deploying-python#how-to-keep-build-artifacts-out-of-git @@ -112,6 +113,6 @@ https://devcenter.heroku.com/articles/deploying-python#how-to-keep-build-artifac 7. Migrate the database: `heroku run python manage.py migrate` -8. Load the data referring to chapter "Scrape and save to the database" above. +8. Load the data referring to chapter "Data import" above. 9. To check the logs: `heroku logs --tail` diff --git a/vulnerablecode/dev.py b/vulnerablecode/dev.py index 15f8a2874..a52f0df75 100644 --- a/vulnerablecode/dev.py +++ b/vulnerablecode/dev.py @@ -1,14 +1,5 @@ import os -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - DEBUG = True ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1'] - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} From 202006a34b747ee70514a9a2491e63babf3deb0b Mon Sep 17 00:00:00 2001 From: Haiko Schol Date: Mon, 16 Sep 2019 16:18:16 +0200 Subject: [PATCH 2/2] Remove hard-coded SECRET_KEY from production settings This change moves the hard-coded key to vulnerablecode/dev.py for use during development (e.g. DJANGO_DEV is set). Otherwise the key is read from an environment variable with the same name. The advantage of this approach over using a gitignored file containing the secret is that it works with deployment environments that use ephemeral filesystems, such as Heroku. closes #46 Signed-off-by: Haiko Schol --- .travis.yml | 3 +++ README.md | 22 +++++++++++++++------- vulnerablecode/dev.py | 2 ++ vulnerablecode/settings.py | 8 +++++--- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index cdf937200..de9364aef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,9 @@ services: install: - pip install -r requirements.txt +env: + - SECRET_KEY="i1bn=oly)w*2yl-5yc&f!vvgt)p)fh3_2$r#spa!*sw36f5ov7" + before_script: - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . - psql -c "CREATE DATABASE vulnerablecode;" -U postgres diff --git a/README.md b/README.md index 251012964..fde92ef3e 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,15 @@ Activate a virtualenv, install dependencies, and run the database migrations: python3 -m venv venv source venv/bin/activate pip install -r requirements.txt -python manage.py migrate +DJANGO_DEV=1 python manage.py migrate ``` +The environment variable `DJANGO_DEV` is used to load settings suitable for development, defined in `vulnerablecode/dev.py`. If you don't want to type +it every time use `export DJANGO_DEV=1` instead. + +When not running in development mode, an environment variable named `SECRET_KEY` needs to be set. The recommended way to generate this key is to use +the code Django includes for this purpose: `SECRET_KEY=$(python -c "from django.core.management import utils; print(utils.get_random_secret_key())")`. + ## Tests ``` @@ -53,13 +59,13 @@ python -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests For Django based tests ``` -python manage.py test vulnerabilities/tests +DJANGO_DEV=1 python manage.py test vulnerabilities/tests ``` ## Data import ``` -python manage.py shell +DJANGO_DEV=1 python manage.py shell ``` ``` @@ -109,10 +115,12 @@ https://devcenter.heroku.com/articles/deploying-python#how-to-keep-build-artifac 5. Create Heroku app: `heroku create` -6. Deploy: `git push heroku :master` +6. Generate a secret key and pass it as an environment variable: `heroku config:set SECRET_KEY=$(python -c "from django.core.management import utils; print(utils.get_random_secret_key())")` + +7. Deploy: `git push heroku :master` -7. Migrate the database: `heroku run python manage.py migrate` +8. Migrate the database: `heroku run python manage.py migrate` -8. Load the data referring to chapter "Data import" above. +9. Load the data referring to chapter "Data import" above. -9. To check the logs: `heroku logs --tail` +10. To check the logs: `heroku logs --tail` diff --git a/vulnerablecode/dev.py b/vulnerablecode/dev.py index a52f0df75..f957c4340 100644 --- a/vulnerablecode/dev.py +++ b/vulnerablecode/dev.py @@ -3,3 +3,5 @@ DEBUG = True ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1'] + +SECRET_KEY = 'hhismo6l@1uj)nr6@o7$b2u68w5*_o^liji+=uzq=954-f$8_1' diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 970744e75..220925f52 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -12,6 +12,8 @@ import os +DEV_MODE = os.environ.get('DJANGO_DEV', False) + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -19,7 +21,8 @@ # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'hhismo6l@1uj)nr6@o7$b2u68w5*_o^liji+=uzq=954-f$8_1' +if not DEV_MODE: + SECRET_KEY = os.environ['SECRET_KEY'] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False @@ -28,7 +31,6 @@ '.herokuapp.com', ] - # Application definition INSTALLED_APPS = [ @@ -159,5 +161,5 @@ # Set `DJANGO_DEV=1` in env to enable dev mode -if os.environ.get('DJANGO_DEV', False): +if DEV_MODE: from vulnerablecode.dev import *