Skip to content

Commit c9904ae

Browse files
authored
Merge pull request #677 from nexB/deployment
Enable deployment Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
2 parents 8da4406 + bf4864b commit c9904ae

51 files changed

Lines changed: 1540 additions & 801 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.dockerignore

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,35 @@
1-
.git
2-
.github
1+
# Ignore the Docker files themselves to avoid that all layers are invalidated if the Dockerfiles are changed during development.
2+
.dockerignore
3+
Dockerfile
4+
docker.env
5+
docker-compose.yml
36

4-
venv
7+
8+
# Ignore Git directory and files and github directory.
9+
**/.git
10+
**/.gitignore
11+
**/.gitattributes
12+
**/.gitmodules
13+
**/.github
14+
15+
16+
# Ignore IDE artifacts and various common tools .dot files
17+
**/.vscode
18+
**/.idea
19+
**/.ssh
20+
**/.DS_Store
21+
22+
23+
# Ignore etc, docs and build dirs
24+
**/docs
25+
**/dist
26+
**/build
27+
**/etc
28+
.readthedocs.yaml
29+
CHANGELOG.rst
30+
README.rst
31+
32+
33+
# Ignore virtualenv and local Python build files
34+
**/venv
35+
**/*.egg-info

.github/workflows/main.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,9 @@ jobs:
4343
- name: Install dependencies
4444
run: make dev envfile
4545

46-
- name: Validate code format
47-
run: make check
46+
# Disable codestyle checks until we have cleaned up the code
47+
# - name: Validate code format
48+
# run: make check
4849

4950
- name: Run tests
5051
run: make test

.gitignore

Lines changed: 5 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ __pycache__/
1111
build/
1212
develop-eggs/
1313
dist/
14-
downloads/
1514
eggs/
1615
.eggs/
1716
lib/
@@ -24,26 +23,20 @@ wheels/
2423
.installed.cfg
2524
*.egg
2625

27-
# PyInstaller
28-
# Usually these files are written by a python script from a template
29-
# before PyInstaller builds the exe, so as to inject date/other infos into it.
30-
*.manifest
31-
*.spec
3226

3327
# Installer logs
3428
pip-log.txt
3529
pip-delete-this-directory.txt
30+
pip-selfcheck.json
3631

3732
# Unit test / coverage reports
3833
htmlcov/
3934
.tox/
4035
.coverage
4136
.coverage.*
4237
.cache
43-
nosetests.xml
4438
coverage.xml
4539
*.cover
46-
.hypothesis/
4740

4841
# Translations
4942
*.mo
@@ -53,13 +46,6 @@ coverage.xml
5346
*.log
5447
local_settings.py
5548

56-
# Flask stuff:
57-
instance/
58-
.webassets-cache
59-
60-
# Scrapy stuff:
61-
.scrapy
62-
6349
# Sphinx documentation
6450
docs/_build/
6551

@@ -69,17 +55,10 @@ target/
6955
# Jupyter Notebook
7056
.ipynb_checkpoints
7157

72-
# pyenv
58+
# pyenv and pip
7359
.python-version
7460
pyvenv.cfg
7561
bin/
76-
pip-selfcheck.json
77-
78-
# celery beat schedule file
79-
celerybeat-schedule
80-
81-
# SageMath parsed files
82-
*.sage.py
8362

8463
# Environments
8564
.env
@@ -88,21 +67,10 @@ env/
8867
venv/
8968
ENV/
9069

91-
# Spyder project settings
92-
.spyderproject
93-
.spyproject
94-
95-
# Rope project settings
96-
.ropeproject
97-
98-
# mkdocs documentation
99-
/site
10070

10171
# mypy
10272
.mypy_cache/
10373

104-
# PyCharm
105-
.idea/
10674

10775
# Database
10876
*.sqlite3*
@@ -118,13 +86,13 @@ share
11886
Pipfile
11987

12088
# editors
121-
.vscode/
89+
.vscode
90+
# PyCharm
91+
.idea/
12292

12393
# pytest
12494
.pytest_cache
12595

126-
# VSCode
127-
.vscode
12896

12997
# Various junk and temp files
13098
.DS_Store

CHANGELOG.rst

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,27 @@
11
Release notes
22
=============
33

4-
vNext
5-
-----
4+
Version v30.0.0
5+
----------------
6+
7+
- Refactor core processing with Importers that import data and Improvers that
8+
get the imported data and convert that in Vulnerabilities and Packages and can
9+
also improve and refine imported and existing data. The migration to this new
10+
architecture is under way.
11+
12+
- Add new data sources including OSV, GitHub and GitLab.
13+
14+
- Improve documentation ncluding adding examples for importers and improvers
15+
16+
- Remove the ability to edit relationships from the UI. The UI is now read-only
17+
and we will need to design a different UI for proper review and curation of
18+
vulnerabilities.
19+
20+
- Add support for nix as a Linux deployemnet target.
621

7-
- Add support for nix
8-
- Improve documentation
922

1023

1124
Version v20.10
12-
--------------
25+
---------------
1326

1427
This release comes with the new calver versioning scheme and an initial data dump.

Dockerfile

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,24 @@
1-
FROM python:3.8
1+
# Copyright (c) nexB Inc. and others. All rights reserved.
2+
# VulnerableCode is a trademark of nexB Inc.
3+
#
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects
28

3-
# Force unbuffered stdout and stderr (i.e. they are flushed to terminal immediately)
9+
FROM python:3.9
10+
11+
WORKDIR /app
12+
13+
# Python settings: Force unbuffered stdout and stderr (i.e. they are flushed to terminal immediately)
414
ENV PYTHONUNBUFFERED 1
15+
# Python settings: do not write pyc files
16+
ENV PYTHONDONTWRITEBYTECODE 1
17+
18+
RUN mkdir -p /var/vulnerablecode/static
19+
20+
# Keep the dependencies installation before the COPY of the app/ for proper caching
21+
COPY setup.cfg setup.py requirements.txt pyproject.toml /app/
22+
RUN pip install . -c requirements.txt
523

6-
RUN mkdir /opt/vulnerablecode && \
7-
mkdir -p /var/vulnerablecode/static/
8-
WORKDIR /opt/vulnerablecode
9-
COPY . .
10-
RUN python -m pip install --upgrade pip && \
11-
pip install -r requirements.txt
24+
COPY . /app

Makefile

Lines changed: 45 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,9 @@
2424
# Python version can be specified with `$ PYTHON_EXE=python3.x make conf`
2525
PYTHON_EXE?=python3
2626
VENV=venv
27+
MANAGE=${VENV}/bin/python manage.py
2728
ACTIVATE?=. ${VENV}/bin/activate;
2829
VIRTUALENV_PYZ=etc/thirdparty/virtualenv.pyz
29-
BLACK_ARGS=-l 100 .
30-
ISORT_ARGS=.
3130
# Do not depend on Python to generate the SECRET_KEY
3231
GET_SECRET_KEY=`base64 /dev/urandom | head -c50`
3332
# Customize with `$ make envfile ENV_FILE=/etc/vulnerablecode/.env`
@@ -49,41 +48,48 @@ virtualenv:
4948

5049
conf: virtualenv
5150
@echo "-> Install dependencies"
52-
@${ACTIVATE} pip install -r requirements.txt
51+
@${ACTIVATE} pip install -e . -c requirements.txt
5352

54-
dev: conf
53+
dev: virtualenv
5554
@echo "-> Configure and install development dependencies"
56-
@${ACTIVATE} pip install -r requirements-dev.txt
55+
@${ACTIVATE} pip install -e .[dev] -c requirements.txt
5756

5857
envfile:
5958
@echo "-> Create the .env file and generate a secret key"
6059
@if test -f ${ENV_FILE}; then echo ".env file exists already"; exit 1; fi
6160
@mkdir -p $(shell dirname ${ENV_FILE}) && touch ${ENV_FILE}
6261
@echo SECRET_KEY=\"${GET_SECRET_KEY}\" > ${ENV_FILE}
6362

64-
check:
65-
@echo "-> Run isort validation"
66-
@${ACTIVATE} isort --check-only ${ISORT_ARGS}
67-
@echo "-> Run black validation"
68-
@${ACTIVATE} black --check ${BLACK_ARGS}
63+
isort:
64+
@echo "-> Apply isort changes to ensure proper imports ordering"
65+
${VENV}/bin/isort .
6966

7067
black:
7168
@echo "-> Apply black code formatter"
72-
${VENV}/bin/black ${BLACK_ARGS}
69+
${VENV}/bin/black .
7370

74-
isort:
75-
@echo "-> Apply isort code formatter"
76-
${VENV}/bin/isort ${ISORT_ARGS}
71+
doc8:
72+
@echo "-> Run doc8 validation"
73+
@${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/
7774

7875
valid: isort black
7976

77+
check:
78+
@echo "-> Run pycodestyle (PEP8) validation"
79+
@${ACTIVATE} pycodestyle --max-line-length=100 --exclude=venv,lib,thirdparty,docs,migrations,settings.py .
80+
@echo "-> Run isort imports ordering validation"
81+
@${ACTIVATE} isort --check-only .
82+
@echo "-> Run black validation"
83+
@${ACTIVATE} black --check ${BLACK_ARGS}
84+
8085
clean:
8186
@echo "-> Clean the Python env"
82-
rm -rf ${VENV}
87+
rm -rf ${VENV} build/ dist/ vulnerablecode.egg-info/ docs/_build/ pip-selfcheck.json
88+
find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
8389

8490
migrate:
8591
@echo "-> Apply database migrations"
86-
${ACTIVATE} ./manage.py migrate
92+
${MANAGE} migrate
8793

8894
postgres:
8995
@echo "-> Configure PostgreSQL database"
@@ -103,26 +109,31 @@ sqlite:
103109
@$(MAKE) migrate
104110

105111
run:
106-
${ACTIVATE} ./manage.py runserver
112+
${MANAGE} runserver 8001 --noreload --insecure
107113

108114
test:
109-
@echo "-> Run offline tests"
110-
${ACTIVATE} ${PYTHON_EXE} -m pytest -v -m "not webtest"
115+
@echo "-> Run the test suite"
116+
${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs -m "not webtest"
111117

112118
webtest:
113119
@echo "-> Run web tests"
114-
${ACTIVATE} ${PYTHON_EXE} -m pytest -v -m "webtest"
115-
116-
package: conf
117-
@echo "-> Create a VulnerableCode package for offline installation"
118-
@echo "-> Fetch dependencies in thirdparty/ for offline installation"
119-
rm -rf thirdparty && mkdir thirdparty
120-
${VENV}/bin/pip download -r requirements.txt --no-cache-dir --dest thirdparty
121-
@echo "-> Create package in dist/ for offline installation"
122-
${VENV}/bin/python setup.py sdist
123-
124-
install: virtualenv
125-
@echo "-> Install and configure the Python env with base dependencies, offline"
126-
${VENV}/bin/pip install --upgrade --no-index --no-cache-dir --find-links=thirdparty -e .
127-
128-
.PHONY: virtualenv conf dev envfile install check valid clean migrate postgres sqlite run test package
120+
${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs -m "webtest"
121+
122+
bump:
123+
@echo "-> Bump the version"
124+
bin/bumpver update --no-fetch --patch
125+
126+
docs:
127+
rm -rf docs/_build/
128+
@${ACTIVATE} sphinx-build docs/ docs/_build/
129+
130+
docker-images:
131+
@echo "-> Build Docker services"
132+
docker-compose build
133+
@echo "-> Pull service images"
134+
docker-compose pull
135+
@echo "-> Save the service images to a compressed tar archive in the dist/ directory"
136+
@mkdir -p dist/
137+
@docker save postgres vulnerablecode_vulnerablecode nginx | gzip > dist/vulnerablecode-images-`git describe --tags`.tar.gz
138+
139+
.PHONY: virtualenv conf dev envfile install check valid isort clean migrate postgres sqlite run test bump docs docker-images

README.gif

-4 MB
Binary file not shown.

0 commit comments

Comments
 (0)