Merge pull request #480 from sbs2001/housekeeping

Remove old code and organize deps

Author: sbs2001

.github/workflows/main.yml

@@ -19,24 +19,24 @@ jobs:
         uses: actions/cache@v2
         with:
           path: .venv
-          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements.txt') }}
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-venv-
 
       - name: Install dependencies
         run: |
           sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev postgresql ncat
-          python -m pip install --upgrade pip 
+          python -m pip install --upgrade pip
 
       - uses: syphar/restore-virtualenv@v1
         id: cache-virtualenv
         with:
-          requirement_files: requirements.txt  # this is optional
+          requirement_files: requirements*.txt  # this is optional
 
       - uses: syphar/restore-pip-download-cache@v1
         if: steps.cache-virtualenv.outputs.cache-hit != 'true'
 
-      - run: pip install -r requirements.txt
+      - run: pip install -r requirements.txt -r requirements-dev.txt
         if: steps.cache-virtualenv.outputs.cache-hit != 'true'
 
       - name: Setup database

.github/workflows/upstream_test.yml

@@ -37,7 +37,7 @@ jobs:
         run: |
           sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
           python -m pip install --upgrade pip 
-          pip install -r requirements.txt
+          pip install -r requirements.txt -r requirements-dev.txt
           pip install pytest-xdist
     
       - name: Run tests

.travis.yml

@@ -5,7 +5,7 @@ services:
   - postgresql
 
 install:
-  - pip install -e . -r requirements.txt
+  - pip install -e . -r requirements.txt -r requirements-dev.txt
 
 env:
   global:

README.rst

@@ -149,13 +149,8 @@ Create a virtualenv, install dependencies, generate static files and run the dat
     python3 -m venv venv
     source venv/bin/activate
     pip install -r requirements.txt
-    DJANGO_DEV=1 python manage.py collectstatic
-    DJANGO_DEV=1 python manage.py migrate
-
-The environment variable ``DJANGO_DEV`` is used to load settings suitable for
-development,  defined in ``vulnerablecode/dev.py``. If you
-don't want to type it every time use ``export DJANGO_DEV=1`` instead.
-Do not use `DJANGO_DEV` in a production environment.
+    python manage.py collectstatic
+    python manage.py migrate
 
 
 For a production mode, an environment variable named ``SECRET_KEY`` needs to be
@@ -211,11 +206,11 @@ Non-Python dependencies are curated in::
 
 Run Tests
 ---------
-
+Make sure to install dev dependencies by running ``pip install -r requirements-dev.txt``
 Use these commands to run code style checks and the test suite::
 
     black -l 100 --check .
-    DJANGO_DEV=1 python -m pytest
+    python -m pytest
 
 
 Data import
@@ -233,23 +228,23 @@ for instructions on how to obtain your GitHub token.
 
 To run all data importers use::
 
-    DJANGO_DEV=1 python manage.py import --all
+    python manage.py import --all
 
 To list available importers use::
 
-    DJANGO_DEV=1 python manage.py import --list
+    python manage.py import --list
 
 To run specific importers::
 
-    DJANGO_DEV=1 python manage.py import rust npm 
+    python manage.py import rust npm 
 
 
 REST API access
 ---------------
 
 Start the webserver::
 
-    DJANGO_DEV=1 python manage.py runserver
+    python manage.py runserver
 
 
 For full documentation about API endpoints use this URL::

etc/nix/flake.nix

@@ -20,6 +20,10 @@
     let
 
       vulnerablecode-src = ./../..;
+      requirements =
+        builtins.readFile (vulnerablecode-src + "/requirements.txt");
+      requirementsDev =
+        builtins.readFile (vulnerablecode-src + "/requirements-dev.txt");
 
       # Extract version from setup.py.
       version = builtins.head (builtins.match ''.*version=["']?([^"',]+).*''
@@ -53,9 +57,9 @@
           # mach-nix release) is usually insufficient. Use
           # ./get-latest-pypi-deps-db.sh to obtain the data rev & hash.
           pypiDataRev =
-            "e9b0fc6b92cd6efbca7ba3b3d4a551bcc13a73c5"; # 2021-03-27T08:13:04Z
+            "8dcec158c51f8a96f316630679222e436c1b078c"; # 2021-06-16T08:41:20Z
           pypiDataSha256 =
-            "1ssa48l2iz8kncby1gfrbds79mg114dkhpxrridwcq6q2c37p62s";
+            "0499zl39aia74f0i7fkn5dsy8244dkmcw4vzd5nf4kai605j2jli";
         });
 
     in {
@@ -65,16 +69,17 @@
         with final.pkgs; {
 
           pythonEnv = machnixFor.${system}.mkPython {
-            requirements =
-              builtins.readFile (vulnerablecode-src + "/requirements.txt");
+            requirements = ''
+              ${requirements}
+            '';
           };
 
           vulnerablecode = stdenv.mkDerivation {
             inherit version;
             name = "vulnerablecode-${version}";
             src = vulnerablecode-src;
             dontConfigure = true; # do not use ./configure
-            propagatedBuildInputs = [ pythonEnv postgresql gitMinimal];
+            propagatedBuildInputs = [ pythonEnv postgresql gitMinimal ];
 
             postPatch = ''
               # Make sure the pycodestyle binary in $PATH is used.
@@ -110,48 +115,53 @@
         forAllSystems (system: self.packages.${system}.vulnerablecode);
 
       # Tests run by 'nix flake check' and by Hydra.
-      checks = forAllSystems (system: {
-        inherit (self.packages.${system}) vulnerablecode;
-
-        vulnerablecode-test = with nixpkgsFor.${system};
-          stdenv.mkDerivation {
-            name = "${vulnerablecode.name}-test";
-
-            buildInputs = [ wget vulnerablecode ];
-
-            # Used by pygit2.
-            # See https://github.com/NixOS/nixpkgs/pull/72544#issuecomment-582674047.
-            SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
-
-            unpackPhase = "true";
-
-            buildPhase = ''
-              source ${libSh}
-              initPostgres $(pwd)
-              export DJANGO_DEV=1
-              ${vulnerablecode}/manage.py migrate
+      checks = forAllSystems (system:
+        let
+          pythonEnvDev = machnixFor.${system}.mkPython {
+            requirements = ''
+              ${requirements}
+              ${requirementsDev}
             '';
-
-            doCheck = true;
-            checkPhase = ''
-              # Run pytest on the installed version. A running postgres
-              # database server is needed.
-              (
-                cd ${vulnerablecode}
-                black -l 100 --check .
-                pytest -m "not webtest"
-              )
-
-              # Launch the webserver and call the API.
-              ${vulnerablecode}/manage.py runserver &
-              sleep 2
-              wget http://127.0.0.1:8000/api/
-              kill %1 # kill background task (i.e. webserver)
-            '';
-
-            installPhase =
-              "mkdir -p $out"; # make this derivation return success
           };
-      });
+
+        in {
+          inherit (self.packages.${system}) vulnerablecode;
+
+          vulnerablecode-test = with nixpkgsFor.${system};
+            stdenv.mkDerivation {
+              name = "${vulnerablecode.name}-test";
+
+              buildInputs = [ wget vulnerablecode pythonEnvDev ];
+
+              unpackPhase = "true";
+
+              buildPhase = ''
+                source ${libSh}
+                initPostgres $(pwd)
+                export DJANGO_DEV=1
+                ${vulnerablecode}/manage.py migrate
+              '';
+
+              doCheck = true;
+              checkPhase = ''
+                # Run pytest on the installed version. A running postgres
+                # database server is needed.
+                (
+                  cd ${vulnerablecode}
+                  black -l 100 --check .
+                  pytest -m "not webtest"
+                )
+
+                # Launch the webserver and call the API.
+                ${vulnerablecode}/manage.py runserver &
+                sleep 2
+                wget http://127.0.0.1:8000/api/
+                kill %1 # kill background task (i.e. webserver)
+              '';
+
+              installPhase =
+                "mkdir -p $out"; # make this derivation return success
+            };
+        });
     };
 }

requirements-dev.txt

@@ -0,0 +1,5 @@
+pytest
+pytest-django
+freezegun
+ipython
+black

requirements.txt

@@ -1,61 +1,17 @@
-aiohttp==3.7.4
-asgiref==3.2.7
-attrs==20.3.0
-backcall==0.1.0
-beautifulsoup4==4.7.1
-binaryornot==0.4.4
-cached-property==1.5.1
-cffi==1.14.0
-contextlib2==0.5.5
-decorator==4.4.2
-univers==21.4.16.6
-dj-database-url==0.4.2
-Django==3.0.14
-django-filter==2.2.0
-djangorestframework==3.11.2
-django-widget-tweaks==1.4.8
-drf-spectacular==0.13.0
-gunicorn==19.7.1
-importlib-metadata==1.3.0
-ipython==7.13.0
-ipython-genutils==0.2.0
-jedi==0.17.0
-lxml==4.6.3
-more-itertools==8.0.2
-packageurl-python==0.9.3
-packaging==20.9
-parso==0.7.0
-pexpect==4.8.0
-pickleshare==0.7.5
-pluggy==0.13.1
-prompt-toolkit==3.0.5
-black==20.8b1
-psycopg2==2.8.4
-ptyprocess==0.6.0
-py==1.10.0
-pycparser==2.20
-gitpython==3.1.14
-Pygments==2.7.4
-pyparsing==2.4.5
-pytest==6.2.3
-pytest-django==4.1.0
-django-widget-tweaks==1.4.8 
-pytest-dependency==0.4.0
-pytest-django==4.1.0
-pytest-mock==1.13.0
-python-dateutil==2.8.1
-pytz==2019.3
-schema==0.7.4
-six==1.13.0
-soupsieve==1.9.5
-sqlparse==0.3.0
-tqdm==4.41.1
-traitlets==4.3.3
-wcwidth==0.1.7
-whitenoise==5.0.1
-zipp==0.6.0
-requests==2.23.0
-toml==0.10.2
-PyYAML==5.4
-freezegun==1.1.0
-saneyaml==0.5.2
+Django>=3.2.4
+psycopg2>=2.8.6
+requests>=2.25.1
+aiohttp>=3.7.4.post0
+djangorestframework>=3.12.4
+django-filter>=2.4.0
+django-widget-tweaks>=1.4.8
+drf-spectacular>=0.17.1
+packageurl-python>=0.9.4
+binaryornot>=0.4.4
+GitPython>=3.1.17
+univers>=21.4.16.6
+saneyaml>=0.5.2
+beautifulsoup4>=4.9.3
+python-dateutil>=2.8.1
+toml>=0.10.2
+lxml>=4.6.3

vulnerabilities/import_runner.py

@@ -24,13 +24,9 @@
 import dataclasses
 import datetime
 import logging
-from collections import Counter
 from itertools import chain
-import traceback
-from typing import Set
 from typing import Tuple
 
-from django.core import serializers
 from django.db import transaction
 
 from vulnerabilities import models