Merge branch 'main' into gitpython

Author: sbs2001

.github/workflows/main.yml

@@ -38,7 +38,7 @@ jobs:
           pip install -r requirements.txt
     
       - name: Run tests
-        run: python -m pytest
+        run: python -m pytest -v -m "not webtest"
         env:
           # The hostname, username used to communicate with the PostgreSQL service container
           POSTGRES_HOST: localhost

.github/workflows/test-import-using-nix.yml

@@ -25,5 +25,6 @@ jobs:
       - name: run checks & test import
         run: |
           cd etc/nix
+          ./get-latest-pypi-deps-db.sh --in-place
           nix --print-build-logs flake check
           ./test-import-using-nix.sh alpine

.github/workflows/upstream_test.yml

@@ -0,0 +1,52 @@
+on:
+  workflow_dispatch: # allow manual execution
+  schedule:
+    # run on every 9 o'clock
+    - cron: '0 9 * * *'
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+
+    services:
+      # Label used to access the service container
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: vulnerablecode
+          POSTGRES_DB: vulnerablecode
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
+          python -m pip install --upgrade pip 
+          pip install -r requirements.txt
+          pip install pytest-xdist
+    
+      - name: Run tests
+        run: |
+          python -m pytest -v vulnerabilities/tests/test_upstream.py -n 2
+        env:
+          # The hostname, username used to communicate with the PostgreSQL service container
+          POSTGRES_HOST: localhost
+          VC_DB_USER: postgres
+          POSTGRES_PORT: 5432
+          DJANGO_DEV: 1
+          GH_TOKEN: 1 

.travis.yml

@@ -18,7 +18,7 @@ before_script:
 
 script:
   - ./manage.py collectstatic
-  - python -m pytest
+  - python -m pytest -v -m "not webtest"
 
 notifications:
   email: false

README.rst

@@ -106,6 +106,9 @@ compose. For this you need to have the following installed.
 Use ``sudo docker-compose up`` to start VulnerableCode. Then access
 VulnerableCode at http://localhost:8000/ or at http://127.0.0.1:8000/
 
+**Important**: Don't forget to run ``sudo docker-compose up -d --no-deps --build web`` to sync your instance after every ``git pull``.
+
+
 Use ``sudo docker-compose exec web bash`` to access the VulnerableCode
 container. From here you can access ``manage.py`` and run management commands
 to import data as specified below.

etc/nix/flake.nix

@@ -53,9 +53,9 @@
           # mach-nix release) is usually insufficient. Use
           # ./get-latest-pypi-deps-db.sh to obtain the data rev & hash.
           pypiDataRev =
-            "499750266bb4b2840cbe856c2cc0e3297685e362"; # 2021-03-06T08:13:08Z
+            "e9b0fc6b92cd6efbca7ba3b3d4a551bcc13a73c5"; # 2021-03-27T08:13:04Z
           pypiDataSha256 =
-            "188g24k8pk4lgqybywimkvwjwh8014v6l2mrkvzv309882i9p5gc";
+            "1ssa48l2iz8kncby1gfrbds79mg114dkhpxrridwcq6q2c37p62s";
         });
 
     in {
@@ -136,7 +136,11 @@
             checkPhase = ''
               # Run pytest on the installed version. A running postgres
               # database server is needed.
-              (cd ${vulnerablecode} && pytest)
+              (
+                cd ${vulnerablecode}
+                black -l 100 --check .
+                pytest -m "not webtest"
+              )
 
               # Launch the webserver and call the API.
               ${vulnerablecode}/manage.py runserver &

etc/nix/get-latest-pypi-deps-db.sh

@@ -7,6 +7,16 @@ COMMIT=$(sed '1q;d' <<< "$DATA")
 DATE=$(sed '2q;d' <<< "$DATA")
 SHA256=$(nix-prefetch-url --unpack --type sha256 "https://github.com/$USER_SLASH_REPO/tarball/$COMMIT" | tail -n 1)
 
+NIX_REV_ATTR="pypiDataRev = \"$COMMIT\"; # $DATE"
+NIX_SHA_ATTR="pypiDataSha256 = \"$SHA256\";"
+
 echo ""
-echo "pypiDataRev = \"$COMMIT\"; # $DATE"
-echo "pypiDataSha256 = \"$SHA256\";"
+echo $NIX_REV_ATTR
+echo $NIX_SHA_ATTR
+
+if [[ "$1" = "--in-place" ]] ; then
+  # Replace the values in the flake.
+  PATTERN="\s*\n?\s*\"[^\n]+" # <space><newline><space>"content...<newline>
+  perl -i.bak1 -0777 -pe "s/pypiDataRev =$PATTERN/$NIX_REV_ATTR/" flake.nix
+  perl -i.bak2 -0777 -pe "s/pypiDataSha256 =$PATTERN/$NIX_SHA_ATTR/" flake.nix
+fi

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+DJANGO_SETTINGS_MODULE = vulnerablecode.settings

requirements.txt

@@ -1,16 +1,16 @@
 aiohttp==3.7.4
 asgiref==3.2.7
-attrs==19.3.0
+attrs==20.3.0
 backcall==0.1.0
 beautifulsoup4==4.7.1
 binaryornot==0.4.4
 cached-property==1.5.1
 cffi==1.14.0
 contextlib2==0.5.5
 decorator==4.4.2
-dephell-specifier==0.2.1
+univers==21.4.16.6
 dj-database-url==0.4.2
-Django==3.0.13
+Django==3.0.14
 django-filter==2.2.0
 djangorestframework==3.11.2
 django-widget-tweaks==1.4.8
@@ -20,10 +20,10 @@ importlib-metadata==1.3.0
 ipython==7.13.0
 ipython-genutils==0.2.0
 jedi==0.17.0
-lxml==4.6.2
+lxml==4.6.3
 more-itertools==8.0.2
 packageurl-python==0.9.3
-packaging==19.2
+packaging==20.9
 parso==0.7.0
 pexpect==4.8.0
 pickleshare==0.7.5
@@ -32,15 +32,16 @@ prompt-toolkit==3.0.5
 black==20.8b1
 psycopg2==2.8.4
 ptyprocess==0.6.0
-py==1.8.0
+py==1.10.0
 pycparser==2.20
 gitpython==3.1.14
 Pygments==2.7.4
 pyparsing==2.4.5
-pytest==5.3.2
+pytest==6.2.3
+pytest-django==4.1.0
 django-widget-tweaks==1.4.8 
 pytest-dependency==0.4.0
-pytest-django==3.7.0
+pytest-django==4.1.0
 pytest-mock==1.13.0
 python-dateutil==2.8.1
 pytz==2019.3

vulnerabilities/admin.py

@@ -51,7 +51,7 @@ class PackageAdmin(admin.ModelAdmin):
 
 @admin.register(PackageRelatedVulnerability)
 class PackageRelatedVulnerabilityAdmin(admin.ModelAdmin):
-    list_filter = ("is_vulnerable", "package__type", "package__namespace")
+    list_filter = ("package__type", "package__namespace")
     search_fields = ["vulnerability__vulnerability_id", "package__name"]