Merge branch 'main' into fix-ubuntu-usn-importer

Author: AmitGupta7580

.github/workflows/upstream_test.yml

@@ -39,9 +39,11 @@ jobs:
           sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
           python -m pip install --upgrade pip 
           pip install -r requirements.txt
+          pip install pytest-xdist
     
       - name: Run tests
-        run: pytest -v -m webtest
+        run: |
+          python -m pytest -v vulnerabilities/tests/test_upstream.py -n 2
         env:
           # The hostname, username used to communicate with the PostgreSQL service container
           POSTGRES_HOST: localhost

README.rst

@@ -106,6 +106,9 @@ compose. For this you need to have the following installed.
 Use ``sudo docker-compose up`` to start VulnerableCode. Then access
 VulnerableCode at http://localhost:8000/ or at http://127.0.0.1:8000/
 
+**Important**: Don't forget to run ``sudo docker-compose up -d --no-deps --build web`` to sync your instance after every ``git pull``.
+
+
 Use ``sudo docker-compose exec web bash`` to access the VulnerableCode
 container. From here you can access ``manage.py`` and run management commands
 to import data as specified below.

etc/nix/flake.nix

@@ -136,7 +136,11 @@
             checkPhase = ''
               # Run pytest on the installed version. A running postgres
               # database server is needed.
-              (cd ${vulnerablecode} && pytest)
+              (
+                cd ${vulnerablecode}
+                black -l 100 --check .
+                pytest -m "not webtest"
+              )
 
               # Launch the webserver and call the API.
               ${vulnerablecode}/manage.py runserver &

requirements.txt

@@ -31,15 +31,16 @@ prompt-toolkit==3.0.5
 black==20.8b1
 psycopg2==2.8.4
 ptyprocess==0.6.0
-py==1.8.0
+py==1.10.0
 pycparser==2.20
 pygit2==1.5.0
 Pygments==2.7.4
 pyparsing==2.4.5
-pytest==5.3.2
+pytest==6.2.3
+pytest-django==4.1.0
 django-widget-tweaks==1.4.8 
 pytest-dependency==0.4.0
-pytest-django==3.7.0
+pytest-django==4.1.0
 pytest-mock==1.13.0
 python-dateutil==2.8.1
 pytz==2019.3

vulnerabilities/helpers.py

@@ -79,7 +79,7 @@ def create_etag(data_src, url, etag_key):
     return True
 
 
-is_cve = re.compile(r"CVE-\d+-\d+", re.IGNORECASE).match
+is_cve = re.compile(r"CVE-\d{4}-\d{4,7}", re.IGNORECASE).match
 
 
 def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5):

vulnerabilities/importers/__init__.py

@@ -47,5 +47,4 @@
 from vulnerabilities.importers.suse_scores import SUSESeverityScoreDataSource
 from vulnerabilities.importers.ubuntu import UbuntuDataSource
 from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource
-from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource
 from vulnerabilities.importers.istio import IstioDataSource

vulnerabilities/tests/test_importer_yielder.py

@@ -58,10 +58,12 @@ class TestImporterYielder(TestCase):
     def setUpClass(cls):
         cls.importer_yielder = importer_yielder
 
+    def setUp(self):
+        Importer.objects.all().delete()
+
     @pytest.mark.django_db
     def test_load_importers_freshdb(self):
         assert Importer.objects.all().count() == 0
-
         with patch(
             "vulnerabilities.importer_yielder.IMPORTER_REGISTRY", new=MOCK_IMPORTER_REGISTRY
         ):  # nopep8
@@ -71,7 +73,6 @@ def test_load_importers_freshdb(self):
     @pytest.mark.django_db
     def test_load_importers_setteddb(self):
         assert Importer.objects.all().count() == 0
-
         mock_existing_importer = Importer.objects.create(**MOCK_IMPORTER_REGISTRY[1])
         mock_existing_importer.data_source_cfg["etags"] = {"url": "0x1234"}
         mock_existing_importer.save()