Merge branch 'main' into patch-1

Author: sify21

.github/workflows/main.yml

@@ -38,7 +38,7 @@ jobs:
           pip install -r requirements.txt
     
       - name: Run tests
-        run: python -m pytest
+        run: python -m pytest -v -m "not webtest"
         env:
           # The hostname, username used to communicate with the PostgreSQL service container
           POSTGRES_HOST: localhost

.github/workflows/upstream_test.yml

@@ -0,0 +1,51 @@
+on:
+  workflow_dispatch: # allow manual execution
+  push:
+  schedule:
+    # run on every 9 o'clock
+    - cron: '0 9 * * *'
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+
+    services:
+      # Label used to access the service container
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: vulnerablecode
+          POSTGRES_DB: vulnerablecode
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
+          python -m pip install --upgrade pip 
+          pip install -r requirements.txt
+    
+      - name: Run tests
+        run: pytest -v -m webtest
+        env:
+          # The hostname, username used to communicate with the PostgreSQL service container
+          POSTGRES_HOST: localhost
+          VC_DB_USER: postgres
+          POSTGRES_PORT: 5432
+          DJANGO_DEV: 1
+          GH_TOKEN: 1 

.travis.yml

@@ -18,7 +18,7 @@ before_script:
 
 script:
   - ./manage.py collectstatic
-  - python -m pytest
+  - python -m pytest -v -m "not webtest"
 
 notifications:
   email: false

README.rst

@@ -106,6 +106,9 @@ compose. For this you need to have the following installed.
 Use ``sudo docker-compose up`` to start VulnerableCode. Then access
 VulnerableCode at http://localhost:8000/ or at http://127.0.0.1:8000/
 
+**Important**: Don't forget to run ``sudo docker-compose up -d --no-deps --build web`` to sync your instance after every ``git pull``.
+
+
 Use ``sudo docker-compose exec web bash`` to access the VulnerableCode
 container. From here you can access ``manage.py`` and run management commands
 to import data as specified below.

vulnerabilities/helpers.py

@@ -25,6 +25,7 @@
 
 import requests
 import toml
+import urllib3
 import yaml
 
 # TODO add logging here
@@ -78,4 +79,22 @@ def create_etag(data_src, url, etag_key):
     return True
 
 
-is_cve = re.compile(r"CVE-\d+-\d+", re.IGNORECASE).match
+is_cve = re.compile(r"CVE-\d{4}-\d{4,7}", re.IGNORECASE).match
+
+
+def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5):
+    """
+    Returns a requests sessions which retries on 5xx errors with
+    a backoff_factor
+    """
+    retries = urllib3.util.Retry(
+        total=max_retries,
+        backoff_factor=backoff_factor,
+        raise_on_status=True,
+        status_forcelist=range(500, 600, 1),
+    )
+    adapter = requests.adapters.HTTPAdapter(max_retries=retries)
+    session = requests.Session()
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session

vulnerabilities/importers/redhat.py

@@ -20,14 +20,15 @@
 #  VulnerableCode is a free software code from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-from packageurl import PackageURL
 import requests
+from packageurl import PackageURL
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
 from vulnerabilities.data_source import DataSourceConfiguration
 from vulnerabilities.data_source import Reference
 from vulnerabilities.data_source import VulnerabilitySeverity
+from vulnerabilities.helpers import requests_with_5xx_retry
 from vulnerabilities.severity_systems import scoring_systems
 
 
@@ -43,6 +44,9 @@ def updated_advisories(self):
         return self.batch_advisories(processed_advisories)
 
 
+requests_session = requests_with_5xx_retry(max_retries=5, backoff_factor=1)
+
+
 def fetch():
     """
     Return a list of CVE data mappings fetched from the RedHat API.
@@ -58,7 +62,7 @@ def fetch():
         current_url = url_template.format(page_no)
         try:
             print(f"Fetching: {current_url}")
-            response = requests.get(current_url)
+            response = requests_session.get(current_url)
             if response.status_code != requests.codes.ok:
                 # TODO: log me
                 print(f"Failed to fetch results from {current_url}")
@@ -90,7 +94,9 @@ def to_advisory(advisory_data):
     bugzilla = advisory_data.get("bugzilla")
     if bugzilla:
         url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla)
-        bugzilla_data = requests.get(f"https://bugzilla.redhat.com/rest/bug/{bugzilla}").json()
+        bugzilla_data = requests_session.get(
+            f"https://bugzilla.redhat.com/rest/bug/{bugzilla}"
+        ).json()
         if (
             bugzilla_data.get("bugs")
             and len(bugzilla_data["bugs"])
@@ -115,18 +121,24 @@ def to_advisory(advisory_data):
         # See https://access.redhat.com/articles/2130961 for more details.
 
         if "RHSA" in rh_adv.upper():
-            rhsa_data = requests.get(
+            rhsa_data = requests_session.get(
                 f"https://access.redhat.com/hydra/rest/securitydata/cvrf/{rh_adv}.json"
             ).json()  # nopep8
-            value = rhsa_data["cvrfdoc"]["aggregate_severity"]
-            rhsa_aggregate_severity = VulnerabilitySeverity(
-                system=scoring_systems["rhas"],
-                value=value,
-            )
+
+            rhsa_aggregate_severities = []
+            if rhsa_data.get("cvrfdoc"):
+                # not all RHSA errata have a corresponding CVRF document
+                value = rhsa_data["cvrfdoc"]["aggregate_severity"]
+                rhsa_aggregate_severities.append(
+                    VulnerabilitySeverity(
+                        system=scoring_systems["rhas"],
+                        value=value,
+                    )
+                )
 
             references.append(
                 Reference(
-                    severities=[rhsa_aggregate_severity],
+                    severities=rhsa_aggregate_severities,
                     url="https://access.redhat.com/errata/{}".format(rh_adv),
                     reference_id=rh_adv,
                 )

vulnerabilities/severity_systems.py

@@ -81,4 +81,10 @@ def as_score(self, value):
         url="https://www.first.org/cvss/specification-document#Qualitative-Severity-Rating-Scale",
         notes="A textual interpretation of severity. Has values like HIGH, MODERATE etc",
     ),
+    "generic_textual": ScoringSystem(
+        identifier="generic_textual",
+        name="Generic textual severity rating",
+        url="",
+        notes="Severity for unknown scoring systems. Contains generic textual values like High, Low etc",
+    ),
 }

vulnerabilities/tests/test_redhat_importer.py

@@ -139,7 +139,7 @@ def test_to_advisory(self):
         }
         for adv in data:
             with unittest.mock.patch(
-                "vulnerabilities.importers.redhat.requests.get", return_value=mock_resp
+                "vulnerabilities.importers.redhat.requests_session.get", return_value=mock_resp
             ):
                 adv = redhat.to_advisory(adv)
                 found_advisories.append(adv)

vulnerabilities/tests/test_upstream.py

@@ -0,0 +1,18 @@
+import pytest
+from vulnerabilities import importers
+from vulnerabilities.importer_yielder import IMPORTER_REGISTRY
+
+
+@pytest.mark.webtest
+@pytest.mark.parametrize(
+    ("data_source", "config"),
+    ((data["data_source"], data["data_source_cfg"]) for data in IMPORTER_REGISTRY),
+)
+def test_updated_advisories(data_source, config):
+
+    if not data_source == "GitHubAPIDataSource":
+        data_src = getattr(importers, data_source)
+        data_src = data_src(batch_size=1, config=config)
+        with data_src:
+            for i in data_src.updated_advisories():
+                pass