Black the test, add separate action for checking upstream data

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

Author: sbs2001

.github/workflows/main.yml

@@ -38,7 +38,7 @@ jobs:
           pip install -r requirements.txt
     
       - name: Run tests
-        run: py.test -v -m webtest
+        run: pytest -v -m "not webtest"
         env:
           # The hostname, username used to communicate with the PostgreSQL service container
           POSTGRES_HOST: localhost

.github/workflows/upstream_test.yml

@@ -0,0 +1,51 @@
+on:
+  workflow_dispatch: # allow manual execution
+  push:
+  schedule:
+    # run on every 9 o'clock
+    - cron: '0 9 * * *'
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+
+    services:
+      # Label used to access the service container
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: vulnerablecode
+          POSTGRES_DB: vulnerablecode
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
+          python -m pip install --upgrade pip 
+          pip install -r requirements.txt
+    
+      - name: Run tests
+        run: pytest -v -m webtest
+        env:
+          # The hostname, username used to communicate with the PostgreSQL service container
+          POSTGRES_HOST: localhost
+          VC_DB_USER: postgres
+          POSTGRES_PORT: 5432
+          DJANGO_DEV: 1
+          GH_TOKEN: 1 

vulnerabilities/importers/project_kb_msr2019.py

@@ -59,7 +59,7 @@ def updated_advisories(self):
 
     def fetch(self):
         response = urllib.request.urlopen(self.url)
-        lines = [l.decode("utf-8") for l in response.readlines()]
+        lines = [line.decode("utf-8") for line in response.readlines()]
         return csv.reader(lines)
 
     @staticmethod

vulnerabilities/tests/test_upstream.py

@@ -2,13 +2,17 @@
 from vulnerabilities import importers
 from vulnerabilities.importer_yielder import IMPORTER_REGISTRY
 
+
 @pytest.mark.webtest
-@pytest.mark.parametrize(("data_source", "config"), ( (data["data_source"] , data["data_source_cfg"]) for data in IMPORTER_REGISTRY),)
+@pytest.mark.parametrize(
+    ("data_source", "config"),
+    ((data["data_source"], data["data_source_cfg"]) for data in IMPORTER_REGISTRY),
+)
 def test_updated_advisories(data_source, config):
 
-    if not data_source == "GitHubAPIDataSource": 
+    if not data_source == "GitHubAPIDataSource":
         data_src = getattr(importers, data_source)
-        data_src = data_src(batch_size=1,config=config)
+        data_src = data_src(batch_size=1, config=config)
         with data_src:
             for i in data_src.updated_advisories():
-                pass
+                pass