From 766dffc310ff674e1f49007595339187d89b3bc2 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 20 Jan 2023 23:36:11 +0530 Subject: [PATCH 1/2] Delete advisories related to importer before using the importer We have a lot of old advisories that doesn't comply with the latest model structure, when we make any changes to importer it makes new advisories. To tackle the problem of advisories that doesn't support the structure, we should clear the advisories imported by an importer before running it again. Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 1 + .../tests/test_deletion_of_advisories.py | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 vulnerabilities/tests/test_deletion_of_advisories.py diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index c57e0eebd..b4716fe41 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -39,6 +39,7 @@ def run(self) -> None: """ importer_name = self.importer_class.qualified_name importer_class = self.importer_class + Advisory.objects.filter(created_by=importer_name).delete() logger.info(f"Starting import for {importer_name}") advisory_datas = importer_class().advisory_data() count = process_advisories(advisory_datas=advisory_datas, importer_name=importer_name) diff --git a/vulnerabilities/tests/test_deletion_of_advisories.py b/vulnerabilities/tests/test_deletion_of_advisories.py new file mode 100644 index 000000000..b9be68519 --- /dev/null +++ b/vulnerabilities/tests/test_deletion_of_advisories.py @@ -0,0 +1,43 @@ +import os +from unittest import mock + +import pytest + +from vulnerabilities.import_runner import ImportRunner +from vulnerabilities.importers.nginx import NginxImporter +from vulnerabilities.improve_runner import ImproveRunner +from vulnerabilities.improvers.default import DefaultImprover +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data", "nginx") + + +@pytest.mark.django_db +@mock.patch("vulnerabilities.importers.nginx.NginxImporter.fetch") +def test_deletion_of_advisories(fetch): + """Test that the deletion of advisories works as expected.""" + with open(os.path.join(TEST_DATA, "security_advisories.html")) as f: + fetch.return_value = f.read() + + ImportRunner(NginxImporter).run() + ImproveRunner(DefaultImprover).run() + packages = Package.objects.all().only("id") + packages_before_deletion = [int(package.id) for package in packages] + vulnerabilities = Vulnerability.objects.all().only("id") + vulnerabilities_before_deletion = [int(vulnerability.id) for vulnerability in vulnerabilities] + references = VulnerabilityReference.objects.all().only("id") + references_before_deletion = [int(reference.id) for reference in references] + ImportRunner(NginxImporter).run() + ImproveRunner(DefaultImprover).run() + packages = Package.objects.all().only("id") + packages_after_deletion = [int(package.id) for package in packages] + vulnerabilities = Vulnerability.objects.all().only("id") + vulnerabilities_after_deletion = [int(vulnerability.id) for vulnerability in vulnerabilities] + references = VulnerabilityReference.objects.all().only("id") + references_after_deletion = [int(reference.id) for reference in references] + assert packages_before_deletion == packages_after_deletion + assert vulnerabilities_before_deletion == vulnerabilities_after_deletion + assert references_before_deletion == references_after_deletion From 64703ac164e9a60734c87ee81e83f4c6f53cc1c1 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 24 Jan 2023 19:24:41 +0530 Subject: [PATCH 2/2] Check if new VCIDs get generated by running importer again Signed-off-by: Tushar Goel --- vulnerabilities/tests/test_deletion_of_advisories.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/tests/test_deletion_of_advisories.py b/vulnerabilities/tests/test_deletion_of_advisories.py index b9be68519..2c86a8126 100644 --- a/vulnerabilities/tests/test_deletion_of_advisories.py +++ b/vulnerabilities/tests/test_deletion_of_advisories.py @@ -26,16 +26,20 @@ def test_deletion_of_advisories(fetch): ImproveRunner(DefaultImprover).run() packages = Package.objects.all().only("id") packages_before_deletion = [int(package.id) for package in packages] - vulnerabilities = Vulnerability.objects.all().only("id") - vulnerabilities_before_deletion = [int(vulnerability.id) for vulnerability in vulnerabilities] + vulnerabilities = Vulnerability.objects.all().only("vulnerability_id") + vulnerabilities_before_deletion = [ + str(vulnerability.vulnerability_id) for vulnerability in vulnerabilities + ] references = VulnerabilityReference.objects.all().only("id") references_before_deletion = [int(reference.id) for reference in references] ImportRunner(NginxImporter).run() ImproveRunner(DefaultImprover).run() packages = Package.objects.all().only("id") packages_after_deletion = [int(package.id) for package in packages] - vulnerabilities = Vulnerability.objects.all().only("id") - vulnerabilities_after_deletion = [int(vulnerability.id) for vulnerability in vulnerabilities] + vulnerabilities = Vulnerability.objects.all().only("vulnerability_id") + vulnerabilities_after_deletion = [ + str(vulnerability.vulnerability_id) for vulnerability in vulnerabilities + ] references = VulnerabilityReference.objects.all().only("id") references_after_deletion = [int(reference.id) for reference in references] assert packages_before_deletion == packages_after_deletion