From c5068971891d569dfd51cc1498784161135c9462 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 28 Aug 2023 20:11:06 +0530 Subject: [PATCH 01/21] Write packages and vulnerabilities at the time of import Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 12 ++++++- vulnerabilities/importers/__init__.py | 12 +++---- vulnerabilities/improve_runner.py | 17 +++++++--- vulnerabilities/improvers/__init__.py | 32 ++----------------- vulnerabilities/improvers/default.py | 10 ++++++ .../management/commands/improve.py | 2 +- vulnerabilities/tests/test_example.py | 4 +-- .../tests/test_importer_specific_improver.py | 2 +- 8 files changed, 45 insertions(+), 46 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 8e6335585..f3fbe44c5 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -14,6 +14,8 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer +from vulnerabilities.improve_runner import ImproveRunner +from vulnerabilities.improvers.default import AdvisoryBasedDefaultImprover from vulnerabilities.models import Advisory logger = logging.getLogger(__name__) @@ -51,6 +53,7 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st Return the number of inserted advisories. """ count = 0 + advisories = [] for data in advisory_datas: # https://nvd.nist.gov/vuln/detail/CVE-2013-4314 # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572 @@ -68,6 +71,7 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st "date_collected": datetime.datetime.now(tz=datetime.timezone.utc), }, ) + advisories.append(obj) except Exception as e: logger.error(f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r}") continue @@ -78,5 +82,11 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st count += 1 else: logger.debug(f"Advisory with aliases: {obj.aliases!r} already exists. Skipped.") - + improve_advisories(importer_name, advisories) return count + +def improve_advisories(importer_name, advisories): + try: + ImproveRunner(improver=AdvisoryBasedDefaultImprover(advisories=advisories)).run() + except Exception as e: + logger.error(f"Error while processing advisories from {importer_name!r}: {e!r}") diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 4a2fd6ac7..80809be62 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -37,21 +37,21 @@ from vulnerabilities.importers import xen IMPORTERS_REGISTRY = [ + nvd.NVDImporter, + github.GitHubAPIImporter, + gitlab.GitLabAPIImporter, + npm.NpmImporter, + pypa.PyPaImporter, nginx.NginxImporter, + pysec.PyPIImporter, alpine_linux.AlpineImporter, - github.GitHubAPIImporter, - nvd.NVDImporter, openssl.OpensslImporter, redhat.RedhatImporter, - pysec.PyPIImporter, debian.DebianImporter, - gitlab.GitLabAPIImporter, postgresql.PostgreSQLImporter, - pypa.PyPaImporter, archlinux.ArchlinuxImporter, ubuntu.UbuntuImporter, debian_oval.DebianOvalImporter, - npm.NpmImporter, retiredotnet.RetireDotnetImporter, apache_httpd.ApacheHTTPDImporter, mozilla.MozillaImporter, diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index a2325bab6..06c92712a 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -15,7 +15,7 @@ from django.core.exceptions import ValidationError from django.db import transaction -from vulnerabilities.improver import Inference +from vulnerabilities.improver import Improver, Inference from vulnerabilities.models import Advisory from vulnerabilities.models import Alias from vulnerabilities.models import Package @@ -36,11 +36,18 @@ class ImproveRunner: improver and parsing the returned Inferences into proper database fields """ - def __init__(self, improver_class): - self.improver_class = improver_class + def __init__(self, improver_class=None, improver: Improver=None): + if improver and improver_class: + raise Exception("Both ``improver`` and ``improver_class`` can't be send as argument") + if not (improver_class or improver): + raise Exception("Send either ``improver`` or ``improver_class`` argument") + if improver_class: + self.improver = improver_class() + else: + self.improver = improver def run(self) -> None: - improver = self.improver_class() + improver = self.improver logger.info(f"Running improver: {improver.qualified_name}") for advisory in improver.interesting_advisories: logger.info(f"Processing advisory: {advisory!r}") @@ -51,7 +58,7 @@ def run(self) -> None: ) except Exception as e: logger.info(f"Failed to process advisory: {advisory!r} with error {e!r}") - logger.info("Finished improving using %s.", self.improver_class.qualified_name) + logger.info("Finished improving using %s.", improver.__class__.qualified_name) @transaction.atomic diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index dd17f2f4e..469048f64 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -11,39 +11,11 @@ from vulnerabilities.improvers import valid_versions IMPROVERS_REGISTRY = [ - importer_specific_improver.NVDImprover, - importer_specific_improver.DebianImprover, - importer_specific_improver.DebianOvalImprover, - importer_specific_improver.AlpineLinuxImprover, - importer_specific_improver.ApacheHTTPDImprover, - importer_specific_improver.ApacheKafkaImprover, - importer_specific_improver.ApacheTomcatImprover, - importer_specific_improver.ArchLinuxImprover, - importer_specific_improver.ElixirSecurityImprover, - importer_specific_improver.FireEyeImprover, - importer_specific_improver.GentooImprover, - importer_specific_improver.GitHubAPIImprover, - importer_specific_improver.GitLabAPIImprover, - importer_specific_improver.IstioImprover, - importer_specific_improver.MozillaImprover, - importer_specific_improver.NginxImprover, - importer_specific_improver.NpmImprover, - importer_specific_improver.OpensslImprover, - importer_specific_improver.PostgreSQLImprover, - importer_specific_improver.ProjectKBMSRImprover, - importer_specific_improver.PyPaImprover, - importer_specific_improver.PyPIImprover, - importer_specific_improver.RedhatImprover, - importer_specific_improver.RetireDotnetImprover, - importer_specific_improver.SUSESeverityScoreImprover, - importer_specific_improver.UbuntuImprover, - importer_specific_improver.UbuntuUSNImprover, - importer_specific_improver.XenImprover, + valid_versions.GitHubBasicImprover, + valid_versions.GitLabBasicImprover, valid_versions.NginxBasicImprover, valid_versions.ApacheHTTPDImprover, valid_versions.DebianBasicImprover, - valid_versions.GitHubBasicImprover, - valid_versions.GitLabBasicImprover, valid_versions.NpmImprover, valid_versions.ElixirImprover, valid_versions.ApacheTomcatImprover, diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py index 03cd4abf8..394bbe63e 100644 --- a/vulnerabilities/improvers/default.py +++ b/vulnerabilities/improvers/default.py @@ -132,3 +132,13 @@ def get_exact_purls(affected_package: AffectedPackage) -> Tuple[List[PackageURL] except Exception as e: logger.error(f"Failed to get exact purls for: {affected_package!r} with error: {e!r}") return [], [] + + +class AdvisoryBasedDefaultImprover(DefaultImprover): + + def __init__(self, advisories) -> None: + self.advisories = advisories + + @property + def interesting_advisories(self) -> QuerySet: + return self.advisories diff --git a/vulnerabilities/management/commands/improve.py b/vulnerabilities/management/commands/improve.py index f1c62d51e..e14c2bacc 100644 --- a/vulnerabilities/management/commands/improve.py +++ b/vulnerabilities/management/commands/improve.py @@ -57,7 +57,7 @@ def improve_data(self, improvers): for improver in improvers: self.stdout.write(f"Improving data using {improver.qualified_name}") try: - ImproveRunner(improver).run() + ImproveRunner(improver_class=improver).run() self.stdout.write( self.style.SUCCESS( f"Successfully improved data using {improver.qualified_name}" diff --git a/vulnerabilities/tests/test_example.py b/vulnerabilities/tests/test_example.py index 539d3bfb3..0fd99c245 100644 --- a/vulnerabilities/tests/test_example.py +++ b/vulnerabilities/tests/test_example.py @@ -79,8 +79,8 @@ def test_import_framework_using_example_importer(self): @pytest.mark.django_db(transaction=True) def test_improve_framework_using_example_improver(self): ImportRunner(ExampleImporter).run() - ImproveRunner(DefaultImprover).run() - ImproveRunner(ExampleAliasImprover).run() + ImproveRunner(improver_class=DefaultImprover).run() + ImproveRunner(improver_class=ExampleAliasImprover).run() assert models.Package.objects.count() == 3 assert models.PackageRelatedVulnerability.objects.filter(fix=True).count() == 1 diff --git a/vulnerabilities/tests/test_importer_specific_improver.py b/vulnerabilities/tests/test_importer_specific_improver.py index 880703ec5..4b8c0d733 100644 --- a/vulnerabilities/tests/test_importer_specific_improver.py +++ b/vulnerabilities/tests/test_importer_specific_improver.py @@ -26,6 +26,6 @@ def test_improvement_of_importer_specific_advisories(): created_by=NVDImporter.qualified_name, date_collected=datetime.datetime.now(tz=datetime.timezone.utc), ) - ImproveRunner(NVDImprover).run() + ImproveRunner(improver_class=NVDImprover).run() alias = Alias.objects.filter(alias="CVE-2021-22").first() assert alias is not None From d083c9759572b7239b8c13073135e3ff73dae82d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 28 Aug 2023 20:27:25 +0530 Subject: [PATCH 02/21] Fix Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 1 + vulnerabilities/improve_runner.py | 7 ++++--- vulnerabilities/improvers/default.py | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index f3fbe44c5..bee2bbedd 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -85,6 +85,7 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st improve_advisories(importer_name, advisories) return count + def improve_advisories(importer_name, advisories): try: ImproveRunner(improver=AdvisoryBasedDefaultImprover(advisories=advisories)).run() diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index 06c92712a..f2e229580 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -15,7 +15,8 @@ from django.core.exceptions import ValidationError from django.db import transaction -from vulnerabilities.improver import Improver, Inference +from vulnerabilities.improver import Improver +from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory from vulnerabilities.models import Alias from vulnerabilities.models import Package @@ -36,12 +37,12 @@ class ImproveRunner: improver and parsing the returned Inferences into proper database fields """ - def __init__(self, improver_class=None, improver: Improver=None): + def __init__(self, improver_class=None, improver: Improver = None): if improver and improver_class: raise Exception("Both ``improver`` and ``improver_class`` can't be send as argument") if not (improver_class or improver): raise Exception("Send either ``improver`` or ``improver_class`` argument") - if improver_class: + if improver_class: self.improver = improver_class() else: self.improver = improver diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py index 394bbe63e..b43acac43 100644 --- a/vulnerabilities/improvers/default.py +++ b/vulnerabilities/improvers/default.py @@ -135,7 +135,6 @@ def get_exact_purls(affected_package: AffectedPackage) -> Tuple[List[PackageURL] class AdvisoryBasedDefaultImprover(DefaultImprover): - def __init__(self, advisories) -> None: self.advisories = advisories From a2e63b066cdd2aa23f6be561a71e675cbfa23593 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 31 Aug 2023 14:29:40 +0530 Subject: [PATCH 03/21] Address review comments Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 5 +++-- vulnerabilities/improve_runner.py | 11 ++--------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index bee2bbedd..dba89c571 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -71,7 +71,8 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st "date_collected": datetime.datetime.now(tz=datetime.timezone.utc), }, ) - advisories.append(obj) + if created: + advisories.append(obj) except Exception as e: logger.error(f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r}") continue @@ -88,6 +89,6 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st def improve_advisories(importer_name, advisories): try: - ImproveRunner(improver=AdvisoryBasedDefaultImprover(advisories=advisories)).run() + ImproveRunner(improver=AdvisoryBasedDefaultImprover, advisories=advisories).run() except Exception as e: logger.error(f"Error while processing advisories from {importer_name!r}: {e!r}") diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index f2e229580..4a94e76e9 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -37,15 +37,8 @@ class ImproveRunner: improver and parsing the returned Inferences into proper database fields """ - def __init__(self, improver_class=None, improver: Improver = None): - if improver and improver_class: - raise Exception("Both ``improver`` and ``improver_class`` can't be send as argument") - if not (improver_class or improver): - raise Exception("Send either ``improver`` or ``improver_class`` argument") - if improver_class: - self.improver = improver_class() - else: - self.improver = improver + def __init__(self, improver_class, **kwargs): + self.improver = improver_class(**kwargs) def run(self) -> None: improver = self.improver From 6c1dc7d8d922867e9cf6c1eacd4f638aa2cc960b Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 31 Aug 2023 14:32:27 +0530 Subject: [PATCH 04/21] Fix Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index dba89c571..865e54ac0 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -9,6 +9,7 @@ import datetime import logging +from traceback import format_exc as traceback_format_exc from typing import Iterable from typing import List @@ -74,7 +75,9 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st if created: advisories.append(obj) except Exception as e: - logger.error(f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r}") + logger.error( + f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r} \n {traceback_format_exc()}" + ) continue if created: logger.info( @@ -89,6 +92,8 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st def improve_advisories(importer_name, advisories): try: - ImproveRunner(improver=AdvisoryBasedDefaultImprover, advisories=advisories).run() + ImproveRunner(improver_class=AdvisoryBasedDefaultImprover, advisories=advisories).run() except Exception as e: - logger.error(f"Error while processing advisories from {importer_name!r}: {e!r}") + logger.error( + f"Error while processing advisories from {importer_name!r}: {e!r} \n {traceback_format_exc()}" + ) From 6b791b50a2ee7a3f77c3621b6a7cce8abd7ff655 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 1 Sep 2023 23:54:30 +0530 Subject: [PATCH 05/21] Address review comments Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 2 - vulnerabilities/improve_runner.py | 30 ++++++-- ..._remove_advisory_date_improved_and_more.py | 74 +++++++++++++++++++ .../migrations/0041_auto_20230901_1543.py | 30 ++++++++ vulnerabilities/models.py | 23 ++++-- 5 files changed, 144 insertions(+), 15 deletions(-) create mode 100644 vulnerabilities/migrations/0040_improver_remove_advisory_date_improved_and_more.py create mode 100644 vulnerabilities/migrations/0041_auto_20230901_1543.py diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 865e54ac0..3c725248d 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -72,8 +72,6 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st "date_collected": datetime.datetime.now(tz=datetime.timezone.utc), }, ) - if created: - advisories.append(obj) except Exception as e: logger.error( f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r} \n {traceback_format_exc()}" diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index 4a94e76e9..1d7cbf4d5 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -19,6 +19,8 @@ from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory from vulnerabilities.models import Alias +from vulnerabilities.models import Improver as ImproverModel +from vulnerabilities.models import ImproverRelatedAdvisory from vulnerabilities.models import Package from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability @@ -43,12 +45,23 @@ def __init__(self, improver_class, **kwargs): def run(self) -> None: improver = self.improver logger.info(f"Running improver: {improver.qualified_name}") + improver_name = improver.qualified_name + advisories = [] for advisory in improver.interesting_advisories: + if improver_name not in advisory.improvers.all(): + advisories.append(advisory) + improver_model_object, _ = ImproverModel.objects.get_or_create( + improver_qualified_name=improver_name + ) + for advisory in advisories: logger.info(f"Processing advisory: {advisory!r}") + improver_name = improver.qualified_name try: inferences = improver.get_inferences(advisory_data=advisory.to_advisory_data()) process_inferences( - inferences=inferences, advisory=advisory, improver_name=improver.qualified_name + inferences=inferences, + advisory=advisory, + improver_model_object=improver_model_object, ) except Exception as e: logger.info(f"Failed to process advisory: {advisory!r} with error {e!r}") @@ -56,7 +69,9 @@ def run(self) -> None: @transaction.atomic -def process_inferences(inferences: List[Inference], advisory: Advisory, improver_name: str): +def process_inferences( + inferences: List[Inference], advisory: Advisory, improver_model_object: ImproverModel +): """ Return number of inferences processed. An atomic transaction that updates both the Advisory (e.g. date_improved) @@ -69,6 +84,8 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver """ inferences_processed_count = 0 + improver_name = improver_model_object.improver_qualified_name + if not inferences: logger.warning(f"Nothing to improve. Source: {improver_name} Advisory id: {advisory.id}") return inferences_processed_count @@ -146,12 +163,9 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver cwe_obj, created = Weakness.objects.get_or_create(cwe_id=cwe_id) cwe_obj.vulnerabilities.add(vulnerability) cwe_obj.save() - - inferences_processed_count += 1 - - advisory.date_improved = datetime.now(timezone.utc) - advisory.save() - return inferences_processed_count + ImproverRelatedAdvisory.objects.create( + improver=improver_model_object, advisory=advisory, date_improved=datetime.now(timezone.utc) + ) def create_valid_vulnerability_reference(url, reference_id=None): diff --git a/vulnerabilities/migrations/0040_improver_remove_advisory_date_improved_and_more.py b/vulnerabilities/migrations/0040_improver_remove_advisory_date_improved_and_more.py new file mode 100644 index 000000000..8300acb72 --- /dev/null +++ b/vulnerabilities/migrations/0040_improver_remove_advisory_date_improved_and_more.py @@ -0,0 +1,74 @@ +# Generated by Django 4.1.7 on 2023-09-01 15:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0039_alter_vulnerabilityseverity_scoring_system"), + ] + + operations = [ + migrations.CreateModel( + name="Improver", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("improver_qualified_name", models.CharField(blank=True, max_length=1000)), + ], + ), + migrations.RemoveField( + model_name="advisory", + name="date_improved", + ), + migrations.CreateModel( + name="ImproverRelatedAdvisory", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "date_improved", + models.DateTimeField( + help_text="Date on which the advisory was improved by an improver", + null=True, + ), + ), + ( + "advisory", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.advisory" + ), + ), + ( + "improver", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.improver" + ), + ), + ], + ), + migrations.AddField( + model_name="improver", + name="advisories", + field=models.ManyToManyField( + through="vulnerabilities.ImproverRelatedAdvisory", to="vulnerabilities.advisory" + ), + ), + migrations.AddField( + model_name="advisory", + name="improvers", + field=models.ManyToManyField( + through="vulnerabilities.ImproverRelatedAdvisory", to="vulnerabilities.improver" + ), + ), + ] diff --git a/vulnerabilities/migrations/0041_auto_20230901_1543.py b/vulnerabilities/migrations/0041_auto_20230901_1543.py new file mode 100644 index 000000000..77e8d009d --- /dev/null +++ b/vulnerabilities/migrations/0041_auto_20230901_1543.py @@ -0,0 +1,30 @@ +# Generated by Django 4.1.7 on 2023-09-01 15:43 + +from django.db import migrations +from vulnerabilities.improver import Improver +from vulnerabilities.improvers import IMPROVERS_REGISTRY +from vulnerabilities.improvers.default import AdvisoryBasedDefaultImprover + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0040_improver_remove_advisory_date_improved_and_more"), + ] + + def register_improvers(apps, _): + Improver = apps.get_model("vulnerabilities", "Improver") + for qual_name, _ in IMPROVERS_REGISTRY.items(): + Improver.objects.create( + improver_qualified_name = qual_name + ) + Improver.objects.create( + improver_qualified_name = AdvisoryBasedDefaultImprover.qualified_name + ) + + dependencies = [ + ("vulnerabilities", "0040_improver_remove_advisory_date_improved_and_more"), + ] + + operations = [ + migrations.RunPython(register_improvers,reverse_code=migrations.RunPython.noop) + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 809b3c4f6..cc034ea02 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -826,17 +826,13 @@ class Advisory(models.Model): ) weaknesses = models.JSONField(blank=True, default=list, help_text="A list of CWE ids") date_collected = models.DateTimeField(help_text="UTC Date on which the advisory was collected") - date_improved = models.DateTimeField( - blank=True, - null=True, - help_text="Latest date on which the advisory was improved by an improver", - ) created_by = models.CharField( max_length=100, help_text="Fully qualified name of the importer prefixed with the" "module name importing the advisory. Eg:" "vulnerabilities.importers.nginx.NginxImporter", ) + improvers = models.ManyToManyField(to="Improver", through="ImproverRelatedAdvisory") objects = AdvisoryQuerySet.as_manager() class Meta: @@ -915,3 +911,20 @@ class ApiUser(UserModel): class Meta: proxy = True + + +class Improver(models.Model): + improver_qualified_name = models.CharField(max_length=1000, blank=True) + advisories = models.ManyToManyField(to="Advisory", through="ImproverRelatedAdvisory") + + +class ImproverRelatedAdvisory(models.Model): + improver = models.ForeignKey( + Improver, + on_delete=models.CASCADE, + ) + advisory = models.ForeignKey(Advisory, on_delete=models.CASCADE) + date_improved = models.DateTimeField( + null=True, + help_text="Date on which the advisory was improved by an improver", + ) From 89f5f0fee02c7278d6b7ac9df5eb0c790007b7b6 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sat, 2 Sep 2023 00:28:26 +0530 Subject: [PATCH 06/21] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/tests/test_improve_runner.py | 29 ++++++++++++++------ 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/tests/test_improve_runner.py b/vulnerabilities/tests/test_improve_runner.py index 9354b4688..05a47970b 100644 --- a/vulnerabilities/tests/test_improve_runner.py +++ b/vulnerabilities/tests/test_improve_runner.py @@ -22,6 +22,7 @@ from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory from vulnerabilities.models import Alias +from vulnerabilities.models import Improver as ImproverModel from vulnerabilities.models import Package from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability @@ -148,16 +149,20 @@ def test_get_or_create_vulnerability_and_aliases_with_existing_vulnerability_and @pytest.mark.django_db def test_process_inferences_with_no_inference(): + DUMMY_ADVISORY.save() + test_improver = ImproverModel.objects.create(improver_qualified_name="test_improver") assert not process_inferences( - inferences=[], advisory=DUMMY_ADVISORY, improver_name="test_improver" + inferences=[], advisory=DUMMY_ADVISORY, improver_model_object=test_improver ) @pytest.mark.django_db def test_process_inferences_with_unknown_but_specified_vulnerability(): inference = Inference(vulnerability_id="VCID-Does-Not-Exist-In-DB", aliases=["MATRIX-Neo"]) + DUMMY_ADVISORY.save() + test_improver = ImproverModel.objects.create(improver_qualified_name="test_improver") assert not process_inferences( - inferences=[inference], advisory=DUMMY_ADVISORY, improver_name="test_improver" + inferences=[inference], advisory=DUMMY_ADVISORY, improver_model_object=test_improver ) @@ -190,17 +195,25 @@ def get_objects_in_all_tables_used_by_process_inferences(): @pytest.mark.django_db def test_process_inferences_idempotency(): - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") + DUMMY_ADVISORY.save() + test_improver = ImproverModel.objects.create(improver_qualified_name="test_improver") + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver) all_objects = get_objects_in_all_tables_used_by_process_inferences() - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver) + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver) assert all_objects == get_objects_in_all_tables_used_by_process_inferences() @pytest.mark.django_db def test_process_inference_idempotency_with_different_improver_names(): - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver_one") + DUMMY_ADVISORY.save() + test_improver_one = ImproverModel.objects.create(improver_qualified_name="test_improver_one") + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver_one) all_objects = get_objects_in_all_tables_used_by_process_inferences() - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver_two") - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver_three") + test_improver_two = ImproverModel.objects.create(improver_qualified_name="test_improver_two") + test_improver_three = ImproverModel.objects.create( + improver_qualified_name="test_improver_three" + ) + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver_two) + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver_three) assert all_objects == get_objects_in_all_tables_used_by_process_inferences() From f447b8ec652fa7d826bcaab73b2b993079ba67ff Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sat, 2 Sep 2023 01:28:45 +0530 Subject: [PATCH 07/21] Optimise queries Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 1 + vulnerabilities/improve_runner.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 3c725248d..4a2350f8b 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -72,6 +72,7 @@ def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: st "date_collected": datetime.datetime.now(tz=datetime.timezone.utc), }, ) + advisories.append(obj) except Exception as e: logger.error( f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r} \n {traceback_format_exc()}" diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index 1d7cbf4d5..445f5b587 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -47,12 +47,16 @@ def run(self) -> None: logger.info(f"Running improver: {improver.qualified_name}") improver_name = improver.qualified_name advisories = [] - for advisory in improver.interesting_advisories: - if improver_name not in advisory.improvers.all(): - advisories.append(advisory) improver_model_object, _ = ImproverModel.objects.get_or_create( improver_qualified_name=improver_name ) + improved_advisories = ImproverRelatedAdvisory.objects.filter( + improver = improver_model_object + ).values('advisory').distinct() + improved_advisories_id = [adv["advisory"] for adv in improved_advisories] + for advisory in improver.interesting_advisories: + if advisory.id not in improved_advisories_id: + advisories.append(advisory) for advisory in advisories: logger.info(f"Processing advisory: {advisory!r}") improver_name = improver.qualified_name From a8848cc5a7bb420d7353598c4821d962d3b215ed Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Sep 2023 15:06:03 +0530 Subject: [PATCH 08/21] Fix Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 296 +++++++++++++++--- vulnerabilities/improve_runner.py | 13 +- ..._remove_advisory_date_improved_and_more.py | 74 ----- .../migrations/0041_auto_20230901_1543.py | 30 -- 4 files changed, 257 insertions(+), 156 deletions(-) delete mode 100644 vulnerabilities/migrations/0040_improver_remove_advisory_date_improved_and_more.py delete mode 100644 vulnerabilities/migrations/0041_auto_20230901_1543.py diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 4a2350f8b..1397f0b78 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -13,11 +13,22 @@ from typing import Iterable from typing import List +from django.core.exceptions import ValidationError +from django.db import transaction + from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer -from vulnerabilities.improve_runner import ImproveRunner +from vulnerabilities.improver import Inference from vulnerabilities.improvers.default import AdvisoryBasedDefaultImprover from vulnerabilities.models import Advisory +from vulnerabilities.models import Alias +from vulnerabilities.models import Package +from vulnerabilities.models import PackageRelatedVulnerability +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import Weakness logger = logging.getLogger(__name__) @@ -44,55 +55,260 @@ def run(self) -> None: importer_class = self.importer_class logger.info(f"Starting import for {importer_name}") advisory_datas = importer_class().advisory_data() - count = process_advisories(advisory_datas=advisory_datas, importer_name=importer_name) + count = self.process_advisories(advisory_datas=advisory_datas, importer_name=importer_name) logger.info(f"Finished import for {importer_name}. Imported {count} advisories.") + def do_import(self, advisories) -> None: + improver = AdvisoryBasedDefaultImprover(advisories=advisories) + logger.info(f"Running improver: {improver.qualified_name}") + improver_name = improver.qualified_name + advisories = [] + for advisory in improver.interesting_advisories.filter(date_imported__is_null=True).exclude( + date_imported__exact="" + ): + logger.info(f"Processing advisory: {advisory!r}") + try: + inferences = improver.get_inferences(advisory_data=advisory.to_advisory_data()) + process_inferences( + inferences=inferences, + advisory=advisory, + improver_name=improver_name, + ) + except Exception as e: + logger.info(f"Failed to process advisory: {advisory!r} with error {e!r}") + logger.info("Finished improving using %s.", improver.__class__.qualified_name) -def process_advisories(advisory_datas: Iterable[AdvisoryData], importer_name: str) -> List: - """ - Insert advisories into the database - Return the number of inserted advisories. - """ - count = 0 - advisories = [] - for data in advisory_datas: - # https://nvd.nist.gov/vuln/detail/CVE-2013-4314 - # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572 + def process_advisories( + self, advisory_datas: Iterable[AdvisoryData], importer_name: str + ) -> List: + """ + Insert advisories into the database + Return the number of inserted advisories. + """ + count = 0 + advisories = [] + for data in advisory_datas: + # https://nvd.nist.gov/vuln/detail/CVE-2013-4314 + # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572 + try: + data.summary = data.summary.replace("\x00", "\uFFFD") + obj, created = Advisory.objects.get_or_create( + aliases=data.aliases, + summary=data.summary, + affected_packages=[pkg.to_dict() for pkg in data.affected_packages], + references=[ref.to_dict() for ref in data.references], + date_published=data.date_published, + weaknesses=data.weaknesses, + defaults={ + "created_by": importer_name, + "date_collected": datetime.datetime.now(tz=datetime.timezone.utc), + }, + ) + if not obj.date_imported: + advisories.append(obj) + except Exception as e: + logger.error( + f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r} \n {traceback_format_exc()}" + ) + continue + if created: + logger.info( + f"[*] New Advisory with aliases: {obj.aliases!r}, created_by: {obj.created_by}" + ) + count += 1 + else: + logger.debug(f"Advisory with aliases: {obj.aliases!r} already exists.") try: - data.summary = data.summary.replace("\x00", "\uFFFD") - obj, created = Advisory.objects.get_or_create( - aliases=data.aliases, - summary=data.summary, - affected_packages=[pkg.to_dict() for pkg in data.affected_packages], - references=[ref.to_dict() for ref in data.references], - date_published=data.date_published, - weaknesses=data.weaknesses, - defaults={ - "created_by": importer_name, - "date_collected": datetime.datetime.now(tz=datetime.timezone.utc), - }, - ) - advisories.append(obj) + self.do_import(advisories) except Exception as e: logger.error( - f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r} \n {traceback_format_exc()}" + f"Error while processing advisories from {importer_name!r}: {e!r} \n {traceback_format_exc()}" ) + return count + + +@transaction.atomic +def process_inferences(inferences: List[Inference], advisory: Advisory, improver_name: str): + """ + Return number of inferences processed. + An atomic transaction that updates both the Advisory (e.g. date_improved) + and processes the given inferences to create or update corresponding + database fields. + + This avoids failing the entire improver when only a single inference is + erroneous. Also, the atomic transaction for every advisory and its + inferences makes sure that date_improved of advisory is consistent. + """ + inferences_processed_count = 0 + + if not inferences: + logger.warning(f"Nothing to improve. Source: {improver_name} Advisory id: {advisory.id}") + return inferences_processed_count + + logger.info(f"Improving advisory id: {advisory.id}") + + for inference in inferences: + vulnerability = get_or_create_vulnerability_and_aliases( + vulnerability_id=inference.vulnerability_id, + alias_names=inference.aliases, + summary=inference.summary, + ) + + if not vulnerability: + logger.warning(f"Unable to get vulnerability for inference: {inference!r}") continue - if created: - logger.info( - f"[*] New Advisory with aliases: {obj.aliases!r}, created_by: {obj.created_by}" + + for ref in inference.references: + + reference = VulnerabilityReference.objects.get_or_none( + reference_id=ref.reference_id, + url=ref.url, ) - count += 1 - else: - logger.debug(f"Advisory with aliases: {obj.aliases!r} already exists. Skipped.") - improve_advisories(importer_name, advisories) - return count + if not reference: + reference = create_valid_vulnerability_reference( + reference_id=ref.reference_id, + url=ref.url, + ) + if not reference: + continue + + VulnerabilityRelatedReference.objects.update_or_create( + reference=reference, + vulnerability=vulnerability, + ) + + for severity in ref.severities: + _vs, updated = VulnerabilitySeverity.objects.update_or_create( + scoring_system=severity.system.identifier, + reference=reference, + defaults={ + "value": str(severity.value), + "scoring_elements": str(severity.scoring_elements), + }, + ) + if updated: + logger.info( + f"Severity updated for reference {ref!r} to value: {severity.value!r} " + f"and scoring_elements: {severity.scoring_elements!r}" + ) + + for affected_purl in inference.affected_purls or []: + vulnerable_package = Package.objects.get_or_create_from_purl(purl=affected_purl) + PackageRelatedVulnerability( + vulnerability=vulnerability, + package=vulnerable_package, + created_by=improver_name, + confidence=inference.confidence, + fix=False, + ).update_or_create() + + if inference.fixed_purl: + fixed_package = Package.objects.get_or_create_from_purl(purl=inference.fixed_purl) + PackageRelatedVulnerability( + vulnerability=vulnerability, + package=fixed_package, + created_by=improver_name, + confidence=inference.confidence, + fix=True, + ).update_or_create() + + if inference.weaknesses and vulnerability: + for cwe_id in inference.weaknesses: + cwe_obj, created = Weakness.objects.get_or_create(cwe_id=cwe_id) + cwe_obj.vulnerabilities.add(vulnerability) + cwe_obj.save() + inferences_processed_count += 1 + advisory.date_imported = datetime.now() + advisory.save() + return inferences_processed_count + + +def create_valid_vulnerability_reference(url, reference_id=None): + """ + Create and return a new validated VulnerabilityReference from a + ``url`` and ``reference_id``. + Return None and log a warning if this is not a valid reference. + """ + reference = VulnerabilityReference( + reference_id=reference_id, + url=url, + ) -def improve_advisories(importer_name, advisories): try: - ImproveRunner(improver_class=AdvisoryBasedDefaultImprover, advisories=advisories).run() - except Exception as e: - logger.error( - f"Error while processing advisories from {importer_name!r}: {e!r} \n {traceback_format_exc()}" + reference.full_clean() + except ValidationError as e: + logger.warning(f"Invalid vulnerability reference: {reference!r}: {e}") + return + + reference.save() + return reference + + +def get_or_create_vulnerability_and_aliases(alias_names, vulnerability_id=None, summary=None): + """ + Get or create vulnerabilitiy and aliases such that all existing and new + aliases point to the same vulnerability + """ + existing_vulns = set() + alias_names = set(alias_names) + new_alias_names = set() + for alias_name in alias_names: + try: + alias = Alias.objects.get(alias=alias_name) + existing_vulns.add(alias.vulnerability) + except Alias.DoesNotExist: + new_alias_names.add(alias_name) + + # If given set of aliases point to different vulnerabilities in the + # database, request is malformed + # TODO: It is possible that all those vulnerabilities are actually + # the same at data level, figure out a way to merge them + if len(existing_vulns) > 1: + logger.warning( + f"Given aliases {alias_names} already exist and do not point " + f"to a single vulnerability. Cannot improve. Skipped." + ) + return + + existing_alias_vuln = existing_vulns.pop() if existing_vulns else None + + if ( + existing_alias_vuln + and vulnerability_id + and existing_alias_vuln.vulnerability_id != vulnerability_id + ): + logger.warning( + f"Given aliases {alias_names!r} already exist and point to existing" + f"vulnerability {existing_alias_vuln}. Unable to create Vulnerability " + f"with vulnerability_id {vulnerability_id}. Skipped" ) + return + + if existing_alias_vuln: + vulnerability = existing_alias_vuln + elif vulnerability_id: + try: + vulnerability = Vulnerability.objects.get(vulnerability_id=vulnerability_id) + except Vulnerability.DoesNotExist: + logger.warning( + f"Given vulnerability_id: {vulnerability_id} does not exist in the database" + ) + return + else: + vulnerability = Vulnerability(summary=summary) + vulnerability.save() + # CHANGELOG: CREATED VULN ON THIS DATE IN VCIO USING ADVISORY SOURCE + + if summary and summary != vulnerability.summary: + logger.warning( + f"Inconsistent summary for {vulnerability!r}. " + f"Existing: {vulnerability.summary}, provided: {summary}" + ) + + for alias_name in new_alias_names: + alias = Alias(alias=alias_name, vulnerability=vulnerability) + alias.save() + logger.info(f"New alias for {vulnerability!r}: {alias_name}") + + return vulnerability diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index 445f5b587..c8e09bbaf 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -46,18 +46,7 @@ def run(self) -> None: improver = self.improver logger.info(f"Running improver: {improver.qualified_name}") improver_name = improver.qualified_name - advisories = [] - improver_model_object, _ = ImproverModel.objects.get_or_create( - improver_qualified_name=improver_name - ) - improved_advisories = ImproverRelatedAdvisory.objects.filter( - improver = improver_model_object - ).values('advisory').distinct() - improved_advisories_id = [adv["advisory"] for adv in improved_advisories] - for advisory in improver.interesting_advisories: - if advisory.id not in improved_advisories_id: - advisories.append(advisory) - for advisory in advisories: + for advisory in improver.interesting_advisories: logger.info(f"Processing advisory: {advisory!r}") improver_name = improver.qualified_name try: diff --git a/vulnerabilities/migrations/0040_improver_remove_advisory_date_improved_and_more.py b/vulnerabilities/migrations/0040_improver_remove_advisory_date_improved_and_more.py deleted file mode 100644 index 8300acb72..000000000 --- a/vulnerabilities/migrations/0040_improver_remove_advisory_date_improved_and_more.py +++ /dev/null @@ -1,74 +0,0 @@ -# Generated by Django 4.1.7 on 2023-09-01 15:42 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ("vulnerabilities", "0039_alter_vulnerabilityseverity_scoring_system"), - ] - - operations = [ - migrations.CreateModel( - name="Improver", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name="ID" - ), - ), - ("improver_qualified_name", models.CharField(blank=True, max_length=1000)), - ], - ), - migrations.RemoveField( - model_name="advisory", - name="date_improved", - ), - migrations.CreateModel( - name="ImproverRelatedAdvisory", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name="ID" - ), - ), - ( - "date_improved", - models.DateTimeField( - help_text="Date on which the advisory was improved by an improver", - null=True, - ), - ), - ( - "advisory", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.advisory" - ), - ), - ( - "improver", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.improver" - ), - ), - ], - ), - migrations.AddField( - model_name="improver", - name="advisories", - field=models.ManyToManyField( - through="vulnerabilities.ImproverRelatedAdvisory", to="vulnerabilities.advisory" - ), - ), - migrations.AddField( - model_name="advisory", - name="improvers", - field=models.ManyToManyField( - through="vulnerabilities.ImproverRelatedAdvisory", to="vulnerabilities.improver" - ), - ), - ] diff --git a/vulnerabilities/migrations/0041_auto_20230901_1543.py b/vulnerabilities/migrations/0041_auto_20230901_1543.py deleted file mode 100644 index 77e8d009d..000000000 --- a/vulnerabilities/migrations/0041_auto_20230901_1543.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 4.1.7 on 2023-09-01 15:43 - -from django.db import migrations -from vulnerabilities.improver import Improver -from vulnerabilities.improvers import IMPROVERS_REGISTRY -from vulnerabilities.improvers.default import AdvisoryBasedDefaultImprover - -class Migration(migrations.Migration): - - dependencies = [ - ("vulnerabilities", "0040_improver_remove_advisory_date_improved_and_more"), - ] - - def register_improvers(apps, _): - Improver = apps.get_model("vulnerabilities", "Improver") - for qual_name, _ in IMPROVERS_REGISTRY.items(): - Improver.objects.create( - improver_qualified_name = qual_name - ) - Improver.objects.create( - improver_qualified_name = AdvisoryBasedDefaultImprover.qualified_name - ) - - dependencies = [ - ("vulnerabilities", "0040_improver_remove_advisory_date_improved_and_more"), - ] - - operations = [ - migrations.RunPython(register_improvers,reverse_code=migrations.RunPython.noop) - ] From 9ec251214b6d2fd8134a89b541aafa712d599511 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Sep 2023 15:14:45 +0530 Subject: [PATCH 09/21] Fix Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 4 ++- vulnerabilities/improve_runner.py | 26 ++++++++----------- ...ry_date_improved_advisory_date_imported.py | 24 +++++++++++++++++ vulnerabilities/models.py | 21 +++------------ 4 files changed, 41 insertions(+), 34 deletions(-) create mode 100644 vulnerabilities/migrations/0040_remove_advisory_date_improved_advisory_date_imported.py diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 1397f0b78..fbc569f16 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -9,6 +9,7 @@ import datetime import logging +from datetime import timezone from traceback import format_exc as traceback_format_exc from typing import Iterable from typing import List @@ -219,7 +220,8 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver cwe_obj.vulnerabilities.add(vulnerability) cwe_obj.save() inferences_processed_count += 1 - advisory.date_imported = datetime.now() + + advisory.date_imported = datetime.now(timezone.utc) advisory.save() return inferences_processed_count diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index c8e09bbaf..5bb35b5d1 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -39,32 +39,28 @@ class ImproveRunner: improver and parsing the returned Inferences into proper database fields """ - def __init__(self, improver_class, **kwargs): - self.improver = improver_class(**kwargs) + def __init__(self, improver_class): + self.improver_class = improver_class def run(self) -> None: - improver = self.improver + improver = self.improver_class() logger.info(f"Running improver: {improver.qualified_name}") - improver_name = improver.qualified_name for advisory in improver.interesting_advisories: logger.info(f"Processing advisory: {advisory!r}") - improver_name = improver.qualified_name try: inferences = improver.get_inferences(advisory_data=advisory.to_advisory_data()) process_inferences( inferences=inferences, advisory=advisory, - improver_model_object=improver_model_object, + improver_model_object=improver.qualified_name, ) except Exception as e: logger.info(f"Failed to process advisory: {advisory!r} with error {e!r}") - logger.info("Finished improving using %s.", improver.__class__.qualified_name) + logger.info("Finished improving using %s.", self.improver_class.qualified_name) @transaction.atomic -def process_inferences( - inferences: List[Inference], advisory: Advisory, improver_model_object: ImproverModel -): +def process_inferences(inferences: List[Inference], advisory: Advisory, improver_name: str): """ Return number of inferences processed. An atomic transaction that updates both the Advisory (e.g. date_improved) @@ -77,8 +73,6 @@ def process_inferences( """ inferences_processed_count = 0 - improver_name = improver_model_object.improver_qualified_name - if not inferences: logger.warning(f"Nothing to improve. Source: {improver_name} Advisory id: {advisory.id}") return inferences_processed_count @@ -156,9 +150,11 @@ def process_inferences( cwe_obj, created = Weakness.objects.get_or_create(cwe_id=cwe_id) cwe_obj.vulnerabilities.add(vulnerability) cwe_obj.save() - ImproverRelatedAdvisory.objects.create( - improver=improver_model_object, advisory=advisory, date_improved=datetime.now(timezone.utc) - ) + inferences_processed_count += 1 + + advisory.date_improved = datetime.now(timezone.utc) + advisory.save() + return inferences_processed_count def create_valid_vulnerability_reference(url, reference_id=None): diff --git a/vulnerabilities/migrations/0040_remove_advisory_date_improved_advisory_date_imported.py b/vulnerabilities/migrations/0040_remove_advisory_date_improved_advisory_date_imported.py new file mode 100644 index 000000000..c609f20f8 --- /dev/null +++ b/vulnerabilities/migrations/0040_remove_advisory_date_improved_advisory_date_imported.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.7 on 2023-09-05 09:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0039_alter_vulnerabilityseverity_scoring_system"), + ] + + operations = [ + migrations.RemoveField( + model_name="advisory", + name="date_improved", + ), + migrations.AddField( + model_name="advisory", + name="date_imported", + field=models.DateTimeField( + blank=True, help_text="UTC Date on which the advisory was imported", null=True + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index cc034ea02..3971651ad 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -26,6 +26,7 @@ from django.db.models.functions import Length from django.db.models.functions import Trim from django.urls import reverse +from django.utils import timezone from packageurl import PackageURL from packageurl.contrib.django.models import PackageURLMixin from packageurl.contrib.django.models import PackageURLQuerySet @@ -39,6 +40,7 @@ from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import build_vcid from vulnerabilities.utils import remove_qualifiers_and_subpath +from vulnerablecode import __version__ as vulnerablecode_version logger = logging.getLogger(__name__) @@ -826,13 +828,13 @@ class Advisory(models.Model): ) weaknesses = models.JSONField(blank=True, default=list, help_text="A list of CWE ids") date_collected = models.DateTimeField(help_text="UTC Date on which the advisory was collected") + date_imported = models.DateTimeField(help_text="UTC Date on which the advisory was imported") created_by = models.CharField( max_length=100, help_text="Fully qualified name of the importer prefixed with the" "module name importing the advisory. Eg:" "vulnerabilities.importers.nginx.NginxImporter", ) - improvers = models.ManyToManyField(to="Improver", through="ImproverRelatedAdvisory") objects = AdvisoryQuerySet.as_manager() class Meta: @@ -911,20 +913,3 @@ class ApiUser(UserModel): class Meta: proxy = True - - -class Improver(models.Model): - improver_qualified_name = models.CharField(max_length=1000, blank=True) - advisories = models.ManyToManyField(to="Advisory", through="ImproverRelatedAdvisory") - - -class ImproverRelatedAdvisory(models.Model): - improver = models.ForeignKey( - Improver, - on_delete=models.CASCADE, - ) - advisory = models.ForeignKey(Advisory, on_delete=models.CASCADE) - date_improved = models.DateTimeField( - null=True, - help_text="Date on which the advisory was improved by an improver", - ) From 88ce081ef6f95a20d993b9b7f3143a67b809c4f1 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Sep 2023 15:16:02 +0530 Subject: [PATCH 10/21] Fix Signed-off-by: Tushar Goel --- vulnerabilities/improve_runner.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index 5bb35b5d1..8de1a1663 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -15,12 +15,9 @@ from django.core.exceptions import ValidationError from django.db import transaction -from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory from vulnerabilities.models import Alias -from vulnerabilities.models import Improver as ImproverModel -from vulnerabilities.models import ImproverRelatedAdvisory from vulnerabilities.models import Package from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability @@ -150,6 +147,7 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver cwe_obj, created = Weakness.objects.get_or_create(cwe_id=cwe_id) cwe_obj.vulnerabilities.add(vulnerability) cwe_obj.save() + inferences_processed_count += 1 advisory.date_improved = datetime.now(timezone.utc) From 84a3e6c1c2d6c4e1a7f0ece1bec9651aa4504972 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Sep 2023 15:29:56 +0530 Subject: [PATCH 11/21] Fix Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 10 ++++------ vulnerabilities/models.py | 4 +++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index fbc569f16..a055197d5 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -9,7 +9,6 @@ import datetime import logging -from datetime import timezone from traceback import format_exc as traceback_format_exc from typing import Iterable from typing import List @@ -64,9 +63,9 @@ def do_import(self, advisories) -> None: logger.info(f"Running improver: {improver.qualified_name}") improver_name = improver.qualified_name advisories = [] - for advisory in improver.interesting_advisories.filter(date_imported__is_null=True).exclude( - date_imported__exact="" - ): + for advisory in improver.interesting_advisories: + if advisory.date_imported: + continue logger.info(f"Processing advisory: {advisory!r}") try: inferences = improver.get_inferences(advisory_data=advisory.to_advisory_data()) @@ -221,7 +220,7 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver cwe_obj.save() inferences_processed_count += 1 - advisory.date_imported = datetime.now(timezone.utc) + advisory.date_imported = datetime.datetime.now(tz=datetime.timezone.utc) advisory.save() return inferences_processed_count @@ -300,7 +299,6 @@ def get_or_create_vulnerability_and_aliases(alias_names, vulnerability_id=None, else: vulnerability = Vulnerability(summary=summary) vulnerability.save() - # CHANGELOG: CREATED VULN ON THIS DATE IN VCIO USING ADVISORY SOURCE if summary and summary != vulnerability.summary: logger.warning( diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 3971651ad..ce682311f 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -828,7 +828,9 @@ class Advisory(models.Model): ) weaknesses = models.JSONField(blank=True, default=list, help_text="A list of CWE ids") date_collected = models.DateTimeField(help_text="UTC Date on which the advisory was collected") - date_imported = models.DateTimeField(help_text="UTC Date on which the advisory was imported") + date_imported = models.DateTimeField( + blank=True, null=True, help_text="UTC Date on which the advisory was imported" + ) created_by = models.CharField( max_length=100, help_text="Fully qualified name of the importer prefixed with the" From 2bbfa43d3c5d8c05e3da026ed140170f314a3ce5 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Sep 2023 15:35:07 +0530 Subject: [PATCH 12/21] Fix Signed-off-by: Tushar Goel --- vulnerabilities/tests/test_improve_runner.py | 29 ++++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/vulnerabilities/tests/test_improve_runner.py b/vulnerabilities/tests/test_improve_runner.py index 05a47970b..9354b4688 100644 --- a/vulnerabilities/tests/test_improve_runner.py +++ b/vulnerabilities/tests/test_improve_runner.py @@ -22,7 +22,6 @@ from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory from vulnerabilities.models import Alias -from vulnerabilities.models import Improver as ImproverModel from vulnerabilities.models import Package from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability @@ -149,20 +148,16 @@ def test_get_or_create_vulnerability_and_aliases_with_existing_vulnerability_and @pytest.mark.django_db def test_process_inferences_with_no_inference(): - DUMMY_ADVISORY.save() - test_improver = ImproverModel.objects.create(improver_qualified_name="test_improver") assert not process_inferences( - inferences=[], advisory=DUMMY_ADVISORY, improver_model_object=test_improver + inferences=[], advisory=DUMMY_ADVISORY, improver_name="test_improver" ) @pytest.mark.django_db def test_process_inferences_with_unknown_but_specified_vulnerability(): inference = Inference(vulnerability_id="VCID-Does-Not-Exist-In-DB", aliases=["MATRIX-Neo"]) - DUMMY_ADVISORY.save() - test_improver = ImproverModel.objects.create(improver_qualified_name="test_improver") assert not process_inferences( - inferences=[inference], advisory=DUMMY_ADVISORY, improver_model_object=test_improver + inferences=[inference], advisory=DUMMY_ADVISORY, improver_name="test_improver" ) @@ -195,25 +190,17 @@ def get_objects_in_all_tables_used_by_process_inferences(): @pytest.mark.django_db def test_process_inferences_idempotency(): - DUMMY_ADVISORY.save() - test_improver = ImproverModel.objects.create(improver_qualified_name="test_improver") - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver) + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") all_objects = get_objects_in_all_tables_used_by_process_inferences() - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver) - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver) + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") assert all_objects == get_objects_in_all_tables_used_by_process_inferences() @pytest.mark.django_db def test_process_inference_idempotency_with_different_improver_names(): - DUMMY_ADVISORY.save() - test_improver_one = ImproverModel.objects.create(improver_qualified_name="test_improver_one") - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver_one) + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver_one") all_objects = get_objects_in_all_tables_used_by_process_inferences() - test_improver_two = ImproverModel.objects.create(improver_qualified_name="test_improver_two") - test_improver_three = ImproverModel.objects.create( - improver_qualified_name="test_improver_three" - ) - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver_two) - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_model_object=test_improver_three) + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver_two") + process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver_three") assert all_objects == get_objects_in_all_tables_used_by_process_inferences() From d62b9e7ae0795419efffb2852149b8f13e3a17c0 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Sep 2023 15:39:42 +0530 Subject: [PATCH 13/21] Fix Signed-off-by: Tushar Goel --- .../improvers/importer_specific_improver.py | 150 ------------------ vulnerabilities/tests/test_import_runner.py | 19 ++- .../tests/test_importer_specific_improver.py | 31 ---- 3 files changed, 9 insertions(+), 191 deletions(-) delete mode 100644 vulnerabilities/improvers/importer_specific_improver.py delete mode 100644 vulnerabilities/tests/test_importer_specific_improver.py diff --git a/vulnerabilities/improvers/importer_specific_improver.py b/vulnerabilities/improvers/importer_specific_improver.py deleted file mode 100644 index e5aa4a482..000000000 --- a/vulnerabilities/improvers/importer_specific_improver.py +++ /dev/null @@ -1,150 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from vulnerabilities.importers.alpine_linux import AlpineImporter -from vulnerabilities.importers.apache_httpd import ApacheHTTPDImporter -from vulnerabilities.importers.apache_kafka import ApacheKafkaImporter -from vulnerabilities.importers.apache_tomcat import ApacheTomcatImporter -from vulnerabilities.importers.archlinux import ArchlinuxImporter -from vulnerabilities.importers.debian import DebianImporter -from vulnerabilities.importers.debian_oval import DebianOvalImporter -from vulnerabilities.importers.elixir_security import ElixirSecurityImporter -from vulnerabilities.importers.fireeye import FireyeImporter -from vulnerabilities.importers.gentoo import GentooImporter -from vulnerabilities.importers.github import GitHubAPIImporter -from vulnerabilities.importers.gitlab import GitLabAPIImporter -from vulnerabilities.importers.istio import IstioImporter -from vulnerabilities.importers.mozilla import MozillaImporter -from vulnerabilities.importers.nginx import NginxImporter -from vulnerabilities.importers.npm import NpmImporter -from vulnerabilities.importers.nvd import NVDImporter -from vulnerabilities.importers.openssl import OpensslImporter -from vulnerabilities.importers.postgresql import PostgreSQLImporter -from vulnerabilities.importers.project_kb_msr2019 import ProjectKBMSRImporter -from vulnerabilities.importers.pypa import PyPaImporter -from vulnerabilities.importers.pysec import PyPIImporter -from vulnerabilities.importers.redhat import RedhatImporter -from vulnerabilities.importers.retiredotnet import RetireDotnetImporter -from vulnerabilities.importers.suse_scores import SUSESeverityScoreImporter -from vulnerabilities.importers.ubuntu import UbuntuImporter -from vulnerabilities.importers.ubuntu_usn import UbuntuUSNImporter -from vulnerabilities.importers.xen import XenImporter -from vulnerabilities.improvers.default import DefaultImprover - - -class NVDImprover(DefaultImprover): - importer = NVDImporter - - -class AlpineLinuxImprover(DefaultImprover): - importer = AlpineImporter - - -class ApacheHTTPDImprover(DefaultImprover): - importer = ApacheHTTPDImporter - - -class ApacheKafkaImprover(DefaultImprover): - importer = ApacheKafkaImporter - - -class ApacheTomcatImprover(DefaultImprover): - importer = ApacheTomcatImporter - - -class ArchLinuxImprover(DefaultImprover): - importer = ArchlinuxImporter - - -class DebianImprover(DefaultImprover): - importer = DebianImporter - - -class DebianOvalImprover(DefaultImprover): - importer = DebianOvalImporter - - -class ElixirSecurityImprover(DefaultImprover): - importer = ElixirSecurityImporter - - -class FireEyeImprover(DefaultImprover): - importer = FireyeImporter - - -class GentooImprover(DefaultImprover): - importer = GentooImporter - - -class GitHubAPIImprover(DefaultImprover): - importer = GitHubAPIImporter - - -class GitLabAPIImprover(DefaultImprover): - importer = GitLabAPIImporter - - -class IstioImprover(DefaultImprover): - importer = IstioImporter - - -class MozillaImprover(DefaultImprover): - importer = MozillaImporter - - -class NginxImprover(DefaultImprover): - importer = NginxImporter - - -class NpmImprover(DefaultImprover): - importer = NpmImporter - - -class OpensslImprover(DefaultImprover): - importer = OpensslImporter - - -class PostgreSQLImprover(DefaultImprover): - importer = PostgreSQLImporter - - -class ProjectKBMSRImprover(DefaultImprover): - importer = ProjectKBMSRImporter - - -class PyPaImprover(DefaultImprover): - importer = PyPaImporter - - -class PyPIImprover(DefaultImprover): - importer = PyPIImporter - - -class RedhatImprover(DefaultImprover): - importer = RedhatImporter - - -class RetireDotnetImprover(DefaultImprover): - importer = RetireDotnetImporter - - -class SUSESeverityScoreImprover(DefaultImprover): - importer = SUSESeverityScoreImporter - - -class UbuntuImprover(DefaultImprover): - importer = UbuntuImporter - - -class UbuntuUSNImprover(DefaultImprover): - importer = UbuntuUSNImporter - - -class XenImprover(DefaultImprover): - importer = XenImporter diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index c10e830a9..c350ad6db 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -14,7 +14,6 @@ from vulnerabilities import models from vulnerabilities.import_runner import ImportRunner -from vulnerabilities.import_runner import process_advisories from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer @@ -53,21 +52,21 @@ def test_import_runner(db): def test_process_advisories_with_no_advisory(db): - process_advisories([], "") + ImportRunner(DummyImporter).process_advisories([], "") assert 0 == models.Advisory.objects.count() def test_process_advisories_with_advisories(db): - process_advisories(ADVISORY_DATAS, "test_importer") + ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") advisories = models.Advisory.objects.all() advisory_datas = [x.to_advisory_data() for x in advisories] assert advisory_datas == ADVISORY_DATAS def test_process_advisories_idempotency(db): - process_advisories(ADVISORY_DATAS, "test_importer") - process_advisories(ADVISORY_DATAS, "test_importer") - process_advisories(ADVISORY_DATAS, "test_importer") + ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") + ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") + ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") advisories = models.Advisory.objects.all() advisory_datas = [x.to_advisory_data() for x in advisories] assert advisory_datas == ADVISORY_DATAS @@ -75,21 +74,21 @@ def test_process_advisories_idempotency(db): def test_process_advisories_idempotency_with_one_new_advisory(db): advisory_datas = ADVISORY_DATAS.copy() - process_advisories(advisory_datas, "test_importer") + ImportRunner(DummyImporter).process_advisories(advisory_datas, "test_importer") advisory_datas.append( AdvisoryData( aliases=["CVE-2022-1337"], ) ) - process_advisories(advisory_datas, "test_importer") + ImportRunner(DummyImporter).process_advisories(advisory_datas, "test_importer") advisories = models.Advisory.objects.all() advisory_datas_in_db = [x.to_advisory_data() for x in advisories] assert advisory_datas_in_db == advisory_datas def test_process_advisories_idempotency_with_different_importer_names(db): - process_advisories(ADVISORY_DATAS, "test_importer_one") - process_advisories(ADVISORY_DATAS, "test_importer_two") + ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer_one") + ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer_two") advisories = models.Advisory.objects.all() advisory_datas = [x.to_advisory_data() for x in advisories] assert advisory_datas == ADVISORY_DATAS diff --git a/vulnerabilities/tests/test_importer_specific_improver.py b/vulnerabilities/tests/test_importer_specific_improver.py deleted file mode 100644 index 4b8c0d733..000000000 --- a/vulnerabilities/tests/test_importer_specific_improver.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import datetime - -import pytest - -from vulnerabilities.importers.nvd import NVDImporter -from vulnerabilities.improve_runner import ImproveRunner -from vulnerabilities.improvers.importer_specific_improver import NVDImprover -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias - - -@pytest.mark.django_db -def test_improvement_of_importer_specific_advisories(): - Advisory.objects.create( - aliases=["CVE-2021-22"], - summary="TEST", - created_by=NVDImporter.qualified_name, - date_collected=datetime.datetime.now(tz=datetime.timezone.utc), - ) - ImproveRunner(improver_class=NVDImprover).run() - alias = Alias.objects.filter(alias="CVE-2021-22").first() - assert alias is not None From 46fdfd8c5bf9b134cc3674ebfe2d1c98ef5626ae Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Sep 2023 15:40:15 +0530 Subject: [PATCH 14/21] Fix Signed-off-by: Tushar Goel --- vulnerabilities/improvers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 469048f64..efbf9d43b 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -7,7 +7,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from vulnerabilities.improvers import importer_specific_improver from vulnerabilities.improvers import valid_versions IMPROVERS_REGISTRY = [ From 301879f505aa0de67e724733948b8e4f8f5db15b Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Sep 2023 15:41:28 +0530 Subject: [PATCH 15/21] remove unused imports Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index ce682311f..02259bba8 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -26,7 +26,6 @@ from django.db.models.functions import Length from django.db.models.functions import Trim from django.urls import reverse -from django.utils import timezone from packageurl import PackageURL from packageurl.contrib.django.models import PackageURLMixin from packageurl.contrib.django.models import PackageURLQuerySet @@ -40,7 +39,6 @@ from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import build_vcid from vulnerabilities.utils import remove_qualifiers_and_subpath -from vulnerablecode import __version__ as vulnerablecode_version logger = logging.getLogger(__name__) From 6cda8db78ed71162a6a2d167b57113bf9170ee0a Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Sep 2023 15:57:04 +0530 Subject: [PATCH 16/21] Add CHANGELOG Signed-off-by: Tushar Goel --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 757969e4b..7f83e390a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,7 +9,7 @@ Version v33.5.0 - We added clickable links to the Essentials tab of the Vulnerability details template that enable the user to navigate to the Fixed by packages tab and the Affected packages tab. - We fixed severity range issue for handling unknown scores. - +- We added support to write packages and vulnerabilities at the time of import. Version v33.4.0 ---------------- From dc7832926b372bb38991bd926ce2dbfba18dcf49 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 11 Sep 2023 17:40:38 +0530 Subject: [PATCH 17/21] Move clean summary function to AdvisoryData class Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 1 - vulnerabilities/importer.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index a055197d5..915966a3f 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -91,7 +91,6 @@ def process_advisories( # https://nvd.nist.gov/vuln/detail/CVE-2013-4314 # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572 try: - data.summary = data.summary.replace("\x00", "\uFFFD") obj, created = Advisory.objects.get_or_create( aliases=data.aliases, summary=data.summary, diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 5a6fdf318..9c36cc63f 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -249,9 +249,14 @@ class AdvisoryData: date_published: Optional[datetime.datetime] = None weaknesses: List[int] = dataclasses.field(default_factory=list) + def __post_init__(self): if self.date_published and not self.date_published.tzinfo: logger.warning(f"AdvisoryData with no tzinfo: {self!r}") + self.clean_summary() + + def clean_summary(self): + self.summary = self.summary.replace("\x00", "\uFFFD") def to_dict(self): return { From 5e1580f4a2f3b3682c16c1dda5fcff060bada76b Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 11 Sep 2023 18:46:06 +0530 Subject: [PATCH 18/21] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 2 -- vulnerabilities/importer.py | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 915966a3f..6970e834d 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -88,8 +88,6 @@ def process_advisories( count = 0 advisories = [] for data in advisory_datas: - # https://nvd.nist.gov/vuln/detail/CVE-2013-4314 - # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572 try: obj, created = Advisory.objects.get_or_create( aliases=data.aliases, diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 9c36cc63f..71c1d4282 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -249,14 +249,16 @@ class AdvisoryData: date_published: Optional[datetime.datetime] = None weaknesses: List[int] = dataclasses.field(default_factory=list) - def __post_init__(self): if self.date_published and not self.date_published.tzinfo: logger.warning(f"AdvisoryData with no tzinfo: {self!r}") self.clean_summary() def clean_summary(self): - self.summary = self.summary.replace("\x00", "\uFFFD") + # https://nvd.nist.gov/vuln/detail/CVE-2013-4314 + # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572 + if self.summary: + self.summary = self.summary.replace("\x00", "\uFFFD") def to_dict(self): return { From fe9df81f8de39e9e4708cd9848d808e9bf746398 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 14 Sep 2023 20:01:06 +0530 Subject: [PATCH 19/21] Address review comments Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 18 ++++++++++-------- vulnerabilities/importer.py | 10 ++++++---- vulnerabilities/improve_runner.py | 6 +++--- vulnerabilities/tests/test_import_runner.py | 16 +++++++++++++++- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 6970e834d..5fd8fc263 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -59,16 +59,18 @@ def run(self) -> None: logger.info(f"Finished import for {importer_name}. Imported {count} advisories.") def do_import(self, advisories) -> None: - improver = AdvisoryBasedDefaultImprover(advisories=advisories) - logger.info(f"Running improver: {improver.qualified_name}") - improver_name = improver.qualified_name + advisory_importer = AdvisoryBasedDefaultImprover(advisories=advisories) + logger.info(f"Running improver: {advisory_importer.qualified_name}") + improver_name = advisory_importer.qualified_name advisories = [] - for advisory in improver.interesting_advisories: + for advisory in advisory_importer.interesting_advisories: if advisory.date_imported: continue logger.info(f"Processing advisory: {advisory!r}") try: - inferences = improver.get_inferences(advisory_data=advisory.to_advisory_data()) + inferences = advisory_importer.get_inferences( + advisory_data=advisory.to_advisory_data() + ) process_inferences( inferences=inferences, advisory=advisory, @@ -76,7 +78,7 @@ def do_import(self, advisories) -> None: ) except Exception as e: logger.info(f"Failed to process advisory: {advisory!r} with error {e!r}") - logger.info("Finished improving using %s.", improver.__class__.qualified_name) + logger.info("Finished improving using %s.", advisory_importer.__class__.qualified_name) def process_advisories( self, advisory_datas: Iterable[AdvisoryData], importer_name: str @@ -128,13 +130,13 @@ def process_advisories( def process_inferences(inferences: List[Inference], advisory: Advisory, improver_name: str): """ Return number of inferences processed. - An atomic transaction that updates both the Advisory (e.g. date_improved) + An atomic transaction that updates both the Advisory (e.g. date_imported) and processes the given inferences to create or update corresponding database fields. This avoids failing the entire improver when only a single inference is erroneous. Also, the atomic transaction for every advisory and its - inferences makes sure that date_improved of advisory is consistent. + inferences makes sure that date_imported of advisory is consistent. """ inferences_processed_count = 0 diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 71c1d4282..179f9f4a3 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -252,13 +252,15 @@ class AdvisoryData: def __post_init__(self): if self.date_published and not self.date_published.tzinfo: logger.warning(f"AdvisoryData with no tzinfo: {self!r}") - self.clean_summary() + self.summary = self.clean_summary(self.summary) - def clean_summary(self): + def clean_summary(self, summary): # https://nvd.nist.gov/vuln/detail/CVE-2013-4314 # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572 - if self.summary: - self.summary = self.summary.replace("\x00", "\uFFFD") + summary = summary.strip() + if summary: + summary = summary.replace("\x00", "\uFFFD") + return summary def to_dict(self): return { diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index 8de1a1663..7d8ad5d77 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -60,13 +60,13 @@ def run(self) -> None: def process_inferences(inferences: List[Inference], advisory: Advisory, improver_name: str): """ Return number of inferences processed. - An atomic transaction that updates both the Advisory (e.g. date_improved) + An atomic transaction that updates both the Advisory (e.g. date_imported) and processes the given inferences to create or update corresponding database fields. This avoids failing the entire improver when only a single inference is erroneous. Also, the atomic transaction for every advisory and its - inferences makes sure that date_improved of advisory is consistent. + inferences makes sure that date_imported of advisory is consistent. """ inferences_processed_count = 0 @@ -150,7 +150,7 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver inferences_processed_count += 1 - advisory.date_improved = datetime.now(timezone.utc) + advisory.date_imported = datetime.now(timezone.utc) advisory.save() return inferences_processed_count diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index c350ad6db..59beacc87 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -10,6 +10,7 @@ from datetime import datetime from datetime import timezone +import pytest from univers.version_range import VersionRange from vulnerabilities import models @@ -43,6 +44,7 @@ def advisory_data(self): return ADVISORY_DATAS +@pytest.mark.django_db(transaction=True) def test_import_runner(db): runner = ImportRunner(DummyImporter) runner.run() @@ -51,11 +53,13 @@ def test_import_runner(db): assert advisory_datas == ADVISORY_DATAS +@pytest.mark.django_db(transaction=True) def test_process_advisories_with_no_advisory(db): ImportRunner(DummyImporter).process_advisories([], "") assert 0 == models.Advisory.objects.count() +@pytest.mark.django_db(transaction=True) def test_process_advisories_with_advisories(db): ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") advisories = models.Advisory.objects.all() @@ -63,6 +67,7 @@ def test_process_advisories_with_advisories(db): assert advisory_datas == ADVISORY_DATAS +@pytest.mark.django_db(transaction=True) def test_process_advisories_idempotency(db): ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") @@ -72,6 +77,7 @@ def test_process_advisories_idempotency(db): assert advisory_datas == ADVISORY_DATAS +@pytest.mark.django_db(transaction=True) def test_process_advisories_idempotency_with_one_new_advisory(db): advisory_datas = ADVISORY_DATAS.copy() ImportRunner(DummyImporter).process_advisories(advisory_datas, "test_importer") @@ -86,9 +92,17 @@ def test_process_advisories_idempotency_with_one_new_advisory(db): assert advisory_datas_in_db == advisory_datas -def test_process_advisories_idempotency_with_different_importer_names(db): +@pytest.mark.django_db(transaction=True) +def test_process_advisories_idempotency_with_different_importer_names(): ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer_one") ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer_two") advisories = models.Advisory.objects.all() advisory_datas = [x.to_advisory_data() for x in advisories] assert advisory_datas == ADVISORY_DATAS + + +def test_advisory_summary_clean_up(): + adv = AdvisoryData( + summary="The X509Extension in pyOpenSSL before 0.13.1 does not properly handle a '\x00' character in a domain name in the Subject Alternative Name field of an X.509 certificate, which allows man-in-the-middle attackers to spoof arbitrary SSL servers via a crafted certificate issued by a legitimate Certification Authority." + ) + assert '\x00' not in adv.summary From 9be3549866b823cc879fa5b4f8b3cf6eba105940 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 14 Sep 2023 20:13:45 +0530 Subject: [PATCH 20/21] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/importer.py | 3 +- .../elixir_security/elixir-expected.json | 2 +- .../elixir-improver-expected.json | 4 +- .../test_data/gentoo/gentoo-expected.json | 2 +- .../test_data/github_api/npm-expected.json | 15 ++++---- .../tests/test_data/mozilla/expected-yml.json | 38 +++++++++---------- vulnerabilities/tests/test_import_runner.py | 2 +- 7 files changed, 33 insertions(+), 33 deletions(-) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 179f9f4a3..36aed107c 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -252,7 +252,8 @@ class AdvisoryData: def __post_init__(self): if self.date_published and not self.date_published.tzinfo: logger.warning(f"AdvisoryData with no tzinfo: {self!r}") - self.summary = self.clean_summary(self.summary) + if self.summary: + self.summary = self.clean_summary(self.summary) def clean_summary(self, summary): # https://nvd.nist.gov/vuln/detail/CVE-2013-4314 diff --git a/vulnerabilities/tests/test_data/elixir_security/elixir-expected.json b/vulnerabilities/tests/test_data/elixir_security/elixir-expected.json index 9f7c500b0..69402de34 100644 --- a/vulnerabilities/tests/test_data/elixir_security/elixir-expected.json +++ b/vulnerabilities/tests/test_data/elixir_security/elixir-expected.json @@ -3,7 +3,7 @@ "aliases": [ "CVE-2018-20301" ], - "summary": "The Coherence library has \"Mass Assignment\"-like vulnerabilities.\n", + "summary": "The Coherence library has \"Mass Assignment\"-like vulnerabilities.", "affected_packages": [ { "package": { diff --git a/vulnerabilities/tests/test_data/elixir_security/elixir-improver-expected.json b/vulnerabilities/tests/test_data/elixir_security/elixir-improver-expected.json index f076bb881..f65ed6074 100644 --- a/vulnerabilities/tests/test_data/elixir_security/elixir-improver-expected.json +++ b/vulnerabilities/tests/test_data/elixir_security/elixir-improver-expected.json @@ -5,7 +5,7 @@ "CVE-2018-20301" ], "confidence": 100, - "summary": "The Coherence library has \"Mass Assignment\"-like vulnerabilities.\n", + "summary": "The Coherence library has \"Mass Assignment\"-like vulnerabilities.", "affected_purls": [ { "type": "hex", @@ -39,7 +39,7 @@ "CVE-2018-20301" ], "confidence": 100, - "summary": "The Coherence library has \"Mass Assignment\"-like vulnerabilities.\n", + "summary": "The Coherence library has \"Mass Assignment\"-like vulnerabilities.", "affected_purls": [], "fixed_purl": null, "references": [ diff --git a/vulnerabilities/tests/test_data/gentoo/gentoo-expected.json b/vulnerabilities/tests/test_data/gentoo/gentoo-expected.json index 4609de554..e9ebb4e84 100644 --- a/vulnerabilities/tests/test_data/gentoo/gentoo-expected.json +++ b/vulnerabilities/tests/test_data/gentoo/gentoo-expected.json @@ -3,7 +3,7 @@ "aliases": [ "CVE-2017-9800" ], - "summary": "A command injection vulnerability in Subversion may allow remote\n attackers to execute arbitrary code.\n ", + "summary": "A command injection vulnerability in Subversion may allow remote\n attackers to execute arbitrary code.", "affected_packages": [ { "package": { diff --git a/vulnerabilities/tests/test_data/github_api/npm-expected.json b/vulnerabilities/tests/test_data/github_api/npm-expected.json index ef115883b..476491f46 100644 --- a/vulnerabilities/tests/test_data/github_api/npm-expected.json +++ b/vulnerabilities/tests/test_data/github_api/npm-expected.json @@ -345,7 +345,6 @@ ], "date_published": "2022-11-28T22:06:24+00:00", "weaknesses": [] - }, { "aliases": [ @@ -584,7 +583,7 @@ "CVE-2022-41940", "GHSA-r7qp-cfhv-p84w" ], - "summary": "Uncaught exception in engine.io ", + "summary": "Uncaught exception in engine.io", "affected_packages": [ { "package": { @@ -646,7 +645,7 @@ "CVE-2022-41940", "GHSA-r7qp-cfhv-p84w" ], - "summary": "Uncaught exception in engine.io ", + "summary": "Uncaught exception in engine.io", "affected_packages": [ { "package": { @@ -708,7 +707,7 @@ "CVE-2022-41919", "GHSA-3fjj-p79j-c9hh" ], - "summary": "Fastify: Incorrect Content-Type parsing can lead to CSRF attack ", + "summary": "Fastify: Incorrect Content-Type parsing can lead to CSRF attack", "affected_packages": [ { "package": { @@ -770,7 +769,7 @@ "CVE-2022-41919", "GHSA-3fjj-p79j-c9hh" ], - "summary": "Fastify: Incorrect Content-Type parsing can lead to CSRF attack ", + "summary": "Fastify: Incorrect Content-Type parsing can lead to CSRF attack", "affected_packages": [ { "package": { @@ -1451,7 +1450,7 @@ "aliases": [ "GHSA-3qmc-2r76-4rqp" ], - "summary": "Redwood is vulnerable to account takeover via dbAuth \"forgot-password\" ", + "summary": "Redwood is vulnerable to account takeover via dbAuth \"forgot-password\"", "affected_packages": [ { "package": { @@ -1517,7 +1516,7 @@ "aliases": [ "GHSA-3qmc-2r76-4rqp" ], - "summary": "Redwood is vulnerable to account takeover via dbAuth \"forgot-password\" ", + "summary": "Redwood is vulnerable to account takeover via dbAuth \"forgot-password\"", "affected_packages": [ { "package": { @@ -4234,7 +4233,7 @@ "CVE-2022-25918", "GHSA-cr84-xvw4-qx3c" ], - "summary": "Inefficient Regular Expression Complexity in shescape ", + "summary": "Inefficient Regular Expression Complexity in shescape", "affected_packages": [ { "package": { diff --git a/vulnerabilities/tests/test_data/mozilla/expected-yml.json b/vulnerabilities/tests/test_data/mozilla/expected-yml.json index b4d7747df..e52d8a7b8 100644 --- a/vulnerabilities/tests/test_data/mozilla/expected-yml.json +++ b/vulnerabilities/tests/test_data/mozilla/expected-yml.json @@ -3,7 +3,7 @@ "aliases": [ "CVE-2022-22746" ], - "summary": "A race condition could have allowed bypassing the fullscreen notification which could have lead to a fullscreen window spoof being unnoticed.*This bug only affects Firefox for Windows. Other operating systems are unaffected.*\n", + "summary": "A race condition could have allowed bypassing the fullscreen notification which could have lead to a fullscreen window spoof being unnoticed.*This bug only affects Firefox for Windows. Other operating systems are unaffected.*", "affected_packages": [ { "package": { @@ -38,7 +38,7 @@ "aliases": [ "CVE-2022-22743" ], - "summary": "When navigating from inside an iframe while requesting fullscreen access, an attacker-controlled tab could have made the browser unable to leave fullscreen mode.\n", + "summary": "When navigating from inside an iframe while requesting fullscreen access, an attacker-controlled tab could have made the browser unable to leave fullscreen mode.", "affected_packages": [], "references": [ { @@ -60,7 +60,7 @@ "aliases": [ "CVE-2022-22742" ], - "summary": "When inserting text while in edit mode, some characters might have lead to out-of-bounds memory access causing a potentially exploitable crash.\n", + "summary": "When inserting text while in edit mode, some characters might have lead to out-of-bounds memory access causing a potentially exploitable crash.", "affected_packages": [], "references": [ { @@ -82,7 +82,7 @@ "aliases": [ "CVE-2022-22741" ], - "summary": "When resizing a popup while requesting fullscreen access, the popup would have become unable to leave fullscreen mode.\n", + "summary": "When resizing a popup while requesting fullscreen access, the popup would have become unable to leave fullscreen mode.", "affected_packages": [], "references": [ { @@ -104,7 +104,7 @@ "aliases": [ "CVE-2022-22740" ], - "summary": "Certain network request objects were freed too early when releasing a network request handle. This could have lead to a use-after-free causing a potentially exploitable crash.\n", + "summary": "Certain network request objects were freed too early when releasing a network request handle. This could have lead to a use-after-free causing a potentially exploitable crash.", "affected_packages": [], "references": [ { @@ -126,7 +126,7 @@ "aliases": [ "CVE-2022-22738" ], - "summary": "Applying a CSS filter effect could have accessed out of bounds memory. This could have lead to a heap-buffer-overflow causing a potentially exploitable crash.\n", + "summary": "Applying a CSS filter effect could have accessed out of bounds memory. This could have lead to a heap-buffer-overflow causing a potentially exploitable crash.", "affected_packages": [], "references": [ { @@ -148,7 +148,7 @@ "aliases": [ "CVE-2022-22737" ], - "summary": "Constructing audio sinks could have lead to a race condition when playing audio files and closing windows. This could have lead to a use-after-free causing a potentially exploitable crash.\n", + "summary": "Constructing audio sinks could have lead to a race condition when playing audio files and closing windows. This could have lead to a use-after-free causing a potentially exploitable crash.", "affected_packages": [], "references": [ { @@ -170,7 +170,7 @@ "aliases": [ "CVE-2021-4140" ], - "summary": "It was possible to construct specific XSLT markup that would be able to bypass an iframe sandbox.\n", + "summary": "It was possible to construct specific XSLT markup that would be able to bypass an iframe sandbox.", "affected_packages": [], "references": [ { @@ -192,7 +192,7 @@ "aliases": [ "CVE-2022-22750" ], - "summary": "By generally accepting and passing resource handles across processes, a compromised content process might have confused higher privileged processes to interact with handles that the unprivileged process should not have access to.*This bug only affects Firefox for Windows and MacOS. Other operating systems are unaffected.*\n", + "summary": "By generally accepting and passing resource handles across processes, a compromised content process might have confused higher privileged processes to interact with handles that the unprivileged process should not have access to.*This bug only affects Firefox for Windows and MacOS. Other operating systems are unaffected.*", "affected_packages": [], "references": [ { @@ -214,7 +214,7 @@ "aliases": [ "CVE-2022-22749" ], - "summary": "When scanning QR codes, Firefox for Android would have allowed navigation to some URLs that do not point to web content.*This bug only affects Firefox for Android. Other operating systems are unaffected.*\n", + "summary": "When scanning QR codes, Firefox for Android would have allowed navigation to some URLs that do not point to web content.*This bug only affects Firefox for Android. Other operating systems are unaffected.*", "affected_packages": [], "references": [ { @@ -236,7 +236,7 @@ "aliases": [ "CVE-2022-22748" ], - "summary": "Malicious websites could have confused Firefox into showing the wrong origin when asking to launch a program and handling an external URL protocol.\n", + "summary": "Malicious websites could have confused Firefox into showing the wrong origin when asking to launch a program and handling an external URL protocol.", "affected_packages": [], "references": [ { @@ -258,7 +258,7 @@ "aliases": [ "CVE-2022-22745" ], - "summary": "Securitypolicyviolation events could have leaked cross-origin information for frame-ancestors violations\n", + "summary": "Securitypolicyviolation events could have leaked cross-origin information for frame-ancestors violations", "affected_packages": [], "references": [ { @@ -280,7 +280,7 @@ "aliases": [ "CVE-2022-22744" ], - "summary": "The constructed curl command from the \"Copy as curl\" feature in DevTools was not properly escaped for PowerShell. This could have lead to command injection if pasted into a Powershell prompt.*This bug only affects Firefox for Windows. Other operating systems are unaffected.*\n", + "summary": "The constructed curl command from the \"Copy as curl\" feature in DevTools was not properly escaped for PowerShell. This could have lead to command injection if pasted into a Powershell prompt.*This bug only affects Firefox for Windows. Other operating systems are unaffected.*", "affected_packages": [], "references": [ { @@ -302,7 +302,7 @@ "aliases": [ "CVE-2022-22763" ], - "summary": "When a worker is shutdown, it was possible to cause script to run late in the lifecycle, at a point after where it should not be possible.\n", + "summary": "When a worker is shutdown, it was possible to cause script to run late in the lifecycle, at a point after where it should not be possible.", "affected_packages": [], "references": [ { @@ -324,7 +324,7 @@ "aliases": [ "CVE-2022-22747" ], - "summary": "After accepting an untrusted certificate, handling an empty pkcs7 sequence as part of the certificate data could have lead to a crash. This crash is believed to be unexploitable.\n", + "summary": "After accepting an untrusted certificate, handling an empty pkcs7 sequence as part of the certificate data could have lead to a crash. This crash is believed to be unexploitable.", "affected_packages": [], "references": [ { @@ -346,7 +346,7 @@ "aliases": [ "CVE-2022-22736" ], - "summary": "If Firefox was installed to a world-writable directory, a local privilege escalation could occur when Firefox searched the current directory for system libraries. However the install directory is not world-writable by default.*This bug only affects Firefox for Windows in a non-default installation. Other operating systems are unaffected.*\n", + "summary": "If Firefox was installed to a world-writable directory, a local privilege escalation could occur when Firefox searched the current directory for system libraries. However the install directory is not world-writable by default.*This bug only affects Firefox for Windows in a non-default installation. Other operating systems are unaffected.*", "affected_packages": [], "references": [ { @@ -368,7 +368,7 @@ "aliases": [ "CVE-2022-22739" ], - "summary": "Malicious websites could have tricked users into accepting launching a program to handle an external URL protocol.\n", + "summary": "Malicious websites could have tricked users into accepting launching a program to handle an external URL protocol.", "affected_packages": [], "references": [ { @@ -390,7 +390,7 @@ "aliases": [ "CVE-2022-22751" ], - "summary": "Mozilla developers Calixte Denizet, Kershaw Chang, Christian Holler, Jason Kratzer, Gabriele Svelto, Tyson Smith, Simon Giesecke, and Steve Fink reported memory safety bugs present in Firefox 95 and Firefox ESR 91.4. Some of these bugs showed evidence of memory corruption and we presume that with enough effort some of these could have been exploited to run arbitrary code.\n", + "summary": "Mozilla developers Calixte Denizet, Kershaw Chang, Christian Holler, Jason Kratzer, Gabriele Svelto, Tyson Smith, Simon Giesecke, and Steve Fink reported memory safety bugs present in Firefox 95 and Firefox ESR 91.4. Some of these bugs showed evidence of memory corruption and we presume that with enough effort some of these could have been exploited to run arbitrary code.", "affected_packages": [], "references": [ { @@ -412,7 +412,7 @@ "aliases": [ "CVE-2022-22752" ], - "summary": "Mozilla developers Christian Holler and Jason Kratzer reported memory safety bugs present in Firefox 95. Some of these bugs showed evidence of memory corruption and we presume that with enough effort some of these could have been exploited to run arbitrary code.\n", + "summary": "Mozilla developers Christian Holler and Jason Kratzer reported memory safety bugs present in Firefox 95. Some of these bugs showed evidence of memory corruption and we presume that with enough effort some of these could have been exploited to run arbitrary code.", "affected_packages": [], "references": [ { diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index 59beacc87..22648a518 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -105,4 +105,4 @@ def test_advisory_summary_clean_up(): adv = AdvisoryData( summary="The X509Extension in pyOpenSSL before 0.13.1 does not properly handle a '\x00' character in a domain name in the Subject Alternative Name field of an X.509 certificate, which allows man-in-the-middle attackers to spoof arbitrary SSL servers via a crafted certificate issued by a legitimate Certification Authority." ) - assert '\x00' not in adv.summary + assert "\x00" not in adv.summary From 0ef58f267a7be784669052d26654cf66cb493050 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 21 Sep 2023 23:20:53 +0530 Subject: [PATCH 21/21] Adress review comments Signed-off-by: Tushar Goel --- vulnerabilities/import_runner.py | 12 ++++++------ vulnerabilities/improvers/default.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 5fd8fc263..91ab75675 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -19,7 +19,7 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.improver import Inference -from vulnerabilities.improvers.default import AdvisoryBasedDefaultImprover +from vulnerabilities.improvers.default import DefaultImporter from vulnerabilities.models import Advisory from vulnerabilities.models import Alias from vulnerabilities.models import Package @@ -59,9 +59,9 @@ def run(self) -> None: logger.info(f"Finished import for {importer_name}. Imported {count} advisories.") def do_import(self, advisories) -> None: - advisory_importer = AdvisoryBasedDefaultImprover(advisories=advisories) - logger.info(f"Running improver: {advisory_importer.qualified_name}") - improver_name = advisory_importer.qualified_name + advisory_importer = DefaultImporter(advisories=advisories) + logger.info(f"Running importer: {advisory_importer.qualified_name}") + importer_name = advisory_importer.qualified_name advisories = [] for advisory in advisory_importer.interesting_advisories: if advisory.date_imported: @@ -74,11 +74,11 @@ def do_import(self, advisories) -> None: process_inferences( inferences=inferences, advisory=advisory, - improver_name=improver_name, + improver_name=importer_name, ) except Exception as e: logger.info(f"Failed to process advisory: {advisory!r} with error {e!r}") - logger.info("Finished improving using %s.", advisory_importer.__class__.qualified_name) + logger.info("Finished importing using %s.", advisory_importer.__class__.qualified_name) def process_advisories( self, advisory_datas: Iterable[AdvisoryData], importer_name: str diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py index b43acac43..3e2f1913f 100644 --- a/vulnerabilities/improvers/default.py +++ b/vulnerabilities/improvers/default.py @@ -134,7 +134,7 @@ def get_exact_purls(affected_package: AffectedPackage) -> Tuple[List[PackageURL] return [], [] -class AdvisoryBasedDefaultImprover(DefaultImprover): +class DefaultImporter(DefaultImprover): def __init__(self, advisories) -> None: self.advisories = advisories