diff --git a/vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py b/vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py new file mode 100644 index 000000000..59f5da903 --- /dev/null +++ b/vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py @@ -0,0 +1,52 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + def remove_duped_changelogs(apps, schema_editor): + PackageChangeLog = apps.get_model("vulnerabilities", "PackageChangeLog") + VulnerabilityChangeLog = apps.get_model("vulnerabilities", "VulnerabilityChangeLog") + + models_list = [PackageChangeLog, VulnerabilityChangeLog] + + for model in models_list: + # Identify duplicate records based on actor_name, action_type, and source_url + duplicate_records = model.objects.values('actor_name', 'action_type', 'source_url').annotate(count=models.Count('id')).filter(count__gt=1) + + to_be_deleted = list() + + for duplicate_set in duplicate_records: + # Get the records for the current duplicate set + records_to_delete = model.objects.filter( + actor_name=duplicate_set['actor_name'], + action_type=duplicate_set['action_type'], + source_url=duplicate_set['source_url'] + ).order_by('-software_version') + + # Keep the record with the older software version + record_to_keep = records_to_delete.last() + + # Delete the records with the newer software version + to_be_deleted.extend(records_to_delete.exclude(id=record_to_keep.id)) + + to_be_deleted = list(set(to_be_deleted)) + to_be_deleted = [rec.id for rec in to_be_deleted] + model.objects.filter(id__in = to_be_deleted).delete() + + dependencies = [ + ("vulnerabilities", "0054_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.RunPython(remove_duped_changelogs, reverse_code=migrations.RunPython.noop), + ] diff --git a/vulnerabilities/migrations/0056_alter_packagechangelog_unique_together_and_more.py b/vulnerabilities/migrations/0056_alter_packagechangelog_unique_together_and_more.py new file mode 100644 index 000000000..3b35e273e --- /dev/null +++ b/vulnerabilities/migrations/0056_alter_packagechangelog_unique_together_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.1.13 on 2024-01-22 09:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0055_remove_changelogs_with_same_data_different_software_version"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="packagechangelog", + unique_together={("action_time", "actor_name", "action_type", "source_url")}, + ), + migrations.AlterUniqueTogether( + name="vulnerabilitychangelog", + unique_together={("action_time", "actor_name", "action_type", "source_url")}, + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 3dca94cc5..cf6f4412a 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -1145,6 +1145,7 @@ def get_iso_time(self): class Meta: abstract = True ordering = ("-action_time",) + unique_together = ("action_time", "actor_name", "action_type", "source_url") class VulnerabilityHistoryManager(models.Manager): diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index 7d8e6b258..2d7521a3f 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -610,3 +610,53 @@ def setUpBeforeMigration(self, apps): def test_removal_of_duped_purls(self): Package = apps.get_model("vulnerabilities", "Package") assert Package.objects.count() == 1 + + +class TestRemoveDupedChangeLogWithSameData(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0054_alter_packagechangelog_software_version_and_more" + migrate_to = "0055_remove_changelogs_with_same_data_different_software_version" + + def setUpBeforeMigration(self, apps): + PackageChangeLog = apps.get_model("vulnerabilities", "PackageChangeLog") + VulnerabilityChangeLog = apps.get_model("vulnerabilities", "VulnerabilityChangeLog") + Package = apps.get_model("vulnerabilities", "Package") + Vulnerability = apps.get_model("vulnerabilities", "Vulnerability") + pkg1 = Package.objects.create(type="nginx", name="nginx", qualifiers={"os": "windows"}) + vuln = Vulnerability.objects.create(summary="NEW") + PackageChangeLog.objects.create( + actor_name="Nginx", + action_type=1, + source_url="test", + software_version="1", + package=pkg1, + related_vulnerability=vuln, + ) + PackageChangeLog.objects.create( + actor_name="Nginx", + action_type=1, + source_url="test", + software_version="2", + package=pkg1, + related_vulnerability=vuln, + ) + VulnerabilityChangeLog.objects.create( + actor_name="Nginx", + action_type=1, + source_url="test", + software_version="2", + vulnerability=vuln, + ) + VulnerabilityChangeLog.objects.create( + actor_name="Nginx", + action_type=1, + source_url="test", + software_version="1", + vulnerability=vuln, + ) + + def test_removal_of_changelog(self): + PackageChangeLog = apps.get_model("vulnerabilities", "PackageChangeLog") + VulnerabilityChangeLog = apps.get_model("vulnerabilities", "VulnerabilityChangeLog") + assert PackageChangeLog.objects.all().count() == 1 + assert VulnerabilityChangeLog.objects.all().count() == 1