From 46056df09f7d4b02c73cf2c9e76006fa43a9da55 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 29 Feb 2024 15:09:09 +0530 Subject: [PATCH] Fix migration 055 Signed-off-by: Tushar Goel --- ...th_same_data_different_software_version.py | 63 +++++++++++++------ ...alter_packagechangelog_options_and_more.py | 58 +++++++++++++++++ ...ckagechangelog_unique_together_and_more.py | 21 ------- ...kagechangelog_software_version_and_more.py | 31 --------- vulnerabilities/tests/test_data_migrations.py | 8 +++ 5 files changed, 109 insertions(+), 72 deletions(-) create mode 100644 vulnerabilities/migrations/0056_alter_packagechangelog_options_and_more.py delete mode 100644 vulnerabilities/migrations/0056_alter_packagechangelog_unique_together_and_more.py delete mode 100644 vulnerabilities/migrations/0057_alter_packagechangelog_software_version_and_more.py diff --git a/vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py b/vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py index 59f5da903..15b6ef233 100644 --- a/vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py +++ b/vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py @@ -7,8 +7,9 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from collections import defaultdict from django.db import migrations -from django.db import models +from django.core.paginator import Paginator class Migration(migrations.Migration): @@ -19,29 +20,36 @@ def remove_duped_changelogs(apps, schema_editor): models_list = [PackageChangeLog, VulnerabilityChangeLog] + common_fields = ('actor_name', 'action_type', 'source_url') for model in models_list: - # Identify duplicate records based on actor_name, action_type, and source_url - duplicate_records = model.objects.values('actor_name', 'action_type', 'source_url').annotate(count=models.Count('id')).filter(count__gt=1) + record_groups = defaultdict(list) + fields = set() + key = tuple() + if model == PackageChangeLog: + fields = common_fields + ('package', 'related_vulnerability') + all_records = model.objects.select_related("package").all() + elif model == VulnerabilityChangeLog: + fields = common_fields + ('vulnerability',) + all_records = model.objects.select_related("vulnerability").all() - to_be_deleted = list() + print("Total number of records", all_records.count()) + for record in paginated(all_records): + print(",", end="") + key = tuple(getattr(record, attr) for attr in fields) + record_groups[key].append(record.id) - for duplicate_set in duplicate_records: - # Get the records for the current duplicate set - records_to_delete = model.objects.filter( - actor_name=duplicate_set['actor_name'], - action_type=duplicate_set['action_type'], - source_url=duplicate_set['source_url'] - ).order_by('-software_version') + to_be_deleted = [] + for record_ids in record_groups.values(): + if len(record_ids) == 1: + continue + print(".", end="") + # We exclude the oldest ID which is the last one based on the standard + # ordering by decreasing the action time + to_be_deleted.extend(record_ids[:-1]) - # Keep the record with the older software version - record_to_keep = records_to_delete.last() - - # Delete the records with the newer software version - to_be_deleted.extend(records_to_delete.exclude(id=record_to_keep.id)) - - to_be_deleted = list(set(to_be_deleted)) - to_be_deleted = [rec.id for rec in to_be_deleted] - model.objects.filter(id__in = to_be_deleted).delete() + chunks = [to_be_deleted[x:x+10000] for x in range(0, len(to_be_deleted), 10000)] + for chunk in chunks: + model.objects.filter(id__in=chunk).delete() dependencies = [ ("vulnerabilities", "0054_alter_packagechangelog_software_version_and_more"), @@ -50,3 +58,18 @@ def remove_duped_changelogs(apps, schema_editor): operations = [ migrations.RunPython(remove_duped_changelogs, reverse_code=migrations.RunPython.noop), ] + + +def paginated(qs, per_page=5000): + """ + Iterate over a (large) QuerySet by chunks of ``per_page`` items. + This technique is essential for preventing memory issues when iterating + See these links for inspiration: + https://nextlinklabs.com/resources/insights/django-big-data-iteration + https://stackoverflow.com/questions/4222176/why-is-iterating-through-a-large-django-queryset-consuming-massive-amounts-of-me/ + """ + paginator = Paginator(qs, per_page=per_page) + for page_number in paginator.page_range: + page = paginator.page(page_number) + for object in page.object_list: + yield object diff --git a/vulnerabilities/migrations/0056_alter_packagechangelog_options_and_more.py b/vulnerabilities/migrations/0056_alter_packagechangelog_options_and_more.py new file mode 100644 index 000000000..a4e66a84b --- /dev/null +++ b/vulnerabilities/migrations/0056_alter_packagechangelog_options_and_more.py @@ -0,0 +1,58 @@ +# Generated by Django 4.1.13 on 2024-02-28 15:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0055_remove_changelogs_with_same_data_different_software_version"), + ] + + operations = [ + migrations.AlterModelOptions( + name="packagechangelog", + options={}, + ), + migrations.AlterModelOptions( + name="vulnerabilitychangelog", + options={}, + ), + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.0.0rc3", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.0.0rc3", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterUniqueTogether( + name="packagechangelog", + unique_together={ + ( + "action_time", + "actor_name", + "action_type", + "source_url", + "related_vulnerability", + "package", + ) + }, + ), + migrations.AlterUniqueTogether( + name="vulnerabilitychangelog", + unique_together={ + ("action_time", "actor_name", "action_type", "source_url", "vulnerability") + }, + ), + ] diff --git a/vulnerabilities/migrations/0056_alter_packagechangelog_unique_together_and_more.py b/vulnerabilities/migrations/0056_alter_packagechangelog_unique_together_and_more.py deleted file mode 100644 index 3b35e273e..000000000 --- a/vulnerabilities/migrations/0056_alter_packagechangelog_unique_together_and_more.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 4.1.13 on 2024-01-22 09:42 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("vulnerabilities", "0055_remove_changelogs_with_same_data_different_software_version"), - ] - - operations = [ - migrations.AlterUniqueTogether( - name="packagechangelog", - unique_together={("action_time", "actor_name", "action_type", "source_url")}, - ), - migrations.AlterUniqueTogether( - name="vulnerabilitychangelog", - unique_together={("action_time", "actor_name", "action_type", "source_url")}, - ), - ] diff --git a/vulnerabilities/migrations/0057_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0057_alter_packagechangelog_software_version_and_more.py deleted file mode 100644 index f12a00e27..000000000 --- a/vulnerabilities/migrations/0057_alter_packagechangelog_software_version_and_more.py +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Django 4.1.13 on 2024-02-26 13:52 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("vulnerabilities", "0056_alter_packagechangelog_unique_together_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="packagechangelog", - name="software_version", - field=models.CharField( - default="34.0.0rc3", - help_text="Version of the software at the time of change", - max_length=100, - ), - ), - migrations.AlterField( - model_name="vulnerabilitychangelog", - name="software_version", - field=models.CharField( - default="34.0.0rc3", - help_text="Version of the software at the time of change", - max_length=100, - ), - ), - ] diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index 2d7521a3f..7d1158dba 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -640,6 +640,14 @@ def setUpBeforeMigration(self, apps): package=pkg1, related_vulnerability=vuln, ) + PackageChangeLog.objects.create( + actor_name="Nginx", + action_type=1, + source_url="test", + software_version="2", + package=pkg1, + related_vulnerability=vuln, + ) VulnerabilityChangeLog.objects.create( actor_name="Nginx", action_type=1,