diff --git a/vulnerabilities/migrations/0016_update_cpe_url.py b/vulnerabilities/migrations/0016_update_cpe_url.py index 16a518b15..b75023751 100644 --- a/vulnerabilities/migrations/0016_update_cpe_url.py +++ b/vulnerabilities/migrations/0016_update_cpe_url.py @@ -4,7 +4,11 @@ class Migration(migrations.Migration): def update_cpe_url(apps, schema_editor): Reference = apps.get_model("vulnerabilities", "VulnerabilityReference") - for reference in Reference.objects.filter(reference_id__startswith="cpe"): + cpe_qs = Reference.objects.filter(reference_id__startswith="cpe") + for index, reference in enumerate(cpe_qs.iterator()): + if index % 100 == 0: + print(f"Processing reference {index}") + cpe = reference.reference_id base_url = 'https://nvd.nist.gov/vuln/search/results' params = '?adv_search=true&isCpeNameSearch=true' diff --git a/vulnerabilities/migrations/0017_delete_reference_to_cpes_with_empty_urls.py b/vulnerabilities/migrations/0017_delete_reference_to_cpes_with_empty_urls.py new file mode 100644 index 000000000..33e8fffd1 --- /dev/null +++ b/vulnerabilities/migrations/0017_delete_reference_to_cpes_with_empty_urls.py @@ -0,0 +1,22 @@ +# Generated by Django 4.0.4 on 2022-08-05 12:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0016_update_cpe_url'), + ] + + def delete_reference_to_cpes_with_empty_urls(apps, _): + """ + Delete references to CPEs with empty URLs. + https://github.com/nexB/vulnerablecode/issues/818#issuecomment-1206437637 + """ + Reference = apps.get_model("vulnerabilities", "VulnerabilityReference") + Reference.objects.filter(reference_id__startswith="cpe", url="").delete() + + operations = [ + migrations.RunPython(delete_reference_to_cpes_with_empty_urls, migrations.RunPython.noop), + ] diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index 0102dd8db..74f1504a6 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -130,7 +130,7 @@ def setUpBeforeMigration(self, apps): reference.save() self.reference = reference - def test_cpe_url_updation(self): + def test_cpe_url_update(self): # using get_model to avoid circular import VulnerabilityReference = self.apps.get_model("vulnerabilities", "VulnerabilityReference") ref = VulnerabilityReference.objects.get(reference_id = self.reference.reference_id)