Delete references to CPEs with empty URLs #818

vulnerabilities/migrations/0016_update_cpe_url.py

@@ -4,7 +4,11 @@ class Migration(migrations.Migration):
 
     def update_cpe_url(apps, schema_editor):
         Reference = apps.get_model("vulnerabilities", "VulnerabilityReference")
-        for reference in Reference.objects.filter(reference_id__startswith="cpe"):
+        cpe_qs = Reference.objects.filter(reference_id__startswith="cpe")
+        for index, reference in enumerate(cpe_qs.iterator()):
+            if index % 100 == 0:
+                print(f"Processing reference {index}")
+
             cpe = reference.reference_id
             base_url = 'https://nvd.nist.gov/vuln/search/results'
             params = '?adv_search=true&isCpeNameSearch=true'

vulnerabilities/migrations/0017_delete_reference_to_cpes_with_empty_urls.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.0.4 on 2022-08-05 12:55
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0016_update_cpe_url'),
+    ]
+
+    def delete_reference_to_cpes_with_empty_urls(apps, _):
+        """
+        Delete references to CPEs with empty URLs.
+        https://github.com/nexB/vulnerablecode/issues/818#issuecomment-1206437637
+        """
+        Reference = apps.get_model("vulnerabilities", "VulnerabilityReference")
+        Reference.objects.filter(reference_id__startswith="cpe", url="").delete()
+
+    operations = [
+        migrations.RunPython(delete_reference_to_cpes_with_empty_urls, migrations.RunPython.noop),
+    ]

vulnerabilities/tests/test_data_migrations.py

@@ -130,7 +130,7 @@ def setUpBeforeMigration(self, apps):
         reference.save()
         self.reference = reference
 
-    def test_cpe_url_updation(self):
+    def test_cpe_url_update(self):
         # using get_model to avoid circular import
         VulnerabilityReference = self.apps.get_model("vulnerabilities", "VulnerabilityReference")
         ref = VulnerabilityReference.objects.get(reference_id = self.reference.reference_id)