Merge pull request #453 from nexB/ui-reduce-queries

Reduce queries in UI

Author: sbs2001

vulnerabilities/templates/package_update.html

@@ -40,7 +40,7 @@ <h1 class="title">
         <div class="tags mx-3">
             {% for vulnerability in impacted_vuln %}
                 <span class="tag is-danger is-medium">
-                  <a href="{% url 'vulnerability_view' vulnerability.pk %}" class="has-text-white">{{vulnerability}}</a>
+                  <a href="{% url 'vulnerability_view' vulnerability.pk %}" class="has-text-white">{{vulnerability.vulnerability_id}}</a>
                   {% if enable_curation %}
                     <form method="POST" action="{% url 'impacted_package_delete' object.pk vulnerability.pk %}"">
                       {% csrf_token %}<button class="delete is-small" type="submit"></button>
@@ -74,7 +74,7 @@ <h1 class="title">
                <div class="tags mx-3">
             {% for vulnerability in resolved_vuln %}
             <span class="tag is-primary is-medium">
-              <a href="{% url 'vulnerability_view' vulnerability.pk %}" class="has-text-white">{{vulnerability}}</a>
+              <a href="{% url 'vulnerability_view' vulnerability.pk %}" class="has-text-white">{{vulnerability.vulnerability_id}}</a>
 
             {% if enable_curation %}
             <form method="POST" action="{% url 'resolved_package_delete' object.pk vulnerability.pk %}">

vulnerabilities/templates/packages.html

@@ -41,9 +41,9 @@ <h1 class="title">
       </tr>
       {% for package in packages %}
       <tr>
-        <td><a href="{% url 'package_view' package.pk %}">{{package}}</a></td>
-        <td>{{package.vulnerable_to.all|length}}</td>
-        <td>{{package.resolved_to.all|length}}</td>
+        <td><a href="{% url 'package_view' package.pk %}">{{package.package_url}}</a></td>
+        <td>{{package.vulnerability_count}}</td>
+        <td>{{package.patched_vulnerability_count}}</td>
       </tr>
       {% endfor %}
     </table>

vulnerabilities/templates/vulnerabilities.html

@@ -37,8 +37,8 @@ <h1 class="title">
           {% for vulnerability in  vulnerabilities %}
               <tr>
                   <td><a href="{% url 'vulnerability_view' vulnerability.pk %}">{{vulnerability.vulnerability_id}}</a></td>
-                  <td>{{vulnerability.vulnerable_to.all|length}}</td>
-                  <td>{{vulnerability.resolved_to.all|length}}</td>
+                  <td>{{vulnerability.vulnerable_package_count}}</td>
+                  <td>{{vulnerability.patched_package_count}}</td>
               </tr>
           {% endfor %}
           </table>

vulnerabilities/views.py

@@ -23,6 +23,7 @@
 from urllib.parse import urlencode
 
 from django.core.paginator import Paginator
+from django.db.models import Count
 from django.http import HttpResponse
 from django.shortcuts import render, redirect
 from django.urls import reverse
@@ -46,7 +47,7 @@ def get(self, request):
 
         if request.GET:
             packages = self.request_to_queryset(request)
-            result_size = packages.count()
+            result_size = len(packages)
             page_no = int(request.GET.get("page", 1))
             packages = Paginator(packages, 50).get_page(page_no)
             context["packages"] = packages
@@ -68,9 +69,14 @@ def request_to_queryset(request):
         if len(request.GET["name"]):
             package_name = request.GET["name"]
 
-        return models.Package.objects.all().filter(
-            name__icontains=package_name,
-            type__icontains=package_type,
+        return list(
+            models.Package.objects.all()
+            .filter(name__icontains=package_name, type__icontains=package_type)
+            .annotate(
+                vulnerability_count=Count("vulnerabilities"),
+                patched_vulnerability_count=Count("resolved_vulnerabilities"),
+            )
+            .prefetch_related()
         )
 
 
@@ -81,8 +87,8 @@ class VulnerabilitySearchView(View):
     def get(self, request):
         context = {"form": forms.CVEForm(request.GET or None)}
         if request.GET:
-            vulnerabilities = self.request_to_queryset(request)
-            result_size = vulnerabilities.count()
+            vulnerabilities = self.request_to_vulnerabilities(request)
+            result_size = len(vulnerabilities)
             pages = Paginator(vulnerabilities, 50)
             vulnerabilities = pages.get_page(int(self.request.GET.get("page", 1)))
             context["vulnerabilities"] = vulnerabilities
@@ -91,9 +97,14 @@ def get(self, request):
         return render(request, self.template_name, context)
 
     @staticmethod
-    def request_to_queryset(request):
+    def request_to_vulnerabilities(request):
         vuln_id = request.GET["vuln_id"]
-        return models.Vulnerability.objects.filter(vulnerability_id__icontains=vuln_id)
+        return list(
+            models.Vulnerability.objects.filter(vulnerability_id__icontains=vuln_id).annotate(
+                vulnerable_package_count=Count("vulnerable_packages"),
+                patched_package_count=Count("patched_packages"),
+            )
+        )
 
 
 class PackageUpdate(UpdateView):
@@ -112,9 +123,11 @@ def get_context_data(self, **kwargs):
         return context
 
     def _package_vulnerabilities(self, package_pk):
-
-        resolved_vuln = [i for i in self.get_object().resolved_to]
-        unresolved_vuln = [i for i in self.get_object().vulnerable_to]
+        # This can be further optimised by caching get_object result first time it
+        # is called
+        package = self.get_object()
+        resolved_vuln = [i for i in package.resolved_to.values("vulnerability_id", "pk")]
+        unresolved_vuln = [i for i in package.vulnerable_to.values("vulnerability_id", "pk")]
 
         return resolved_vuln, unresolved_vuln
 
@@ -214,9 +227,10 @@ def relationship_already_exists(relationship):
     @staticmethod
     def create_relationship_instance(vulnerability_id, package_id, is_vulnerable):
         package = models.Package.objects.get(id=package_id)
+        # FIXME: Handle the case when  vuln_created=True
         vulnerability, vuln_created = models.Vulnerability.objects.get_or_create(
             vulnerability_id=vulnerability_id
-        )  # nopep8
+        )
         return models.PackageRelatedVulnerability(
             vulnerability=vulnerability, package=package, is_vulnerable=is_vulnerable
         )