From 3d28b8c9372304dc4fe7bfe16a5fc9d70ebb10e6 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sun, 16 May 2021 19:28:54 +0530 Subject: [PATCH 1/2] Reduce queries in vulnerability search Signed-off-by: Shivam Sandbhor --- vulnerabilities/templates/vulnerabilities.html | 4 ++-- vulnerabilities/views.py | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/templates/vulnerabilities.html b/vulnerabilities/templates/vulnerabilities.html index b6e633eef..06d885cff 100644 --- a/vulnerabilities/templates/vulnerabilities.html +++ b/vulnerabilities/templates/vulnerabilities.html @@ -37,8 +37,8 @@

{% for vulnerability in vulnerabilities %} {{vulnerability.vulnerability_id}} - {{vulnerability.vulnerable_to.all|length}} - {{vulnerability.resolved_to.all|length}} + {{vulnerability.vulnerable_package_count}} + {{vulnerability.patched_package_count}} {% endfor %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index b10853f51..0f1bad0a7 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -23,6 +23,7 @@ from urllib.parse import urlencode from django.core.paginator import Paginator +from django.db.models import Count from django.http import HttpResponse from django.shortcuts import render, redirect from django.urls import reverse @@ -81,8 +82,8 @@ class VulnerabilitySearchView(View): def get(self, request): context = {"form": forms.CVEForm(request.GET or None)} if request.GET: - vulnerabilities = self.request_to_queryset(request) - result_size = vulnerabilities.count() + vulnerabilities = self.request_to_vulnerabilities(request) + result_size = len(vulnerabilities) pages = Paginator(vulnerabilities, 50) vulnerabilities = pages.get_page(int(self.request.GET.get("page", 1))) context["vulnerabilities"] = vulnerabilities @@ -91,9 +92,14 @@ def get(self, request): return render(request, self.template_name, context) @staticmethod - def request_to_queryset(request): + def request_to_vulnerabilities(request): vuln_id = request.GET["vuln_id"] - return models.Vulnerability.objects.filter(vulnerability_id__icontains=vuln_id) + return list( + models.Vulnerability.objects.filter(vulnerability_id__icontains=vuln_id).annotate( + vulnerable_package_count=Count("vulnerable_packages"), + patched_package_count=Count("patched_packages"), + ) + ) class PackageUpdate(UpdateView): @@ -214,9 +220,10 @@ def relationship_already_exists(relationship): @staticmethod def create_relationship_instance(vulnerability_id, package_id, is_vulnerable): package = models.Package.objects.get(id=package_id) + # FIXME: Handle the case when vuln_created=True vulnerability, vuln_created = models.Vulnerability.objects.get_or_create( vulnerability_id=vulnerability_id - ) # nopep8 + ) return models.PackageRelatedVulnerability( vulnerability=vulnerability, package=package, is_vulnerable=is_vulnerable ) From c3c92954727e53791e42f310b4a3b340448aba5c Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sun, 16 May 2021 20:51:03 +0530 Subject: [PATCH 2/2] Reduce db queries in package search view Signed-off-by: Shivam Sandbhor --- vulnerabilities/templates/package_update.html | 4 ++-- vulnerabilities/templates/packages.html | 6 +++--- vulnerabilities/views.py | 21 ++++++++++++------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/templates/package_update.html b/vulnerabilities/templates/package_update.html index 3ca45dfa9..9198582be 100644 --- a/vulnerabilities/templates/package_update.html +++ b/vulnerabilities/templates/package_update.html @@ -40,7 +40,7 @@

{% for vulnerability in impacted_vuln %} - {{vulnerability}} + {{vulnerability.vulnerability_id}} {% if enable_curation %}
{% csrf_token %} @@ -74,7 +74,7 @@

{% for vulnerability in resolved_vuln %} - {{vulnerability}} + {{vulnerability.vulnerability_id}} {% if enable_curation %} diff --git a/vulnerabilities/templates/packages.html b/vulnerabilities/templates/packages.html index 1e22668c9..9cebe4264 100644 --- a/vulnerabilities/templates/packages.html +++ b/vulnerabilities/templates/packages.html @@ -41,9 +41,9 @@

{% for package in packages %} - {{package}} - {{package.vulnerable_to.all|length}} - {{package.resolved_to.all|length}} + {{package.package_url}} + {{package.vulnerability_count}} + {{package.patched_vulnerability_count}} {% endfor %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 0f1bad0a7..8757239d7 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -47,7 +47,7 @@ def get(self, request): if request.GET: packages = self.request_to_queryset(request) - result_size = packages.count() + result_size = len(packages) page_no = int(request.GET.get("page", 1)) packages = Paginator(packages, 50).get_page(page_no) context["packages"] = packages @@ -69,9 +69,14 @@ def request_to_queryset(request): if len(request.GET["name"]): package_name = request.GET["name"] - return models.Package.objects.all().filter( - name__icontains=package_name, - type__icontains=package_type, + return list( + models.Package.objects.all() + .filter(name__icontains=package_name, type__icontains=package_type) + .annotate( + vulnerability_count=Count("vulnerabilities"), + patched_vulnerability_count=Count("resolved_vulnerabilities"), + ) + .prefetch_related() ) @@ -118,9 +123,11 @@ def get_context_data(self, **kwargs): return context def _package_vulnerabilities(self, package_pk): - - resolved_vuln = [i for i in self.get_object().resolved_to] - unresolved_vuln = [i for i in self.get_object().vulnerable_to] + # This can be further optimised by caching get_object result first time it + # is called + package = self.get_object() + resolved_vuln = [i for i in package.resolved_to.values("vulnerability_id", "pk")] + unresolved_vuln = [i for i in package.vulnerable_to.values("vulnerability_id", "pk")] return resolved_vuln, unresolved_vuln