optimizations

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

policy/tasks.py

@@ -33,10 +33,10 @@ def evaluate_product_rules_task(product_uuid):
         logger.error(f"evaluate_product_rules_task: product {product_uuid} not found, skipping.")
         return
 
-    logger.info(f"Evaluating policy rules for product {product}")
+    logger.info(f"Evaluating policy rules for product id={product.id}")
     new_violations, resolved_count = evaluate_rules(product)
     logger.info(
-        f"Policy rules evaluated for {product}: "
+        f"Policy rules evaluated for product id={product.id}: "
         f"{len(new_violations)} new violation(s), {resolved_count} resolved."
     )
 
@@ -59,14 +59,16 @@ def evaluate_all_products_rules_task(include_locked=False, product_uuids=None):
     logger.info(f"Starting policy rule evaluation for {count} product(s).")
 
     for product in products:
-        logger.info(f"Evaluating policy rules for product {product}")
+        logger.info(f"Evaluating policy rules for product id={product.id}")
         try:
             new_violations, resolved_count = evaluate_rules(product)
         except Exception:
-            logger.exception(f"Policy rule evaluation failed for product {product}, skipping.")
+            logger.exception(
+                f"Policy rule evaluation failed for product id={product.id}, skipping."
+            )
             continue
         logger.info(
-            f"Policy rules evaluated for {product}: "
+            f"Policy rules evaluated for product id={product.id}: "
             f"{len(new_violations)} new violation(s), {resolved_count} resolved."
         )
 

product_portfolio/admin.py

@@ -59,6 +59,8 @@
 from product_portfolio.importers import CodebaseResourceImporter
 from product_portfolio.importers import ProductComponentImporter
 from product_portfolio.importers import ProductPackageImporter
+from product_portfolio.importers import paused_product_package_reevaluation
+from product_portfolio.importers import reevaluate_products
 from product_portfolio.inlines import CodebaseResourceUsageDeployedFromInline
 from product_portfolio.inlines import CodebaseResourceUsageDeployedToInline
 from product_portfolio.inlines import ProductComponentInline
@@ -72,7 +74,6 @@
 from product_portfolio.models import ProductRelationStatus
 from product_portfolio.models import ProductStatus
 from reporting.filters import ReportingQueryListFilter
-from vulnerabilities.triage.engine import reevaluate_product_rulesets
 from vulnerabilities.triage.models import ProductTriageRuleset
 
 
@@ -242,13 +243,14 @@ def save_model(self, request, obj, form, change):
         if "_saveasnew" in request.POST:
             old_product_id = request.resolver_match.kwargs.get("object_id")
             old_product = self.get_object(request, old_product_id)
-            for model_class in [ProductComponent, ProductPackage, ProductTriageRuleset]:
-                for relationship in model_class.objects.filter(product=old_product):
-                    relationship.id = None
-                    relationship.uuid = uuid.uuid4()
-                    relationship.product = obj
-                    relationship.save()
-            reevaluate_product_rulesets(obj)
+            with paused_product_package_reevaluation():
+                for model_class in [ProductComponent, ProductPackage, ProductTriageRuleset]:
+                    for relationship in model_class.objects.filter(product=old_product):
+                        relationship.id = None
+                        relationship.uuid = uuid.uuid4()
+                        relationship.product = obj
+                        relationship.save()
+            reevaluate_products([obj])
 
     def get_obj_perms_user_select_form(self, request):
         """

vulnerabilities/triage/engine.py

@@ -6,6 +6,8 @@
 # See https://aboutcode.org for more information about AboutCode FOSS projects.
 #
 
+import logging
+
 from django.apps import apps
 from django.utils import timezone
 
@@ -14,6 +16,8 @@
 from vulnerabilities.triage.rules import RULE_REGISTRY
 from vulnerabilities.triage.rules import rule_parameters_from_config
 
+logger = logging.getLogger(__name__)
+
 
 def collect_matches(ruleset, product):
     """
@@ -243,6 +247,27 @@ def evaluate_ruleset(ruleset, product, apply_preset=True):
     )
 
 
+def evaluate_assignments(assignments):
+    """
+    Evaluate each ProductTriageRuleset assignment in the given queryset, skipping and
+    logging any that raise instead of aborting the whole batch.
+
+    Returns the number of assignments successfully evaluated.
+    """
+    evaluated_count = 0
+    for assignment in assignments:
+        try:
+            evaluate_ruleset(ruleset=assignment.ruleset, product=assignment.product)
+        except Exception:
+            logger.exception(
+                f"Triage evaluation failed for ruleset id={assignment.ruleset_id} /"
+                f" product id={assignment.product_id}, skipping."
+            )
+            continue
+        evaluated_count += 1
+    return evaluated_count
+
+
 def reevaluate_product_rulesets(product, apply_preset=True):
     """Re-evaluate all enabled triage rulesets currently assigned to the product."""
     assignments = ProductTriageRuleset.objects.filter(

vulnerabilities/triage/management/commands/evaluate_triage.py

@@ -10,7 +10,7 @@
 from django.core.management.base import CommandError
 
 from dje.models import Dataspace
-from vulnerabilities.triage.engine import evaluate_ruleset
+from vulnerabilities.triage.engine import evaluate_assignments
 from vulnerabilities.triage.models import ProductTriageRuleset
 from vulnerabilities.triage.models import TriageRecord
 
@@ -45,9 +45,8 @@ def handle(self, *args, **options):
             self.stdout.write("No active ruleset assignments found.")
             return
 
-        for assignment in assignments:
-            self.stdout.write(f"  {assignment.product} / {assignment.ruleset}")
-            evaluate_ruleset(ruleset=assignment.ruleset, product=assignment.product)
+        evaluated_count = evaluate_assignments(assignments)
+        self.stdout.write(f"Evaluated: {evaluated_count}/{assignment_count}")
 
         total = TriageRecord.objects.filter(dataspace=dataspace).count()
         self.stdout.write(

vulnerabilities/triage/tasks.py

@@ -13,7 +13,7 @@
 from django_rq import job
 
 from dje.models import get_unsecured_manager
-from vulnerabilities.triage.engine import evaluate_ruleset
+from vulnerabilities.triage.engine import evaluate_assignments
 from vulnerabilities.triage.engine import reevaluate_product_rulesets
 from vulnerabilities.triage.models import ProductTriageRuleset
 
@@ -34,7 +34,7 @@ def reevaluate_product_triage_rulesets_task(product_uuid):
         )
         return
 
-    logger.info(f"Evaluating triage rulesets for product {product}")
+    logger.info(f"Evaluating triage rulesets for product id={product.id}")
     reevaluate_product_rulesets(product)
 
 
@@ -50,15 +50,5 @@ def evaluate_all_products_vulnerability_triage_task():
     count = assignments.count()
     logger.info(f"Starting triage evaluation for {count} ruleset assignment(s).")
 
-    for assignment in assignments:
-        logger.info(f"Evaluating triage ruleset {assignment.ruleset} for {assignment.product}")
-        try:
-            evaluate_ruleset(ruleset=assignment.ruleset, product=assignment.product)
-        except Exception:
-            logger.exception(
-                f"Triage evaluation failed for {assignment.ruleset} / {assignment.product},"
-                " skipping."
-            )
-            continue
-
-    logger.info("Triage evaluation complete.")
+    evaluated_count = evaluate_assignments(assignments)
+    logger.info(f"Triage evaluation complete: {evaluated_count}/{count} evaluated.")