Merge pull request #2344 from aboutcode-org/pipeline-frequency-config

Increase frequency and priority for critical pipelines

Author: keshav-space

vulnerabilities/api_v2.py

@@ -788,7 +788,7 @@ def get_latest_run(self, schedule):
 
     def to_representation(self, schedule):
         representation = super().to_representation(schedule)
-        representation["run_interval"] = f"{schedule.run_interval}hr"
+        representation["run_interval"] = f"{schedule.run_interval}min"
         representation["execution_timeout"] = f"{schedule.execution_timeout}hr"
         return representation
 

vulnerabilities/migrations/0137_alter_pipelineschedule_run_interval.py

@@ -0,0 +1,46 @@
+# Generated by Django 5.2.11 on 2026-06-16 12:29
+
+import django.core.validators
+from django.db import migrations
+from django.db import models
+from django.db.models import F
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0136_advisorysetmember_unique_advisory_per_set"),
+    ]
+
+    def convert_hours_to_minutes(apps, schema_editor):
+        PipelineSchedule = apps.get_model("vulnerabilities", "PipelineSchedule")
+        PipelineSchedule.objects.update(run_interval=F("run_interval") * 60)
+
+    def revert_convert_hours_to_minutes(apps, schema_editor):
+        PipelineSchedule = apps.get_model("vulnerabilities", "PipelineSchedule")
+        for schedule in PipelineSchedule.objects.all():
+            schedule.run_interval = min(8760, max(1, schedule.run_interval // 60))
+            schedule.save(update_fields=["run_interval"])
+
+    operations = [
+        migrations.AlterField(
+            model_name="pipelineschedule",
+            name="run_interval",
+            field=models.PositiveSmallIntegerField(
+                default=1440,
+                help_text="Number of minutes to wait between run of this pipeline.",
+                validators=[
+                    django.core.validators.MinValueValidator(
+                        5, message="Interval must be at least 5 minutes."
+                    ),
+                    django.core.validators.MaxValueValidator(
+                        43200, message="Interval must be at most 43200 minutes (i.e 30 days)."
+                    ),
+                ],
+            ),
+        ),
+        migrations.RunPython(
+            convert_hours_to_minutes,
+            revert_convert_hours_to_minutes,
+        ),
+    ]

vulnerabilities/models.py

@@ -2302,13 +2302,15 @@ class ExecutionPriority(models.IntegerChoices):
         ),
     )
 
-    run_interval = models.PositiveSmallIntegerField(
+    run_interval = models.IntegerField(
         validators=[
-            MinValueValidator(1, message="Interval must be at least 1 hour."),
-            MaxValueValidator(8760, message="Interval must be at most 8760 hours."),
+            MinValueValidator(5, message="Interval must be at least 5 minutes."),
+            MaxValueValidator(
+                43200, message="Interval must be at most 43200 minutes (i.e 30 days)."
+            ),
         ],
-        default=24,
-        help_text=("Number of hours to wait between run of this pipeline."),
+        default=1440,
+        help_text=("Number of minutes to wait between run of this pipeline."),
     )
 
     run_priority = models.IntegerField(

vulnerabilities/pipelines/__init__.py

@@ -145,8 +145,8 @@ class VulnerableCodePipeline(PipelineDefinition, BasePipelineRun):
     # When set to true pipeline is run only once.
     # To rerun onetime pipeline reset is_active field to True via migration.
     run_once = False
-    # Interval between runs in hour.
-    run_interval = 24
+    # Interval between runs in minutes.
+    run_interval = 1440
     run_priority = PipelineSchedule.ExecutionPriority.DEFAULT
 
     def on_failure(self):
@@ -180,8 +180,8 @@ class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline):
     # When set to true pipeline is run only once.
     # To rerun onetime pipeline reset is_active field to True via migration.
     run_once = False
-    # Interval between runs in hour.
-    run_interval = 24
+    # Interval between runs in minutes.
+    run_interval = 1440
     run_priority = PipelineSchedule.ExecutionPriority.DEFAULT
 
     @classmethod
@@ -290,8 +290,8 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
     # When set to true pipeline is run only once.
     # To rerun onetime pipeline reset is_active field to True via migration.
     run_once = False
-    # Interval between runs in hour.
-    run_interval = 24
+    # Interval between runs in minutes.
+    run_interval = 1440
     run_priority = PipelineSchedule.ExecutionPriority.DEFAULT
 
     @classmethod

vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py

@@ -16,6 +16,7 @@
 from vulnerabilities.models import SSVC
 from vulnerabilities.models import AdvisorySeverity
 from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import PipelineSchedule
 from vulnerabilities.pipelines import VulnerableCodePipeline
 from vulnerabilities.severity_systems import SCORING_SYSTEMS
 
@@ -31,6 +32,10 @@ class CollectSSVCPipeline(VulnerableCodePipeline):
 
     pipeline_id = "collect_ssvc_trees"
 
+    # Run pipeline every 30 minutes.
+    run_interval = 30
+    run_priority = PipelineSchedule.ExecutionPriority.HIGH
+
     @classmethod
     def steps(cls):
         return (cls.collect_ssvc_data,)

vulnerabilities/pipelines/v2_improvers/compute_package_risk.py

@@ -15,6 +15,7 @@
 from vulnerabilities.models import AdvisorySeverity
 from vulnerabilities.models import AdvisoryV2
 from vulnerabilities.models import PackageV2
+from vulnerabilities.models import PipelineSchedule
 from vulnerabilities.pipelines import VulnerableCodePipeline
 from vulnerabilities.pipes.risk_score import bulk_update
 from vulnerabilities.risk import compute_vulnerability_risk_factors
@@ -30,6 +31,10 @@ class ComputePackageRiskPipeline(VulnerableCodePipeline):
     pipeline_id = "compute_package_risk_v2"
     license_expression = None
 
+    # Run pipeline every 30 minutes.
+    run_interval = 30
+    run_priority = PipelineSchedule.ExecutionPriority.HIGH
+
     @classmethod
     def steps(cls):
         return (

vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py

@@ -17,6 +17,7 @@
 from dateutil import parser as dateparser
 
 from vulnerabilities.models import AdvisoryExploit
+from vulnerabilities.models import PipelineSchedule
 from vulnerabilities.pipelines import VulnerableCodePipeline
 from vulnerabilities.utils import build_alias_to_advisory_map
 
@@ -30,6 +31,10 @@ class ExploitDBImproverPipeline(VulnerableCodePipeline):
     pipeline_id = "enhance_with_exploitdb_v2"
     spdx_license_expression = "GPL-2.0"
 
+    # Run pipeline every 30 minutes.
+    run_interval = 30
+    run_priority = PipelineSchedule.ExecutionPriority.HIGH
+
     @classmethod
     def steps(cls):
         return (

vulnerabilities/pipelines/v2_improvers/enhance_with_github_poc.py

@@ -13,9 +13,7 @@
 from aboutcode.pipeline import LoopProgress
 from fetchcode.vcs import fetch_via_vcs
 
-from vulnerabilities.models import AdvisoryAlias
 from vulnerabilities.models import AdvisoryPOC
-from vulnerabilities.models import AdvisoryV2
 from vulnerabilities.pipelines import VulnerableCodePipeline
 from vulnerabilities.utils import relate_aliases_with_advisories
 

vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py

@@ -14,6 +14,7 @@
 from aboutcode.pipeline import LoopProgress
 
 from vulnerabilities.models import AdvisoryExploit
+from vulnerabilities.models import PipelineSchedule
 from vulnerabilities.pipelines import VulnerableCodePipeline
 from vulnerabilities.utils import build_alias_to_advisory_map
 
@@ -27,6 +28,10 @@ class VulnerabilityKevPipeline(VulnerableCodePipeline):
     pipeline_id = "enhance_with_kev_v2"
     license_expression = None
 
+    # Run pipeline every 30 minutes.
+    run_interval = 30
+    run_priority = PipelineSchedule.ExecutionPriority.HIGH
+
     @classmethod
     def steps(cls):
         return (

vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py

@@ -16,6 +16,7 @@
 from dateutil import parser as dateparser
 
 from vulnerabilities.models import AdvisoryExploit
+from vulnerabilities.models import PipelineSchedule
 from vulnerabilities.pipelines import VulnerableCodePipeline
 from vulnerabilities.utils import build_alias_to_advisory_map
 
@@ -29,6 +30,10 @@ class MetasploitImproverPipeline(VulnerableCodePipeline):
     pipeline_id = "enhance_with_metasploit_v2"
     spdx_license_expression = "BSD-3-clause"
 
+    # Run pipeline every 30 minutes.
+    run_interval = 30
+    run_priority = PipelineSchedule.ExecutionPriority.HIGH
+
     @classmethod
     def steps(cls):
         return (