fix: run "Improve from PurlDB" asynchronously (#570)

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

Makefile

@@ -23,6 +23,10 @@ start:
 	@echo "-> Start the Docker compose services in background"
 	${COMPOSE} up -d
 
+stop:
+	@echo "-> Stop the Docker compose services"
+	${COMPOSE} stop
+
 # make logs TAIL=100 SERVICE=db
 logs:
 	${COMPOSE} logs -f --tail=${TAIL:-50} ${SERVICE}

product_portfolio/templates/product_portfolio/tabs/tab_activity.html

@@ -6,8 +6,8 @@
     {% include 'product_portfolio/tabs/tab_activity_summary.html' %}
   {% endblock %}
 
-  {% block imports %}
-    {% include 'product_portfolio/tabs/tab_activity_imports.html' %}
+  {% block actions %}
+    {% include 'product_portfolio/tabs/tab_activity_actions.html' %}
   {% endblock %}
 
   {% block requests %}

…portfolio/tabs/tab_activity_imports.html …portfolio/tabs/tab_activity_actions.htmlproduct_portfolio/templates/product_portfolio/tabs/tab_activity_imports.html renamed to product_portfolio/templates/product_portfolio/tabs/tab_activity_actions.html product_portfolio/templates/product_portfolio/tabs/tab_activity_imports.html renamed to product_portfolio/templates/product_portfolio/tabs/tab_activity_actions.html

@@ -4,15 +4,15 @@
 {% if has_projects_in_progress %}
   <div class="alert alert-primary" role="alert">
     <i class="fas fa-spinner fa-spin"></i>
-    Imports are currently in progress. This view will automatically refresh upon
-    completion of the import process.
+    Actions are currently in progress.
+    This view will automatically refresh upon completion.
   </div>
 {% endif %}
 
-<div class="border rounded-3 p-3 mb-4" id="activity-imports">
+<div class="border rounded-3 p-3 mb-4" id="activity-actions">
   <div class="d-flex justify-content-between align-items-center mb-3">
     <h3 class="fs-5 fw-medium mb-0">
-      Imports
+      Actions
       {% if scancode_projects %}<span class="text-body-tertiary fw-normal ms-1">{{ scancode_projects|length }}</span>{% endif %}
     </h3>
     {% if has_projects_in_progress %}
@@ -27,9 +27,9 @@ <h3 class="fs-5 fw-medium mb-0">
       <table class="table table-sm mb-0 align-middle">
         <thead>
           <tr>
-            <th class="fw-medium">Type</th>
+            <th class="fw-medium" style="min-width: 200px;">Type</th>
             <th class="fw-medium" style="width: 200px;">Status</th>
-            <th class="fw-medium">Input</th>
+            <th class="fw-medium" style="min-width: 200px;">Input</th>
             <th class="fw-medium">Log</th>
           </tr>
         </thead>
@@ -79,7 +79,7 @@ <h3 class="fs-5 fw-medium mb-0">
     </div>
   {% else %}
     <div class="text-body-tertiary small">
-      No imports yet
+      No actions yet
     </div>
   {% endif %}
 </div>

product_portfolio/templates/product_portfolio/tabs/tab_activity_summary.html

@@ -1,8 +1,8 @@
 <div class="row g-3 mb-4">
 
   <div class="col-6 col-md-4">
-    <div class="bg-body-secondary rounded-3 p-3 cursor-pointer" role="button" data-scroll-to="activity-imports">
-      <div class="small text-body-secondary mb-1">Imports</div>
+    <div class="bg-body-secondary rounded-3 p-3 cursor-pointer" role="button" data-scroll-to="activity-actions">
+      <div class="small text-body-secondary mb-1">Actions</div>
       <div class="fs-4 fw-medium lh-sm {% if has_projects_in_progress %}text-primary{% endif %}">{{ scancode_projects|length }}</div>
     </div>
   </div>

product_portfolio/tests/test_views.py

@@ -205,7 +205,7 @@ def test_product_portfolio_detail_view_tab_activit_view(self):
         url = self.product1.get_url("tab_activity")
 
         response = self.client.get(url)
-        self.assertContains(response, "No imports yet")
+        self.assertContains(response, "No actions yet")
         self.assertContains(response, "No requests yet")
         self.assertContains(response, "No changes yet")
 
@@ -220,7 +220,7 @@ def test_product_portfolio_detail_view_tab_activit_view(self):
         self.assertTrue(response.context["has_projects_in_progress"])
         htmx_refresh = 'hx-trigger="load delay:10s" hx-swap="outerHTML"'
         self.assertContains(response, htmx_refresh)
-        self.assertContains(response, "Imports are currently in progress.")
+        self.assertContains(response, "Actions are currently in progress.")
         self.assertContains(response, "Import SBOM")
 
         project.status = ScanCodeProject.Status.SUCCESS
@@ -229,7 +229,7 @@ def test_product_portfolio_detail_view_tab_activit_view(self):
         self.assertFalse(response.context["has_projects_in_progress"])
         self.assertContains(response, "Import SBOM")
         self.assertNotContains(response, "hx-trigger")
-        self.assertNotContains(response, "Imports are currently in progress.")
+        self.assertNotContains(response, "Actions are currently in progress.")
 
         expected = "File:"
         download_url = reverse(
@@ -243,6 +243,28 @@ def test_product_portfolio_detail_view_tab_activit_view(self):
         self.assertContains(response, expected)
         self.assertContains(response, download_url)
 
+    def test_product_portfolio_detail_view_tab_activity_in_progress_any_type(self):
+        """has_projects_in_progress is not limited to the ScanCode.io submitted types."""
+        self.client.login(username="nexb_user", password="secret")
+        url = self.product1.get_url("tab_activity")
+
+        project = ScanCodeProject.objects.create(
+            product=self.product1,
+            dataspace=self.product1.dataspace,
+            type=ScanCodeProject.ProjectType.IMPROVE_FROM_PURLDB,
+            status=ScanCodeProject.Status.IMPORT_STARTED,
+        )
+
+        response = self.client.get(url)
+        self.assertTrue(response.context["has_projects_in_progress"])
+        self.assertContains(response, "Actions are currently in progress.")
+
+        project.status = ScanCodeProject.Status.SUCCESS
+        project.save()
+        response = self.client.get(url)
+        self.assertFalse(response.context["has_projects_in_progress"])
+        self.assertNotContains(response, "Actions are currently in progress.")
+
     def test_product_portfolio_detail_view_tab_dependency_view(self):
         self.client.login(username="nexb_user", password="secret")
         url = self.product1.get_url("tab_dependencies")

product_portfolio/views.py

@@ -1492,13 +1492,7 @@ class ProductTabActivityView(
     def get_context_data(self, **kwargs):
         context_data = super().get_context_data(**kwargs)
         scancode_projects = self.object.scancodeprojects.all()
-        submitted_projects = self.get_submitted_projects(scancode_projects)
-
-        # Check the status of the "submitted" projects on ScanCode.io and update the
-        # local ScanCodeProject instances accordingly.
-        scancodeio = ScanCodeIO(self.request.user.dataspace)
-        for submitted_project in submitted_projects:
-            self.synchronize(scancodeio=scancodeio, project=submitted_project)
+        self.synchronize_scancodeio_projects(scancode_projects)
 
         history_entries = (
             History.objects.get_for_object(self.object)
@@ -1509,9 +1503,9 @@ def get_context_data(self, **kwargs):
         context_data.update(
             {
                 "tab_view_url": self.object.get_url("tab_activity"),
-                # Imports
+                # Actions
                 "scancode_projects": scancode_projects,
-                "has_projects_in_progress": bool(submitted_projects),
+                "has_projects_in_progress": scancode_projects.in_progress().exists(),
                 # Requests
                 "requests": self.object.get_requests(self.request.user),
                 # History
@@ -1521,20 +1515,32 @@ def get_context_data(self, **kwargs):
 
         return context_data
 
-    @staticmethod
-    def get_submitted_projects(scancode_projects):
-        submitted_types = [
+    def synchronize_scancodeio_projects(self, scancode_projects):
+        """
+        Poll ScanCode.io for the run status of the projects submitted to it as
+        external pipeline runs (SBOM and manifest imports), and update the
+        local ScanCodeProject status accordingly.
+        Other action types are handled entirely by local RQ tasks and have no
+        external run to poll.
+        """
+        scancodeio_project_types = [
             ScanCodeProject.ProjectType.LOAD_SBOMS,
             ScanCodeProject.ProjectType.IMPORT_FROM_MANIFEST,
         ]
-        return [
+        pending_scancodeio_projects = [
             project
             for project in scancode_projects
             if project.status == ScanCodeProject.Status.SUBMITTED
-            and project.type in submitted_types
+            and project.type in scancodeio_project_types
         ]
+        if not pending_scancodeio_projects:
+            return
+
+        scancodeio = ScanCodeIO(self.request.user.dataspace)
+        for project in pending_scancodeio_projects:
+            self.synchronize_scancodeio_project_status(scancodeio, project)
 
-    def synchronize(self, scancodeio, project):
+    def synchronize_scancodeio_project_status(self, scancodeio, project):
         scan_detail_url = scancodeio.get_scan_detail_url(project.project_uuid)
         scan_data = scancodeio.fetch_scan_data(scan_detail_url)
         if not scan_data:
@@ -2899,7 +2905,7 @@ def improve_packages_from_purldb_view(request, dataspace, name, version=""):
         messages.error(request, "Improve Packages already in progress...")
     else:
         transaction.on_commit(
-            lambda: improve_packages_from_purldb_task(
+            lambda: improve_packages_from_purldb_task.delay(
                 product_uuid=product.uuid,
                 user_uuid=user.uuid,
             )