feat: auto-fill project PURL from single input URL (#2185)

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

scanpipe/api/serializers.py

@@ -39,6 +39,7 @@
 from scanpipe.models import Run
 from scanpipe.models import WebhookSubscription
 from scanpipe.pipes import count_group_by
+from scanpipe.pipes.fetch import set_project_purl_from_input_url
 
 scanpipe_app = apps.get_app_config("scanpipe")
 
@@ -320,6 +321,9 @@ def create(self, validated_data):
         for url in input_urls:
             project.add_input_source(download_url=url)
 
+        if not upload_file:
+            set_project_purl_from_input_url(project, input_urls)
+
         for pipeline in pipelines:
             pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline)
             pipeline_name = scanpipe_app.get_new_pipeline_name(pipeline_name)

scanpipe/forms.py

@@ -124,6 +124,9 @@ def handle_inputs(self, project):
         for url in input_urls:
             project.add_input_source(download_url=url)
 
+        if not input_files:
+            fetch.set_project_purl_from_input_url(project, input_urls)
+
 
 class CheckboxChoiceField(forms.MultipleChoiceField):
     widget = forms.CheckboxSelectMultiple

scanpipe/management/commands/__init__.py

@@ -38,6 +38,7 @@
 from scanpipe.models import Project
 from scanpipe.models import ProjectMessage
 from scanpipe.pipes import count_group_by
+from scanpipe.pipes.fetch import set_project_purl_from_input_url
 
 scanpipe_app = apps.get_app_config("scanpipe")
 
@@ -389,6 +390,9 @@ def add_project_inputs(
     if input_urls:
         handle_input_urls(project=project, input_urls=input_urls, command=command)
 
+    if not input_files_data:
+        set_project_purl_from_input_url(project, input_urls)
+
     if copy_from:
         handle_copy_codebase(project=project, copy_from=copy_from, command=command)
 

scanpipe/pipes/fetch.py

@@ -43,6 +43,7 @@
 from fetchcode.pypi import Pypi as PyPIFetcher
 from packageurl import PackageURL
 from packageurl.contrib import purl2url
+from packageurl.contrib import url2purl
 from plugincode.location_provider import get_location
 from requests import auth as request_auth
 
@@ -459,3 +460,10 @@ def check_urls_availability(urls):
     """Check the safety and accessibility of a list of URLs."""
     errors = [url for url in urls if not check_url(url) if url.startswith("http")]
     return errors
+
+
+def set_project_purl_from_input_url(project, input_urls):
+    """Auto-fill the project PURL when the sole input is a single resolvable URL."""
+    if not project.purl and input_urls and len(input_urls) == 1:
+        if purl := url2purl.get_purl(input_urls[0]):
+            project.update(purl=str(purl))

scanpipe/tests/pipes/test_fetch.py

@@ -30,6 +30,7 @@
 import requests
 from requests import auth as request_auth
 
+from scanpipe.models import Project
 from scanpipe.pipes import fetch
 from scanpipe.tests import make_mock_response
 
@@ -351,3 +352,38 @@ def test_scanpipe_pipes_fetch_check_urls_availability(
         # All URLs fail
         mock_head.side_effect = requests.exceptions.RequestException
         self.assertEqual(http_urls, fetch.check_urls_availability(urls))
+
+    def test_scanpipe_pipes_fetch_set_project_purl_from_input_url(self):
+        project = Project.objects.create(name="purl_from_url")
+
+        # Single resolvable HTTP URL -> purl auto-filled
+        fetch.set_project_purl_from_input_url(
+            project, ["https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"]
+        )
+        project.refresh_from_db()
+        self.assertEqual("pkg:npm/lodash@4.17.21", project.purl)
+
+        # Existing purl is not overwritten
+        fetch.set_project_purl_from_input_url(
+            project, ["https://registry.npmjs.org/react/-/react-18.0.0.tgz"]
+        )
+        project.refresh_from_db()
+        self.assertEqual("pkg:npm/lodash@4.17.21", project.purl)
+
+        # Multiple URLs -> purl not set
+        project2 = Project.objects.create(name="purl_from_url2")
+        fetch.set_project_purl_from_input_url(
+            project2,
+            [
+                "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+                "https://registry.npmjs.org/react/-/react-18.0.0.tgz",
+            ],
+        )
+        project2.refresh_from_db()
+        self.assertEqual("", project2.purl)
+
+        # Bad input -> no crash, no purl set
+        project3 = Project.objects.create(name="purl_from_url3")
+        fetch.set_project_purl_from_input_url(project3, ["not-a-url"])
+        project3.refresh_from_db()
+        self.assertEqual("", project3.purl)

scanpipe/tests/test_api.py

@@ -421,6 +421,27 @@ def test_scanpipe_api_project_create_input_urls(self):
         self.assertEqual(status.HTTP_201_CREATED, response.status_code)
         self.assertEqual(3, len(response.data["input_sources"]))
 
+    def test_scanpipe_api_project_create_purl_from_input_url(self):
+        lodash_url = "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"
+
+        # Single URL -> purl auto-filled
+        data = {"name": "purl project", "input_urls": [lodash_url]}
+        response = self.csrf_client.post(self.project_list_url, data)
+        self.assertEqual(status.HTTP_201_CREATED, response.status_code)
+        self.assertEqual("pkg:npm/lodash@4.17.21", response.data["purl"])
+
+        # Multiple URLs -> no auto-fill
+        data = {
+            "name": "multi url project",
+            "input_urls": [
+                lodash_url,
+                "https://registry.npmjs.org/react/-/react-18.0.0.tgz",
+            ],
+        }
+        response = self.csrf_client.post(self.project_list_url, data)
+        self.assertEqual(status.HTTP_201_CREATED, response.status_code)
+        self.assertEqual("", response.data["purl"])
+
     def test_scanpipe_api_project_create_multiple_pipelines(self):
         data = {
             "name": "Single string",

scanpipe/tests/test_commands.py

@@ -172,6 +172,26 @@ def test_scanpipe_management_command_create_project_inputs(self):
         tagged_source = project.inputsources.get(filename="test_models.py")
         self.assertEqual("tag", tagged_source.tag)
 
+    def test_scanpipe_management_command_create_project_purl_from_input_url(self):
+        lodash_url = "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"
+
+        # Single URL -> purl auto-filled
+        call_command("create-project", "purl-project", "--input-url", lodash_url)
+        project = Project.objects.get(name="purl-project")
+        self.assertEqual("pkg:npm/lodash@4.17.21", project.purl)
+
+        # Multiple URLs -> no auto-fill
+        call_command(
+            "create-project",
+            "multi-url-project",
+            "--input-url",
+            lodash_url,
+            "--input-url",
+            "https://registry.npmjs.org/react/-/react-18.0.0.tgz",
+        )
+        project2 = Project.objects.get(name="multi-url-project")
+        self.assertEqual("", project2.purl)
+
     def test_scanpipe_management_command_create_project_execute(self):
         options = ["--execute"]
         expected = "The --execute option requires one or more pipelines."

scanpipe/tests/test_forms.py

@@ -252,6 +252,26 @@ def test_scanpipe_forms_project_settings_form_purl(self):
         obj = form2.save()
         self.assertEqual("pkg:npm/lodash@4.17.21", obj.purl)
 
+    @mock.patch("requests.sessions.Session.head")
+    def test_scanpipe_forms_handle_inputs_auto_fill_purl(self, mock_head):
+        mock_head.return_value = mock.Mock(headers={}, status_code=200)
+        lodash_url = "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"
+
+        # Single URL -> purl auto-filled
+        form = InputsBaseForm(data={"input_urls": lodash_url})
+        self.assertTrue(form.is_valid())
+        form.handle_inputs(project=self.project1)
+        self.project1.refresh_from_db()
+        self.assertEqual("pkg:npm/lodash@4.17.21", self.project1.purl)
+
+        # Multiple URLs -> no auto-fill
+        project2 = Project.objects.create(name="Analysis2")
+        form = InputsBaseForm(data={"input_urls": f"{lodash_url} {lodash_url}"})
+        self.assertTrue(form.is_valid())
+        form.handle_inputs(project=project2)
+        project2.refresh_from_db()
+        self.assertEqual("", project2.purl)
+
     def test_scanpipe_forms_edit_input_source_tag_form(self):
         data = {}
         form = EditInputSourceTagForm(data=data)