From 1d545e7ce471437a53f4a14d4ed7e3b63e960bb9 Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 19 Mar 2024 09:47:59 +0100 Subject: [PATCH 01/11] Base layout for "Import manifests" view #65 Signed-off-by: tdruez --- product_portfolio/models.py | 3 +++ .../templates/product_portfolio/product_details.html | 1 + product_portfolio/urls.py | 1 + 3 files changed, 5 insertions(+) diff --git a/product_portfolio/models.py b/product_portfolio/models.py index baa72ac2..1d516fa3 100644 --- a/product_portfolio/models.py +++ b/product_portfolio/models.py @@ -254,6 +254,9 @@ def get_check_package_version_url(self): def get_load_sboms_url(self): return self.get_url("load_sboms") + def get_import_manifests_url(self): + return self.get_url("import_manifests") + def get_pull_project_data_url(self): return self.get_url("pull_project_data") diff --git a/product_portfolio/templates/product_portfolio/product_details.html b/product_portfolio/templates/product_portfolio/product_details.html index c68411d0..da4c0ae4 100644 --- a/product_portfolio/templates/product_portfolio/product_details.html +++ b/product_portfolio/templates/product_portfolio/product_details.html @@ -39,6 +39,7 @@ {% trans 'Import data from Scan' %} {% if request.user.dataspace.enable_package_scanning %} {% trans 'Load Packages from SBOMs' %} + {% trans 'Import Packages from manifests' %} {% endif %} {% if pull_project_data_form %} {% trans 'Pull ScanCode.io Project data' %} diff --git a/product_portfolio/urls.py b/product_portfolio/urls.py index e10f81ac..14f8dc67 100644 --- a/product_portfolio/urls.py +++ b/product_portfolio/urls.py @@ -88,6 +88,7 @@ def product_path(path_segment, view): *product_path("license_summary", license_summary_view), *product_path("check_package_version", check_package_version_ajax_view), *product_path("load_sboms", LoadSBOMsView.as_view()), + *product_path("import_manifests", LoadSBOMsView.as_view()), *product_path("tab_codebase", ProductTabCodebaseView.as_view()), *product_path("tab_imports", ProductTabImportsView.as_view()), *product_path("pull_project_data", PullProjectDataFromScanCodeIOView.as_view()), From fa5268c4d16f5cf9971c79bdca48e27cd71138b6 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 20 Mar 2024 12:09:36 +0100 Subject: [PATCH 02/11] Refactor the LoadSBOMsView class for re-usability #65 Signed-off-by: tdruez --- product_portfolio/urls.py | 3 +- product_portfolio/views.py | 78 ++++++++++++++++---------------------- 2 files changed, 34 insertions(+), 47 deletions(-) diff --git a/product_portfolio/urls.py b/product_portfolio/urls.py index 14f8dc67..c63227c9 100644 --- a/product_portfolio/urls.py +++ b/product_portfolio/urls.py @@ -9,6 +9,7 @@ from django.urls import path from product_portfolio.views import AttributionView +from product_portfolio.views import ImportManifestsView from product_portfolio.views import LoadSBOMsView from product_portfolio.views import ManageComponentGridView from product_portfolio.views import ManagePackageGridView @@ -88,7 +89,7 @@ def product_path(path_segment, view): *product_path("license_summary", license_summary_view), *product_path("check_package_version", check_package_version_ajax_view), *product_path("load_sboms", LoadSBOMsView.as_view()), - *product_path("import_manifests", LoadSBOMsView.as_view()), + *product_path("import_manifests", ImportManifestsView.as_view()), *product_path("tab_codebase", ProductTabCodebaseView.as_view()), *product_path("tab_imports", ProductTabImportsView.as_view()), *product_path("pull_project_data", PullProjectDataFromScanCodeIOView.as_view()), diff --git a/product_portfolio/views.py b/product_portfolio/views.py index b15a68a9..03ad61aa 100644 --- a/product_portfolio/views.py +++ b/product_portfolio/views.py @@ -1834,7 +1834,7 @@ def get_latest_version_entry(current_uuid): return JsonResponse({"success": "success", "upgrade_available": upgrade_available}) -class LoadSBOMsView( +class BaseProductImportFormView( LoginRequiredMixin, PermissionRequiredMixin, GetDataspacedObjectMixin, @@ -1842,9 +1842,8 @@ class LoadSBOMsView( BaseProductView, FormView, ): - template_name = "product_portfolio/load_sboms_form.html" permission_required = "product_portfolio.change_product" - form_class = LoadSBOMsForm + success_msg = "" def get_queryset(self): return self.model.objects.get_queryset( @@ -1871,7 +1870,36 @@ def get_success_url(self): def form_valid(self, form): self.object = self.get_object() form.submit(product=self.object, user=self.request.user) - msg = "SBOM file submitted to ScanCode.io for inspection." + if self.success_msg: + messages.success(self.request, self.success_msg) + return super().form_valid(form) + + +class LoadSBOMsView(BaseProductImportFormView): + template_name = "product_portfolio/load_sboms_form.html" + form_class = LoadSBOMsForm + + +class ImportManifestsView(BaseProductImportFormView): + template_name = "product_portfolio/load_sboms_form.html" + form_class = LoadSBOMsForm + + +@method_decorator(require_POST, name="dispatch") +class PullProjectDataFromScanCodeIOView(BaseProductImportFormView): + form_class = PullProjectDataForm + + def form_invalid(self, form): + raise Http404 + + def form_valid(self, form): + try: + form.submit(product=self.object, user=self.request.user) + except ValidationError as error: + messages.error(self.request, error) + return redirect(self.object.get_absolute_url()) + + msg = "Packages import from ScanCode.io in progress..." messages.success(self.request, msg) return super().form_valid(form) @@ -1931,45 +1959,3 @@ def scancodeio_project_status_view(request, scancodeproject_uuid): raise Http404 return TemplateResponse(request, template, context) - - -@method_decorator(require_POST, name="dispatch") -class PullProjectDataFromScanCodeIOView( - LoginRequiredMixin, - PermissionRequiredMixin, - GetDataspacedObjectMixin, - DataspacedModelFormMixin, - BaseProductView, - FormView, -): - permission_required = "product_portfolio.change_product" - form_class = PullProjectDataForm - - def get_queryset(self): - return self.model.objects.get_queryset( - user=self.request.user, - perms="change_product", - ) - - def post(self, request, *args, **kwargs): - self.object = self.get_object() - return super().post(request, *args, **kwargs) - - def form_invalid(self, form): - raise Http404 - - def get_success_url(self): - return f"{self.object.get_absolute_url()}#imports" - - def form_valid(self, form): - self.object = self.get_object() - - try: - form.submit(product=self.object, user=self.request.user) - except ValidationError as error: - messages.error(self.request, error) - return redirect(self.object.get_absolute_url()) - - msg = "Packages import from ScanCode.io in progress..." - messages.success(self.request, msg) - return super().form_valid(form) From b90d290072cf635df8120369ecc650d8a4b095c2 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 20 Mar 2024 17:11:36 +0100 Subject: [PATCH 03/11] Add the ImportManifestsForm logic #65 Signed-off-by: tdruez --- dejacode_toolkit/scancodeio.py | 8 +- dje/tasks.py | 15 ++-- product_portfolio/forms.py | 29 ++++++- product_portfolio/models.py | 1 - .../import_manifests_form.html | 78 +++++++++++++++++++ product_portfolio/tests/test_views.py | 14 ++-- product_portfolio/views.py | 5 +- 7 files changed, 126 insertions(+), 24 deletions(-) create mode 100644 product_portfolio/templates/product_portfolio/import_manifests_form.html diff --git a/dejacode_toolkit/scancodeio.py b/dejacode_toolkit/scancodeio.py index 962043e5..332217a8 100644 --- a/dejacode_toolkit/scancodeio.py +++ b/dejacode_toolkit/scancodeio.py @@ -76,10 +76,12 @@ def submit_scan(self, uri, user_uuid, dataspace_uuid): logger.debug(f'{self.label}: submit scan uri="{uri}" webhook_url="{webhook_url}"') return self.request_post(url=self.project_api_url, json=data) - def submit_load_sbom(self, project_name, file_location, user_uuid, execute_now=False): + def submit_project( + self, project_name, pipeline_name, file_location, user_uuid, execute_now=False + ): data = { "name": project_name, - "pipeline": "load_sbom", + "pipeline": pipeline_name, "execute_now": execute_now, } files = { @@ -92,7 +94,7 @@ def submit_load_sbom(self, project_name, file_location, user_uuid, execute_now=F data["webhook_url"] = webhook_url logger.debug( - f"{self.label}: submit load sbom " + f"{self.label}: submit pipeline={pipeline_name} " f'project_name="{project_name}" webhook_url="{webhook_url}"' ) return self.request_post(url=self.project_api_url, data=data, files=files) diff --git a/dje/tasks.py b/dje/tasks.py index 3f7c810d..7464b1f6 100644 --- a/dje/tasks.py +++ b/dje/tasks.py @@ -115,13 +115,13 @@ def scancodeio_submit_scan(uris, user_uuid, dataspace_uuid): @job -def scancodeio_submit_load_sbom(scancodeproject_uuid, user_uuid): +def scancodeio_submit_project(scancodeproject_uuid, user_uuid, pipeline_name): """Submit the provided SBOM file to ScanCode.io as an asynchronous task.""" from dje.models import DejacodeUser logger.info( - f"Entering scancodeio_submit_load_sbom task with " - f"scancodeproject_uuid={scancodeproject_uuid} user_uuid={user_uuid}" + f"Entering scancodeio_submit_project task with " + f"scancodeproject_uuid= {scancodeproject_uuid} user_uuid={user_uuid}" ) ScanCodeProject = apps.get_model("product_portfolio", "scancodeproject") @@ -137,24 +137,25 @@ def scancodeio_submit_load_sbom(scancodeproject_uuid, user_uuid): # Create a Project instance on ScanCode.io without immediate execution of the # pipeline. This allows to get instant feedback from ScanCode.io about the Project # creation status and its related data, even in SYNC mode. - response = scancodeio.submit_load_sbom( + response = scancodeio.submit_project( project_name=scancodeproject_uuid, + pipeline_name=pipeline_name, file_location=scancode_project.input_file.path, user_uuid=user_uuid, execute_now=False, ) if not response: - logger.info("Error submitting the SBOM file to ScanCode.io server") + logger.info("Error submitting the file to ScanCode.io server") scancode_project.status = ScanCodeProject.Status.FAILURE - msg = "- Error: SBOM could not be submitted to ScanCode.io" + msg = "- Error: File could not be submitted to ScanCode.io" scancode_project.append_to_log(msg, save=True) return logger.info("Update the ScanCodeProject instance") scancode_project.status = ScanCodeProject.Status.SUBMITTED scancode_project.project_uuid = response.get("uuid") - msg = "- SBOM file submitted to ScanCode.io for inspection" + msg = "- File submitted to ScanCode.io for inspection" scancode_project.append_to_log(msg, save=True) # Delay the execution of the pipeline after the ScancodeProject instance was diff --git a/product_portfolio/forms.py b/product_portfolio/forms.py index 2b901568..29d3681a 100644 --- a/product_portfolio/forms.py +++ b/product_portfolio/forms.py @@ -587,11 +587,15 @@ def save(self, product): return warnings, created_counts -class LoadSBOMsForm(forms.Form): +class BaseProductImportFormView(forms.Form): + project_type = None + input_label = "" + input_file = SmartFileField( - label=_("SBOM file or zip archive"), + label=_("file or zip archive"), required=True, ) + update_existing_packages = forms.BooleanField( label=_("Update existing packages with discovered packages data"), required=False, @@ -614,6 +618,10 @@ class LoadSBOMsForm(forms.Form): ), ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["input_file"].label = _(f"{self.input_label} file or zip archive") + @property def helper(self): helper = FormHelper() @@ -627,7 +635,7 @@ def submit(self, product, user): scancode_project = ScanCodeProject.objects.create( product=product, dataspace=product.dataspace, - type=ScanCodeProject.ProjectType.LOAD_SBOMS, + type=self.project_type, input_file=self.cleaned_data.get("input_file"), update_existing_packages=self.cleaned_data.get("update_existing_packages"), scan_all_packages=self.cleaned_data.get("scan_all_packages"), @@ -635,13 +643,26 @@ def submit(self, product, user): ) transaction.on_commit( - lambda: tasks.scancodeio_submit_load_sbom.delay( + lambda: tasks.scancodeio_submit_project.delay( scancodeproject_uuid=scancode_project.uuid, user_uuid=user.uuid, + pipeline_name=self.pipeline_name, ) ) +class LoadSBOMsForm(BaseProductImportFormView): + project_type = ScanCodeProject.ProjectType.LOAD_SBOMS + input_label = "SBOM" + pipeline_name = "load_sbom" + + +class ImportManifestsForm(BaseProductImportFormView): + project_type = ScanCodeProject.ProjectType.IMPORT_FROM_MANIFEST + input_label = "Manifest" + pipeline_name = "resolve_dependencies" + + class StrongTextWidget(forms.Widget): def render(self, name, value, attrs=None, renderer=None): if value: diff --git a/product_portfolio/models.py b/product_portfolio/models.py index 1d516fa3..61f6573f 100644 --- a/product_portfolio/models.py +++ b/product_portfolio/models.py @@ -1122,7 +1122,6 @@ class ScanCodeProject(HistoryFieldsMixin, DataspacedModel): """Wrap a ScanCode.io Project.""" class ProjectType(models.TextChoices): - # This type was replaced by LOAD_SBOMS but is kept for backward compatibility IMPORT_FROM_MANIFEST = "IMPORT_FROM_MANIFEST", _("Import from Manifest") LOAD_SBOMS = "LOAD_SBOMS", _("Load SBOMs") PULL_FROM_SCANCODEIO = "PULL_FROM_SCANCODEIO", _("Pull from ScanCode.io") diff --git a/product_portfolio/templates/product_portfolio/import_manifests_form.html b/product_portfolio/templates/product_portfolio/import_manifests_form.html new file mode 100644 index 00000000..b1e89c61 --- /dev/null +++ b/product_portfolio/templates/product_portfolio/import_manifests_form.html @@ -0,0 +1,78 @@ +{% extends "bootstrap_base.html" %} +{% load i18n static crispy_forms_tags %} +{% load inject_preserved_filters from dje_tags %} + +{% block page_title %}{% trans "Import Packages from manifests" %}{% endblock %} + +{% block content %} +
+
+
+
+
+ {% trans "Products" %} + / {{ object.get_absolute_link }} +
+

+ {% trans "Import Packages from manifests" %} +

+
+
+
+
+ + {% include 'includes/messages_alert.html' %} + +
+
+ The list of supported package manifests and dependency lockfiles is available at + + https://scancode-toolkit.readthedocs.io/en/stable/reference/available_package_parsers.html + +
+
+ Multiple Manifests: + You can provide multiple Manifests by packaging them into a zip archive. + DejaCode will handle and process them accordingly. +
+ + + +
+
+ {{ form.errors }} + {% crispy form %} +
+
+{% endblock %} + +{% block javascripts %} + +{% endblock %} \ No newline at end of file diff --git a/product_portfolio/tests/test_views.py b/product_portfolio/tests/test_views.py index 9ca75abc..ee35ac1a 100644 --- a/product_portfolio/tests/test_views.py +++ b/product_portfolio/tests/test_views.py @@ -34,7 +34,7 @@ from dje.models import History from dje.tasks import logger as tasks_logger from dje.tasks import pull_project_data_from_scancodeio -from dje.tasks import scancodeio_submit_load_sbom +from dje.tasks import scancodeio_submit_project from dje.tests import add_perms from dje.tests import create_superuser from dje.tests import create_user @@ -2693,8 +2693,8 @@ def test_product_portfolio_product_export_cyclonedx_view(self): self.assertEqual(expected, content) - @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.submit_load_sbom") - def test_scancodeio_submit_load_sbom_task(self, mock_submit_sbom): + @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.submit_project") + def test_scancodeio_submit_project_task(self, mock_submit_project): scancodeproject = ScanCodeProject.objects.create( product=self.product1, dataspace=self.product1.dataspace, @@ -2702,8 +2702,8 @@ def test_scancodeio_submit_load_sbom_task(self, mock_submit_sbom): input_file=ContentFile("Data", name="data.json"), ) - mock_submit_sbom.return_value = None - scancodeio_submit_load_sbom( + mock_submit_project.return_value = None + scancodeio_submit_project( scancodeproject_uuid=scancodeproject.uuid, user_uuid=self.super_user.uuid, ) @@ -2719,8 +2719,8 @@ def test_scancodeio_submit_load_sbom_task(self, mock_submit_sbom): scancodeproject.save() project_uuid = uuid.uuid4() - mock_submit_sbom.return_value = {"uuid": project_uuid} - scancodeio_submit_load_sbom( + mock_submit_project.return_value = {"uuid": project_uuid} + scancodeio_submit_project( scancodeproject_uuid=scancodeproject.uuid, user_uuid=self.super_user.uuid, ) diff --git a/product_portfolio/views.py b/product_portfolio/views.py index 03ad61aa..dd529862 100644 --- a/product_portfolio/views.py +++ b/product_portfolio/views.py @@ -99,6 +99,7 @@ from product_portfolio.forms import BaseProductRelationshipInlineFormSet from product_portfolio.forms import ComparisonExcludeFieldsForm from product_portfolio.forms import ImportFromScanForm +from product_portfolio.forms import ImportManifestsForm from product_portfolio.forms import LoadSBOMsForm from product_portfolio.forms import ProductComponentForm from product_portfolio.forms import ProductComponentInlineForm @@ -1881,8 +1882,8 @@ class LoadSBOMsView(BaseProductImportFormView): class ImportManifestsView(BaseProductImportFormView): - template_name = "product_portfolio/load_sboms_form.html" - form_class = LoadSBOMsForm + template_name = "product_portfolio/import_manifests_form.html" + form_class = ImportManifestsForm @method_decorator(require_POST, name="dispatch") From 637640a37be682c5e1af0507449d54f5131cb0f0 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 20 Mar 2024 21:10:31 +0100 Subject: [PATCH 04/11] Complete implementation and fix unit tests #65 Signed-off-by: tdruez --- dje/tasks.py | 3 +- .../import_manifests_form.html | 9 +++--- product_portfolio/tests/test_views.py | 29 +++++++++++++++++-- product_portfolio/views.py | 23 +++++++-------- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/dje/tasks.py b/dje/tasks.py index 7464b1f6..74bf39ef 100644 --- a/dje/tasks.py +++ b/dje/tasks.py @@ -121,7 +121,8 @@ def scancodeio_submit_project(scancodeproject_uuid, user_uuid, pipeline_name): logger.info( f"Entering scancodeio_submit_project task with " - f"scancodeproject_uuid= {scancodeproject_uuid} user_uuid={user_uuid}" + f"scancodeproject_uuid={scancodeproject_uuid} user_uuid={user_uuid} " + f"pipeline_name={pipeline_name}" ) ScanCodeProject = apps.get_model("product_portfolio", "scancodeproject") diff --git a/product_portfolio/templates/product_portfolio/import_manifests_form.html b/product_portfolio/templates/product_portfolio/import_manifests_form.html index b1e89c61..f806f77a 100644 --- a/product_portfolio/templates/product_portfolio/import_manifests_form.html +++ b/product_portfolio/templates/product_portfolio/import_manifests_form.html @@ -25,12 +25,11 @@

- The list of supported package manifests and dependency lockfiles is available at - - https://scancode-toolkit.readthedocs.io/en/stable/reference/available_package_parsers.html - + Supports resolving packages for: +
    +
  • Python: requirements.txt and setup.py manifest files.
    • +
-
Multiple Manifests: You can provide multiple Manifests by packaging them into a zip archive. DejaCode will handle and process them accordingly. diff --git a/product_portfolio/tests/test_views.py b/product_portfolio/tests/test_views.py index ee35ac1a..823bbf94 100644 --- a/product_portfolio/tests/test_views.py +++ b/product_portfolio/tests/test_views.py @@ -2706,11 +2706,12 @@ def test_scancodeio_submit_project_task(self, mock_submit_project): scancodeio_submit_project( scancodeproject_uuid=scancodeproject.uuid, user_uuid=self.super_user.uuid, + pipeline_name="load_sboms", ) scancodeproject.refresh_from_db() self.assertEqual("failure", scancodeproject.status) self.assertIsNone(scancodeproject.project_uuid) - expected = ["- Error: SBOM could not be submitted to ScanCode.io"] + expected = ["- Error: File could not be submitted to ScanCode.io"] self.assertEqual(expected, scancodeproject.import_log) # Reset the instance values @@ -2723,15 +2724,16 @@ def test_scancodeio_submit_project_task(self, mock_submit_project): scancodeio_submit_project( scancodeproject_uuid=scancodeproject.uuid, user_uuid=self.super_user.uuid, + pipeline_name="load_sboms", ) scancodeproject.refresh_from_db() self.assertEqual("submitted", scancodeproject.status) self.assertEqual(project_uuid, scancodeproject.project_uuid) - expected = ["- SBOM file submitted to ScanCode.io for inspection"] + expected = ["- File submitted to ScanCode.io for inspection"] self.assertEqual(expected, scancodeproject.import_log) - @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.submit_load_sbom") + @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.submit_project") def test_product_portfolio_load_sbom_view(self, mock_submit): mock_submit.return_value = None self.client.login(username=self.super_user.username, password="secret") @@ -2752,6 +2754,27 @@ def test_product_portfolio_load_sbom_view(self, mock_submit): self.client.post(url, data=data, follow=True) scan.assert_called_once() + @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.submit_project") + def test_product_portfolio_mport_manifest_view(self, mock_submit): + mock_submit.return_value = None + self.client.login(username=self.super_user.username, password="secret") + url = self.product1.get_import_manifests_url() + response = self.client.get(url) + expected = "Import Packages from manifests" + self.assertContains(response, expected) + + data = {"input_file": ContentFile("Data")} + response = self.client.post(url, data=data, follow=True) + expected = "Manifest file submitted to ScanCode.io for inspection." + self.assertContains(response, expected) + self.assertEqual(1, ScanCodeProject.objects.count()) + + with override_settings(CLAMD_ENABLED=True): + with mock.patch("dje.fields.SmartFileField.scan_file_for_virus") as scan: + data = {"input_file": ContentFile("Data")} + self.client.post(url, data=data, follow=True) + scan.assert_called_once() + @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_project_packages") def test_product_portfolio_import_packages_from_scancodeio_view(self, mock_fetch_packages): self.client.login(username=self.super_user.username, password="secret") diff --git a/product_portfolio/views.py b/product_portfolio/views.py index dd529862..2f0dd763 100644 --- a/product_portfolio/views.py +++ b/product_portfolio/views.py @@ -1870,40 +1870,39 @@ def get_success_url(self): def form_valid(self, form): self.object = self.get_object() - form.submit(product=self.object, user=self.request.user) + + try: + form.submit(product=self.object, user=self.request.user) + except ValidationError as error: + messages.error(self.request, error) + return redirect(self.object.get_absolute_url()) + if self.success_msg: messages.success(self.request, self.success_msg) + return super().form_valid(form) class LoadSBOMsView(BaseProductImportFormView): template_name = "product_portfolio/load_sboms_form.html" form_class = LoadSBOMsForm + success_msg = "SBOM file submitted to ScanCode.io for inspection." class ImportManifestsView(BaseProductImportFormView): template_name = "product_portfolio/import_manifests_form.html" form_class = ImportManifestsForm + success_msg = "Manifest file submitted to ScanCode.io for inspection." @method_decorator(require_POST, name="dispatch") class PullProjectDataFromScanCodeIOView(BaseProductImportFormView): form_class = PullProjectDataForm + success_msg = "Packages import from ScanCode.io in progress..." def form_invalid(self, form): raise Http404 - def form_valid(self, form): - try: - form.submit(product=self.object, user=self.request.user) - except ValidationError as error: - messages.error(self.request, error) - return redirect(self.object.get_absolute_url()) - - msg = "Packages import from ScanCode.io in progress..." - messages.success(self.request, msg) - return super().form_valid(form) - @require_POST @csrf_exempt From 7a65b59453c2a6549a01cd990d5d2a4961a75ab0 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 21 Mar 2024 13:00:58 +0400 Subject: [PATCH 05/11] Add the IMPORT_FROM_MANIFEST type in synchronize #65 Signed-off-by: tdruez --- product_portfolio/views.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/product_portfolio/views.py b/product_portfolio/views.py index 2f0dd763..efbf968d 100644 --- a/product_portfolio/views.py +++ b/product_portfolio/views.py @@ -781,7 +781,7 @@ class ProductTabImportsView( def get_context_data(self, **kwargs): context_data = super().get_context_data(**kwargs) scancode_projects = self.object.scancodeprojects.all() - submitted_projects = self.get_submitted_load_sboms_projects(scancode_projects) + submitted_projects = self.get_submitted_projects(scancode_projects) # Check the status of the "submitted" projects on ScanCode.io and update the # local ScanCodeProject instances accordingly. @@ -800,12 +800,16 @@ def get_context_data(self, **kwargs): return context_data @staticmethod - def get_submitted_load_sboms_projects(scancode_projects): + def get_submitted_projects(scancode_projects): + submitted_types = [ + ScanCodeProject.ProjectType.LOAD_SBOMS, + ScanCodeProject.ProjectType.IMPORT_FROM_MANIFEST, + ] return [ project for project in scancode_projects if project.status == ScanCodeProject.Status.SUBMITTED - and project.type == ScanCodeProject.ProjectType.LOAD_SBOMS + and project.type in submitted_types ] def synchronize(self, scancodeio, project): From aa3e751026a15a49522f504d63d97f48904b9f89 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 21 Mar 2024 19:43:05 +0400 Subject: [PATCH 06/11] Add REST API action for the import manifest feature #65 Signed-off-by: tdruez --- CHANGELOG.rst | 3 +++ product_portfolio/api.py | 37 +++++++++++++++++++++++++++++ product_portfolio/tests/test_api.py | 29 ++++++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3ba33617..bcb1ba30 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -31,6 +31,9 @@ Release notes - Refactor the "Import manifest" feature as "Load SBOMs". https://github.com/nexB/dejacode/issues/61 +- Add support to import packages from manifest. + https://github.com/nexB/dejacode/issues/65 + ### Version 5.0.1 - Improve the stability of the "Check for new Package versions" feature. diff --git a/product_portfolio/api.py b/product_portfolio/api.py index c8c4a1d8..45a22304 100644 --- a/product_portfolio/api.py +++ b/product_portfolio/api.py @@ -35,6 +35,7 @@ from dje.permissions import assign_all_object_permissions from product_portfolio.filters import ComponentCompletenessAPIFilter from product_portfolio.forms import ImportFromScanForm +from product_portfolio.forms import ImportManifestsForm from product_portfolio.forms import LoadSBOMsForm from product_portfolio.forms import PullProjectDataForm from product_portfolio.models import CodebaseResource @@ -216,6 +217,25 @@ class LoadSBOMsFormSerializer(serializers.Serializer): ) +class ImportManifestsFormSerializer(serializers.Serializer): + """Serializer equivalent of ImportManifestsForm, used for API documentation.""" + + input_file = serializers.FileField( + required=True, + help_text=ImportManifestsForm.base_fields["input_file"].label, + ) + update_existing_packages = serializers.BooleanField( + required=False, + default=False, + help_text=ImportManifestsForm.base_fields["update_existing_packages"].help_text, + ) + scan_all_packages = serializers.BooleanField( + required=False, + default=False, + help_text=ImportManifestsForm.base_fields["scan_all_packages"].help_text, + ) + + class ImportFromScanSerializer(serializers.Serializer): """Serializer equivalent of ImportFromScanForm, used for API documentation.""" @@ -319,6 +339,23 @@ def load_sboms(self, request, *args, **kwargs): form.submit(product=product, user=request.user) return Response({"status": "SBOM file submitted to ScanCode.io for inspection."}) + @action(detail=True, methods=["post"], serializer_class=ImportManifestsFormSerializer) + def import_manifests(self, request, *args, **kwargs): + """ + Import Packages from Manifests. + + Multiple Manifests: You can provide multiple files by packaging them into a zip + archive. DejaCode will handle and process them accordingly. + """ + product = self.get_object() + + form = ImportManifestsForm(data=request.POST, files=request.FILES) + if not form.is_valid(): + return Response(form.errors, status=status.HTTP_400_BAD_REQUEST) + + form.submit(product=product, user=request.user) + return Response({"status": "Manifest file submitted to ScanCode.io for inspection."}) + @action(detail=True, methods=["post"], serializer_class=ImportFromScanSerializer) def import_from_scan(self, request, *args, **kwargs): """ diff --git a/product_portfolio/tests/test_api.py b/product_portfolio/tests/test_api.py index 9001aabf..9c6bd1c3 100644 --- a/product_portfolio/tests/test_api.py +++ b/product_portfolio/tests/test_api.py @@ -379,6 +379,35 @@ def test_api_product_endpoint_load_sboms_action(self): self.assertEqual(expected, response.data) self.assertEqual(1, ScanCodeProject.objects.count()) + def test_api_product_endpoint_import_manifests_action(self): + url = reverse("api_v2:product-import-manifests", args=[self.product1.uuid]) + + self.client.login(username=self.base_user.username, password="secret") + response = self.client.get(url) + self.assertEqual(status.HTTP_405_METHOD_NOT_ALLOWED, response.status_code) + response = self.client.post(url, data={}) + self.assertEqual(status.HTTP_403_FORBIDDEN, response.status_code) + + # Required permissions + add_perm(self.base_user, "add_product") + assign_perm("view_product", self.base_user, self.product1) + + response = self.client.post(url, data={}) + self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) + expected = {"input_file": ["This field is required."]} + self.assertEqual(expected, response.data) + + data = { + "input_file": ContentFile("Content", name="requirements.txt"), + "update_existing_packages": False, + "scan_all_packages": False, + } + response = self.client.post(url, data) + self.assertEqual(status.HTTP_200_OK, response.status_code) + expected = {"status": "Manifest file submitted to ScanCode.io for inspection."} + self.assertEqual(expected, response.data) + self.assertEqual(1, ScanCodeProject.objects.count()) + def test_api_product_endpoint_import_from_scan_action(self): url = reverse("api_v2:product-import-from-scan", args=[self.product1.uuid]) From df1c11e9c497e843861912b02341f718424247d2 Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 26 Mar 2024 11:50:42 +0400 Subject: [PATCH 07/11] Simplify the scancodeio_project_status_view #65 Signed-off-by: tdruez --- .../scancodeio_project_status.html | 35 +++++++++++++++++-- .../scancodeio_pull_data_status.html | 28 --------------- product_portfolio/views.py | 19 ++++------ 3 files changed, 40 insertions(+), 42 deletions(-) delete mode 100644 product_portfolio/templates/product_portfolio/scancodeio_pull_data_status.html diff --git a/product_portfolio/templates/product_portfolio/scancodeio_project_status.html b/product_portfolio/templates/product_portfolio/scancodeio_project_status.html index 822d746b..e3e03309 100644 --- a/product_portfolio/templates/product_portfolio/scancodeio_project_status.html +++ b/product_portfolio/templates/product_portfolio/scancodeio_project_status.html @@ -1,6 +1,37 @@ +{% load humanize %} + +{% if scancode_project.results.errors %} +
{{ scancode_project.results.errors|length }} errors:
+
    + {% for error in scancode_project.results.errors %} +
  • {{ error }}
    • + {% endfor %} +
+
+{% endif %} +{% if scancode_project.results.created %} +
{{ scancode_project.results.created|length }} packages created:
+
    + {% for package in scancode_project.results.created %} +
  • {{ package }}
    • + {% endfor %} +
+
+{% endif %} +{% if scancode_project.results.existing %} +
{{ scancode_project.results.existing|length }} packages updated/existing:
+
    + {% for package in scancode_project.results.existing %} +
  • {{ package }}
    • + {% endfor %} +
+
+{% endif %} +
    -
  • Package count: {{ scan_data.package_count }}
    • -
  • Resource count: {{ scan_data.resource_count }}
    • +
  • Package count: {{ scan_data.package_count|intcomma }}
    • +
  • Dependency count: {{ scan_data.dependency_count|intcomma }}
    • +
  • Resource count: {{ scan_data.resource_count|intcomma }}
{% for run in scan_data.runs %}
    diff --git a/product_portfolio/templates/product_portfolio/scancodeio_pull_data_status.html b/product_portfolio/templates/product_portfolio/scancodeio_pull_data_status.html deleted file mode 100644 index cb4cd092..00000000 --- a/product_portfolio/templates/product_portfolio/scancodeio_pull_data_status.html +++ /dev/null @@ -1,28 +0,0 @@ -{% if scancode_project.results.errors %} -
    {{ scancode_project.results.errors|length }} errors:
    -
      - {% for error in scancode_project.results.errors %} -
    • {{ error }}
      • - {% endfor %} -
    -
    -{% endif %} - -{% if scancode_project.results.created %} -
    {{ scancode_project.results.created|length }} packages created:
    -
      - {% for package in scancode_project.results.created %} -
    • {{ package }}
      • - {% endfor %} -
    -
    -{% endif %} - -{% if scancode_project.results.existing %} -
    {{ scancode_project.results.existing|length }} packages updated/existing:
    -
      - {% for package in scancode_project.results.existing %} -
    • {{ package }}
      • - {% endfor %} -
    -{% endif %} \ No newline at end of file diff --git a/product_portfolio/views.py b/product_portfolio/views.py index efbf968d..2cd95eaf 100644 --- a/product_portfolio/views.py +++ b/product_portfolio/views.py @@ -1943,23 +1943,18 @@ def import_packages_from_scancodeio_view(request, key): @login_required def scancodeio_project_status_view(request, scancodeproject_uuid): + template = "product_portfolio/scancodeio_project_status.html" user = request.user base_qs = ScanCodeProject.objects.scope(user.dataspace) scancode_project = get_object_or_404(base_qs, uuid=scancodeproject_uuid) - project_types = ScanCodeProject.ProjectType scancodeio = ScanCodeIO(user) - if scancode_project.type in [project_types.LOAD_SBOMS, project_types.IMPORT_FROM_MANIFEST]: - template = "product_portfolio/scancodeio_project_status.html" - scan_detail_url = scancodeio.get_scan_detail_url(scancode_project.project_uuid) - scan_data = scancodeio.fetch_scan_data(scan_detail_url) - context = {"scan_data": scan_data} - - elif scancode_project.type == project_types.PULL_FROM_SCANCODEIO: - template = "product_portfolio/scancodeio_pull_data_status.html" - context = {"scancode_project": scancode_project} + scan_detail_url = scancodeio.get_scan_detail_url(scancode_project.project_uuid) + scan_data = scancodeio.fetch_scan_data(scan_detail_url) - else: - raise Http404 + context = { + "scancode_project": scancode_project, + "scan_data": scan_data, + } return TemplateResponse(request, template, context) From 14b6dc77bb0fb0b69e6a5a14f157dda08a2e6ada Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 26 Mar 2024 13:00:13 +0400 Subject: [PATCH 08/11] Workaround the error_dict containing too much fields #65 Signed-off-by: tdruez --- dje/models.py | 3 +++ product_portfolio/importers.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dje/models.py b/dje/models.py index 1d908cd0..6e12184a 100644 --- a/dje/models.py +++ b/dje/models.py @@ -745,6 +745,9 @@ def create_from_data(cls, user, data, validate=False): ) if validate: + # Run a pre-clean with `from_api=True` to get simplified messages, + # Note that the clean() is run again in the full_clean call. + instance.clean(from_api=True) instance.full_clean() instance.save() diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index bbc8cde1..907d387b 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -683,8 +683,9 @@ def import_package(self, package_data): if not package: try: package = Package.create_from_data(self.user, package_data, validate=True) - except ValidationError as e: - self.errors.append(e) + except ValidationError as errors: + errors = self.simplify_errors(errors) + self.errors.append(errors) return self.created.append(package) @@ -698,3 +699,11 @@ def import_package(self, package_data): "created_by": self.user, }, ) + + @staticmethod + def simplify_errors(errors): + error_dict = getattr(errors, "error_dict", None) + if error_dict: + if len(error_dict.get("error", [])) == 1 and len(error_dict.get("copy_url", [])): + errors = error_dict["error"][0] + return errors From 76154154710fa31005c9906a8cf52e9ed18225a2 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 27 Mar 2024 18:49:54 +0400 Subject: [PATCH 09/11] Copy objects from reference during import #65 Signed-off-by: tdruez --- dje/models.py | 2 +- product_portfolio/importers.py | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/dje/models.py b/dje/models.py index 6e12184a..6d06b7fb 100644 --- a/dje/models.py +++ b/dje/models.py @@ -747,7 +747,7 @@ def create_from_data(cls, user, data, validate=False): if validate: # Run a pre-clean with `from_api=True` to get simplified messages, # Note that the clean() is run again in the full_clean call. - instance.clean(from_api=True) + # instance.clean(from_api=True) instance.full_clean() instance.save() diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index 907d387b..d831f1bc 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -13,6 +13,7 @@ from django.core.exceptions import MultipleObjectsReturned from django.core.exceptions import ValidationError from django.core.validators import EMPTY_VALUES +from django.db import IntegrityError from django.db import transaction from django.db.models import ObjectDoesNotExist from django.db.models import Q @@ -26,11 +27,13 @@ from component_catalog.models import Component from component_catalog.models import Package from dejacode_toolkit.scancodeio import ScanCodeIO +from dje.copier import copy_object from dje.importers import BaseImporter from dje.importers import BaseImportModelForm from dje.importers import BaseImportModelFormSet from dje.importers import ComponentRelatedFieldImportMixin from dje.importers import ModelChoiceFieldForImport +from dje.models import Dataspace from dje.utils import get_help_text from dje.utils import is_uuid4 from product_portfolio.forms import ProductComponentLicenseExpressionFormMixin @@ -667,12 +670,26 @@ def import_package(self, package_data): if (value := package_data.get(field)) } + # Check if the Package already exists in the local Dataspace try: package = Package.objects.scope(self.user.dataspace).get(**unique_together_lookups) self.existing.append(package) except (ObjectDoesNotExist, MultipleObjectsReturned): package = None + # Check if the Package already exists in the reference Dataspace + reference_dataspace = Dataspace.objects.get_reference() + user_dataspace = self.user.dataspace + if not package and user_dataspace != reference_dataspace: + qs = Package.objects.scope(reference_dataspace).filter(**unique_together_lookups) + if qs.exists(): + reference_object = qs.first() + try: + package = copy_object(reference_object, user_dataspace, self.user, update=False) + self.created.append(package) + except IntegrityError as error: + self.errors.append(error) + if license_expression := package_data.get("declared_license_expression"): license_expression = str(self.licensing.dedup(license_expression)) package_data["license_expression"] = license_expression @@ -684,7 +701,6 @@ def import_package(self, package_data): try: package = Package.create_from_data(self.user, package_data, validate=True) except ValidationError as errors: - errors = self.simplify_errors(errors) self.errors.append(errors) return self.created.append(package) @@ -699,11 +715,3 @@ def import_package(self, package_data): "created_by": self.user, }, ) - - @staticmethod - def simplify_errors(errors): - error_dict = getattr(errors, "error_dict", None) - if error_dict: - if len(error_dict.get("error", [])) == 1 and len(error_dict.get("copy_url", [])): - errors = error_dict["error"][0] - return errors From 20a82c315a30dbbffbc5cf835a8999d955d23e71 Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 29 Mar 2024 15:38:58 +0400 Subject: [PATCH 10/11] Remove comment #65 Signed-off-by: tdruez --- dje/models.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dje/models.py b/dje/models.py index 6d06b7fb..1d908cd0 100644 --- a/dje/models.py +++ b/dje/models.py @@ -745,9 +745,6 @@ def create_from_data(cls, user, data, validate=False): ) if validate: - # Run a pre-clean with `from_api=True` to get simplified messages, - # Note that the clean() is run again in the full_clean call. - # instance.clean(from_api=True) instance.full_clean() instance.save() From 0e305124779e109173e5101960b79be05a2886cc Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 29 Mar 2024 15:41:52 +0400 Subject: [PATCH 11/11] Raise timeout for the pull_project_data task #71 Signed-off-by: tdruez --- dje/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dje/tasks.py b/dje/tasks.py index 74bf39ef..724fd458 100644 --- a/dje/tasks.py +++ b/dje/tasks.py @@ -166,7 +166,7 @@ def scancodeio_submit_project(scancodeproject_uuid, user_uuid, pipeline_name): transaction.on_commit(lambda: scancodeio.start_pipeline(run_url=runs[0]["url"])) -@job +@job("default", timeout=1200) def pull_project_data_from_scancodeio(scancodeproject_uuid): """ Pull Project data from ScanCode.io as an asynchronous task for the provided