From 07ce56de8519f59b02647d37c010835a1c0f8694 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 31 Aug 2026 18:42:54 +0200 Subject: [PATCH 1/4] add ability to configure content of the JSON output Signed-off-by: tdruez --- scanpipe/api/views.py | 11 ++++- scanpipe/management/commands/output.py | 21 +++++++- scanpipe/pipes/output.py | 43 +++++++++++++---- .../dropdowns/project_download_dropdown.html | 8 +++- .../scanpipe/includes/project_downloads.html | 9 +++- .../modals/project_json_download_modal.html | 48 +++++++++++++++++++ .../templates/scanpipe/project_detail.html | 1 + scanpipe/templates/scanpipe/project_list.html | 1 + scanpipe/views.py | 13 +++-- 9 files changed, 137 insertions(+), 18 deletions(-) create mode 100644 scanpipe/templates/scanpipe/modals/project_json_download_modal.html diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py index d58f978726..7c8b0b828b 100644 --- a/scanpipe/api/views.py +++ b/scanpipe/api/views.py @@ -161,8 +161,12 @@ def results(self, request, *args, **kwargs): Return the results compatible with ScanCode data format. The content is returned as a stream of JSON content using the JSONResultsGenerator class. + Optionally restrict the packages/dependencies/files/relations arrays + included using one or more `?sections=` query parameters, e.g. + `?sections=packages§ions=dependencies`. Defaults to all sections. """ - return project_results_json_response(self.get_object()) + sections = request.query_params.getlist("sections") or None + return project_results_json_response(self.get_object(), sections=sections) @action(detail=True, name="Results (download)") def results_download(self, request, *args, **kwargs): @@ -176,7 +180,10 @@ def results_download(self, request, *args, **kwargs): output_kwargs["version"] = version if format == "json": - return project_results_json_response(project, as_attachment=True) + sections = request.query_params.getlist("sections") or None + return project_results_json_response( + project, as_attachment=True, sections=sections + ) elif format == "xlsx": output_file = output.to_xlsx(project) elif format == "spdx": diff --git a/scanpipe/management/commands/output.py b/scanpipe/management/commands/output.py index 8d3d78f50a..f9681ae974 100644 --- a/scanpipe/management/commands/output.py +++ b/scanpipe/management/commands/output.py @@ -35,6 +35,8 @@ "ort-package-list", ] +JSON_SECTIONS = ["packages", "dependencies", "files", "relations"] + class Command(ProjectCommand): help = "Output project results as JSON, XLSX, Attribution, SPDX, and CycloneDX." @@ -58,11 +60,20 @@ def add_arguments(self, parser): action="store_true", help="Print the output to stdout.", ) + parser.add_argument( + "--sections", + nargs="+", + choices=JSON_SECTIONS, + metavar=f"{{{','.join(JSON_SECTIONS)}}}", + help="Restrict the json output to the given sections. Only " + "supported for the json format, and defaults to all sections.", + ) def handle(self, *args, **options): super().handle(*args, **options) self.print_to_stdout = options["print"] formats = options["format"] + sections = options["sections"] if self.print_to_stdout and len(formats) > 1: raise CommandError( @@ -72,10 +83,13 @@ def handle(self, *args, **options): if self.print_to_stdout and ("xlsx" in formats or "csv" in formats): raise CommandError("--print is not compatible with xlsx and csv formats.") + if sections and formats != ["json"]: + raise CommandError("--sections is only supported for the json format.") + for output_format in formats: - self.handle_output(output_format) + self.handle_output(output_format, sections=sections) - def handle_output(self, output_format): + def handle_output(self, output_format, sections=None): output_kwargs = {} if ":" in output_format: output_format, version = output_format.split(":", maxsplit=1) @@ -85,6 +99,9 @@ def handle_output(self, output_format): ) output_kwargs["version"] = version + if output_format == "json" and sections: + output_kwargs["sections"] = sections + output_function = { "json": output.to_json, "csv": output.to_csv, diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 5f3b46cef6..a8cea2eaa7 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -194,18 +194,39 @@ class JSONResultsGenerator: issues. """ - def __init__(self, project): + def __init__(self, project, sections=None): + """ + `sections` is an optional iterable restricting which of the + packages/dependencies/files/relations arrays are included. + Defaults to including all of them. + """ self.project = project + self.sections = sections def __iter__(self): yield "{\n" - yield from self.serialize(label="headers", generator=self.get_headers) - yield from self.serialize(label="packages", generator=self.get_packages) - yield from self.serialize(label="dependencies", generator=self.get_dependencies) - yield from self.serialize(label="files", generator=self.get_files) + + sections = [ + ("packages", self.get_packages), + ("dependencies", self.get_dependencies), + ("files", self.get_files), + ("relations", self.get_relations), + ] + if self.sections is not None: + sections = [ + (label, generator) + for label, generator in sections + if label in self.sections + ] + yield from self.serialize( - label="relations", generator=self.get_relations, latest=True + label="headers", generator=self.get_headers, latest=not sections ) + for index, (label, generator) in enumerate(sections): + yield from self.serialize( + label=label, generator=generator, latest=index == len(sections) - 1 + ) + yield "}" def serialize(self, label, generator, latest=False): @@ -257,7 +278,9 @@ def get_headers(self, project): def encode_queryset(self, project, model_name, serializer): queryset = get_queryset(project, model_name) - for obj in queryset.iterator(chunk_size=2000): + # A larger chunk_size reduces how often prefetch_related() re-runs its + # queries, since iterator() re-executes prefetching once per chunk. + for obj in queryset.iterator(chunk_size=10000): yield self.encode(serializer(obj).data) def get_packages(self, project): @@ -289,13 +312,15 @@ def get_relations(self, project): ) -def to_json(project): +def to_json(project, sections=None): """ Generate output for the provided `project` in JSON format. The output file is created in the `project` output/ directory. Return the path of the generated output file. + `sections` is an optional iterable restricting which of the + packages/dependencies/files/relations arrays are included. """ - results_generator = JSONResultsGenerator(project) + results_generator = JSONResultsGenerator(project, sections=sections) output_file = project.get_output_file_path("results", "json") with output_file.open("w") as file: diff --git a/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html b/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html index ed36748fbb..9a8ca526d3 100644 --- a/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html +++ b/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html @@ -1,3 +1,4 @@ +{% load humanize %}
@@ -9,7 +10,12 @@ Download results as: - + JSON diff --git a/scanpipe/templates/scanpipe/includes/project_downloads.html b/scanpipe/templates/scanpipe/includes/project_downloads.html index 0b7514d5cc..5c97958f10 100644 --- a/scanpipe/templates/scanpipe/includes/project_downloads.html +++ b/scanpipe/templates/scanpipe/includes/project_downloads.html @@ -1,8 +1,15 @@ +{% load humanize %}
Download results: - + JSON diff --git a/scanpipe/templates/scanpipe/modals/project_json_download_modal.html b/scanpipe/templates/scanpipe/modals/project_json_download_modal.html new file mode 100644 index 0000000000..10b0b60ce0 --- /dev/null +++ b/scanpipe/templates/scanpipe/modals/project_json_download_modal.html @@ -0,0 +1,48 @@ + diff --git a/scanpipe/templates/scanpipe/project_detail.html b/scanpipe/templates/scanpipe/project_detail.html index f844167aae..e76d5de2d6 100644 --- a/scanpipe/templates/scanpipe/project_detail.html +++ b/scanpipe/templates/scanpipe/project_detail.html @@ -205,6 +205,7 @@ {% include 'scanpipe/modals/clone_modal.html' %} {% include "scanpipe/modals/add_labels_modal.html" %} {% include "scanpipe/modals/edit_input_tag_modal.html" %} + {% include "scanpipe/modals/project_json_download_modal.html" %} {% endblock %} {% block scripts %} diff --git a/scanpipe/templates/scanpipe/project_list.html b/scanpipe/templates/scanpipe/project_list.html index b00652bf12..00018bc7d7 100644 --- a/scanpipe/templates/scanpipe/project_list.html +++ b/scanpipe/templates/scanpipe/project_list.html @@ -58,6 +58,7 @@ {% block modals %} {% include 'scanpipe/modals/run_modal.html' %} {% include "scanpipe/modals/projects_download_modal.html" %} + {% include "scanpipe/modals/project_json_download_modal.html" %} {% include "scanpipe/modals/projects_report_modal.html" %} {% include "scanpipe/modals/projects_archive_modal.html" %} {% include "scanpipe/modals/projects_delete_modal.html" %} diff --git a/scanpipe/views.py b/scanpipe/views.py index 962fde218c..c76e32744d 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1554,14 +1554,16 @@ def delete_label_view(request, slug, label_name): return JsonResponse({}) -def project_results_json_response(project, as_attachment=False): +def project_results_json_response(project, as_attachment=False, sections=None): """ Return the results as JSON compatible with ScanCode data format. The content is returned as a stream of JSON content using the JSONResultsGenerator class. If `as_attachment` is True, the response will force the download of the file. + `sections` is an optional iterable restricting which of the + packages/dependencies/files/relations arrays are included. """ - results_generator = output.JSONResultsGenerator(project) + results_generator = output.JSONResultsGenerator(project, sections=sections) response = FileResponse( streaming_content=results_generator, content_type="application/json", @@ -1588,7 +1590,12 @@ def get(self, request, *args, **kwargs): output_kwargs["version"] = version if format == "json": - return project_results_json_response(project, as_attachment=True) + sections = None + if "filtered" in request.GET: + sections = request.GET.getlist("sections") + return project_results_json_response( + project, as_attachment=True, sections=sections + ) elif format == "xlsx": output_file = output.to_xlsx(project) elif format == "spdx": From d8a707149f93bafcb63279a12065a3342e998806 Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 1 Sep 2026 10:54:35 +0200 Subject: [PATCH 2/4] refine the UI for content selection Signed-off-by: tdruez --- scancodeio/static/main.css | 19 ++++++ .../dropdowns/project_download_dropdown.html | 11 +--- .../scanpipe/includes/project_downloads.html | 9 +-- .../modals/project_json_download_modal.html | 62 +++++++++++-------- scanpipe/templates/scanpipe/project_list.html | 1 - 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/scancodeio/static/main.css b/scancodeio/static/main.css index aba13f3d19..f85577d7d1 100644 --- a/scancodeio/static/main.css +++ b/scancodeio/static/main.css @@ -122,6 +122,25 @@ .modal.is-medium-size .modal-card { width: 800px; } +.json-section-option { + cursor: pointer; + border: 2px solid transparent; + transition: + border-color 0.15s ease, + background-color 0.15s ease; +} +.json-section-option:hover { + border-color: var(--bulma-success); +} +.json-section-option:has(input:checked) { + border-color: var(--bulma-success); + background-color: hsla( + var(--bulma-success-h), + var(--bulma-success-s), + var(--bulma-success-l), + 0.08 + ); +} .modal.is-desktop-size .modal-card { width: 960px; } diff --git a/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html b/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html index 9a8ca526d3..af787c4e71 100644 --- a/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html +++ b/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html @@ -1,4 +1,3 @@ -{% load humanize %}
@@ -10,12 +9,7 @@ Download results as: - + JSON @@ -52,4 +46,5 @@
-
\ No newline at end of file +
+{% include "scanpipe/modals/project_json_download_modal.html" with project=project only %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/includes/project_downloads.html b/scanpipe/templates/scanpipe/includes/project_downloads.html index 5c97958f10..077da46141 100644 --- a/scanpipe/templates/scanpipe/includes/project_downloads.html +++ b/scanpipe/templates/scanpipe/includes/project_downloads.html @@ -1,15 +1,8 @@ -{% load humanize %}
Download results: - + JSON diff --git a/scanpipe/templates/scanpipe/modals/project_json_download_modal.html b/scanpipe/templates/scanpipe/modals/project_json_download_modal.html index 10b0b60ce0..cb97249e4f 100644 --- a/scanpipe/templates/scanpipe/modals/project_json_download_modal.html +++ b/scanpipe/templates/scanpipe/modals/project_json_download_modal.html @@ -1,37 +1,45 @@ -