diff --git a/aboutcode/pipeline/README.md b/aboutcode/pipeline/README.md index 13ea051d30..01199c399e 100644 --- a/aboutcode/pipeline/README.md +++ b/aboutcode/pipeline/README.md @@ -2,7 +2,7 @@ Define and run pipelines. -### Install +## Install ```bash pip install aboutcode.pipeline @@ -13,6 +13,7 @@ pip install aboutcode.pipeline ```python from aboutcode.pipeline import BasePipeline + class PrintMessages(BasePipeline): @classmethod def steps(cls): @@ -21,6 +22,7 @@ class PrintMessages(BasePipeline): def step1(self): print("Message from step1") + PrintMessages().execute() ``` diff --git a/scancodeio/static/main.css b/scancodeio/static/main.css index aba13f3d19..f85577d7d1 100644 --- a/scancodeio/static/main.css +++ b/scancodeio/static/main.css @@ -122,6 +122,25 @@ .modal.is-medium-size .modal-card { width: 800px; } +.json-section-option { + cursor: pointer; + border: 2px solid transparent; + transition: + border-color 0.15s ease, + background-color 0.15s ease; +} +.json-section-option:hover { + border-color: var(--bulma-success); +} +.json-section-option:has(input:checked) { + border-color: var(--bulma-success); + background-color: hsla( + var(--bulma-success-h), + var(--bulma-success-s), + var(--bulma-success-l), + 0.08 + ); +} .modal.is-desktop-size .modal-card { width: 960px; } diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py index d58f978726..7c8b0b828b 100644 --- a/scanpipe/api/views.py +++ b/scanpipe/api/views.py @@ -161,8 +161,12 @@ def results(self, request, *args, **kwargs): Return the results compatible with ScanCode data format. The content is returned as a stream of JSON content using the JSONResultsGenerator class. + Optionally restrict the packages/dependencies/files/relations arrays + included using one or more `?sections=` query parameters, e.g. + `?sections=packages§ions=dependencies`. Defaults to all sections. """ - return project_results_json_response(self.get_object()) + sections = request.query_params.getlist("sections") or None + return project_results_json_response(self.get_object(), sections=sections) @action(detail=True, name="Results (download)") def results_download(self, request, *args, **kwargs): @@ -176,7 +180,10 @@ def results_download(self, request, *args, **kwargs): output_kwargs["version"] = version if format == "json": - return project_results_json_response(project, as_attachment=True) + sections = request.query_params.getlist("sections") or None + return project_results_json_response( + project, as_attachment=True, sections=sections + ) elif format == "xlsx": output_file = output.to_xlsx(project) elif format == "spdx": diff --git a/scanpipe/management/commands/output.py b/scanpipe/management/commands/output.py index 8d3d78f50a..f9681ae974 100644 --- a/scanpipe/management/commands/output.py +++ b/scanpipe/management/commands/output.py @@ -35,6 +35,8 @@ "ort-package-list", ] +JSON_SECTIONS = ["packages", "dependencies", "files", "relations"] + class Command(ProjectCommand): help = "Output project results as JSON, XLSX, Attribution, SPDX, and CycloneDX." @@ -58,11 +60,20 @@ def add_arguments(self, parser): action="store_true", help="Print the output to stdout.", ) + parser.add_argument( + "--sections", + nargs="+", + choices=JSON_SECTIONS, + metavar=f"{{{','.join(JSON_SECTIONS)}}}", + help="Restrict the json output to the given sections. Only " + "supported for the json format, and defaults to all sections.", + ) def handle(self, *args, **options): super().handle(*args, **options) self.print_to_stdout = options["print"] formats = options["format"] + sections = options["sections"] if self.print_to_stdout and len(formats) > 1: raise CommandError( @@ -72,10 +83,13 @@ def handle(self, *args, **options): if self.print_to_stdout and ("xlsx" in formats or "csv" in formats): raise CommandError("--print is not compatible with xlsx and csv formats.") + if sections and formats != ["json"]: + raise CommandError("--sections is only supported for the json format.") + for output_format in formats: - self.handle_output(output_format) + self.handle_output(output_format, sections=sections) - def handle_output(self, output_format): + def handle_output(self, output_format, sections=None): output_kwargs = {} if ":" in output_format: output_format, version = output_format.split(":", maxsplit=1) @@ -85,6 +99,9 @@ def handle_output(self, output_format): ) output_kwargs["version"] = version + if output_format == "json" and sections: + output_kwargs["sections"] = sections + output_function = { "json": output.to_json, "csv": output.to_csv, diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 5f3b46cef6..a8cea2eaa7 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -194,18 +194,39 @@ class JSONResultsGenerator: issues. """ - def __init__(self, project): + def __init__(self, project, sections=None): + """ + `sections` is an optional iterable restricting which of the + packages/dependencies/files/relations arrays are included. + Defaults to including all of them. + """ self.project = project + self.sections = sections def __iter__(self): yield "{\n" - yield from self.serialize(label="headers", generator=self.get_headers) - yield from self.serialize(label="packages", generator=self.get_packages) - yield from self.serialize(label="dependencies", generator=self.get_dependencies) - yield from self.serialize(label="files", generator=self.get_files) + + sections = [ + ("packages", self.get_packages), + ("dependencies", self.get_dependencies), + ("files", self.get_files), + ("relations", self.get_relations), + ] + if self.sections is not None: + sections = [ + (label, generator) + for label, generator in sections + if label in self.sections + ] + yield from self.serialize( - label="relations", generator=self.get_relations, latest=True + label="headers", generator=self.get_headers, latest=not sections ) + for index, (label, generator) in enumerate(sections): + yield from self.serialize( + label=label, generator=generator, latest=index == len(sections) - 1 + ) + yield "}" def serialize(self, label, generator, latest=False): @@ -257,7 +278,9 @@ def get_headers(self, project): def encode_queryset(self, project, model_name, serializer): queryset = get_queryset(project, model_name) - for obj in queryset.iterator(chunk_size=2000): + # A larger chunk_size reduces how often prefetch_related() re-runs its + # queries, since iterator() re-executes prefetching once per chunk. + for obj in queryset.iterator(chunk_size=10000): yield self.encode(serializer(obj).data) def get_packages(self, project): @@ -289,13 +312,15 @@ def get_relations(self, project): ) -def to_json(project): +def to_json(project, sections=None): """ Generate output for the provided `project` in JSON format. The output file is created in the `project` output/ directory. Return the path of the generated output file. + `sections` is an optional iterable restricting which of the + packages/dependencies/files/relations arrays are included. """ - results_generator = JSONResultsGenerator(project) + results_generator = JSONResultsGenerator(project, sections=sections) output_file = project.get_output_file_path("results", "json") with output_file.open("w") as file: diff --git a/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html b/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html index ed36748fbb..af787c4e71 100644 --- a/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html +++ b/scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html @@ -9,7 +9,7 @@ Download results as: - + JSON @@ -46,4 +46,5 @@ - \ No newline at end of file + +{% include "scanpipe/modals/project_json_download_modal.html" with project=project only %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/includes/project_downloads.html b/scanpipe/templates/scanpipe/includes/project_downloads.html index 0b7514d5cc..077da46141 100644 --- a/scanpipe/templates/scanpipe/includes/project_downloads.html +++ b/scanpipe/templates/scanpipe/includes/project_downloads.html @@ -2,7 +2,7 @@
Download results: - + JSON diff --git a/scanpipe/templates/scanpipe/modals/project_json_download_modal.html b/scanpipe/templates/scanpipe/modals/project_json_download_modal.html new file mode 100644 index 0000000000..5134ab40bf --- /dev/null +++ b/scanpipe/templates/scanpipe/modals/project_json_download_modal.html @@ -0,0 +1,55 @@ +{% load humanize %} + diff --git a/scanpipe/templates/scanpipe/project_detail.html b/scanpipe/templates/scanpipe/project_detail.html index f844167aae..e76d5de2d6 100644 --- a/scanpipe/templates/scanpipe/project_detail.html +++ b/scanpipe/templates/scanpipe/project_detail.html @@ -205,6 +205,7 @@ {% include 'scanpipe/modals/clone_modal.html' %} {% include "scanpipe/modals/add_labels_modal.html" %} {% include "scanpipe/modals/edit_input_tag_modal.html" %} + {% include "scanpipe/modals/project_json_download_modal.html" %} {% endblock %} {% block scripts %} diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index 5c89d67028..56a05e5e44 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -208,6 +208,22 @@ def test_scanpipe_pipes_outputs_to_json(self): output_file = output.to_json(project=project) self.assertIn(output_file.name, project.output_root) + def test_scanpipe_pipes_outputs_to_json_with_sections(self): + fixtures = self.data / "asgiref" / "asgiref-3.3.0_fixtures.json" + call_command("loaddata", fixtures, **{"verbosity": 0}) + project = Project.objects.get(name="asgiref") + + output_file = output.to_json(project=project, sections=["packages"]) + with output_file.open() as f: + results = json.loads(f.read()) + self.assertEqual(["headers", "packages"], sorted(results.keys())) + self.assertEqual(2, len(results["packages"])) + + output_file = output.to_json(project=project, sections=[]) + with output_file.open() as f: + results = json.loads(f.read()) + self.assertEqual(["headers"], sorted(results.keys())) + def test_scanpipe_pipes_outputs_to_xlsx(self): fixtures = self.data / "asgiref" / "asgiref-3.3.0_fixtures.json" call_command("loaddata", fixtures, **{"verbosity": 0}) diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 3fb58b3f7f..15efb12599 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -644,6 +644,15 @@ def test_scanpipe_api_project_action_results(self): self.assertEqual(1, len(results["files"])) self.assertEqual(1, len(results["packages"])) + def test_scanpipe_api_project_action_results_with_sections(self): + url = reverse("project-results", args=[self.project1.uuid]) + data = {"sections": ["packages", "dependencies"]} + response = self.csrf_client.get(url, data=data) + results = json.loads(response.getvalue()) + self.assertEqual( + ["dependencies", "headers", "packages"], sorted(results.keys()) + ) + def test_scanpipe_api_project_action_results_download(self): url = reverse("project-results-download", args=[self.project1.uuid]) response = self.csrf_client.get(url) @@ -657,6 +666,13 @@ def test_scanpipe_api_project_action_results_download(self): expected = ["dependencies", "files", "headers", "packages", "relations"] self.assertEqual(expected, sorted(results.keys())) + def test_scanpipe_api_project_action_results_download_with_sections(self): + url = reverse("project-results-download", args=[self.project1.uuid]) + data = {"output_format": "json", "sections": "files"} + response = self.csrf_client.get(url, data=data) + results = json.loads(response.getvalue()) + self.assertEqual(["files", "headers"], sorted(results.keys())) + @mock.patch("scanpipe.pipes.datetime", mocked_now) def test_scanpipe_api_project_action_results_download_output_formats(self): url = reverse("project-results-download", args=[self.project1.uuid]) diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 499c8d2100..8cfb4fde54 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -788,6 +788,29 @@ def test_scanpipe_management_command_output(self): self.assertIn('"bomFormat": "CycloneDX"', out_value) self.assertIn('"specVersion": "1.5",', out_value) + def test_scanpipe_management_command_output_with_sections(self): + project = make_project(name="my_project") + make_package(project, package_url="pkg:generic/name@1.0") + + out = StringIO() + options = ["--project", project.name, "--no-color"] + options.extend(["--format", "json", "--print", "--sections", "packages"]) + call_command("output", *options, stdout=out) + results = json.loads(out.getvalue().strip()) + self.assertEqual(["headers", "packages"], sorted(results.keys())) + + options = ["--project", project.name, "--no-color"] + options.extend(["--format", "xlsx", "--sections", "packages"]) + message = "--sections is only supported for the json format." + with self.assertRaisesMessage(CommandError, message): + call_command("output", *options) + + options = ["--project", project.name, "--no-color"] + options.extend(["--format", "json", "--sections", "bogus"]) + message = "Error: argument --sections: invalid choice: 'bogus'" + with self.assertRaisesMessage(CommandError, message): + call_command("output", *options) + def test_scanpipe_management_command_delete_project(self): project = make_project(name="my_project") work_path = project.work_path diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 43ae6bd5af..e87f5ff07f 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -386,6 +386,20 @@ def test_scanpipe_views_project_details_download_output_view(self): response.headers["Content-Disposition"], ) + def test_scanpipe_views_project_results_json_view(self): + make_package(self.project1, package_url="pkg:generic/name@1.0") + + url = reverse("project_results", args=[self.project1.slug, "json"]) + response = self.client.get(url) + results = json.loads(response.getvalue()) + expected = ["dependencies", "files", "headers", "packages", "relations"] + self.assertEqual(expected, sorted(results.keys())) + self.assertEqual(1, len(results["packages"])) + + response = self.client.get(url, data={"sections": "packages"}) + results = json.loads(response.getvalue()) + self.assertEqual(["headers", "packages"], sorted(results.keys())) + def test_scanpipe_views_project_details_delete_input_view(self): random_uuid = str(uuid.uuid4()) url = reverse("project_delete_input", args=[self.project1.slug, random_uuid]) diff --git a/scanpipe/views.py b/scanpipe/views.py index 962fde218c..95c0f27a5c 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1554,14 +1554,21 @@ def delete_label_view(request, slug, label_name): return JsonResponse({}) -def project_results_json_response(project, as_attachment=False): +def get_project_results_sections(request): + """Return the requested JSON `sections` filter from the request, or None.""" + return request.GET.getlist("sections") or None + + +def project_results_json_response(project, as_attachment=False, sections=None): """ Return the results as JSON compatible with ScanCode data format. The content is returned as a stream of JSON content using the JSONResultsGenerator class. If `as_attachment` is True, the response will force the download of the file. + `sections` is an optional iterable restricting which of the + packages/dependencies/files/relations arrays are included. """ - results_generator = output.JSONResultsGenerator(project) + results_generator = output.JSONResultsGenerator(project, sections=sections) response = FileResponse( streaming_content=results_generator, content_type="application/json", @@ -1588,7 +1595,10 @@ def get(self, request, *args, **kwargs): output_kwargs["version"] = version if format == "json": - return project_results_json_response(project, as_attachment=True) + sections = get_project_results_sections(request) + return project_results_json_response( + project, as_attachment=True, sections=sections + ) elif format == "xlsx": output_file = output.to_xlsx(project) elif format == "spdx":