feat: add ability to configure content of the JSON output (#2222)

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

aboutcode/pipeline/README.md

@@ -2,7 +2,7 @@
 
 Define and run pipelines.
 
-### Install
+## Install
 
 ```bash
 pip install aboutcode.pipeline
@@ -13,6 +13,7 @@ pip install aboutcode.pipeline
 ```python
 from aboutcode.pipeline import BasePipeline
 
+
 class PrintMessages(BasePipeline):
     @classmethod
     def steps(cls):
@@ -21,6 +22,7 @@ class PrintMessages(BasePipeline):
     def step1(self):
         print("Message from step1")
 
+
 PrintMessages().execute()
 ```
 

scancodeio/static/main.css

@@ -122,6 +122,25 @@
 .modal.is-medium-size .modal-card {
   width: 800px;
 }
+.json-section-option {
+  cursor: pointer;
+  border: 2px solid transparent;
+  transition:
+    border-color 0.15s ease,
+    background-color 0.15s ease;
+}
+.json-section-option:hover {
+  border-color: var(--bulma-success);
+}
+.json-section-option:has(input:checked) {
+  border-color: var(--bulma-success);
+  background-color: hsla(
+    var(--bulma-success-h),
+    var(--bulma-success-s),
+    var(--bulma-success-l),
+    0.08
+  );
+}
 .modal.is-desktop-size .modal-card {
   width: 960px;
 }

scanpipe/api/views.py

@@ -161,8 +161,12 @@ def results(self, request, *args, **kwargs):
         Return the results compatible with ScanCode data format.
         The content is returned as a stream of JSON content using the
         JSONResultsGenerator class.
+        Optionally restrict the packages/dependencies/files/relations arrays
+        included using one or more `?sections=` query parameters, e.g.
+        `?sections=packages&sections=dependencies`. Defaults to all sections.
         """
-        return project_results_json_response(self.get_object())
+        sections = request.query_params.getlist("sections") or None
+        return project_results_json_response(self.get_object(), sections=sections)
 
     @action(detail=True, name="Results (download)")
     def results_download(self, request, *args, **kwargs):
@@ -176,7 +180,10 @@ def results_download(self, request, *args, **kwargs):
             output_kwargs["version"] = version
 
         if format == "json":
-            return project_results_json_response(project, as_attachment=True)
+            sections = request.query_params.getlist("sections") or None
+            return project_results_json_response(
+                project, as_attachment=True, sections=sections
+            )
         elif format == "xlsx":
             output_file = output.to_xlsx(project)
         elif format == "spdx":

scanpipe/management/commands/output.py

@@ -35,6 +35,8 @@
     "ort-package-list",
 ]
 
+JSON_SECTIONS = ["packages", "dependencies", "files", "relations"]
+
 
 class Command(ProjectCommand):
     help = "Output project results as JSON, XLSX, Attribution, SPDX, and CycloneDX."
@@ -58,11 +60,20 @@ def add_arguments(self, parser):
             action="store_true",
             help="Print the output to stdout.",
         )
+        parser.add_argument(
+            "--sections",
+            nargs="+",
+            choices=JSON_SECTIONS,
+            metavar=f"{{{','.join(JSON_SECTIONS)}}}",
+            help="Restrict the json output to the given sections. Only "
+            "supported for the json format, and defaults to all sections.",
+        )
 
     def handle(self, *args, **options):
         super().handle(*args, **options)
         self.print_to_stdout = options["print"]
         formats = options["format"]
+        sections = options["sections"]
 
         if self.print_to_stdout and len(formats) > 1:
             raise CommandError(
@@ -72,10 +83,13 @@ def handle(self, *args, **options):
         if self.print_to_stdout and ("xlsx" in formats or "csv" in formats):
             raise CommandError("--print is not compatible with xlsx and csv formats.")
 
+        if sections and formats != ["json"]:
+            raise CommandError("--sections is only supported for the json format.")
+
         for output_format in formats:
-            self.handle_output(output_format)
+            self.handle_output(output_format, sections=sections)
 
-    def handle_output(self, output_format):
+    def handle_output(self, output_format, sections=None):
         output_kwargs = {}
         if ":" in output_format:
             output_format, version = output_format.split(":", maxsplit=1)
@@ -85,6 +99,9 @@ def handle_output(self, output_format):
                 )
             output_kwargs["version"] = version
 
+        if output_format == "json" and sections:
+            output_kwargs["sections"] = sections
+
         output_function = {
             "json": output.to_json,
             "csv": output.to_csv,

scanpipe/pipes/output.py

@@ -194,18 +194,39 @@ class JSONResultsGenerator:
     issues.
     """
 
-    def __init__(self, project):
+    def __init__(self, project, sections=None):
+        """
+        `sections` is an optional iterable restricting which of the
+        packages/dependencies/files/relations arrays are included.
+        Defaults to including all of them.
+        """
         self.project = project
+        self.sections = sections
 
     def __iter__(self):
         yield "{\n"
-        yield from self.serialize(label="headers", generator=self.get_headers)
-        yield from self.serialize(label="packages", generator=self.get_packages)
-        yield from self.serialize(label="dependencies", generator=self.get_dependencies)
-        yield from self.serialize(label="files", generator=self.get_files)
+
+        sections = [
+            ("packages", self.get_packages),
+            ("dependencies", self.get_dependencies),
+            ("files", self.get_files),
+            ("relations", self.get_relations),
+        ]
+        if self.sections is not None:
+            sections = [
+                (label, generator)
+                for label, generator in sections
+                if label in self.sections
+            ]
+
         yield from self.serialize(
-            label="relations", generator=self.get_relations, latest=True
+            label="headers", generator=self.get_headers, latest=not sections
         )
+        for index, (label, generator) in enumerate(sections):
+            yield from self.serialize(
+                label=label, generator=generator, latest=index == len(sections) - 1
+            )
+
         yield "}"
 
     def serialize(self, label, generator, latest=False):
@@ -257,7 +278,9 @@ def get_headers(self, project):
 
     def encode_queryset(self, project, model_name, serializer):
         queryset = get_queryset(project, model_name)
-        for obj in queryset.iterator(chunk_size=2000):
+        # A larger chunk_size reduces how often prefetch_related() re-runs its
+        # queries, since iterator() re-executes prefetching once per chunk.
+        for obj in queryset.iterator(chunk_size=10000):
             yield self.encode(serializer(obj).data)
 
     def get_packages(self, project):
@@ -289,13 +312,15 @@ def get_relations(self, project):
         )
 
 
-def to_json(project):
+def to_json(project, sections=None):
     """
     Generate output for the provided `project` in JSON format.
     The output file is created in the `project` output/ directory.
     Return the path of the generated output file.
+    `sections` is an optional iterable restricting which of the
+    packages/dependencies/files/relations arrays are included.
     """
-    results_generator = JSONResultsGenerator(project)
+    results_generator = JSONResultsGenerator(project, sections=sections)
     output_file = project.get_output_file_path("results", "json")
 
     with output_file.open("w") as file:

scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html

@@ -9,7 +9,7 @@
       <span class="dropdown-item">
         Download results as:
       </span>
-      <a href="{% url 'project_results' project.slug 'json' %}" class="dropdown-item">
+      <a href="#" class="dropdown-item modal-button" data-target="modal-json-download-{{ project.slug }}">
         <strong>JSON</strong>
       </a>
       <a href="{% url 'project_results' project.slug 'xlsx' %}" class="dropdown-item">
@@ -46,4 +46,5 @@
       </a>
     </div>
   </div>
-</div>
+</div>
+{% include "scanpipe/modals/project_json_download_modal.html" with project=project only %}

scanpipe/templates/scanpipe/includes/project_downloads.html

@@ -2,7 +2,7 @@
   <div class="message-body p-3">
     <span class="icon"><i class="fa-solid fa-download"></i></span>
     Download results:
-    <a class="tag is-success is-medium ml-2" href="{% url 'project_results' project.slug 'json' %}">
+    <a href="#" class="tag is-success is-medium ml-2 modal-button" data-target="modal-json-download-{{ project.slug }}" aria-haspopup="true">
       JSON
     </a>
     <a class="tag is-success is-medium" href="{% url 'project_results' project.slug 'xlsx' %}">

scanpipe/templates/scanpipe/modals/project_json_download_modal.html

@@ -0,0 +1,55 @@
+{% load humanize %}
+<div class="modal" id="modal-json-download-{{ project.slug }}">
+  <div class="modal-background"></div>
+  <div class="modal-card">
+    <header class="modal-card-head">
+      <p class="modal-card-title">
+        <span class="icon-text">
+          <span class="icon has-text-success"><i class="fa-solid fa-download"></i></span>
+          <span>Download JSON results</span>
+        </span>
+      </p>
+      <button class="delete" aria-label="close"></button>
+    </header>
+    <form action="{% url 'project_results' project.slug 'json' %}" method="get">
+      <section class="modal-card-body">
+        <p class="has-text-grey mb-4">Select the sections to include in the JSON output.</p>
+        <label class="json-section-option box has-border-radius p-3 mb-2 is-flex is-align-items-center">
+          <span class="icon has-text-success is-size-4 mr-3"><i class="fa-solid fa-box"></i></span>
+          <span class="has-text-weight-semibold is-flex-grow-1">Packages</span>
+          <span class="tag is-light has-text-weight-bold mr-2">{{ project.package_count|intcomma }}</span>
+          <input type="checkbox" class="width-1 height-1" name="sections" value="packages" checked>
+        </label>
+        <label class="json-section-option box has-border-radius p-3 mb-2 is-flex is-align-items-center">
+          <span class="icon has-text-success is-size-4 mr-3"><i class="fa-solid fa-layer-group"></i></span>
+          <span class="has-text-weight-semibold is-flex-grow-1">Dependencies</span>
+          <span class="tag is-light has-text-weight-bold mr-2">{{ project.dependency_count|intcomma }}</span>
+          <input type="checkbox" class="width-1 height-1" name="sections" value="dependencies" checked>
+        </label>
+        <label class="json-section-option box has-border-radius p-3 mb-2 is-flex is-align-items-center">
+          <span class="icon has-text-success is-size-4 mr-3"><i class="fa-solid fa-folder-open"></i></span>
+          <span class="has-text-weight-semibold is-flex-grow-1">Resources</span>
+          <span class="tag is-light has-text-weight-bold mr-2">{{ project.resource_count|intcomma }}</span>
+          <input type="checkbox" class="width-1 height-1" name="sections" value="files" checked>
+        </label>
+        {% if project.relation_count %}
+          <label class="json-section-option box has-border-radius p-3 mb-0 is-flex is-align-items-center">
+            <span class="icon has-text-success is-size-4 mr-3"><i class="fa-solid fa-link"></i></span>
+            <span class="has-text-weight-semibold is-flex-grow-1">Relations</span>
+            <span class="tag is-light has-text-weight-bold mr-2">{{ project.relation_count|intcomma }}</span>
+            <input type="checkbox" class="width-1 height-1" name="sections" value="relations" checked>
+          </label>
+        {% endif %}
+      </section>
+      <footer class="modal-card-foot is-justify-content-flex-end">
+        <div class="buttons">
+          <button class="button has-text-weight-semibold" type="reset">Cancel</button>
+          <button class="button is-success" type="submit">
+            <span class="icon mr-1"><i class="fa-solid fa-download"></i></span>
+            Download
+          </button>
+        </div>
+      </footer>
+    </form>
+  </div>
+</div>

scanpipe/templates/scanpipe/project_detail.html

@@ -205,6 +205,7 @@
   {% include 'scanpipe/modals/clone_modal.html' %}
   {% include "scanpipe/modals/add_labels_modal.html" %}
   {% include "scanpipe/modals/edit_input_tag_modal.html" %}
+  {% include "scanpipe/modals/project_json_download_modal.html" %}
 {% endblock %}
 
 {% block scripts %}

scanpipe/tests/pipes/test_output.py

@@ -208,6 +208,22 @@ def test_scanpipe_pipes_outputs_to_json(self):
             output_file = output.to_json(project=project)
         self.assertIn(output_file.name, project.output_root)
 
+    def test_scanpipe_pipes_outputs_to_json_with_sections(self):
+        fixtures = self.data / "asgiref" / "asgiref-3.3.0_fixtures.json"
+        call_command("loaddata", fixtures, **{"verbosity": 0})
+        project = Project.objects.get(name="asgiref")
+
+        output_file = output.to_json(project=project, sections=["packages"])
+        with output_file.open() as f:
+            results = json.loads(f.read())
+        self.assertEqual(["headers", "packages"], sorted(results.keys()))
+        self.assertEqual(2, len(results["packages"]))
+
+        output_file = output.to_json(project=project, sections=[])
+        with output_file.open() as f:
+            results = json.loads(f.read())
+        self.assertEqual(["headers"], sorted(results.keys()))
+
     def test_scanpipe_pipes_outputs_to_xlsx(self):
         fixtures = self.data / "asgiref" / "asgiref-3.3.0_fixtures.json"
         call_command("loaddata", fixtures, **{"verbosity": 0})