merge: link the BOM subject, stamp Tools metadata, clean up duplicated metadata/empty lists

src/cyclonedx/Commands/MergeCommand.cs

@@ -19,6 +19,7 @@
 using System.Diagnostics.Contracts;
 using System.CommandLine;
 using System.CommandLine.Invocation;
+using System.IO;
 using System.Threading.Tasks;
 using CycloneDX.Models;
 using CycloneDX.Utils;
@@ -34,14 +35,19 @@ public static void Configure(RootCommand rootCommand)
             var subCommand = new System.CommandLine.Command("merge", "Merge two or more BOMs")
             {
                 new Option<List<string>>("--input-files", "Input BOM filenames (separate filenames with a space).") { AllowMultipleArgumentsPerToken = true },
+                new Option<List<string>>("--input-files-list", "One or more text file(s) with input BOM filenames (one per line). Combined with --input-files, useful to exceed OS/shell command-line length limits when merging many BOMs.") { AllowMultipleArgumentsPerToken = true },
+                new Option<List<string>>("--input-files-nul-list", "One or more text-like file(s) with input BOM filenames (separated by 0x00 characters, e.g. from `find -print0`).") { AllowMultipleArgumentsPerToken = true },
                 new Option<string>("--output-file", "Output BOM filename, will write to stdout if no value provided."),
                 new Option<CycloneDXBomFormat>("--input-format", "Specify input file format."),
                 new Option<CycloneDXBomFormat>("--output-format", "Specify output file format."),
                 new Option<SpecificationVersion>("--output-version", "Specify output BOM specification version."),
                 new Option<bool>("--hierarchical", "Perform a hierarchical merge."),
                 new Option<string>("--group", "Provide the group of software the merged BOM describes."),
                 new Option<string>("--name", "Provide the name of software the merged BOM describes (required for hierarchical merging)."),
-                new Option<string>("--version", "Provide the version of software the merged BOM describes (required for hierarchical merging).")
+                new Option<string>("--version", "Provide the version of software the merged BOM describes (required for hierarchical merging)."),
+#if NET8_0_OR_GREATER
+                new Option<ComponentConflictResolution>("--component-conflict-resolution", "How to resolve two equivalent (same type/name/version/group/purl) but not-identical Components, e.g. differing only by Scope. Default: squash, preferring the more permissive Scope."),
+#endif
             };
             subCommand.Handler = CommandHandler.Create<MergeCommandOptions>(Merge);
             rootCommand.Add(subCommand);
@@ -65,7 +71,7 @@ public static async Task<int> Merge(MergeCommandOptions options)
                 return (int)ExitCode.ParameterValidationError;
             }
 
-            var inputBoms = await InputBoms(options.InputFiles, options.InputFormat, outputToConsole).ConfigureAwait(false);
+            var inputBoms = await InputBoms(DetermineInputFiles(options), options.InputFormat, outputToConsole).ConfigureAwait(false);
 
             Component bomSubject = null;
             if (options.Group != null || options.Name != null || options.Version != null)
@@ -77,23 +83,37 @@ public static async Task<int> Merge(MergeCommandOptions options)
                     Version = options.Version,
                 };
 
+#if NET8_0_OR_GREATER
+            var mergeStrategy = MergeStrategy.Default();
+            if (options.ComponentConflictResolution.HasValue)
+            {
+                mergeStrategy.ComponentConflictResolution = options.ComponentConflictResolution.Value;
+            }
+#endif
+
             Bom outputBom;
             if (options.Hierarchical)
             {
+#if NET8_0_OR_GREATER
+                outputBom = CycloneDXUtils.HierarchicalMerge(inputBoms, bomSubject, mergeStrategy);
+#else
                 outputBom = CycloneDXUtils.HierarchicalMerge(inputBoms, bomSubject);
+#endif
             }
             else
             {
-                outputBom = CycloneDXUtils.FlatMerge(inputBoms);
+#if NET8_0_OR_GREATER
+                outputBom = CycloneDXUtils.FlatMerge(inputBoms, bomSubject, mergeStrategy);
+#else
+                outputBom = CycloneDXUtils.FlatMerge(inputBoms, bomSubject);
+#endif
                 if (outputBom.Metadata is null) outputBom.Metadata = new Metadata();
-                if (bomSubject != null)
+                if (bomSubject is null)
                 {
-                    // use the params provided if possible
-                    outputBom.Metadata.Component = bomSubject;
-                }
-                else
-                {
-                    // otherwise use the first non-null component from the input BOMs as the default
+                    // otherwise use the first non-null component from the input
+                    // BOMs as the default; note CleanupMetadataComponent below,
+                    // since that same component may also already be present
+                    // in outputBom.Components.
                     foreach (var bom in inputBoms)
                     {
                         if(bom.Metadata != null && bom.Metadata.Component != null)
@@ -105,6 +125,18 @@ public static async Task<int> Merge(MergeCommandOptions options)
                 }
             }
 
+#if NET8_0_OR_GREATER
+            outputBom = CycloneDXUtils.CleanupMetadataComponent(outputBom, mergeStrategy);
+            outputBom = CycloneDXUtils.CleanupEmptyLists(outputBom);
+#endif
+
+            // Ensure that the merged document has its own identity (new
+            // SerialNumber, Version=1, Timestamp...) and that its Tools
+            // collection records the library and program that produced it.
+#if NET8_0_OR_GREATER
+            outputBom.BomMetadataUpdate(true);
+            outputBom.BomMetadataReferThisToolkit();
+#else
             outputBom.Version = 1;
             outputBom.SerialNumber = "urn:uuid:" + System.Guid.NewGuid().ToString();
             if (outputBom.Metadata == null)
@@ -115,6 +147,7 @@ public static async Task<int> Merge(MergeCommandOptions options)
             {
                 outputBom.Metadata.Timestamp = DateTime.Now;
             }
+#endif
 
             if (!outputToConsole)
             {
@@ -125,6 +158,60 @@ public static async Task<int> Merge(MergeCommandOptions options)
             return await CliUtils.OutputBomHelper(outputBom, (ConvertFormat)options.OutputFormat, options.OutputVersion, options.OutputFile).ConfigureAwait(false);
         }
 
+        /// <summary>
+        /// Combines --input-files with any filenames listed inside
+        /// --input-files-list (one per line) and --input-files-nul-list
+        /// (0x00-separated) files, deduplicating as it goes. Lets callers
+        /// exceed OS/shell command-line length or argument-count limits
+        /// when merging many BOMs, by passing a generated list file
+        /// instead of one --input-files argument per BOM.
+        /// </summary>
+        private static List<string> DetermineInputFiles(MergeCommandOptions options)
+        {
+            var inputFiles = options.InputFiles != null ? new List<string>(options.InputFiles) : new List<string>();
+
+            if (options.InputFilesList != null)
+            {
+                foreach (var oneList in options.InputFilesList)
+                {
+                    Console.WriteLine($"Adding to input file list from {oneList}");
+                    var count = 0;
+                    foreach (var line in File.ReadAllLines(oneList))
+                    {
+                        if (string.IsNullOrEmpty(line) || inputFiles.Contains(line))
+                        {
+                            continue;
+                        }
+                        inputFiles.Add(line);
+                        count++;
+                    }
+                    Console.WriteLine($"Got {count} new entries from {oneList}");
+                }
+            }
+
+            if (options.InputFilesNulList != null)
+            {
+                foreach (var oneList in options.InputFilesNulList)
+                {
+                    Console.WriteLine($"Adding to input file list from {oneList}");
+                    var count = 0;
+                    foreach (var line in File.ReadAllText(oneList).Split('\0'))
+                    {
+                        if (string.IsNullOrEmpty(line) || inputFiles.Contains(line))
+                        {
+                            continue;
+                        }
+                        inputFiles.Add(line);
+                        count++;
+                    }
+                    Console.WriteLine($"Got {count} new entries from {oneList}");
+                }
+            }
+
+            Console.WriteLine($"Determined {inputFiles.Count} input file(s) to merge");
+            return inputFiles;
+        }
+
         private static async Task<IEnumerable<Bom>> InputBoms(IEnumerable<string> inputFilenames, CycloneDXBomFormat inputFormat, bool outputToConsole)
         {
             var boms = new List<Bom>();

src/cyclonedx/Commands/MergeCommandOptions.cs

@@ -15,12 +15,15 @@
 // SPDX-License-Identifier: Apache-2.0
 // Copyright (c) OWASP Foundation. All Rights Reserved.
 using System.Collections.Generic;
+using CycloneDX.Models;
 
 namespace CycloneDX.Cli.Commands
 {
     internal class MergeCommandOptions
     {
         public IList<string> InputFiles { get; set; }
+        public IList<string> InputFilesList { get; set; }
+        public IList<string> InputFilesNulList { get; set; }
         public string OutputFile { get; set; }
         public CycloneDXBomFormat InputFormat { get; set; }
         public CycloneDXBomFormat OutputFormat { get; set; }
@@ -29,5 +32,8 @@ internal class MergeCommandOptions
         public string Group { get; set; }
         public string Name { get; set; }
         public string Version { get; set; }
+#if NET8_0_OR_GREATER
+        public ComponentConflictResolution? ComponentConflictResolution { get; set; }
+#endif
     }
 }

src/cyclonedx/Commands/RenameEntityCommand.cs

@@ -0,0 +1,103 @@
+// This file is part of CycloneDX CLI Tool
+//
+// Licensed under the Apache License, Version 2.0 (the “License”);
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an “AS IS” BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) OWASP Foundation. All Rights Reserved.
+#if NET8_0_OR_GREATER
+using System;
+using System.CommandLine;
+using System.CommandLine.NamingConventionBinder;
+using System.Diagnostics.Contracts;
+using System.Threading.Tasks;
+
+namespace CycloneDX.Cli.Commands
+{
+    internal static class RenameEntityCommand
+    {
+        internal static void Configure(RootCommand rootCommand)
+        {
+            Contract.Requires(rootCommand != null);
+            var subCommand = new Command("rename-entity", "Rename an entity identified by a \"bom-ref\" (including back-references to it) in the BOM document");
+            subCommand.Add(new Option<string>("--input-file", "Input BOM filename."));
+            subCommand.Add(new Option<string>("--output-file", "Output BOM filename, will write to stdout if no value provided."));
+            subCommand.Add(new Option<string>("--old-ref", "Old value of \"bom-ref\" entity identifier (or \"ref\" values or certain list items pointing to it)."));
+            subCommand.Add(new Option<string>("--new-ref", "New value of \"bom-ref\" entity identifier (or \"ref\" values or certain list items pointing to it)."));
+            subCommand.Add(new Option<CycloneDXBomFormat>("--input-format", "Specify input file format."));
+            subCommand.Add(new Option<CycloneDXBomFormat>("--output-format", "Specify output file format."));
+            subCommand.Handler = CommandHandler.Create<RenameEntityCommandOptions>(RenameEntity);
+            rootCommand.Add(subCommand);
+        }
+
+        public static async Task<int> RenameEntity(RenameEntityCommandOptions options)
+        {
+            Contract.Requires(options != null);
+            var outputToConsole = string.IsNullOrEmpty(options.OutputFile);
+
+            if (options.OutputFormat == CycloneDXBomFormat.autodetect)
+            {
+                options.OutputFormat = CliUtils.AutoDetectBomFormat(options.OutputFile);
+                if (options.OutputFormat == CycloneDXBomFormat.autodetect)
+                {
+                    Console.WriteLine($"Unable to auto-detect output format");
+                    return (int)ExitCode.ParameterValidationError;
+                }
+            }
+
+            Console.WriteLine($"Loading input document...");
+            if (!outputToConsole) Console.WriteLine($"Processing input file {options.InputFile}");
+            var bom = await CliUtils.InputBomHelper(options.InputFile, options.InputFormat).ConfigureAwait(false);
+
+            if (bom is null)
+            {
+                Console.WriteLine($"Empty or absent input document");
+                return (int)ExitCode.ParameterValidationError;
+            }
+
+            Console.WriteLine($"Renaming \"{options.OldRef}\" to \"{options.NewRef}\" (this can take a while)");
+            try
+            {
+                if (bom.RenameRef(options.OldRef, options.NewRef))
+                {
+                    Console.WriteLine($"Did not encounter any issues during the rename operation");
+                }
+                else
+                {
+                    Console.WriteLine($"Rename operation found nothing to do (e.g. old ref name not mentioned in the Bom document)");
+                }
+            }
+            catch (InvalidOperationException ex)
+            {
+                Console.WriteLine($"Rename operation refused: {ex.Message}");
+                return (int)ExitCode.ParameterValidationError;
+            }
+
+            // Ensure that the modified document has its own identity
+            // (new SerialNumber, Version=1, Timestamp...) and its Tools
+            // collection refers to this library and the program/tool
+            // like cyclonedx-cli which consumes it:
+            bom.BomMetadataUpdate(true);
+            bom.BomMetadataReferThisToolkit();
+
+            if (!outputToConsole)
+            {
+                Console.WriteLine("Writing output file...");
+                Console.WriteLine($"    Total {bom.Components?.Count ?? 0} components, {bom.Dependencies?.Count ?? 0} dependencies");
+            }
+
+            int res = await CliUtils.OutputBomHelper(bom, options.OutputFormat, options.OutputFile).ConfigureAwait(false);
+            return res;
+        }
+    }
+}
+#endif

src/cyclonedx/Commands/RenameEntityCommandOptions.cs

@@ -0,0 +1,31 @@
+// This file is part of CycloneDX CLI Tool
+//
+// Licensed under the Apache License, Version 2.0 (the “License”);
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an “AS IS” BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) OWASP Foundation. All Rights Reserved.
+
+#if NET8_0_OR_GREATER
+namespace CycloneDX.Cli.Commands
+{
+    internal class RenameEntityCommandOptions
+    {
+        public string InputFile { get; set; }
+        public string OutputFile { get; set; }
+        public string OldRef { get; set; }
+        public string NewRef { get; set; }
+        public CycloneDXBomFormat InputFormat { get; set; }
+        public CycloneDXBomFormat OutputFormat { get; set; }
+    }
+}
+#endif

src/cyclonedx/Program.cs

@@ -47,6 +47,9 @@ public static async Task<int> Main(string[] args)
             DiffCommand.Configure(rootCommand);
             KeyGenCommand.Configure(rootCommand);
             MergeCommand.Configure(rootCommand);
+#if NET8_0_OR_GREATER
+            RenameEntityCommand.Configure(rootCommand);
+#endif
             SignCommand.Configure(rootCommand);
             ValidateCommand.Configure(rootCommand);
             VerifyCommand.Configure(rootCommand);

tests/cyclonedx.tests/MergeTests.cs

@@ -79,6 +79,12 @@ public async Task Merge(
                 bom = Regex.Replace(bom, @"\s+serialNumber="".*?""", ""); // xml
                 bom = Regex.Replace(bom, @"\s*""timestamp"": "".*?"",\r?\n", ""); // json
                 bom = Regex.Replace(bom, @"\s+<timestamp>.*?</timestamp>", ""); // xml
+                // The tools list embeds this build's assembly names/versions
+                // (e.g. "testhost" under `dotnet test` vs. the real CLI
+                // executable otherwise), which are environment-specific --
+                // strip the whole block before snapshotting.
+                bom = Regex.Replace(bom, @"\s*""tools"":\s*\[.*?\],?", "", RegexOptions.Singleline); // json
+                bom = Regex.Replace(bom, @"\s*<tools>.*?</tools>", "", RegexOptions.Singleline); // xml
                 Snapshot.Match(bom, SnapshotNameExtension.Create(hierarchical ? "Hierarchical" : "Flat", snapshotInputFilenames, inputFormat, outputFilename, outputFormat, outputVersion));
             }
         }