Merge alternate release changes

build.proj

@@ -2,6 +2,7 @@
 
   <ItemGroup >
     <ProjectReference Include="src/dirs.proj" />
+    <ProjectReference Include="eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj" />
     <ProjectReference Include="src/tests/dirs.proj" Condition="'$(SkipTests)' != 'true'" />
   </ItemGroup>
 

diagnostics.yml

@@ -371,6 +371,8 @@ extends:
                 }
               displayName: 'Setup layout for packaging'
 
+            - template: /eng/common/core-templates/steps/enable-internal-sources.yml@self
+
             # Create nuget packages, sign binaries and publish to blob feed
             - script: $(Build.SourcesDirectory)\eng\ci-prepare-artifacts.cmd /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
               displayName: Package, Sign, and Publish
@@ -403,6 +405,7 @@ extends:
         isOfficialBuild: ${{ variables['isOfficialBuild'] }}
         repoAllowList: https://github.com/dotnet/diagnostics https://dev.azure.com/dnceng/internal/_git/dotnet-diagnostics
         productAllowList: diagnostics dotnet-diagnostics
+        skipFileList: MergedManifest.xml
         ${{ if eq(variables['isOfficialBuild'], 'true') }}:
           dependsOn: [ publish_using_darc ]
         ${{ else }}:

eng/Version.Details.props

@@ -29,9 +29,9 @@ This file should be imported by eng/Versions.props
     <runtimeosxx64MicrosoftDotNetCdacTransportPackageVersion>11.0.0-preview.6.26305.101</runtimeosxx64MicrosoftDotNetCdacTransportPackageVersion>
     <runtimewinarm64MicrosoftDotNetCdacTransportPackageVersion>11.0.0-preview.6.26305.101</runtimewinarm64MicrosoftDotNetCdacTransportPackageVersion>
     <runtimewinx64MicrosoftDotNetCdacTransportPackageVersion>11.0.0-preview.6.26305.101</runtimewinx64MicrosoftDotNetCdacTransportPackageVersion>
-    <!-- microsoft-clrmd dependencies -->
-    <MicrosoftDiagnosticsRuntimePackageVersion>4.0.727102</MicrosoftDiagnosticsRuntimePackageVersion>
-    <MicrosoftDiagnosticsRuntimeUtilitiesPackageVersion>4.0.727102</MicrosoftDiagnosticsRuntimeUtilitiesPackageVersion>
+    <!-- Microsoft-clrmd dependencies -->
+    <MicrosoftDiagnosticsRuntimePackageVersion>4.0.743101</MicrosoftDiagnosticsRuntimePackageVersion>
+    <MicrosoftDiagnosticsRuntimeUtilitiesPackageVersion>4.0.743101</MicrosoftDiagnosticsRuntimeUtilitiesPackageVersion>
   </PropertyGroup>
   <!--Property group for alternate package version names-->
   <PropertyGroup>
@@ -59,7 +59,7 @@ This file should be imported by eng/Versions.props
     <runtimeosxx64MicrosoftDotNetCdacTransportVersion>$(runtimeosxx64MicrosoftDotNetCdacTransportPackageVersion)</runtimeosxx64MicrosoftDotNetCdacTransportVersion>
     <runtimewinarm64MicrosoftDotNetCdacTransportVersion>$(runtimewinarm64MicrosoftDotNetCdacTransportPackageVersion)</runtimewinarm64MicrosoftDotNetCdacTransportVersion>
     <runtimewinx64MicrosoftDotNetCdacTransportVersion>$(runtimewinx64MicrosoftDotNetCdacTransportPackageVersion)</runtimewinx64MicrosoftDotNetCdacTransportVersion>
-    <!-- microsoft-clrmd dependencies -->
+    <!-- Microsoft-clrmd dependencies -->
     <MicrosoftDiagnosticsRuntimeVersion>$(MicrosoftDiagnosticsRuntimePackageVersion)</MicrosoftDiagnosticsRuntimeVersion>
     <MicrosoftDiagnosticsRuntimeUtilitiesVersion>$(MicrosoftDiagnosticsRuntimeUtilitiesPackageVersion)</MicrosoftDiagnosticsRuntimeUtilitiesVersion>
   </PropertyGroup>

eng/Version.Details.xml

@@ -1,13 +1,13 @@
 <Dependencies>
   <Source Uri="https://github.com/dotnet/dotnet" Mapping="diagnostics" Sha="025cac314d7303a35dcc77dd4ef2b209ab98c641" BarId="317483" />
   <ProductDependencies>
-    <Dependency Name="Microsoft.Diagnostics.Runtime" Version="4.0.727102">
-      <Uri>https://github.com/microsoft/clrmd</Uri>
-      <Sha>86762cb7283a30fbb99e87008114be6be4a4f4dd</Sha>
+    <Dependency Name="Microsoft.Diagnostics.Runtime" Version="4.0.743101">
+      <Uri>https://dev.azure.com/dnceng/internal/_git/Microsoft-clrmd</Uri>
+      <Sha>04c417f99d7724630eef6c3f737f7b7356b40a7a</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.Diagnostics.Runtime.Utilities" Version="4.0.727102">
-      <Uri>https://github.com/microsoft/clrmd</Uri>
-      <Sha>86762cb7283a30fbb99e87008114be6be4a4f4dd</Sha>
+    <Dependency Name="Microsoft.Diagnostics.Runtime.Utilities" Version="4.0.743101">
+      <Uri>https://dev.azure.com/dnceng/internal/_git/Microsoft-clrmd</Uri>
+      <Sha>04c417f99d7724630eef6c3f737f7b7356b40a7a</Sha>
     </Dependency>
     <Dependency Name="System.CommandLine" Version="2.0.0-beta5.25210.1">
       <Uri>https://github.com/dotnet/command-line-api</Uri>

eng/pipelines/build.yml

@@ -184,6 +184,8 @@ jobs:
             targetPath: '$(Build.SourcesDirectory)/artifacts'
             checkDownloadedFiles: true
 
+      - template: /eng/common/core-templates/steps/enable-internal-sources.yml@self
+
       - script: $(_buildScript)
           -ci
           -binaryLog

eng/pipelines/prepare-release.yml

@@ -8,6 +8,9 @@ parameters:
 - name: productAllowList
   type: string
   default: ''
+- name: skipFileList
+  type: string
+  default: ''
 - name: dependsOn
   type: object
   default: [ publish_using_darc ]
@@ -60,7 +63,8 @@ stages:
               -BarBuildId "$(BARBuildId)" `
               -ReleaseVersion "$(Build.Repository.Name)_$(Build.BuildNumber)" `
               -DownloadTargetPath "$(System.ArtifactsDirectory)\ReleaseTarget" `
-              -AzdoToken $azdoToken
+              -AzdoToken $azdoToken `
+              -IncludeNonShipping
           workingDirectory: '$(Build.Repository.LocalPath)'
       - task: AzureCLI@2
         displayName: 'Manifest generation and asset publishing'
@@ -84,4 +88,5 @@ stages:
                 --account-name "$(dotnet-diagnostics-storage-accountname)" `
                 --client-id "$env:servicePrincipalId" `
                 --container-name "$(dotnet-diagnostics-container-name)" `
+                --skip-files ${{ parameters.skipFileList }} `
                 -v True

eng/release/DiagnosticsReleaseTool/Config.cs

@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
+using System.Collections.Frozen;
 using System.IO;
 
 namespace DiagnosticsReleaseTool.Impl
@@ -17,6 +19,7 @@ internal sealed class Config
         public string ContainerName { get; }
         public string[] ReleaseProductAllowList { get; }
         public string[] ReleaseRepoAllowList { get; }
+        public FrozenSet<string> SkipFiles { get; }
 
         public Config(
             FileInfo toolManifest,
@@ -28,7 +31,8 @@ public Config(
             string[] releaseRepoAllowList,
             string accountName,
             string clientId,
-            string containerName)
+            string containerName,
+            string[] skipFiles)
         {
             ToolManifest = toolManifest;
             ShouldVerifyManifest = verifyToolManifest;
@@ -40,6 +44,7 @@ public Config(
             ContainerName = containerName;
             ReleaseProductAllowList = releaseProductAllowList;
             ReleaseRepoAllowList = releaseRepoAllowList;
+            SkipFiles = (skipFiles ?? []).ToFrozenSet(StringComparer.OrdinalIgnoreCase);
         }
     }
 }

eng/release/DiagnosticsReleaseTool/Core/FileReleaseData.cs

@@ -5,18 +5,20 @@ namespace ReleaseTool.Core
 {
     public class FileReleaseData
     {
-        public FileReleaseData(FileMapping fileMap, FileMetadata fileMetadata)
-            : this(fileMap, fileMetadata, null) { }
+        public FileReleaseData(FileMapping fileMap, FileMetadata fileMetadata, bool isAssetForPublicRelease = true)
+            : this(fileMap, fileMetadata, isAssetForPublicRelease, null) { }
 
-        private FileReleaseData(FileMapping fileMap, FileMetadata fileMetadata, string publishUri)
+        private FileReleaseData(FileMapping fileMap, FileMetadata fileMetadata, bool isAssetForPublicRelease, string publishUri)
         {
             FileMap = fileMap;
             FileMetadata = fileMetadata;
+            IsAssetForPublicRelease = isAssetForPublicRelease;
             PublishUri = publishUri;
         }
 
         public FileMapping FileMap { get; }
         public FileMetadata FileMetadata { get; }
+        public bool IsAssetForPublicRelease { get; }
         public string PublishUri { get; internal set; }
     }
 }

eng/release/DiagnosticsReleaseTool/Core/Release.cs

@@ -13,24 +13,26 @@ namespace ReleaseTool.Core
 {
     public class Release : IDisposable
     {
-        // TODO: there might be a need to expose this for multiple product roots.
-        private readonly DirectoryInfo _productBuildPath;
+        private readonly DirectoryInfo _publicReleasePath;
+        private readonly DirectoryInfo _internalOnlyReleasePath;
         private readonly List<ILayoutWorker> _layoutWorkers;
         private readonly List<IReleaseVerifier> _verifiers;
         private readonly IPublisher _publisher;
         private readonly IManifestGenerator _manifestGenerator;
         private readonly string _manifestSavePath;
+        private readonly Func<FileInfo, bool> _shouldSkipFile;
 
         private readonly List<FileReleaseData> _filesToRelease;
         private ILogger _logger;
 
-        public Release(DirectoryInfo productBuildPath,
+        public Release(DirectoryInfo publicReleasePath, DirectoryInfo internalOnlyReleasePath,
             List<ILayoutWorker> layoutWorkers, List<IReleaseVerifier> verifiers,
-            IPublisher publisher, IManifestGenerator manifestGenerator, string manifestSavePath)
+            IPublisher publisher, IManifestGenerator manifestGenerator, string manifestSavePath,
+            Func<FileInfo, bool> shouldSkipFile = null)
         {
-            if (productBuildPath is null)
+            if (publicReleasePath is null)
             {
-                throw new ArgumentException("Product build path can't be empty or null.");
+                throw new ArgumentException("Public release path can't be empty or null.");
             }
 
             if (layoutWorkers is null)
@@ -48,12 +50,14 @@ public Release(DirectoryInfo productBuildPath,
                 throw new ArgumentException($"{nameof(manifestGenerator)} can't be null.");
             }
 
-            _productBuildPath = productBuildPath;
+            _publicReleasePath = publicReleasePath;
+            _internalOnlyReleasePath = internalOnlyReleasePath;
             _layoutWorkers = layoutWorkers;
             _verifiers = verifiers;
             _publisher = publisher;
             _manifestGenerator = manifestGenerator;
             _manifestSavePath = manifestSavePath ?? Path.Join(Path.GetTempPath(), Path.GetRandomFileName(), "releaseManifest");
+            _shouldSkipFile = shouldSkipFile ?? (_ => false);
             _filesToRelease = new List<FileReleaseData>();
             _logger = null;
             // TODO: Validate drop to publish exists.
@@ -71,15 +75,34 @@ public async Task<int> RunAsync(CancellationToken ct)
             int unusedFiles;
             try
             {
-                unusedFiles = await LayoutFilesAsync(ct);
+                HashSet<string> relativePublishPathsUsed = new();
+                unusedFiles = await LayoutFilesAsync(
+                    _publicReleasePath,
+                    isAssetForPublicRelease: true,
+                    relativePublishPathsUsed,
+                    ct);
 
-                // TODO: Implement switch to ignore files that are not used as option.
                 if (unusedFiles != 0)
                 {
-                    _logger.LogError("{UnusedFiles} files were not handled for release.", unusedFiles);
+                    _logger.LogError("{UnusedFiles} public release files were not handled for release.", unusedFiles);
                     return unusedFiles;
                 }
 
+                if (_internalOnlyReleasePath is not null)
+                {
+                    unusedFiles = await LayoutFilesAsync(
+                        _internalOnlyReleasePath,
+                        isAssetForPublicRelease: false,
+                        relativePublishPathsUsed,
+                        ct);
+
+                    if (unusedFiles != 0)
+                    {
+                        _logger.LogError("{UnusedFiles} internal-only release files were not handled for release.", unusedFiles);
+                        return unusedFiles;
+                    }
+                }
+
                 // TODO: Verification
 
                 unusedFiles = await PublishFiles(ct);
@@ -181,21 +204,32 @@ private async Task<int> PublishFiles(CancellationToken ct)
             return unpublishedFiles;
         }
 
-        private async Task<int> LayoutFilesAsync(CancellationToken ct)
+        private async Task<int> LayoutFilesAsync(
+            DirectoryInfo buildPath,
+            bool isAssetForPublicRelease,
+            HashSet<string> relativePublishPathsUsed,
+            CancellationToken ct)
         {
             int unhandledFiles = 0;
-            HashSet<string> relativePublishPathsUsed = new();
 
             using IDisposable scope = _logger.BeginScope("Laying out files");
 
-            _logger.LogInformation("Laying out files from {_productBuildPath}", _productBuildPath.FullName);
+            _logger.LogInformation("Laying out {ReleaseType} files from {BuildPath}",
+                isAssetForPublicRelease ? "public release" : "internal-only release",
+                buildPath.FullName);
 
             // TODO: Make this parallel using Task.Run + semaphore to batch process files. Need to make collections concurrent or have single
             //       queue to aggregate results.
             // TODO: The file enumeration should have the possibility to inject a custom enumerator. Useful in case there's only subsets of files.
             //       For example, shipping only files.
-            foreach (FileInfo file in _productBuildPath.EnumerateFiles("*", SearchOption.AllDirectories))
+            foreach (FileInfo file in buildPath.EnumerateFiles("*", SearchOption.AllDirectories))
             {
+                if (_shouldSkipFile(file))
+                {
+                    _logger.LogTrace("Skipping file {File}", file);
+                    continue;
+                }
+
                 bool isProcessed = false;
                 foreach (ILayoutWorker worker in _layoutWorkers)
                 {
@@ -236,8 +270,13 @@ private async Task<int> LayoutFilesAsync(CancellationToken ct)
                                 return -1;
                             }
                             relativePublishPathsUsed.Add(dstPath);
-                            _logger.LogTrace("{SrcPath} -> {DstPath} [{FileMetadata}]", srcPath, dstPath, fileMetadata);
-                            _filesToRelease.Add(new FileReleaseData(fileMap, fileMetadata));
+                            _logger.LogTrace(
+                                "{SrcPath} -> {DstPath} [{FileMetadata}] [IsAssetForPublicRelease: {IsAssetForPublicRelease}]",
+                                srcPath,
+                                dstPath,
+                                fileMetadata,
+                                isAssetForPublicRelease);
+                            _filesToRelease.Add(new FileReleaseData(fileMap, fileMetadata, isAssetForPublicRelease));
                         }
                     }
                 }

eng/release/DiagnosticsReleaseTool/DarcHelpers.cs

@@ -77,7 +77,7 @@ internal ReleaseMetadata GetDropMetadataForSingleRepoVariants(IEnumerable<string
             }
         }
 
-        internal DirectoryInfo GetShippingDirectoryForSingleProjectVariants(IEnumerable<string> projectNames)
+        internal (DirectoryInfo ShippingDirectory, DirectoryInfo NonShippingDirectory) GetShippingDirectoryForSingleProjectVariants(IEnumerable<string> projectNames)
         {
             using (Stream darcManifest = File.OpenRead(ReleaseFilePath))
             using (JsonDocument jsonDoc = JsonDocument.Parse(darcManifest))
@@ -95,7 +95,19 @@ internal DirectoryInfo GetShippingDirectoryForSingleProjectVariants(IEnumerable<
                         $"There's {matchingProducts.Count()} products that could be released in the release manifest. Expected 1");
                 }
 
-                return new DirectoryInfo(matchingProducts.First().GetProperty("fileshare").GetString());
+                DirectoryInfo shippingDirectory = new(matchingProducts.First().GetProperty("fileshare").GetString());
+
+                if (!shippingDirectory.Name.Equals("shipping", StringComparison.OrdinalIgnoreCase) ||
+                    shippingDirectory.Parent is null)
+                {
+                    throw new InvalidOperationException(
+                        $"Shipping directory '{shippingDirectory.FullName}' does not end in the expected 'shipping' directory.");
+                }
+
+                string nonShippingPath = Path.Combine(shippingDirectory.Parent.FullName, "nonshipping");
+                DirectoryInfo nonShippingDirectory = Directory.Exists(nonShippingPath) ? new(nonShippingPath) : null;
+
+                return (shippingDirectory, nonShippingDirectory);
             }
         }
     }

eng/release/DiagnosticsReleaseTool/DiagnosticsManifestGenerator.cs

@@ -81,6 +81,7 @@ private static void WriteBundledTools(Utf8JsonWriter writer, IEnumerable<FileRel
                 writer.WriteString("PublishRelativePath", fileToRelease.FileMap.RelativeOutputPath);
                 writer.WriteString("PublishedPath", fileToRelease.PublishUri);
                 writer.WriteString("Sha512", fileToRelease.FileMetadata.Sha512);
+                writer.WriteBoolean("IsAssetForPublicRelease", fileToRelease.IsAssetForPublicRelease);
                 writer.WriteEndObject();
             }
 
@@ -102,6 +103,7 @@ private static void WriteNugetShippingPackages(
                 writer.WriteString("PublishRelativePath", fileToRelease.FileMap.RelativeOutputPath);
                 writer.WriteString("PublishedPath", fileToRelease.PublishUri);
                 writer.WriteString("Sha512", fileToRelease.FileMetadata.Sha512);
+                writer.WriteBoolean("IsAssetForPublicRelease", fileToRelease.IsAssetForPublicRelease);
                 writer.WriteEndObject();
             }
 

eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs

@@ -30,6 +30,8 @@ public static Command PrepareRelease()
             {
                 // Inputs
                 InputDropPathOption, ToolManifestPathOption, ReleaseNameOption, ReleaseProductAllowListOption, ReleaseRepoAllowListOption,
+                // Filters
+                SkipFilesOption,
                 // Toggles
                 ToolManifestVerificationOption, DiagnosticLoggingOption,
                 // Outputs
@@ -49,7 +51,8 @@ public static Command PrepareRelease()
                     releaseRepoAllowList: parseResult.GetValue(ReleaseRepoAllowListOption),
                     accountName: parseResult.GetValue(AzureStorageAccountNameOption),
                     clientId: parseResult.GetValue(AzureStorageAccountKeyOption),
-                    containerName: parseResult.GetValue(AzureStorageContainerNameOption)
+                    containerName: parseResult.GetValue(AzureStorageContainerNameOption),
+                    skipFiles: parseResult.GetValue(SkipFilesOption)
                 );
 
                 bool isVerbose = parseResult.GetValue(DiagnosticLoggingOption);
@@ -110,6 +113,14 @@ public static Command PrepareRelease()
                 Required = true
             };
 
+        private static Option<string[]> SkipFilesOption =
+            new("--skip-files")
+            {
+                Arity = ArgumentArity.ZeroOrMore,
+                AllowMultipleArgumentsPerToken = true,
+                Description = "List of file names to skip during release processing."
+            };
+
         private static readonly Option<DirectoryInfo> StagingPathOption = InitStagingPath();
 
         private static Option<DirectoryInfo> InitStagingPath()