Merge pull request #98 from Microsoft365DSC/dev

Release 3.1.0.1

Author: FabienTschanz

.github/workflows/Build and Test.yml

@@ -137,3 +137,57 @@ jobs:
           }
 
           Write-Host "Class-based discovery and parsing validated against $($classBasedModule.Name) ($($classBased.Count) class-based resources)."
+
+          # Regression where the engine wipes its DSC caches when a Configuration is compiled to MOF.
+          # Test against that behavior by simulating a wipe of the engine's caches and asserting that both entry points self-heal.
+          $reflection = [DSCParser.PSDSC.DscResourceService].Assembly.GetType('DSCParser.PSDSC.DscClassCacheReflection')
+          $reflection.GetMethod('ClearCache').Invoke($null, $null)
+          $reflection.GetMethod('ResetDynamicKeywords').Invoke($null, $null)
+
+          $parsed = @(ConvertTo-DSCObject -Content $config -ErrorAction Stop)
+          if ($parsed.Count -ne 1 -or $parsed[0].ResourceName -ne $resource.Name)
+          {
+              throw "ConvertTo-DSCObject did not recover after the engine DSC caches were cleared."
+          }
+
+          $resourcesAfterWipe = @(Get-DscResourceV2 -Module $classBasedModule.Name -ErrorAction Stop)
+          if ($resourcesAfterWipe.Count -ne $resources.Count)
+          {
+              throw "Get-DscResourceV2 returned $($resourcesAfterWipe.Count) resources after the engine DSC caches were cleared, expected $($resources.Count)."
+          }
+
+          Write-Host "Stale engine cache recovery validated (deterministic wipe)."
+
+          # Same regression through the real trigger with compiling a Configuration to MOF.
+          if (Get-Module -ListAvailable -Name PSDesiredStateConfiguration)
+          {
+              Import-Module PSDesiredStateConfiguration -ErrorAction Stop
+              $wipeOutput = Join-Path ([System.IO.Path]::GetTempPath()) 'dscparser-wipe'
+              # Compile in this process; even a failing compile wipes the caches.
+              try
+              {
+                  Invoke-Expression "Configuration DscParserWipeTest { Node localhost { } }`nDscParserWipeTest -OutputPath '$wipeOutput'" | Out-Null
+              }
+              catch
+              {
+                  throw "Compiling a Configuration after using DSCParser failed: $_"
+              }
+
+              $parsed = @(ConvertTo-DSCObject -Content $config -ErrorAction Stop)
+              if ($parsed.Count -ne 1 -or $parsed[0].ResourceName -ne $resource.Name)
+              {
+                  throw "ConvertTo-DSCObject did not recover after a Configuration was compiled to MOF."
+              }
+
+              $resourcesAfterCompile = @(Get-DscResourceV2 -Module $classBasedModule.Name -ErrorAction Stop)
+              if ($resourcesAfterCompile.Count -ne $resources.Count)
+              {
+                  throw "Get-DscResourceV2 returned $($resourcesAfterCompile.Count) resources after a MOF compile, expected $($resources.Count)."
+              }
+
+              Write-Host "Stale engine cache recovery validated (real MOF compile)."
+          }
+          else
+          {
+              Write-Host "::warning::PSDesiredStateConfiguration is not installed; skipping the MOF-compile recovery validation."
+          }

Modules/DSCParser/DSCParser.psd1

@@ -12,7 +12,7 @@
     RootModule = 'DSCParser.psm1'
 
     # Version number of this module.
-    ModuleVersion = '3.1.0.0'
+    ModuleVersion = '3.1.0.1'
 
     # ID used to uniquely identify this module
     GUID = 'e168239a-233d-468d-9025-d6dfc0e4e2b6'
@@ -110,7 +110,7 @@
             IconUri = 'https://github.com/Microsoft365DSC/DSCParser/blob/master/Images/DSCParser.png?raw=true'
 
             # ReleaseNotes of this module
-            ReleaseNotes = '* Refactored the module and fixed an issue with wildcard pattern matching in the Get-DscResourceV2 cmdlet.'
+            ReleaseNotes = '* Fixed a regression where compiling a Configuration to MOF cleared the engine DSC caches underneath DSCParser, after which Get-DscResourceV2 returned no resources and ConvertTo-DSCObject failed. The keyword registry now detects the wipe and re-imports automatically.'
         } # End of PSData hashtable
 
     } # End of PrivateData hashtable

Modules/DSCParser/Modules/DSCParser.psm1

@@ -155,9 +155,11 @@ function ConvertTo-DSCObject
         }
 
         # Buffer diagnostics and emit them after the call, so Write-Warning runs on the pipeline
-        # thread and honours the caller's $WarningPreference. The sink is a process-wide static.
+        # thread and honours the caller's $WarningPreference. The sinks are process-wide statics.
         $parserWarnings = [System.Collections.Generic.List[System.String]]::new()
-        [DSCParser.CSharp.DscParser]::WarningSink = [System.Action[System.String]] { param($Message) $parserWarnings.Add($Message) }
+        $warningSink = [System.Action[System.String]] { param($Message) $parserWarnings.Add($Message) }
+        [DSCParser.CSharp.DscParser]::WarningSink = $warningSink
+        [DSCParser.PSDSC.DscResourceService]::WarningSink = $warningSink
 
         try
         {
@@ -174,6 +176,7 @@ function ConvertTo-DSCObject
         finally
         {
             [DSCParser.CSharp.DscParser]::WarningSink = $null
+            [DSCParser.PSDSC.DscResourceService]::WarningSink = $null
             foreach ($parserWarning in $parserWarnings)
             {
                 Write-Warning -Message $parserWarning

src/DSCParser.CSharp/DscParser.cs

@@ -133,12 +133,25 @@ public static List<DscResourceInstance> ConvertToDscObject(string? path = null,
             dscContent = RemoveModuleVersionInfo(dscContent, modulesToRemoveVersionFrom);
 
             List<ModuleReference> modulesToLoad = GetModulesToLoad(dscContent);
-            RegisterKeywords(modulesToLoad, errorPrefix);
 
             // Parse the DSC configuration using PowerShell AST instead of with "Import-DscResource"
-            // to avoid loading modules from disk, which may be very slow with many class-based resources
-            ScriptBlockAst ast = Parser.ParseInput(
-                RemoveImportDscResourceStatements(dscContent), out Token[] tokens, out ParseError[] parseErrors);
+            // to avoid loading modules from disk, which may be very slow with many class-based resources.
+            // The DynamicKeyword table only exists for the duration of the parse because leaving it populated
+            // breaks the engine's own Configuration-to-MOF compilation afterwards.
+            ScriptBlockAst ast;
+            Token[] tokens;
+            ParseError[] parseErrors;
+            try
+            {
+                RegisterKeywords(modulesToLoad, errorPrefix);
+                DscKeywordRegistry.MaterializeKeywordTable();
+                ast = Parser.ParseInput(
+                    RemoveImportDscResourceStatements(dscContent), out tokens, out parseErrors);
+            }
+            finally
+            {
+                DscKeywordRegistry.ClearKeywordTable();
+            }
 
             // Find the Configuration definition
             ConfigurationDefinitionAst? configAst = ast.Find(a => a is ConfigurationDefinitionAst, false) as ConfigurationDefinitionAst;

src/DSCParser.PSDSC/DscClassCacheReflection.cs

@@ -49,6 +49,8 @@ internal static class DscClassCacheReflection
         private static readonly MethodInfo? ResetDynamicKeywordsMethod =
             typeof(DynamicKeyword).GetMethod("Reset", BindingFlags.Public | BindingFlags.Static);
 
+        public static bool IsDscClassCacheAvailable => CacheType is not null;
+
         public static void LoadDefaultCimKeywords()
         {
             try
@@ -106,6 +108,25 @@ public static List<CimClass> GetCachedClassByFileName(string fileName)
             return GetFileDefiningClassMethod?.Invoke(null, [className]) as List<string>;
         }
 
+        /// <summary>
+        /// Whether the engine's class cache currently holds a definition for the given class.
+        /// Returns false when the lookup method is unavailable or throws. Staleness probes must
+        /// pair it with <see cref="IsDscClassCacheAvailable"/> to avoid misreading an unsupported
+        /// host as a wiped cache.
+        /// </summary>
+        public static bool HasCachedClass(string className)
+        {
+            try
+            {
+                return GetFileDefiningClassMethod is not null
+                    && GetFileDefiningClass(className) is { Count: > 0 };
+            }
+            catch
+            {
+                return false;
+            }
+        }
+
         public static void ClearCache()
         {
             try
@@ -123,6 +144,7 @@ public static void ResetDynamicKeywords()
             _ = ResetDynamicKeywordsMethod?.Invoke(null, null);
         }
 
+
         private static Dictionary<string, ScriptBlock> NewFunctionTable()
         {
             return new Dictionary<string, ScriptBlock>(StringComparer.OrdinalIgnoreCase);

src/DSCParser.PSDSC/DscKeywordRegistry.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Management.Automation;
+using System.Management.Automation.Language;
 
 namespace DSCParser.PSDSC
 {
@@ -14,14 +15,57 @@ namespace DSCParser.PSDSC
     /// PowerShell rebuilds this registration from scratch every time it parses a configuration that
     /// contains an Import-DSCResource statement, probing the file system for a schema file per
     /// resource. On a module the size of Microsoft365DSC that dominates the cost of parsing. Keeping
-    /// the registration for the lifetime of the process lets callers strip the Import-DSCResource
-    /// statement and skip that work entirely.
+    /// the registration alive lets callers strip the Import-DSCResource statement and skip that
+    /// work entirely.
     /// </remarks>
     public static class DscKeywordRegistry
     {
-        private static readonly HashSet<string> ImportedModules = new(StringComparer.OrdinalIgnoreCase);
+        [ThreadStatic]
+        private static HashSet<string>? t_importedModules;
 
-        private static bool _defaultKeywordsLoaded;
+        private static HashSet<string> ImportedModules => t_importedModules ??= new(StringComparer.OrdinalIgnoreCase);
+
+        [ThreadStatic]
+        private static bool t_defaultKeywordsLoaded;
+
+        [ThreadStatic]
+        private static bool t_engineUnsupported;
+
+        [ThreadStatic]
+        private static List<DynamicKeyword>? t_defaultTableKeywords;
+
+        [ThreadStatic]
+        private static bool t_staleWarningIssued;
+
+        [ThreadStatic]
+        private static int t_expectedCachedKeywordCount;
+
+        // Registered into the class cache by LoadDefaultCimKeywords and dies with it, so its
+        // absence while the bookkeeping claims otherwise means the engine wiped the cache.
+        private const string ClassCacheSentinel = "OMI_ConfigurationDocument";
+
+        // Lives in the DynamicKeyword table (not the class cache) and is re-added by
+        // LoadDefaultCimKeywords, so it tells whether the table currently holds the defaults.
+        private const string NodeKeyword = "Node";
+
+        private static bool EngineStateIsFresh
+        {
+            get
+            {
+                if (!DscClassCacheReflection.HasCachedClass(ClassCacheSentinel))
+                {
+                    return false;
+                }
+
+                return t_expectedCachedKeywordCount == 0
+                    || CurrentCachedKeywordCount() >= t_expectedCachedKeywordCount;
+            }
+        }
+
+        private static int CurrentCachedKeywordCount()
+        {
+            return DscClassCacheReflection.GetCachedKeywords()?.Count() ?? 0;
+        }
 
         /// <summary>
         /// Registers the resources of the supplied modules, skipping modules already registered.
@@ -47,6 +91,8 @@ public static void ImportModules(IEnumerable<PSModuleInfo> modules)
                 // A later request that names no version is satisfied by any registered version.
                 _ = ImportedModules.Add(GetModuleKey(module.Name, null));
             }
+
+            t_expectedCachedKeywordCount = CurrentCachedKeywordCount();
         }
 
         /// <summary>
@@ -62,6 +108,9 @@ public static bool EnsureRegistered(string moduleName, Version? version)
                 return false;
             }
 
+            // Heals a wiped engine cache before the fast path below can return a stale answer.
+            EnsureDefaultKeywordsLoaded();
+
             if (ImportedModules.Contains(GetModuleKey(moduleName, version)))
             {
                 return true;
@@ -79,25 +128,132 @@ public static bool EnsureRegistered(string moduleName, Version? version)
         }
 
         /// <summary>
-        /// Drops every registered keyword and the underlying class cache.
+        /// Drops every registered keyword and the underlying class cache on the current thread.
         /// </summary>
         public static void Reset()
         {
             ImportedModules.Clear();
-            _defaultKeywordsLoaded = false;
+            t_defaultKeywordsLoaded = false;
+            t_engineUnsupported = false;
+            t_defaultTableKeywords = null;
+            t_expectedCachedKeywordCount = 0;
             DscClassCacheReflection.ResetDynamicKeywords();
             DscClassCacheReflection.ClearCache();
         }
 
         internal static void EnsureDefaultKeywordsLoaded()
         {
-            if (_defaultKeywordsLoaded)
+            _ = HandleExternalCacheReset();
+
+            if (t_defaultKeywordsLoaded)
             {
                 return;
             }
 
             DscClassCacheReflection.LoadDefaultCimKeywords();
-            _defaultKeywordsLoaded = true;
+            t_defaultKeywordsLoaded = true;
+            t_engineUnsupported = !EngineStateIsFresh;
+
+            List<DynamicKeyword> snapshot = [];
+            IEnumerable<string> defaultNames =
+                (DscClassCacheReflection.GetCachedKeywords()?.Select(k => k.Keyword) ?? [])
+                .Concat([NodeKeyword, "Import-DscResource"]);
+            foreach (string name in defaultNames.Distinct(StringComparer.OrdinalIgnoreCase))
+            {
+                if (DynamicKeyword.GetKeyword(name) is { } keyword)
+                {
+                    snapshot.Add(keyword);
+                }
+            }
+
+            t_defaultTableKeywords = snapshot;
+            t_expectedCachedKeywordCount = CurrentCachedKeywordCount();
+        }
+
+        /// <summary>
+        /// The DSC engine clears its internal class cache and DynamicKeyword table whenever a
+        /// Configuration block is compiled to MOF in this process. This means that this class
+        /// then no longer matches engine state, resulting in no resource results although
+        /// shown as imported.
+        /// </summary>
+        public static bool HandleExternalCacheReset()
+        {
+            if (t_engineUnsupported || !DscClassCacheReflection.IsDscClassCacheAvailable)
+            {
+                return false;
+            }
+
+            if (ImportedModules.Count == 0 && !t_defaultKeywordsLoaded)
+            {
+                return false;
+            }
+
+            if (EngineStateIsFresh)
+            {
+                return false;
+            }
+
+            if (!t_staleWarningIssued)
+            {
+                t_staleWarningIssued = true;
+                DscResourceService.ReportWarning(
+                    "The PowerShell engine cleared its internal DSC caches (typically caused by compiling a "
+                    + "Configuration to MOF). DSCParser re-imported its DSC resource keywords automatically.");
+            }
+
+            Reset();
+            return true;
+        }
+
+        /// <summary>
+        /// Fills the engine's DynamicKeyword table with the default CIM keywords (including Node)
+        /// and every keyword the class cache holds, so a configuration can be parsed without any
+        /// Import-DscResource statement. Pair with <see cref="ClearKeywordTable"/> once parsing is
+        /// done.
+        /// </summary>
+        public static void MaterializeKeywordTable()
+        {
+            EnsureDefaultKeywordsLoaded();
+
+            List<DynamicKeyword>? cachedKeywords = DscClassCacheReflection.GetCachedKeywords()?.ToList();
+
+            if (!DynamicKeyword.ContainsKeyword(NodeKeyword))
+            {
+                if (t_defaultTableKeywords is { Count: > 0 } defaults)
+                {
+                    foreach (DynamicKeyword keyword in defaults)
+                    {
+                        if (!DynamicKeyword.ContainsKeyword(keyword.Keyword))
+                        {
+                            DynamicKeyword.AddKeyword(keyword);
+                        }
+                    }
+                }
+                else if (ImportedModules.Count == 0)
+                {
+                    DscClassCacheReflection.LoadDefaultCimKeywords();
+                }
+            }
+
+            if (cachedKeywords is not null)
+            {
+                foreach (DynamicKeyword keyword in cachedKeywords)
+                {
+                    if (!DynamicKeyword.ContainsKeyword(keyword.Keyword))
+                    {
+                        DynamicKeyword.AddKeyword(keyword);
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Empties the engine's DynamicKeyword table while keeping the class cache. Every public
+        /// operation must end with this.
+        /// </summary>
+        public static void ClearKeywordTable()
+        {
+            DscClassCacheReflection.ResetDynamicKeywords();
         }
 
         private static string GetModuleKey(string moduleName, Version? version)