From d08ac37bbe468a5af80709fda1db4e14b5498aa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sat, 21 Dec 2024 22:31:20 +0100 Subject: [PATCH 1/2] input: parse recipes after all plugins were loaded Plugins might define new recipe/class keywords. Because recipes might use classes from other layers, there could be a difference of the additional keywords when the different layers are parsed. To prevent inconsistencies, defer the parsing of recipes and classes until after all plugins were loaded. Previously Bob could crash when resolving the inherited classes. --- pym/bob/input.py | 63 ++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/pym/bob/input.py b/pym/bob/input.py index 4c83d427..4d205a30 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -3524,7 +3524,35 @@ def __parse(self, envOverrides, platform, recipesRoot=""): os.path.join(os.path.expanduser("~"), '.config')), 'bob', 'default.yaml')) # Begin with root layer - self.__parseLayer(LayerSpec(""), "9999", recipesRoot, None) + allLayers = self.__parseLayer(LayerSpec(""), "9999", recipesRoot, None) + + # Parse all recipes and classes of all layers. Need to be done last + # because only by now we have loaded all plugins. + for layer, rootDir, scriptLanguage in allLayers: + classesDir = os.path.join(rootDir, 'classes') + for root, dirnames, filenames in os.walk(classesDir): + for path in fnmatch.filter(filenames, "[!.]*.yaml"): + try: + [r] = Recipe.loadFromFile(self, layer, classesDir, + os.path.relpath(os.path.join(root, path), classesDir), + self.__properties, self.__classSchema, False) + self.__addClass(r) + except ParseError as e: + e.pushFrame(path) + raise + + recipesDir = os.path.join(rootDir, 'recipes') + for root, dirnames, filenames in os.walk(recipesDir): + for path in fnmatch.filter(filenames, "[!.]*.yaml"): + try: + recipes = Recipe.loadFromFile(self, layer, recipesDir, + os.path.relpath(os.path.join(root, path), recipesDir), + self.__properties, self.__recipeSchema, True, scriptLanguage) + for r in recipes: + self.__addRecipe(r) + except ParseError as e: + e.pushFrame(path) + raise # Out-of-tree builds may have a dedicated default.yaml if recipesRoot: @@ -3614,7 +3642,7 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer): layer = upperLayer + "/" + layer if layer in self.__layers: - return + return [] if managedLayers: # SCM backed layers are in build dir, regular layers are in @@ -3656,8 +3684,9 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer): # First parse any sub-layers. Their settings have a lower precedence # and may be overwritten by higher layers. + allLayers = [] for l in config.get("layers", []): - self.__parseLayer(l, maxVer, recipesRoot, layer) + allLayers.extend(self.__parseLayer(l, maxVer, recipesRoot, layer)) # Load plugins and re-create schemas as new keys may have been added self.__loadPlugins(rootDir, layer, config.get("plugins", [])) @@ -3671,32 +3700,8 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer): setColorMode(self._colorModeConfig or self.__uiConfig.get('color', 'auto')) setParallelTUIThreshold(self.__uiConfig.get('parallelTUIThreshold', 16)) - # finally parse recipes - classesDir = os.path.join(rootDir, 'classes') - for root, dirnames, filenames in os.walk(classesDir): - for path in fnmatch.filter(filenames, "[!.]*.yaml"): - try: - [r] = Recipe.loadFromFile(self, layer, classesDir, - os.path.relpath(os.path.join(root, path), classesDir), - self.__properties, self.__classSchema, False) - self.__addClass(r) - except ParseError as e: - e.pushFrame(path) - raise - - scriptLanguage = config["scriptLanguage"] - recipesDir = os.path.join(rootDir, 'recipes') - for root, dirnames, filenames in os.walk(recipesDir): - for path in fnmatch.filter(filenames, "[!.]*.yaml"): - try: - recipes = Recipe.loadFromFile(self, layer, recipesDir, - os.path.relpath(os.path.join(root, path), recipesDir), - self.__properties, self.__recipeSchema, True, scriptLanguage) - for r in recipes: - self.__addRecipe(r) - except ParseError as e: - e.pushFrame(path) - raise + allLayers.append((layer, rootDir, config["scriptLanguage"])) + return allLayers def __parseUserConfig(self, fileName): cfg = self.loadYaml(fileName, self.__userConfigSchema) From bc8b9d8c252cfd17a1c19162f764ba605742fc2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sat, 21 Dec 2024 22:38:32 +0100 Subject: [PATCH 2/2] doc: clarify the scope of defaulLanguage in config.yaml --- doc/manual/configuration.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index aaec1cb7..9d99dc60 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -2050,6 +2050,11 @@ Bob will either invoke ``bash`` or ``pwsh``/``powershell`` as script interpreter. In either case the command must be present in ``$PATH``/``%PATH%``. +.. important:: + Each layer configures the default language individually. That is, layers + with higher precedence will not override the setting of layers with lower + precedence. + .. _configuration-config-usr: User configuration (default.yaml)