diff --git a/pym/bob/audit.py b/pym/bob/audit.py index 74ee3d72..998f25e1 100644 --- a/pym/bob/audit.py +++ b/pym/bob/audit.py @@ -42,6 +42,8 @@ def digestData(d, h): elif isinstance(d, bytes): h.update(struct.pack("
-
Recipes: """ + runInEventLoop(RecipeSet().getScmStatus()) + """
+
Recipes: """ + runInEventLoop(recipes.getScmStatus()) + """
Bob version: """ + BOB_VERSION +"""
Generated using D3JS
@@ -584,6 +584,6 @@ def doGraph(argv, bobRoot): print(colorize(" GRAPH {} ({})".format(p, args.type), "32")) if args.type == 'd3': - makeD3Graph(packages, p, os.path.join(destination, filename), options, excludes, highlights, args.max_depth) + makeD3Graph(recipes, packages, p, os.path.join(destination, filename), options, excludes, highlights, args.max_depth) elif args.type == 'dot': makeDotGraph(packages, p, os.path.join(destination, filename), excludes, highlights, args.max_depth) diff --git a/pym/bob/cmds/jenkins/exec.py b/pym/bob/cmds/jenkins/exec.py index 8c82d437..b07f42f4 100644 --- a/pym/bob/cmds/jenkins/exec.py +++ b/pym/bob/cmds/jenkins/exec.py @@ -86,7 +86,7 @@ def recipesAudit(self): if self.__recipesAudit: return json.loads("".join(self.__recipesAudit)) else: - return None + return {} @property def execIR(self): diff --git a/pym/bob/cmds/jenkins/intermediate.py b/pym/bob/cmds/jenkins/intermediate.py index 2d6aa90f..92c2bbd3 100644 --- a/pym/bob/cmds/jenkins/intermediate.py +++ b/pym/bob/cmds/jenkins/intermediate.py @@ -74,7 +74,8 @@ class PartialRecipeSet(PartialIRBase, RecipeSetIR): @classmethod def fromData(cls, data, scmAudit): self = super(PartialRecipeSet, cls).fromData(data) - self.__scmAudit = scmAudit and auditFromData(scmAudit) + self.__scmAudit = scmAudit and { name : (audit and auditFromData(audit)) + for name, audit in scmAudit.items() } return self async def getScmAudit(self): @@ -94,7 +95,7 @@ def __init__(self): self.packages = {} self.recipes = {} self.recipeSet = None - self.scmAudit = None + self.scmAudit = {} @classmethod def fromData(cls, data): diff --git a/pym/bob/cmds/jenkins/jenkins.py b/pym/bob/cmds/jenkins/jenkins.py index 3dc64f29..c112293c 100644 --- a/pym/bob/cmds/jenkins/jenkins.py +++ b/pym/bob/cmds/jenkins/jenkins.py @@ -1281,8 +1281,10 @@ def doJenkinsPush(recipes, argv): date = str(datetime.datetime.now()) recipesAudit = runInEventLoop(recipes.getScmAudit()) - if recipesAudit is not None: - recipesAudit = json.dumps(recipesAudit.dump(), sort_keys=True) + if recipesAudit: + recipesAudit = json.dumps({ name : (audit and audit.dump()) + for name, audit in recipesAudit.items() }, + sort_keys=True) def printLine(level, job, *args): if level <= verbose: diff --git a/pym/bob/input.py b/pym/bob/input.py index ad729b02..4c83d427 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -8,7 +8,7 @@ from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage from .pathspec import PackageSet from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \ - auditFromDir, getScm, SYNTHETIC_SCM_PROPS + auditFromDir, auditFromProperties, getScm, SYNTHETIC_SCM_PROPS from .state import BobState from .stringparser import checkGlobList, Env, DEFAULT_STRING_FUNS, IfExpression from .tty import InfoOnce, Warn, WarnOnce, setColorMode, setParallelTUIThreshold @@ -1739,13 +1739,28 @@ def getScm(self): return self.__scm class LayerValidator: + @staticmethod + def __validateName(name): + if name == "": + raise schema.SchemaError("Layer name must not be empty") + if any((c in name) for c in '\\/'): + raise schema.SchemaError("Invalid character in layer name") + def validate(self, data): - if isinstance(data,str): + if isinstance(data, str): + self.__validateName(data) return LayerSpec(data) + elif not isinstance(data, dict): + raise schema.SchemaUnexpectedTypeError("Layer entry must be a string or a dict", None) + if 'name' not in data: raise schema.SchemaMissingKeyError("Missing 'name' key in {}".format(data), None) + elif not isinstance(data['name'], str): + raise schema.SchemaUnexpectedTypeError("Layer name must be a string", None) + _data = data.copy(); name = _data.get('name') + self.__validateName(name) del _data['name'] return LayerSpec(name, RecipeSet.LAYERS_SCM_SCHEMA.validate(_data)[0]) @@ -2957,11 +2972,12 @@ class RecipeSet: }) # We do not support the "import" SCM for layers. It just makes no sense. + # Also, all SCMs lack the "dir" and "if" attributes. LAYERS_SCM_SCHEMA = ScmValidator({ - 'git' : GitScm.SCHEMA, - 'svn' : SvnScm.SCHEMA, - 'cvs' : CvsScm.SCHEMA, - 'url' : UrlScm.SCHEMA, + 'git' : GitScm.LAYERS_SCHEMA, + 'svn' : SvnScm.LAYERS_SCHEMA, + 'cvs' : CvsScm.LAYERS_SCHEMA, + 'url' : UrlScm.LAYERS_SCHEMA, }) SCM_SCHEMA = ScmValidator({ @@ -3101,7 +3117,7 @@ def __init__(self): self.__commandConfig = {} self.__uiConfig = {} self.__shareConfig = {} - self.__layers = [] + self.__layers = {} self.__buildHooks = {} self.__sandboxOpts = {} self.__scmDefaults = {} @@ -3424,20 +3440,34 @@ async def getScmAudit(self): try: ret = self.__recipeScmAudit except AttributeError: + ret = {} try: - ret = await auditFromDir(".") + ret[""] = await auditFromDir(".") except BobError as e: Warn("could not determine recipes state").warn(e.slogan) - ret = None + + # We look into every layers directory. If the Bob state knows it, + # use the SCM information from there. Otherwise make a best guess. + for layer, path in sorted(self.__layers.items()): + state = None + try: + scmProps = (BobState().getLayerState(path) or {}).get("prop") + if scmProps is None: + state = await auditFromDir(path) + else: + state = await auditFromProperties(path, scmProps) + except BobError as e: + Warn(f"could not determine layer '{layer}' state").warn(e.slogan) + ret[layer] = state + self.__recipeScmAudit = ret return ret async def getScmStatus(self): - audit = await self.getScmAudit() - if audit is None: - return "unknown" - else: - return audit.getStatusLine() + scmAudit = await self.getScmAudit() + return ", ".join(( ((f"layer {name}: " if name else "") + + ("unknown" if audit is None else audit.getStatusLine())) + for name, audit in sorted(scmAudit.items()) )) def getBuildHook(self, name): return self.__buildHooks.get(name) @@ -3481,7 +3511,7 @@ def __parse(self, envOverrides, platform, recipesRoot=""): if platform not in ('cygwin', 'darwin', 'linux', 'msys', 'win32'): raise ParseError("Invalid platform: " + platform) self.__platform = platform - self.__layers = [] + self.__layers = {} self.__whiteList = getPlatformEnvWhiteList(platform) self.__pluginPropDeps = b'' self.__pluginSettingsDeps = b'' @@ -3585,7 +3615,6 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer): if layer in self.__layers: return - self.__layers.append(layer) if managedLayers: # SCM backed layers are in build dir, regular layers are in @@ -3602,6 +3631,8 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer): for l in layer.split("/") )) if not os.path.isdir(rootDir): raise ParseError(f"Layer '{layer}' does not exist!") + + self.__layers[layer] = rootDir else: rootDir = recipesRoot diff --git a/pym/bob/layers.py b/pym/bob/layers.py index 29095466..91bcd391 100644 --- a/pym/bob/layers.py +++ b/pym/bob/layers.py @@ -100,7 +100,6 @@ def __init__(self, name, upperConfig, defines, projectRoot, scm=None): async def __checkoutTask(self, verbose, attic): if self.__scm is None: return - dir = self.__scm.getProperties(False).get("dir") invoker = Invoker(spec=LayerStepSpec(self.__layerDir, self.__upperConfig.envWhiteList()), preserveEnv= False, diff --git a/pym/bob/scm/__init__.py b/pym/bob/scm/__init__.py index 0788a6f3..c6bc8842 100644 --- a/pym/bob/scm/__init__.py +++ b/pym/bob/scm/__init__.py @@ -21,6 +21,21 @@ async def auditFromDir(dir): else: return None +async def auditFromProperties(baseDir, props): + auditSpec = getScm(props).getAuditSpec() + if auditSpec is None: + return None + + SCMS = { + 'git' : GitAudit, + 'svn' : SvnAudit, + 'url' : UrlAudit, + 'import' : ImportAudit, + } + + (typ, subDir, extra) = auditSpec + return await SCMS[typ].fromDir(baseDir, subDir, extra) + def auditFromData(data): typ = data.get("type") if typ == "git": diff --git a/pym/bob/scm/cvs.py b/pym/bob/scm/cvs.py index dbc335f4..2c36d07a 100644 --- a/pym/bob/scm/cvs.py +++ b/pym/bob/scm/cvs.py @@ -19,11 +19,17 @@ class CvsScm(Scm): 'scm' : 'cvs', 'cvsroot' : str, 'module' : str, - schema.Optional('if') : str, schema.Optional('rev') : str } - SCHEMA = schema.Schema({**__SCHEMA, **DEFAULTS}) + SCHEMA = schema.Schema({ + **__SCHEMA, + **DEFAULTS, + schema.Optional('if') : str, + }) + + # Layers have no "dir" and no "if" + LAYERS_SCHEMA = schema.Schema({ **__SCHEMA }) # Checkout using CVS # - mandatory parameters: cvsroot, module diff --git a/pym/bob/scm/git.py b/pym/bob/scm/git.py index 53b22bfc..dedf9b58 100644 --- a/pym/bob/scm/git.py +++ b/pym/bob/scm/git.py @@ -57,7 +57,7 @@ def getBranchTagCommit(spec): class GitScm(Scm): - DEFAULTS = { + __DEFAULTS = { schema.Optional('branch') : str, schema.Optional('sslVerify') : bool, schema.Optional('singleBranch') : bool, @@ -65,7 +65,6 @@ class GitScm(Scm): schema.Optional('recurseSubmodules') : bool, schema.Optional('shallowSubmodules') : bool, schema.Optional('shallow') : schema.Or(int, str), - schema.Optional('dir') : str, schema.Optional('references') : schema.Schema([schema.Or(str, { schema.Optional('url') : str, @@ -80,14 +79,26 @@ class GitScm(Scm): __SCHEMA = { 'scm' : 'git', 'url' : str, - schema.Optional('if') : schema.Or(str, IfExpression), schema.Optional('tag') : str, schema.Optional('commit') : str, schema.Optional('rev') : str, schema.Optional(schema.Regex('^remote-.*')) : str, } - SCHEMA = schema.Schema({**__SCHEMA, **DEFAULTS}) + DEFAULTS = { + **__DEFAULTS, + schema.Optional('dir') : str, + } + + SCHEMA = schema.Schema({ + **__SCHEMA, + **DEFAULTS, + schema.Optional('if') : schema.Or(str, IfExpression), + }) + + # Layers have no "dir" and no "if" + LAYERS_SCHEMA = schema.Schema({**__SCHEMA, **__DEFAULTS}) + REMOTE_PREFIX = "remote-" def __init__(self, spec, overrides=[], stripUser=None, useBranchAndCommit=False): @@ -897,11 +908,10 @@ def __statusSubmodule(self, workspacePath, status, shouldExist, base = "."): def getAuditSpec(self): - extra = {} - if self.__submodules: - extra['submodules'] = self.__submodules - if self.__recurseSubmodules: - extra['recurseSubmodules'] = True + extra = { + 'submodules' : self.__submodules, + 'recurseSubmodules' : self.__recurseSubmodules, + } return ("git", self.__dir, extra) def hasLiveBuildId(self): @@ -976,8 +986,11 @@ class GitAudit(ScmAudit): async def _scanDir(self, workspace, dir, extra): self.__dir = dir - self.__submodules = extra.get('submodules', False) - self.__recurseSubmodules = extra.get('recurseSubmodules', False) + # In case we scan an unkown directry, the `extra` dict will be empty. + # In this case we assume submodules exist and are checked our + # recursively. + self.__submodules = extra.get('submodules') + self.__recurseSubmodules = extra.get('recurseSubmodules') dir = os.path.join(workspace, dir) try: remotes = (await check_output(["git", "remote", "-v"], @@ -1026,7 +1039,8 @@ async def __scanSubmodules(self, dir, shouldExist, base = "."): # Normalize subset of submodules if isinstance(shouldExist, list): shouldExist = set(normPath(p) for p in shouldExist) - elif shouldExist: + elif shouldExist or shouldExist is None: + # If unspecified, we expect all submodules to be present. shouldExist = set(normPath(p) for p in allPaths.keys()) else: shouldExist = set() @@ -1069,8 +1083,8 @@ def _load(self, data): self.__commit = data["commit"] self.__description = data["description"] self.__dirty = data["dirty"] - self.__submodules = data.get("submodules", False) - self.__recurseSubmodules = data.get("recurseSubmodules", False) + self.__submodules = data.get("submodules") + self.__recurseSubmodules = data.get("recurseSubmodules") def dump(self): ret = { @@ -1081,10 +1095,10 @@ def dump(self): "description" : self.__description, "dirty" : self.__dirty, } - if self.__submodules: + if self.__submodules is not None: ret["submodules"] = self.__submodules - if self.__recurseSubmodules: - ret["recurseSubmodules"] = True + if self.__recurseSubmodules is not None: + ret["recurseSubmodules"] = self.__recurseSubmodules return ret def getStatusLine(self): diff --git a/pym/bob/scm/svn.py b/pym/bob/scm/svn.py index 7068a30e..1ff35d34 100644 --- a/pym/bob/scm/svn.py +++ b/pym/bob/scm/svn.py @@ -16,19 +16,32 @@ class SvnScm(Scm): - DEFAULTS = { - schema.Optional('dir') : str, + __DEFAULTS = { schema.Optional('sslVerify') : bool, }; __SCHEMA = { 'scm' : 'svn', 'url' : str, - schema.Optional('if') : schema.Or(str, IfExpression), schema.Optional('revision') : schema.Or(int, str), } - SCHEMA = schema.Schema({**__SCHEMA, **DEFAULTS}) + DEFAULTS = { + **__DEFAULTS, + schema.Optional('dir') : str, + } + + SCHEMA = schema.Schema({ + **__SCHEMA, + **DEFAULTS, + schema.Optional('if') : schema.Or(str, IfExpression), + }) + + # Layers have no "dir" and no "if" + LAYERS_SCHEMA = schema.Schema({ + **__SCHEMA, + **__DEFAULTS, + }) def __init__(self, spec, overrides=[]): super().__init__(spec, overrides) diff --git a/pym/bob/scm/url.py b/pym/bob/scm/url.py index e155242a..4cb4fb52 100644 --- a/pym/bob/scm/url.py +++ b/pym/bob/scm/url.py @@ -157,7 +157,7 @@ def dumpMode(mode): class UrlScm(Scm): - DEFAULTS = { + __DEFAULTS = { schema.Optional('extract') : schema.Or(bool, str), schema.Optional('fileName') : str, schema.Optional('stripComponents') : int, @@ -169,14 +169,27 @@ class UrlScm(Scm): __SCHEMA = { 'scm' : 'url', 'url' : str, - schema.Optional('dir') : str, - schema.Optional('if') : schema.Or(str, IfExpression), schema.Optional('digestSHA1') : str, schema.Optional('digestSHA256') : str, schema.Optional('digestSHA512') : str, } - SCHEMA = schema.Schema({**__SCHEMA, **DEFAULTS}) + DEFAULTS = { + **__DEFAULTS, + schema.Optional('dir') : str, + } + + SCHEMA = schema.Schema({ + **__SCHEMA, + **DEFAULTS, + schema.Optional('if') : schema.Or(str, IfExpression), + }) + + # Layers have no "dir" and no "if" + LAYERS_SCHEMA = schema.Schema({ + **__SCHEMA, + **__DEFAULTS, + }) MIRRORS_SCHEMA = schema.Schema({ 'scm' : 'url',