audit: add layers to audit

Now that we support managed layers, its time to add the information to
the audit trail too.

pym/bob/audit.py

@@ -42,6 +42,8 @@ def digestData(d, h):
     elif isinstance(d, bytes):
         h.update(struct.pack("<BI", 6, len(d)))
         h.update(d)
+    elif d is None:
+        h.update(struct.pack("<B", 7))
     else:
         assert False, "Cannot digest " + str(type(d))
 
@@ -80,6 +82,7 @@ class Artifact:
         schema.Optional('metaEnv') : { schema.Optional(str) : str },
         "scms" : [ dict ],
         schema.Optional("recipes") : dict,
+        schema.Optional("layers") : dict,
         "dependencies" : {
             schema.Optional('args') : [ HexValidator() ],
             schema.Optional('tools') : { str : HexValidator() },
@@ -122,6 +125,7 @@ def reset(self, variantId, buildId, resultHash):
         self.__buildId = buildId
         self.__resultHash = resultHash
         self.__recipes = None
+        self.__layers = {}
         self.__defines = {}
         u = platform.uname()
         self.__build = {
@@ -155,6 +159,13 @@ def load(self, data):
         else:
             self.__recipes = None
 
+        layers = data.get("layers")
+        if layers:
+            self.__layers = { name : (audit and auditFromData(audit))
+                              for name, audit in layers.items() }
+        else:
+            self.__layers = {}
+
         self.__defines = data["meta"]
         self.__build = data["build"]
         self.__env = data["env"]
@@ -206,11 +217,18 @@ def __dump(self):
         if self.__recipes is not None:
             ret["recipes"] = self.__recipes.dump()
 
+        if self.__layers: # Explicitly filter empty layers
+            ret["layers"] = { name : (audit and audit.dump())
+                              for name, audit in self.__layers.items() }
+
         return ret
 
     def setRecipes(self, recipes):
         self.__recipes = recipes
 
+    def setLayers(self, layers):
+        self.__layers = layers
+
     def setEnv(self, env):
         try:
             with open(env) as f:
@@ -386,8 +404,11 @@ def getReferencedBuildIds(self):
                 refs.update(artifact.getReferences())
         return sorted(ret)
 
-    def setRecipesAudit(self, recipes):
-        self.__artifact.setRecipes(recipes)
+    def setRecipesAudit(self, recipesAudit):
+        self.__artifact.setRecipes(recipesAudit.get(""))
+        self.__artifact.setLayers({
+            layer : audit for layer, audit in recipesAudit.items() if layer != ""
+        })
 
     def setEnv(self, env):
         self.__artifact.setEnv(env)

pym/bob/cmds/jenkins/exec.py

@@ -86,7 +86,7 @@ def recipesAudit(self):
         if self.__recipesAudit:
             return json.loads("".join(self.__recipesAudit))
         else:
-            return None
+            return {}
 
     @property
     def execIR(self):

pym/bob/cmds/jenkins/intermediate.py

@@ -74,7 +74,8 @@ class PartialRecipeSet(PartialIRBase, RecipeSetIR):
     @classmethod
     def fromData(cls, data, scmAudit):
         self = super(PartialRecipeSet, cls).fromData(data)
-        self.__scmAudit = scmAudit and auditFromData(scmAudit)
+        self.__scmAudit = scmAudit and { name : (audit and auditFromData(audit))
+                                         for name, audit in scmAudit.items() }
         return self
 
     async def getScmAudit(self):
@@ -94,7 +95,7 @@ def __init__(self):
         self.packages = {}
         self.recipes = {}
         self.recipeSet = None
-        self.scmAudit = None
+        self.scmAudit = {}
 
     @classmethod
     def fromData(cls, data):

pym/bob/cmds/jenkins/jenkins.py

@@ -1281,8 +1281,10 @@ def doJenkinsPush(recipes, argv):
     date = str(datetime.datetime.now())
 
     recipesAudit = runInEventLoop(recipes.getScmAudit())
-    if recipesAudit is not None:
-        recipesAudit = json.dumps(recipesAudit.dump(), sort_keys=True)
+    if recipesAudit:
+        recipesAudit = json.dumps({ name : (audit and audit.dump())
+                                    for name, audit in recipesAudit.items() },
+                                  sort_keys=True)
 
     def printLine(level, job, *args):
         if level <= verbose:

pym/bob/input.py

@@ -8,7 +8,7 @@
 from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage
 from .pathspec import PackageSet
 from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \
-    auditFromDir, getScm, SYNTHETIC_SCM_PROPS
+    auditFromDir, auditFromProperties, getScm, SYNTHETIC_SCM_PROPS
 from .state import BobState
 from .stringparser import checkGlobList, Env, DEFAULT_STRING_FUNS, IfExpression
 from .tty import InfoOnce, Warn, WarnOnce, setColorMode, setParallelTUIThreshold
@@ -3102,7 +3102,7 @@ def __init__(self):
         self.__commandConfig = {}
         self.__uiConfig = {}
         self.__shareConfig = {}
-        self.__layers = []
+        self.__layers = {}
         self.__buildHooks = {}
         self.__sandboxOpts = {}
         self.__scmDefaults = {}
@@ -3425,20 +3425,34 @@ async def getScmAudit(self):
         try:
             ret = self.__recipeScmAudit
         except AttributeError:
+            ret = {}
             try:
-                ret = await auditFromDir(".")
+                ret[""] = await auditFromDir(".")
             except BobError as e:
                 Warn("could not determine recipes state").warn(e.slogan)
-                ret = None
+
+            # We look into every layers directory. If the Bob state knows it,
+            # use the SCM information from there. Otherwise make a best guess.
+            for layer, path in sorted(self.__layers.items()):
+                state = None
+                try:
+                    scmProps = (BobState().getLayerState(path) or {}).get("prop")
+                    if scmProps is None:
+                        state = await auditFromDir(path)
+                    else:
+                        state = await auditFromProperties(path, scmProps)
+                except BobError as e:
+                    Warn(f"could not determine layer '{layer}' state").warn(e.slogan)
+                ret[layer] = state
+
             self.__recipeScmAudit = ret
         return ret
 
     async def getScmStatus(self):
-        audit = await self.getScmAudit()
-        if audit is None:
-            return "unknown"
-        else:
-            return audit.getStatusLine()
+        scmAudit = await self.getScmAudit()
+        return ", ".join(( ((f"layer {name}: " if name else "")
+                            + ("unknown" if audit is None else audit.getStatusLine()))
+                           for name, audit in sorted(scmAudit.items()) ))
 
     def getBuildHook(self, name):
         return self.__buildHooks.get(name)
@@ -3482,7 +3496,7 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
         if platform not in ('cygwin', 'darwin', 'linux', 'msys', 'win32'):
             raise ParseError("Invalid platform: " + platform)
         self.__platform = platform
-        self.__layers = []
+        self.__layers = {}
         self.__whiteList = getPlatformEnvWhiteList(platform)
         self.__pluginPropDeps = b''
         self.__pluginSettingsDeps = b''
@@ -3586,7 +3600,6 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer):
 
             if layer in self.__layers:
                 return
-            self.__layers.append(layer)
 
             if managedLayers:
                 # SCM backed layers are in build dir, regular layers are in
@@ -3603,6 +3616,8 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer):
                                                        for l in layer.split("/") ))
                 if not os.path.isdir(rootDir):
                     raise ParseError(f"Layer '{layer}' does not exist!")
+
+            self.__layers[layer] = rootDir
         else:
             rootDir = recipesRoot
 

pym/bob/scm/__init__.py

@@ -21,6 +21,21 @@ async def auditFromDir(dir):
     else:
         return None
 
+async def auditFromProperties(baseDir, props):
+    auditSpec = getScm(props).getAuditSpec()
+    if auditSpec is None:
+        return None
+
+    SCMS = {
+        'git' : GitAudit,
+        'svn' : SvnAudit,
+        'url' : UrlAudit,
+        'import' : ImportAudit,
+    }
+
+    (typ, subDir, extra) = auditSpec
+    return await SCMS[typ].fromDir(baseDir, subDir, extra)
+
 def auditFromData(data):
     typ = data.get("type")
     if typ == "git":