Merge pull request #543 from jkloetzke/nested-scms

Add support for nested SCMs

doc/manual/configuration.rst

@@ -868,6 +868,14 @@ be given as IfExpression (see :ref:`configuration-principle-booleans`). By
 default the SCMs check out to the root of the workspace. You may specify any
 relative path in ``dir`` to checkout to this directory.
 
+Special care must be taken if SCMs are nested, that is the ``dir`` attribute of
+one SCM is a subdirectory of another. Bob requires that the SCM with the upper
+directory has to be in the list before the SCMs that are checked out into
+subdirectories. Additionally, SCMs that are natively supported by Jenkins
+plugins (git, svn), cannot be nested into the other SCMs (cvs, import, url).
+The reason is that Jenkins SCM plugins always execute before anything else in a
+Jenkins job.
+
 By using ``if`` you can selectively enable or disable a particular SCM using
 either a string or a expression. In case a string is given to the ``if``-keyword
 it is substituted according to :ref:`configuration-principle-subst` and the final

pym/bob/builder.py

@@ -88,8 +88,12 @@ def compareDirectoryState(left, right):
     return left == right
 
 def checkoutsFromState(state):
-    """Return only the tuples related to SCMs from the checkout state."""
-    return [ (k, v) for k, v in state.items() if k not in CHECKOUT_NON_DIR_KEYS ]
+    """Return only the tuples related to SCMs from the checkout state.
+
+    The list is sorted so that SCM operation can safely be done in list order.
+    """
+    return sorted(( (d, v) for d, v in state.items() if d not in CHECKOUT_NON_DIR_KEYS ),
+                  key=lambda i: os.path.normcase(os.path.normpath(i[0])) )
 
 def checkoutBuildOnlyState(checkoutStep, inputHashes):
     """Obtain state for build-only checkout updates.
@@ -263,6 +267,38 @@ def addOverrides(self, overrides):
     def getActiveOverrides(self):
         return self.__activeOverrides
 
+class AtticTracker:
+    """Track directories that were moved to attic."""
+
+    def __init__(self):
+        self.__paths = {}
+
+    def add(self, scmPath, atticPath):
+        """Add directory that was moved to attic."""
+        scmPath = os.path.normpath(scmPath)
+        self.__paths[scmPath + os.sep] = atticPath
+
+    def affected(self, nestedScmPath):
+        """Check if a directory is affected by a directory that was previously
+           moved to attic.
+        """
+        nestedScmPath = os.path.normpath(nestedScmPath)
+        return self.__match(nestedScmPath) is not None
+
+    def getAtticPath(self, nestedScmPath):
+        """Get attic path for a directory that was affected."""
+        nestedScmPath = os.path.normpath(nestedScmPath)
+        prefix = self.__match(nestedScmPath)
+        subDir = nestedScmPath[len(prefix):]
+        return os.path.join(self.__paths[prefix], subDir)
+
+    def __match(self, nestedScmPath):
+        nestedScmPath += os.sep
+        for p in self.__paths:
+            if nestedScmPath.startswith(p):
+                return p
+        return None
+
 class LocalBuilder:
 
     RUN_TEMPLATE_POSIX = """#!/bin/bash
@@ -1106,10 +1142,23 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                 (BobState().getResultHash(prettySrcPath) is None) or
                 not compareDirectoryState(checkoutState, oldCheckoutState) or
                 (checkoutInputHashes != BobState().getInputHashes(prettySrcPath))):
-                # Switch or move away old or changed source directories
+                # Switch or move away old or changed source directories. In
+                # case of nested SCMs the loop relies on the property of
+                # checkoutsFromState() to return the directories in a top-down
+                # order.
+                atticPaths = AtticTracker()
                 for (scmDir, (scmDigest, scmSpec)) in checkoutsFromState(oldCheckoutState):
-                    if scmDigest != checkoutState.get(scmDir, (None, None))[0]:
-                        scmPath = os.path.normpath(os.path.join(prettySrcPath, scmDir))
+                    scmPath = os.path.normpath(os.path.join(prettySrcPath, scmDir))
+                    if atticPaths.affected(scmPath):
+                        # An SCM in the hierarchy above this one was moved to
+                        # attic. This implies that all nested SCMs are affected
+                        # too...
+                        atticPath = atticPaths.getAtticPath(scmPath)
+                        if os.path.exists(atticPath):
+                            BobState().setAtticDirectoryState(atticPath, scmSpec)
+                        del oldCheckoutState[scmDir]
+                        BobState().setDirectoryState(prettySrcPath, oldCheckoutState)
+                    elif scmDigest != checkoutState.get(scmDir, (None, None))[0]:
                         canSwitch = (scmDir in scmMap) and scmDigest and \
                                      scmSpec is not None and \
                                      scmMap[scmDir].canSwitch(scmSpec) and \
@@ -1137,6 +1186,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                             atticPath = os.path.join(atticPath, atticName)
                             os.rename(scmPath, atticPath)
                             BobState().setAtticDirectoryState(atticPath, scmSpec)
+                            atticPaths.add(scmPath, atticPath)
                         del oldCheckoutState[scmDir]
                         BobState().setDirectoryState(prettySrcPath, oldCheckoutState)
 

pym/bob/input.py

@@ -40,14 +40,21 @@
 warnDeprecatedPluginState = Warn("Plugin uses deprecated 'bob.input.PluginState' API!")
 warnDeprecatedStringFn = Warn("Plugin uses deprecated 'stringFunctions' API!")
 
-def overlappingPaths(p1, p2):
+def isPrefixPath(p1, p2):
+    """Check if the initial elements of ``p2`` equal ``p1``.
+
+    :return: True if ``p2`` is a subdirectory or file inside ``p1``, otherwise
+             False.
+    """
     p1 = os.path.normcase(os.path.normpath(p1)).split(os.sep)
     if p1 == ["."]: p1 = []
     p2 = os.path.normcase(os.path.normpath(p2)).split(os.sep)
     if p2 == ["."]: p2 = []
-    for i in range(min(len(p1), len(p2))):
-        if p1[i] != p2[i]: return False
-    return True
+
+    if len(p1) > len(p2):
+        return False
+
+    return p2[:len(p1)] == p1
 
 def __maybeGlob(pred):
     if pred.startswith("!"):
@@ -1271,17 +1278,30 @@ def __init__(self, corePackage, checkout=None, checkoutSCMs=[],
                 if fullEnv.evaluate(scm.get("if"), "checkoutSCM") ]
             isValid = (checkout[1] is not None) or bool(self.scmList)
 
-            # Validate that SCM paths do not overlap
+            # Validate all SCM paths. Only relative paths are allowed. If they
+            # overlap, the following rules must apply:
+            #  1. Deeper paths must be checked out later.
+            #     -> required for initial checkout to work
+            #  2. A Jenkins native SCM (e.g. git) must not overlap with a
+            #     previous non-native SCM.
+            #     -> required because SCM plugins are running before scripts on
+            #        Jenkins jobs
             knownPaths = []
             for s in self.scmList:
                 p = s.getDirectory()
                 if os.path.isabs(p):
                     raise ParseError("SCM paths must be relative! Offending path: " + p)
-                for known in knownPaths:
-                    if overlappingPaths(known, p):
-                        raise ParseError("SCM paths '{}' and '{}' overlap."
-                                            .format(known, p))
-                knownPaths.append(p)
+                for knownPath, knownHasJenkins in knownPaths:
+                    if isPrefixPath(p, knownPath):
+                        raise ParseError("SCM path '{}' obstructs '{}'."
+                                            .format(p, knownPath),
+                                         help="Nested SCMs must be specified in a upper-to-lower directory order.")
+                    if isPrefixPath(knownPath, p) and s.hasJenkinsPlugin() and not knownHasJenkins:
+                        raise ParseError("SCM path '{}' cannot be checked out after '{}' in Jenkins jobs"
+                                            .format(p, knownPath),
+                                         help="SCM plugins always run first in a Jenkins job but the SCM in '{}' does not have a native Jenkins plugin."
+                                                .format(knownPath))
+                knownPaths.append((p, s.hasJenkinsPlugin()))
         else:
             isValid = False
             self.scmList = []

test/black-box/nested-scms/config.yaml

@@ -0,0 +1 @@
+bobMinimumVersion: "0.23"

test/black-box/nested-scms/recipes/root.yaml

@@ -0,0 +1,16 @@
+root: True
+
+checkoutSCM:
+    - if: "${GIT_1_ENABLE:-true}"
+      scm: git
+      url: file://${GIT_1_DIR}
+      rev: ${GIT_1_REV:-refs/heads/master}
+      dir: "foo"
+    - if: "${GIT_2_ENABLE:-true}"
+      scm: git
+      url: file://${GIT_2_DIR}
+      rev: ${GIT_2_REV:-refs/heads/master}
+      dir: "foo/bar"
+
+buildScript: "true"
+packageScript: "true"

test/black-box/nested-scms/run.sh

@@ -0,0 +1,81 @@
+#!/bin/bash -e
+#
+# Verify checkouts and updates of nested SCMs.
+
+. ../../test-lib.sh 2>/dev/null || { echo "Must run in script directory!" ; exit 1 ; }
+
+git_dir1=$(mktemp -d)
+git_dir2=$(mktemp -d)
+trap 'rm -rf "$git_dir1" "$git_dir2"' EXIT
+cleanup
+
+# Prepare git repositories
+
+pushd "$git_dir1"
+git init .
+git config user.email "[email protected]"
+git config user.name test
+echo "commit-1" > git1.txt
+git add git1.txt
+git commit -m "initial commit"
+git tag -a -m "First Tag" tag1
+echo "commit-2" > git1.txt
+git commit -a -m "second commit"
+git tag -a -m "Second Tag" tag2
+popd
+
+pushd "$git_dir2"
+git init .
+git config user.email "[email protected]"
+git config user.name test
+echo "commit-1" > git2.txt
+git add git2.txt
+git commit -m "first commit"
+git tag -a -m "First Tag" tag1
+echo "commit-2" > git2.txt
+git commit -a -m "second commit"
+git tag -a -m "Second Tag" tag2
+popd
+
+
+# First a simple checkout. We put a canary there to detect attic moves.
+run_bob dev -DGIT_1_DIR="$git_dir1" -DGIT_2_DIR="$git_dir2" root
+expect_output "commit-2" cat dev/src/root/1/workspace/foo/git1.txt
+echo canary > dev/src/root/1/workspace/foo/canary.txt
+expect_output "commit-2" cat dev/src/root/1/workspace/foo/bar/git2.txt
+echo canary > dev/src/root/1/workspace/foo/bar/canary.txt
+
+# Change tag on nested SCM
+run_bob dev -DGIT_1_DIR="$git_dir1" -DGIT_2_DIR="$git_dir2" \
+	-DGIT_2_REV="refs/tags/tag1" root
+expect_exist dev/src/root/1/workspace/foo/canary.txt
+expect_exist dev/src/root/1/workspace/foo/bar/canary.txt
+expect_output "commit-2" cat dev/src/root/1/workspace/foo/git1.txt
+expect_output "commit-1" cat dev/src/root/1/workspace/foo/bar/git2.txt
+
+# Change tag on upper SCM
+run_bob dev -DGIT_1_DIR="$git_dir1" -DGIT_2_DIR="$git_dir2" \
+	-DGIT_1_REV="refs/tags/tag1" \
+	-DGIT_2_REV="refs/tags/tag1" root
+expect_exist dev/src/root/1/workspace/foo/canary.txt
+expect_exist dev/src/root/1/workspace/foo/bar/canary.txt
+expect_output "commit-1" cat dev/src/root/1/workspace/foo/git1.txt
+expect_output "commit-1" cat dev/src/root/1/workspace/foo/bar/git2.txt
+
+# Remove upper SCM. The upper SCM and the nested one are both moved to the
+# attic and can be found there. The "nested" SCM is then checked out again.
+run_bob dev -DGIT_1_DIR="$git_dir1" -DGIT_1_ENABLE=0 -DGIT_2_DIR="$git_dir2" root
+expect_not_exist dev/src/root/1/workspace/foo/canary.txt
+expect_not_exist dev/src/root/1/workspace/foo/git1.txt
+expect_not_exist dev/src/root/1/workspace/foo/bar/canary.txt
+expect_output "commit-2" cat dev/src/root/1/workspace/foo/bar/git2.txt
+expect_exist dev/src/root/1/attic
+expect_exist dev/src/root/1/attic/*foo
+expect_exist dev/src/root/1/attic/*foo/bar
+
+status=$(run_bob status -DGIT_1_DIR="$git_dir1" -DGIT_1_ENABLE=0 -DGIT_2_DIR="$git_dir2" \
+		--attic --show-clean)
+# On Windows "bob status" will output native paths. Needs to be substituted
+# before comparison.
+[[ ${status//\\//} == *dev/src/root/1/attic/*foo* ]]
+[[ ${status//\\//} == *dev/src/root/1/attic/*foo/bar ]]