From ce2e1e9df37a76c05d2675b41dad0e74abe12002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 22 Oct 2023 21:38:14 +0200 Subject: [PATCH 1/4] Support nested SCMs Add support to checkout an SCM into a subdirectory of another SCM. This requires that the SCM with the upper directory is first in the list before any other SCMs in subdirectories. Additionally, SCMs that are natively supported by Jenkins plugins (git, svn), cannot be nested into the other SCMs (cvs, import, url). The reason is that Jenkins SCM plugins always execute before anything else in a Jenkins job. This relaxes the rules that did not allow any nesting before. There are some drawbacks of such setups. Namely, if an SCM is moved to the attic, all their nested SCMs are affected too. --- pym/bob/builder.py | 60 ++++++++++++++++++++++++++++++++++++++++++---- pym/bob/input.py | 40 +++++++++++++++++++++++-------- 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/pym/bob/builder.py b/pym/bob/builder.py index 28b13f5ba..02792a95b 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -88,8 +88,12 @@ def compareDirectoryState(left, right): return left == right def checkoutsFromState(state): - """Return only the tuples related to SCMs from the checkout state.""" - return [ (k, v) for k, v in state.items() if k not in CHECKOUT_NON_DIR_KEYS ] + """Return only the tuples related to SCMs from the checkout state. + + The list is sorted so that SCM operation can safely be done in list order. + """ + return sorted(( (d, v) for d, v in state.items() if d not in CHECKOUT_NON_DIR_KEYS ), + key=lambda i: os.path.normcase(os.path.normpath(i[0])) ) def checkoutBuildOnlyState(checkoutStep, inputHashes): """Obtain state for build-only checkout updates. @@ -263,6 +267,38 @@ def addOverrides(self, overrides): def getActiveOverrides(self): return self.__activeOverrides +class AtticTracker: + """Track directories that were moved to attic.""" + + def __init__(self): + self.__paths = {} + + def add(self, scmPath, atticPath): + """Add directory that was moved to attic.""" + scmPath = os.path.normpath(scmPath) + self.__paths[scmPath + os.sep] = atticPath + + def affected(self, nestedScmPath): + """Check if a directory is affected by a directory that was previously + moved to attic. + """ + nestedScmPath = os.path.normpath(nestedScmPath) + return self.__match(nestedScmPath) is not None + + def getAtticPath(self, nestedScmPath): + """Get attic path for a directory that was affected.""" + nestedScmPath = os.path.normpath(nestedScmPath) + prefix = self.__match(nestedScmPath) + subDir = nestedScmPath[len(prefix):] + return os.path.join(self.__paths[prefix], subDir) + + def __match(self, nestedScmPath): + nestedScmPath += os.sep + for p in self.__paths: + if nestedScmPath.startswith(p): + return p + return None + class LocalBuilder: RUN_TEMPLATE_POSIX = """#!/bin/bash @@ -1106,10 +1142,23 @@ async def _cookCheckoutStep(self, checkoutStep, depth): (BobState().getResultHash(prettySrcPath) is None) or not compareDirectoryState(checkoutState, oldCheckoutState) or (checkoutInputHashes != BobState().getInputHashes(prettySrcPath))): - # Switch or move away old or changed source directories + # Switch or move away old or changed source directories. In + # case of nested SCMs the loop relies on the property of + # checkoutsFromState() to return the directories in a top-down + # order. + atticPaths = AtticTracker() for (scmDir, (scmDigest, scmSpec)) in checkoutsFromState(oldCheckoutState): - if scmDigest != checkoutState.get(scmDir, (None, None))[0]: - scmPath = os.path.normpath(os.path.join(prettySrcPath, scmDir)) + scmPath = os.path.normpath(os.path.join(prettySrcPath, scmDir)) + if atticPaths.affected(scmPath): + # An SCM in the hierarchy above this one was moved to + # attic. This implies that all nested SCMs are affected + # too... + atticPath = atticPaths.getAtticPath(scmPath) + if os.path.exists(atticPath): + BobState().setAtticDirectoryState(atticPath, scmSpec) + del oldCheckoutState[scmDir] + BobState().setDirectoryState(prettySrcPath, oldCheckoutState) + elif scmDigest != checkoutState.get(scmDir, (None, None))[0]: canSwitch = (scmDir in scmMap) and scmDigest and \ scmSpec is not None and \ scmMap[scmDir].canSwitch(scmSpec) and \ @@ -1137,6 +1186,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth): atticPath = os.path.join(atticPath, atticName) os.rename(scmPath, atticPath) BobState().setAtticDirectoryState(atticPath, scmSpec) + atticPaths.add(scmPath, atticPath) del oldCheckoutState[scmDir] BobState().setDirectoryState(prettySrcPath, oldCheckoutState) diff --git a/pym/bob/input.py b/pym/bob/input.py index 4af9d8b66..90e44b36d 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -40,14 +40,21 @@ warnDeprecatedPluginState = Warn("Plugin uses deprecated 'bob.input.PluginState' API!") warnDeprecatedStringFn = Warn("Plugin uses deprecated 'stringFunctions' API!") -def overlappingPaths(p1, p2): +def isPrefixPath(p1, p2): + """Check if the initial elements of ``p2`` equal ``p1``. + + :return: True if ``p2`` is a subdirectory or file inside ``p1``, otherwise + False. + """ p1 = os.path.normcase(os.path.normpath(p1)).split(os.sep) if p1 == ["."]: p1 = [] p2 = os.path.normcase(os.path.normpath(p2)).split(os.sep) if p2 == ["."]: p2 = [] - for i in range(min(len(p1), len(p2))): - if p1[i] != p2[i]: return False - return True + + if len(p1) > len(p2): + return False + + return p2[:len(p1)] == p1 def __maybeGlob(pred): if pred.startswith("!"): @@ -1271,17 +1278,30 @@ def __init__(self, corePackage, checkout=None, checkoutSCMs=[], if fullEnv.evaluate(scm.get("if"), "checkoutSCM") ] isValid = (checkout[1] is not None) or bool(self.scmList) - # Validate that SCM paths do not overlap + # Validate all SCM paths. Only relative paths are allowed. If they + # overlap, the following rules must apply: + # 1. Deeper paths must be checked out later. + # -> required for initial checkout to work + # 2. A Jenkins native SCM (e.g. git) must not overlap with a + # previous non-native SCM. + # -> required because SCM plugins are running before scripts on + # Jenkins jobs knownPaths = [] for s in self.scmList: p = s.getDirectory() if os.path.isabs(p): raise ParseError("SCM paths must be relative! Offending path: " + p) - for known in knownPaths: - if overlappingPaths(known, p): - raise ParseError("SCM paths '{}' and '{}' overlap." - .format(known, p)) - knownPaths.append(p) + for knownPath, knownHasJenkins in knownPaths: + if isPrefixPath(p, knownPath): + raise ParseError("SCM path '{}' obstructs '{}'." + .format(p, knownPath), + help="Nested SCMs must be specified in a upper-to-lower directory order.") + if isPrefixPath(knownPath, p) and s.hasJenkinsPlugin() and not knownHasJenkins: + raise ParseError("SCM path '{}' cannot be checked out after '{}' in Jenkins jobs" + .format(p, knownPath), + help="SCM plugins always run first in a Jenkins job but the SCM in '{}' does not have a native Jenkins plugin." + .format(knownPath)) + knownPaths.append((p, s.hasJenkinsPlugin())) else: isValid = False self.scmList = [] From 3573bc243bc198ac0076ccb3ad6ef038ddaf6216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 10 Dec 2023 17:41:23 +0100 Subject: [PATCH 2/4] test: make recipe name optional --- test/unit/test_input_recipe.py | 48 +++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/unit/test_input_recipe.py b/test/unit/test_input_recipe.py index c38244962..77f29f86d 100644 --- a/test/unit/test_input_recipe.py +++ b/test/unit/test_input_recipe.py @@ -194,7 +194,7 @@ def applyRecipeDefaults(self, recipe): r.setdefault("checkoutUpdateIf", False) return r - def parseAndPrepare(self, name, recipe, classes={}, allRelocatable=None): + def parseAndPrepare(self, recipe, classes={}, allRelocatable=None, name="foo"): cwd = os.getcwd() recipeSet = MagicMock() @@ -223,7 +223,7 @@ def testNormalRelocatable(self): recipe = { "packageScript" : "asdf" } - p = self.parseAndPrepare("foo", recipe) + p = self.parseAndPrepare(recipe) self.assertTrue(p.isRelocatable()) def testToolsNonRelocatable(self): @@ -235,7 +235,7 @@ def testToolsNonRelocatable(self): "foo" : "bar" } } - p = self.parseAndPrepare("foo", recipe) + p = self.parseAndPrepare(recipe) self.assertFalse(p.isRelocatable()) def testCheckoutAndBuildStep(self): @@ -246,7 +246,7 @@ def testCheckoutAndBuildStep(self): "buildScript" : "asdf", "packageScript" : "asdf", } - p = self.parseAndPrepare("foo", recipe) + p = self.parseAndPrepare(recipe) self.assertFalse(p.getCheckoutStep().isRelocatable()) self.assertFalse(p.getBuildStep().isRelocatable()) self.assertTrue(p.getPackageStep().isRelocatable()) @@ -261,7 +261,7 @@ def testToolRelocatable(self): }, "relocatable" : True } - p = self.parseAndPrepare("foo", recipe) + p = self.parseAndPrepare(recipe) self.assertTrue(p.isRelocatable()) def testNotRelocatable(self): @@ -271,7 +271,7 @@ def testNotRelocatable(self): "packageScript" : "asdf", "relocatable" : False } - p = self.parseAndPrepare("foo", recipe) + p = self.parseAndPrepare(recipe) self.assertFalse(p.isRelocatable()) def testClassCanSetRelocatable(self): @@ -286,7 +286,7 @@ def testClassCanSetRelocatable(self): "relocatable" : False } } - p = self.parseAndPrepare("foo", recipe, classes) + p = self.parseAndPrepare(recipe, classes) self.assertFalse(p.isRelocatable()) # two-stage inheritence @@ -298,7 +298,7 @@ def testClassCanSetRelocatable(self): "relocatable" : False, } } - p = self.parseAndPrepare("foo", recipe, classes) + p = self.parseAndPrepare(recipe, classes) self.assertFalse(p.isRelocatable()) def testClassOverride(self): @@ -317,7 +317,7 @@ def testClassOverride(self): "relocatable" : True, } } - p = self.parseAndPrepare("foo", recipe, classes) + p = self.parseAndPrepare(recipe, classes) self.assertFalse(p.isRelocatable()) # recipe overrides classes @@ -325,7 +325,7 @@ def testClassOverride(self): "inherit" : [ "bar" ], "relocatable" : True, } - p = self.parseAndPrepare("foo", recipe, classes) + p = self.parseAndPrepare(recipe, classes) self.assertTrue(p.isRelocatable()) def testAllRelocatablePolicy(self): @@ -335,7 +335,7 @@ def testAllRelocatablePolicy(self): recipe = { "packageScript" : "asdf" } - p = self.parseAndPrepare("foo", recipe, allRelocatable=True) + p = self.parseAndPrepare(recipe, allRelocatable=True) self.assertTrue(p.isRelocatable()) # tool package @@ -345,7 +345,7 @@ def testAllRelocatablePolicy(self): "foo" : "bar" } } - p = self.parseAndPrepare("foo", recipe, allRelocatable=True) + p = self.parseAndPrepare(recipe, allRelocatable=True) self.assertTrue(p.isRelocatable()) @@ -359,7 +359,7 @@ def testDefault(self): "buildScript" : "asdf", "packageScript" : "asdf", } - c = self.parseAndPrepare("foo", recipe).getCheckoutStep() + c = self.parseAndPrepare(recipe).getCheckoutStep() self.assertFalse(c.getUpdateScript()) self.assertTrue(c.isUpdateDeterministic()) @@ -371,7 +371,7 @@ def testSimpleBool(self): "buildScript" : "asdf", "packageScript" : "asdf", } - c = self.parseAndPrepare("foo", recipe).getCheckoutStep() + c = self.parseAndPrepare(recipe).getCheckoutStep() self.assertIn("asdf", c.getUpdateScript()) def testSimpleIfString(self): @@ -382,11 +382,11 @@ def testSimpleIfString(self): "buildScript" : "asdf", "packageScript" : "asdf", } - c = self.parseAndPrepare("foo", recipe).getCheckoutStep() + c = self.parseAndPrepare(recipe).getCheckoutStep() self.assertNotIn("asdf", c.getUpdateScript()) recipe["checkoutUpdateIf"] = "$(eq,a,a)" - c = self.parseAndPrepare("foo", recipe).getCheckoutStep() + c = self.parseAndPrepare(recipe).getCheckoutStep() self.assertIn("asdf", c.getUpdateScript()) def testSimpleIfExpr(self): @@ -397,11 +397,11 @@ def testSimpleIfExpr(self): "buildScript" : "asdf", "packageScript" : "asdf", } - c = self.parseAndPrepare("foo", recipe).getCheckoutStep() + c = self.parseAndPrepare(recipe).getCheckoutStep() self.assertNotIn("asdf", c.getUpdateScript()) recipe["checkoutUpdateIf"] = IfExpression('"a" == "a"') - c = self.parseAndPrepare("foo", recipe).getCheckoutStep() + c = self.parseAndPrepare(recipe).getCheckoutStep() self.assertIn("asdf", c.getUpdateScript()) def testDeterministicDefault(self): @@ -412,11 +412,11 @@ def testDeterministicDefault(self): "buildScript" : "asdf", "packageScript" : "asdf", } - c = self.parseAndPrepare("foo", recipe).getCheckoutStep() + c = self.parseAndPrepare(recipe).getCheckoutStep() self.assertFalse(c.isUpdateDeterministic()) recipe["checkoutDeterministic"] = True - c = self.parseAndPrepare("foo", recipe).getCheckoutStep() + c = self.parseAndPrepare(recipe).getCheckoutStep() self.assertTrue(c.isUpdateDeterministic()) def testDeterministicInherit(self): @@ -438,7 +438,7 @@ def testDeterministicInherit(self): }, } - c = self.parseAndPrepare("foo", recipe, classes).getCheckoutStep() + c = self.parseAndPrepare(recipe, classes).getCheckoutStep() self.assertTrue(c.isUpdateDeterministic()) def testInherit(self): @@ -467,7 +467,7 @@ def testInherit(self): }, } - c = self.parseAndPrepare("foo", recipe, classes).getCheckoutStep() + c = self.parseAndPrepare(recipe, classes).getCheckoutStep() self.assertFalse(c.isUpdateDeterministic()) self.assertIn("asdf", c.getUpdateScript()) self.assertNotIn("qwer", c.getUpdateScript()) @@ -488,7 +488,7 @@ def testInheritNullNotEnabled(self): }, } - c = self.parseAndPrepare("foo", recipe, classes).getCheckoutStep() + c = self.parseAndPrepare(recipe, classes).getCheckoutStep() self.assertNotIn("asdf", c.getUpdateScript()) self.assertNotIn("qwer", c.getUpdateScript()) @@ -508,6 +508,6 @@ def testInheritNullEnabled(self): }, } - c = self.parseAndPrepare("foo", recipe, classes).getCheckoutStep() + c = self.parseAndPrepare(recipe, classes).getCheckoutStep() self.assertIn("asdf", c.getUpdateScript()) self.assertIn("qwer", c.getUpdateScript()) From 03a807996273ee2438bb50bcb00738487dd7cbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 10 Dec 2023 17:42:31 +0100 Subject: [PATCH 3/4] test: add tests for nested SCMs --- test/black-box/nested-scms/config.yaml | 1 + test/black-box/nested-scms/recipes/root.yaml | 16 ++ test/black-box/nested-scms/run.sh | 81 ++++++++++ test/unit/test_input_recipe.py | 157 +++++++++++++++++++ 4 files changed, 255 insertions(+) create mode 100644 test/black-box/nested-scms/config.yaml create mode 100644 test/black-box/nested-scms/recipes/root.yaml create mode 100755 test/black-box/nested-scms/run.sh diff --git a/test/black-box/nested-scms/config.yaml b/test/black-box/nested-scms/config.yaml new file mode 100644 index 000000000..1e430156b --- /dev/null +++ b/test/black-box/nested-scms/config.yaml @@ -0,0 +1 @@ +bobMinimumVersion: "0.23" diff --git a/test/black-box/nested-scms/recipes/root.yaml b/test/black-box/nested-scms/recipes/root.yaml new file mode 100644 index 000000000..47693eaef --- /dev/null +++ b/test/black-box/nested-scms/recipes/root.yaml @@ -0,0 +1,16 @@ +root: True + +checkoutSCM: + - if: "${GIT_1_ENABLE:-true}" + scm: git + url: file://${GIT_1_DIR} + rev: ${GIT_1_REV:-refs/heads/master} + dir: "foo" + - if: "${GIT_2_ENABLE:-true}" + scm: git + url: file://${GIT_2_DIR} + rev: ${GIT_2_REV:-refs/heads/master} + dir: "foo/bar" + +buildScript: "true" +packageScript: "true" diff --git a/test/black-box/nested-scms/run.sh b/test/black-box/nested-scms/run.sh new file mode 100755 index 000000000..3dbdbe86d --- /dev/null +++ b/test/black-box/nested-scms/run.sh @@ -0,0 +1,81 @@ +#!/bin/bash -e +# +# Verify checkouts and updates of nested SCMs. + +. ../../test-lib.sh 2>/dev/null || { echo "Must run in script directory!" ; exit 1 ; } + +git_dir1=$(mktemp -d) +git_dir2=$(mktemp -d) +trap 'rm -rf "$git_dir1" "$git_dir2"' EXIT +cleanup + +# Prepare git repositories + +pushd "$git_dir1" +git init . +git config user.email "bob@bob.bob" +git config user.name test +echo "commit-1" > git1.txt +git add git1.txt +git commit -m "initial commit" +git tag -a -m "First Tag" tag1 +echo "commit-2" > git1.txt +git commit -a -m "second commit" +git tag -a -m "Second Tag" tag2 +popd + +pushd "$git_dir2" +git init . +git config user.email "bob@bob.bob" +git config user.name test +echo "commit-1" > git2.txt +git add git2.txt +git commit -m "first commit" +git tag -a -m "First Tag" tag1 +echo "commit-2" > git2.txt +git commit -a -m "second commit" +git tag -a -m "Second Tag" tag2 +popd + + +# First a simple checkout. We put a canary there to detect attic moves. +run_bob dev -DGIT_1_DIR="$git_dir1" -DGIT_2_DIR="$git_dir2" root +expect_output "commit-2" cat dev/src/root/1/workspace/foo/git1.txt +echo canary > dev/src/root/1/workspace/foo/canary.txt +expect_output "commit-2" cat dev/src/root/1/workspace/foo/bar/git2.txt +echo canary > dev/src/root/1/workspace/foo/bar/canary.txt + +# Change tag on nested SCM +run_bob dev -DGIT_1_DIR="$git_dir1" -DGIT_2_DIR="$git_dir2" \ + -DGIT_2_REV="refs/tags/tag1" root +expect_exist dev/src/root/1/workspace/foo/canary.txt +expect_exist dev/src/root/1/workspace/foo/bar/canary.txt +expect_output "commit-2" cat dev/src/root/1/workspace/foo/git1.txt +expect_output "commit-1" cat dev/src/root/1/workspace/foo/bar/git2.txt + +# Change tag on upper SCM +run_bob dev -DGIT_1_DIR="$git_dir1" -DGIT_2_DIR="$git_dir2" \ + -DGIT_1_REV="refs/tags/tag1" \ + -DGIT_2_REV="refs/tags/tag1" root +expect_exist dev/src/root/1/workspace/foo/canary.txt +expect_exist dev/src/root/1/workspace/foo/bar/canary.txt +expect_output "commit-1" cat dev/src/root/1/workspace/foo/git1.txt +expect_output "commit-1" cat dev/src/root/1/workspace/foo/bar/git2.txt + +# Remove upper SCM. The upper SCM and the nested one are both moved to the +# attic and can be found there. The "nested" SCM is then checked out again. +run_bob dev -DGIT_1_DIR="$git_dir1" -DGIT_1_ENABLE=0 -DGIT_2_DIR="$git_dir2" root +expect_not_exist dev/src/root/1/workspace/foo/canary.txt +expect_not_exist dev/src/root/1/workspace/foo/git1.txt +expect_not_exist dev/src/root/1/workspace/foo/bar/canary.txt +expect_output "commit-2" cat dev/src/root/1/workspace/foo/bar/git2.txt +expect_exist dev/src/root/1/attic +expect_exist dev/src/root/1/attic/*foo +expect_exist dev/src/root/1/attic/*foo/bar + +status=$(run_bob status -DGIT_1_DIR="$git_dir1" -DGIT_1_ENABLE=0 -DGIT_2_DIR="$git_dir2" \ + --attic --show-clean) +# On Windows "bob status" will output native paths. Needs to be substituted +# before comparison. +[[ ${status//\\//} == *dev/src/root/1/attic/*foo* ]] +[[ ${status//\\//} == *dev/src/root/1/attic/*foo/bar ]] diff --git a/test/unit/test_input_recipe.py b/test/unit/test_input_recipe.py index 77f29f86d..fba385734 100644 --- a/test/unit/test_input_recipe.py +++ b/test/unit/test_input_recipe.py @@ -511,3 +511,160 @@ def testInheritNullEnabled(self): c = self.parseAndPrepare(recipe, classes).getCheckoutStep() self.assertIn("asdf", c.getUpdateScript()) self.assertIn("qwer", c.getUpdateScript()) + + +class TestSCMs(RecipeCommon, TestCase): + + def testAbsPath(self): + """Absolute SCM paths are rejected""" + recipe = { + "checkoutSCM" : [ + { + "scm" : "git", + "url" : "http://bob.test/test.git", + "dir" : "/absolute", + }, + ], + "buildScript" : "true", + } + with self.assertRaises(ParseError): + self.parseAndPrepare(recipe) + + def testDifferentPath(self): + """Multple SCMs in different directories are fine""" + recipe = { + "checkoutSCM" : [ + { + "scm" : "git", + "url" : "http://bob.test/test.git", + "dir" : "foo", + }, + { + "scm" : "svn", + "url" : "http://bob.test/test.svn", + "dir" : "bar", + }, + ], + "buildScript" : "true", + } + c = self.parseAndPrepare(recipe).getCheckoutStep() + l = c.getScmList() + self.assertEqual(len(l), 2) + self.assertEqual(l[0].getProperties(False)["scm"], "git") + self.assertEqual(l[0].getDirectory(), "foo") + self.assertEqual(l[1].getProperties(False)["scm"], "svn") + self.assertEqual(l[1].getDirectory(), "bar") + + def testSamePath(self): + """Multiple SCMs on same directory are rejected""" + recipe = { + "checkoutSCM" : [ + { + "scm" : "git", + "url" : "http://bob.test/test.git", + "dir" : "same", + }, + { + "scm" : "svn", + "url" : "http://bob.test/test.svn", + "dir" : "same", + }, + ], + "buildScript" : "true", + } + with self.assertRaises(ParseError): + self.parseAndPrepare(recipe) + + def testNested(self): + """Nested SCMs in upper-to-lower order are accepted""" + recipe = { + "checkoutSCM" : [ + { + "scm" : "git", + "url" : "http://bob.test/test.git", + "dir" : "foo", + }, + { + "scm" : "git", + "url" : "http://bob.test/test.git", + "dir" : "foo/bar", + }, + ], + "buildScript" : "true", + } + c = self.parseAndPrepare(recipe).getCheckoutStep() + l = c.getScmList() + self.assertEqual(len(l), 2) + self.assertEqual(l[0].getDirectory(), "foo") + self.assertEqual(l[1].getDirectory(), "foo/bar") + + def testNestedObstructs(self): + """Nested SCMs that obstruct deeper SCMs are rejected""" + recipe = { + "checkoutSCM" : [ + { + "scm" : "git", + "url" : "http://bob.test/test.git", + "dir" : "foo/bar", + }, + { + "scm" : "git", + "url" : "http://bob.test/test.git", + "dir" : "foo", + }, + ], + "buildScript" : "true", + } + with self.assertRaises(ParseError): + self.parseAndPrepare(recipe) + + def testNestedJenkinsMixedPluginOk(self): + """Nested non-plugin SCMs inside plugin SCMs are ok""" + recipe = { + "checkoutSCM" : [ + { + "scm" : "git", + "url" : "http://bob.test/test.git", + "dir" : "foo", + }, + { + "scm" : "cvs", + "url" : "http://bob.test/test.cvs", + "cvsroot" : "cvsroot", + "module" : "module", + "dir" : "foo/bar", + }, + ], + "buildScript" : "true", + } + c = self.parseAndPrepare(recipe).getCheckoutStep() + l = c.getScmList() + self.assertEqual(len(l), 2) + self.assertEqual(l[0].getProperties(True)["scm"], "git") + self.assertEqual(l[0].getDirectory(), "foo") + self.assertTrue(l[0].hasJenkinsPlugin()) + self.assertEqual(l[1].getProperties(True)["scm"], "cvs") + self.assertEqual(l[1].getDirectory(), "foo/bar") + self.assertFalse(l[1].hasJenkinsPlugin()) + + def testNestedJenkinsMixedPluginBad(self): + """Nested plugin SCMs inside non-plugin SCMs are rejected""" + recipe = { + "checkoutSCM" : [ + { + "scm" : "cvs", + "url" : "http://bob.test/test.cvs", + "cvsroot" : "cvsroot", + "module" : "module", + "dir" : "foo", + }, + { + "scm" : "git", + "url" : "http://bob.test/test.git", + "dir" : "foo/bar", + }, + ], + "buildScript" : "true", + } + with self.assertRaises(ParseError): + self.parseAndPrepare(recipe) From 289a6e01df8511443a83e7bdf12f01c78d5e0e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 10 Dec 2023 17:43:18 +0100 Subject: [PATCH 4/4] doc: add chapter about nested SCMs --- doc/manual/configuration.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index d70c958d9..c8ec36386 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -868,6 +868,14 @@ be given as IfExpression (see :ref:`configuration-principle-booleans`). By default the SCMs check out to the root of the workspace. You may specify any relative path in ``dir`` to checkout to this directory. +Special care must be taken if SCMs are nested, that is the ``dir`` attribute of +one SCM is a subdirectory of another. Bob requires that the SCM with the upper +directory has to be in the list before the SCMs that are checked out into +subdirectories. Additionally, SCMs that are natively supported by Jenkins +plugins (git, svn), cannot be nested into the other SCMs (cvs, import, url). +The reason is that Jenkins SCM plugins always execute before anything else in a +Jenkins job. + By using ``if`` you can selectively enable or disable a particular SCM using either a string or a expression. In case a string is given to the ``if``-keyword it is substituted according to :ref:`configuration-principle-subst` and the final