From 8502cf9c72920a9cc660e47e2091367aff540a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Tue, 7 Jan 2025 22:48:47 +0100 Subject: [PATCH 1/4] input: do not substitute internal SCM properties The '__source' and 'recipe' string properties are created internally. They should not undergo any string substitution because it may alter their values. --- pym/bob/input.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pym/bob/input.py b/pym/bob/input.py index 4d205a30..c627d3a5 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -370,7 +370,9 @@ def validate(self, data): def Scm(spec, env, overrides, recipeSet): # resolve with environment - spec = { k : ( env.substitute(v, "checkoutSCM::"+k) if isinstance(v, str) else v) + spec = { k : ( env.substitute(v, "checkoutSCM::"+k) + if isinstance(v, str) and k not in ('__source', 'recipe') + else v ) for (k, v) in spec.items() } # apply overrides before creating scm instances. It's possible to switch the Scm type with an override.. From e39254a59438434e5c99b5dfe0b7209cf1b288ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Fri, 27 Dec 2024 22:11:21 +0100 Subject: [PATCH 2/4] import: add recipe relative urls The import SCM base directory is the project root directory. This is useless for recipes in layers because they don't know their location in the project. The new "recipeRelative" property can pivot the base directory to the one where the recipe is located. --- pym/bob/scm/imp.py | 11 +++++++++-- pym/bob/scm/scm.py | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pym/bob/scm/imp.py b/pym/bob/scm/imp.py index 076171f6..43b4a917 100644 --- a/pym/bob/scm/imp.py +++ b/pym/bob/scm/imp.py @@ -116,6 +116,7 @@ class ImportScm(Scm): DEFAULTS = { schema.Optional('dir') : str, schema.Optional('prune') : bool, + schema.Optional('recipeRelative') : bool, } __SCHEMA = { @@ -134,6 +135,11 @@ def __init__(self, spec, overrides=[], pruneDefault=None, fixDigestBug=False, pr self.__data = spec.get("__data") self.__projectRoot = spec.get("__projectRoot", projectRoot) self.__fixDigestBug = fixDigestBug + self.__recipeRelative = spec.get("recipeRelative", False) + + def _getSrcDir(self): + rootDir = os.path.dirname(self._getRecipe()) if self.__recipeRelative else self.__projectRoot + return os.path.join(rootDir, self.__url) def getProperties(self, isJenkins, pretty=False): ret = super().getProperties(isJenkins) @@ -142,9 +148,10 @@ def getProperties(self, isJenkins, pretty=False): 'url' : self.__url, 'dir' : self.__dir, 'prune' : self.__prune, + 'recipeRelative' : self.__recipeRelative, }) if isJenkins: - ret['__data'] = packTree(self.__url) + ret['__data'] = packTree(self._getSrcDir()) else: ret['__projectRoot'] = self.__projectRoot return ret @@ -154,7 +161,7 @@ async def invoke(self, invoker): os.makedirs(dest, exist_ok=True) if self.__prune: emptyDirectory(dest) if self.__data is None: - src = os.path.join(self.__projectRoot, self.__url) + src = self._getSrcDir() if not os.path.isdir(src): invoker.fail("Cannot import '{}': not a directory!".format(src)) copyTree(src, dest, invoker) diff --git a/pym/bob/scm/scm.py b/pym/bob/scm/scm.py index abbd2eb0..bef6adbd 100644 --- a/pym/bob/scm/scm.py +++ b/pym/bob/scm/scm.py @@ -231,6 +231,9 @@ def _diffSpec(self, oldScm): ret -= {"if"} return ret + def _getRecipe(self): + return self.__recipe + def getSource(self): return self.__source From 56cd6b1a1789859df88e4cd7d2a0395f726d5902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Fri, 27 Dec 2024 22:13:52 +0100 Subject: [PATCH 3/4] doc: document recipeRelative import scm property --- doc/manual/configuration.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index 15f69d6c..7f9dc90f 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -957,6 +957,7 @@ git | ``url``: URL of remote repository | ``dissociate``: (Boolean, default false). Dissociate the reference (see man git-clone). import | ``url``: Directory path relative to project root. | ``prune`` (\*): Delete destination directory before importing files. + | ``recipeRelative`` (\*): Whether ``url`` is relative to recipe or project root. (optional) svn | ``url``: URL of SVN module | ``revision``: Optional revision number (optional) | ``sslVerify`` (\*): Whether to verify the SSL certificate when fetching (optional) @@ -1109,15 +1110,15 @@ git import The ``import`` SCM copies the directory specified in ``url`` to the - workspace. By default the destination is always overwritten and obsolete + workspace. By default, the destination is always overwritten and obsolete files are deleted. Set ``prune`` to ``False`` to only overwrite if the source file was changed more recently than the exiting destination in the - workspace. Before Bob 0.18 the default was the other way around (see + workspace. Before Bob 0.18, the default was the other way around (see :ref:`policies-pruneImportScm`). - In contrast to the other SCMs that fetch across the network the ``import`` + In contrast to the other SCMs that fetch across the network, the ``import`` SCM is always updated, even if ``--build-only`` is used. Because only local - files are imported there is no possibility to inadvertely fetch unwanted + files are imported, there is no possibility to inadvertently fetch unwanted changes from other users. The files should thus always be edited at the import source location and not in the workspace. @@ -1126,6 +1127,11 @@ import content is included in the job configuration that will get too large otherwise. + By default, the directory given in ``url`` is interpreted relative to the + project root. Alternatively, ``url`` can be made relative to the recipe + itself if ``recipeRelative`` is set to ``True``. This is recommended + especially for recipes that are included as layers into other projects. + svn The `Svn`_ SCM, like git, requires the ``url`` attribute too. If you specify a numeric ``revision`` Bob considers the SCM as deterministic. From 4a6ef0c41cd447057759c980f44bf3b7593ce387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Fri, 27 Dec 2024 22:14:41 +0100 Subject: [PATCH 4/4] test: add recipeRelative import scm test --- test/black-box/import-scm-relative/config.yaml | 1 + .../recipes/sub/data/data.txt | 1 + .../import-scm-relative/recipes/sub/root.yaml | 7 +++++++ test/black-box/import-scm-relative/run.sh | 18 ++++++++++++++++++ 4 files changed, 27 insertions(+) create mode 100644 test/black-box/import-scm-relative/config.yaml create mode 100644 test/black-box/import-scm-relative/recipes/sub/data/data.txt create mode 100644 test/black-box/import-scm-relative/recipes/sub/root.yaml create mode 100755 test/black-box/import-scm-relative/run.sh diff --git a/test/black-box/import-scm-relative/config.yaml b/test/black-box/import-scm-relative/config.yaml new file mode 100644 index 00000000..703af468 --- /dev/null +++ b/test/black-box/import-scm-relative/config.yaml @@ -0,0 +1 @@ +bobMinimumVersion: "0.25" diff --git a/test/black-box/import-scm-relative/recipes/sub/data/data.txt b/test/black-box/import-scm-relative/recipes/sub/data/data.txt new file mode 100644 index 00000000..557db03d --- /dev/null +++ b/test/black-box/import-scm-relative/recipes/sub/data/data.txt @@ -0,0 +1 @@ +Hello World diff --git a/test/black-box/import-scm-relative/recipes/sub/root.yaml b/test/black-box/import-scm-relative/recipes/sub/root.yaml new file mode 100644 index 00000000..6c74f2d0 --- /dev/null +++ b/test/black-box/import-scm-relative/recipes/sub/root.yaml @@ -0,0 +1,7 @@ +root: True +checkoutSCM: + scm: import + url: data + recipeRelative: True +buildScript: cp -a "$1/"* . +packageScript: cp -a "$1/"* . diff --git a/test/black-box/import-scm-relative/run.sh b/test/black-box/import-scm-relative/run.sh new file mode 100755 index 00000000..f601e73f --- /dev/null +++ b/test/black-box/import-scm-relative/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e +# +# Test recipeRelative "import" SCM property +# +. ../../test-lib.sh 2>/dev/null || { echo "Must run in script directory!" ; exit 1 ; } + +cleanup + +# First try in-tree build +run_bob dev sub::root +diff -Nrq recipes/sub/data dev/dist/sub/root/1/workspace + +# Out of tree builds should work as well +build="$(mktemp -d)" +trap 'rm -rf "$build"' EXIT +run_bob init . "$build" +run_bob -C "$build" dev sub::root +diff -Nrq recipes/sub/data "$build/dev/dist/sub/root/1/workspace"