Merge pull request #546 from jkloetzke/mirrors

Add mirrors support for URL SCMs

doc/manual/configuration.rst

@@ -1093,8 +1093,15 @@ svn
    numeric ``revision`` Bob considers the SCM as deterministic.
 
 url
-   The ``url`` SCM naturally needs an ``url`` attribute. If a SHA digest is
-   given with ``digestSHA1``, ``digestSHA256`` and/or ``digestSHA512`` the
+   The ``url`` SCM naturally needs an ``url`` attribute. This might be a proper
+   URL (e.g. ``http://foo.bar/baz.tgz``) or a file name. The supported URL
+   schemas depend on Pythons ``urllib`` module but ``http``, ``https``, ``ftp``
+   and ``file`` should work. If a bare file name is specified, tilde expansion
+   is performed. This replaces the initial ``~`` or ``~user`` by the *user*’s
+   home directory.
+
+   If a SHA digest is
+   given with ``digestSHA1``, ``digestSHA256`` and/or ``digestSHA512``, the
    downloaded file will be checked for a matching hash sum. This also makes the
    URL deterministic for Bob. Otherwise the URL will be checked in each build
    for updates. Based on the file name ending, Bob will try to extract the
@@ -2392,6 +2399,52 @@ postBuildHook
 
 .. _configuration-config-rootFilter:
 
+preMirror / fallbackMirror
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Type: Mirror-entry or list of mirror-entries
+
+Define alternate URLs that are checked either before (``preMirror``) or as a
+fallback (``fallbackMirror``) to the primary URL as defined in the SCM.
+Optionally, these mirrors can be populated during the build process. This is
+primarily useful for local ``preMirror``'s so that the files are available on
+the next build and the original server does not need to be used any more,
+saving build time and bandwidth on the upstream server.
+
+Mirrors are only used for fully deterministic SCMs. The reason is that
+otherwise the URL would not be interchangeable because every server could
+provide a different result. For the same reason it is not possible to mirror
+between different methods e.g., use a HTTP URL SCM mirror for a git SCM. It is
+not possible to independently verify the equivalence of the mirror in such a
+case.
+
+Each mirror entry specifies the SCM type (``scm``), a regular expression to
+match the URL (``url``) and a replacement URL (``mirror``). Optionally, it is
+possible to upload to the mirror (``upload``). Currently only the URL SCM is
+supported for mirrors.
+
+Examples::
+
+    fallbackMirror:
+        - scm: url
+          url: "https?://ftp.gnu.org/(pub/)?gnu/(.*)"
+          mirror: "http://mirror.netcologne.de/gnu/\\2"
+        - scm: url
+          url: "https?://ftp.gnu.org/(pub/)?gnu/(.*)"
+          mirror: "http://www.mirrorservice.org/sites/ftp.gnu.org/gnu/\\2"
+
+A typical mirror configuration for the global user configuration could look
+like the following. It mirrors all remote URLs to a local directory::
+
+    preMirror:
+        scm: url
+        url: "https?://.*/(.*)"
+        mirror: "~/.cache/bob/mirror/\\1"
+        upload: True
+
+This will put all downloaded files into a caching directory of the current
+user.
+
 rootFilter
 ~~~~~~~~~~
 
@@ -2446,7 +2499,7 @@ from sharing such shared packages on the same machine.
 Example::
 
     share:
-        path: ~/.cache/bob
+        path: ~/.cache/bob/pkgs
         quota: "5G"
         autoClean: True
 

pym/bob/input.py

@@ -1781,7 +1781,7 @@ def validate(self, data):
         if isinstance(data, dict):
             data = [self.__validateScm(data)]
         elif isinstance(data, list):
-            for i in data: self.__validateScm(i)
+            data = [ self.__validateScm(i) for i in data ]
         else:
             raise schema.SchemaUnexpectedTypeError(
                 'checkoutSCM must be a SCM spec or a list threreof',
@@ -3023,6 +3023,10 @@ class RecipeSet:
         'import' : ImportScm.SCHEMA,
     })
 
+    MIRRORS_SCHEMA = ScmValidator({
+        'url' : UrlScm.MIRRORS_SCHEMA,
+    })
+
     _ignoreCmdConfig = False
     @classmethod
     def ignoreCommandCfg(cls):
@@ -3142,7 +3146,12 @@ def __init__(self):
         self.__buildHooks = {}
         self.__sandboxOpts = {}
         self.__scmDefaults = {}
+        self.__preMirrors = []
+        self.__fallbackMirrors = []
+
         def updateArchive(x): self.__archive = x
+        def updatePreMirror(x) : self.__preMirrors = x
+        def updateFallbackMirror(x) : self.__fallbackMirrors = x
 
         def updateWhiteList(x):
             if self.__platform == "win32":
@@ -3176,13 +3185,23 @@ def updateWhiteList(x):
                 VarDefineValidator("environment"),
                 lambda x: self.__defaultEnv.update(x)
             ),
+            "fallbackMirror" : BuiltinSetting(
+                self.MIRRORS_SCHEMA,
+                updateFallbackMirror,
+                True
+            ),
             "hooks" : BuiltinSetting(
                 schema.Schema({
                     schema.Optional('preBuildHook') : str,
                     schema.Optional('postBuildHook') : str,
                 }),
                 lambda x: self.__buildHooks.update(x)
             ),
+            "preMirror" : BuiltinSetting(
+                self.MIRRORS_SCHEMA,
+                updatePreMirror,
+                True
+            ),
             "rootFilter" : BuiltinSetting(
                 schema.Schema([str]),
                 lambda x: self.__rootFilter.extend(x)
@@ -3884,6 +3903,12 @@ def getProjectRoot(self):
         """
         return self.__projectRoot
 
+    def getPreMirrors(self):
+        return self.__preMirrors
+
+    def getFallbackMirrors(self):
+        return self.__fallbackMirrors
+
 
 class YamlCache:
     def __if_expression_constructor(loader, node):

pym/bob/intermediate.py

@@ -569,6 +569,8 @@ def fromRecipeSet(cls, recipeSet):
         self.__data['archiveSpec'] = recipeSet.archiveSpec()
         self.__data['envWhiteList'] = sorted(recipeSet.envWhiteList())
         self.__data['projectRoot'] = recipeSet.getProjectRoot()
+        self.__data['preMirrors'] = recipeSet.getPreMirrors()
+        self.__data['fallbackMirrors'] = recipeSet.getFallbackMirrors()
         return self
 
     @classmethod
@@ -599,3 +601,9 @@ def getPolicy(self, name, location=None):
 
     def getProjectRoot(self):
         return self.__data['projectRoot']
+
+    def getPreMirrors(self):
+        return self.__data['preMirrors']
+
+    def getFallbackMirrors(self):
+        return self.__data['fallbackMirrors']

pym/bob/scm/__init__.py

@@ -59,6 +59,8 @@ def getScm(spec, overrides=[], recipeSet=None):
         return CvsScm(spec, overrides)
     elif scm == "url":
         return UrlScm(spec, overrides, recipeSet and recipeSet.getPolicy('tidyUrlScm'),
-            recipeSet and recipeSet.getPolicy('scmIgnoreUser'))
+            recipeSet and recipeSet.getPolicy('scmIgnoreUser'),
+            recipeSet and recipeSet.getPreMirrors(),
+            recipeSet and recipeSet.getFallbackMirrors())
     else:
         raise ParseError("Unknown SCM '{}'".format(scm))