git: add generic git refs support

So far, Bob could only handle branches (refs/heads/...), tags
(refs/tags/...) and plain commits. But Git allows other "refs" (git
references) as well. One example is Gerrit where changes can be
downloaded as /refs/changes/... refs.

A generic ref is handled like a tag. It is checked out once and the
workspace will be in detached head state. Bob will not expect the ref to
change its targeted commit by default. In case the ref does change, the
user can enable the "rebase" property.

Fixes #601.

pym/bob/scm/git.py

@@ -28,7 +28,8 @@ def dirIsEmpty(p):
         return False
     return os.listdir(p) == []
 
-def getBranchTagCommit(spec):
+def getRefBranchTagCommit(spec):
+    ref = None
     branch = None
     tag = None
     commit = None
@@ -38,6 +39,8 @@ def getBranchTagCommit(spec):
             branch = rev[11:]
         elif rev.startswith("refs/tags/"):
             tag = rev[10:]
+        elif rev.startswith("refs/"):
+            ref = rev
         elif len(rev) == 40:
             commit = rev
         else:
@@ -49,11 +52,11 @@ def getBranchTagCommit(spec):
         # validate commit
         if re.match("^[0-9a-f]{40}$", commit) is None:
             raise ParseError("Invalid commit id: " + str(commit))
-    elif not branch and not tag:
+    elif not ref and not branch and not tag:
         # nothing secified at all -> master branch
         branch = "master"
 
-    return (branch, tag, commit)
+    return (ref, branch, tag, commit)
 
 class GitScm(Scm):
 
@@ -105,7 +108,7 @@ def __init__(self, spec, overrides=[], stripUser=None, useBranchAndCommit=False)
         super().__init__(spec, overrides)
         self.__url = spec["url"]
         self.__dir = spec.get("dir", ".")
-        self.__branch, self.__tag, self.__commit = getBranchTagCommit(spec)
+        self.__ref, self.__branch, self.__tag, self.__commit = getRefBranchTagCommit(spec)
         self.__remotes = {}
         # convert remotes into separate dictionary
         for key, val in spec.items():
@@ -147,17 +150,24 @@ def __resolveReferences(self, alt):
 
     def getProperties(self, isJenkins, pretty=False):
         properties = super().getProperties(isJenkins)
+
+        if self.__commit:
+            rev = self.__commit
+        elif self.__tag:
+            rev = "refs/tags/" + self.__tag
+        elif self.__branch:
+            rev = "refs/heads/" + self.__branch
+        else:
+            rev = self.__ref
+
         properties.update({
             'scm' : 'git',
             'url' : self.__url,
             'branch' : self.__branch,
             'tag' : self.__tag,
             'commit' : self.__commit,
             'dir' : self.__dir,
-            'rev' : ( self.__commit if self.__commit else
-                (("refs/tags/" + self.__tag) if self.__tag else
-                    ("refs/heads/" + self.__branch))
-            ),
+            'rev' : rev,
             'retries' : self.__retries,
             'sslVerify' : self.__sslVerify,
             'singleBranch' : self.__singleBranch,
@@ -243,6 +253,8 @@ async def invoke(self, invoker, switch=False):
         # Calculate appropriate refspec (all/singleBranch/tag)
         if singleBranch:
             fetchCmd += ["+refs/heads/{0}:refs/remotes/origin/{0}".format(self.__branch)]
+        elif self.__ref:
+            fetchCmd += [self.__ref]
         else:
             fetchCmd += ["+refs/heads/*:refs/remotes/origin/*"]
         if self.__tag:
@@ -253,6 +265,8 @@ async def invoke(self, invoker, switch=False):
             await self.__checkoutTagOnBranch(invoker, fetchCmd, switch)
         elif self.__tag or self.__commit:
             await self.__checkoutTag(invoker, fetchCmd, switch)
+        elif self.__ref:
+            await self.__checkoutRef(invoker, fetchCmd, switch)
         else:
             await self.__checkoutBranch(invoker, fetchCmd, switch)
 
@@ -383,6 +397,36 @@ async def __checkoutTag(self, invoker, fetchCmd, switch):
             # FIXME: will not be called again if interrupted!
             await self.__checkoutSubmodules(invoker)
 
+    async def __checkoutRef(self, invoker, fetchCmd, switch):
+        oldUpstreamCommit = None
+        if self.__rebase:
+            # The user apparently expects the upstream ref to change. We
+            # remember the last fetched commit in a bob specific config entry.
+            oldBobHead = await invoker.runCommand(["git", "config", "remote.origin.bob-head"],
+                                                  stdout=True, stderr=False, cwd=self.__dir)
+            if oldBobHead.returncode == 0:
+                oldUpstreamCommit = oldBobHead.stdout.rstrip()
+
+        head = await invoker.callCommand(["git", "rev-parse", "--verify", "-q", "HEAD"],
+            stdout=False, cwd=self.__dir)
+        if head != 0 or switch or self.__rebase:
+            await invoker.checkCommand(fetchCmd, retries=self.__retries, cwd=self.__dir)
+            fetchHead = await invoker.checkOutputCommand(["git", "rev-parse", "FETCH_HEAD"],
+                                                         cwd=self.__dir)
+            if self.__rebase and oldUpstreamCommit:
+                await invoker.checkCommand(
+                    ["git", "-c", "submodule.recurse=0", "rebase", "--onto",
+                     fetchHead, oldUpstreamCommit],
+                    cwd=self.__dir)
+            else:
+                await invoker.checkCommand(["git", "checkout", "-q", "--no-recurse-submodules",
+                    fetchHead], cwd=self.__dir)
+
+            # Remember old fetched commit in case we need to rebase later.
+            await invoker.checkCommand(["git", "config", "remote.origin.bob-head", fetchHead],
+                                       cwd=self.__dir)
+            await self.__checkoutSubmodules(invoker)
+
     async def __checkoutBranch(self, invoker, fetchCmd, switch):
         oldUpstreamCommit = None
         if self.__rebase:
@@ -619,15 +663,21 @@ async def switch(self, invoker, oldScm):
             # was checked out and the repo is still at this commit.
             oldTag = oldScm.__tag
             oldCommit = oldScm.__commit
+            oldBranch = oldScm.__branch
             if oldCommit:
                 pass # just compare this commit
             elif oldTag:
                 # Convert tag to commit. Beware of annotated commits!
                 oldCommit = await invoker.checkOutputCommand(["git",
                     "rev-parse", "tags/"+oldTag+"^0"], cwd=self.__dir)
-            else:
+            elif oldBranch:
                 # User moved from branch to detached HEAD
                 invoker.fail("Cannot switch: detached HEAD state")
+            else:
+                oldBobHead = await invoker.runCommand(["git", "config", "remote.origin.bob-head"],
+                                                      stdout=True, stderr=False, cwd=self.__dir)
+                if oldBobHead.returncode == 0:
+                    oldCommit = oldBobHead.stdout.rstrip()
 
             curCommit = await invoker.checkOutputCommand(["git", "rev-parse",
                 "HEAD"], cwd=self.__dir)
@@ -653,8 +703,10 @@ def asDigestScript(self):
             ret = self.__commit + " " + self.__dir
         elif self.__tag:
             ret = filt(self.__url) + " refs/tags/" + self.__tag + " " + self.__dir
-        else:
+        elif self.__branch:
             ret = filt(self.__url) + " refs/heads/" + self.__branch + " " + self.__dir
+        else:
+            ret = filt(self.__url) + " " + self.__ref + " " + self.__dir
 
         if self.__submodules:
             ret += " submodules"
@@ -696,8 +748,10 @@ def asJenkins(self, workPath, config):
             branch.text = self.__commit
         elif self.__tag:
             branch.text = "refs/tags/" + self.__tag
-        else:
+        elif self.__branch:
             branch.text = "refs/heads/" + self.__branch
+        else:
+            branch.text = self.__ref
 
         ElementTree.SubElement(scm, "doGenerateSubmoduleConfigurations").text = "false"
         ElementTree.SubElement(scm, "submoduleCfg", attrib={"class" : "list"})
@@ -759,7 +813,8 @@ def getDirectory(self):
         return self.__dir
 
     def isDeterministic(self):
-        return bool(self.__tag) or bool(self.__commit)
+        return bool(self.__tag) or bool(self.__commit) or \
+                bool(not self.__rebase and self.__ref)
 
     def hasJenkinsPlugin(self):
         # Cloning a subset of submodules is not supported by the Jenkins
@@ -820,6 +875,13 @@ def status(self, workspacePath):
                             joinLines("> unpushed commits on {}:".format(self.__branch),
                                 indent(output, '   ')))
                     onCorrectBranch = True
+            else:
+                currentCommit = self.callGit(workspacePath, 'rev-parse', 'HEAD')
+                refCommit = self.callGit(workspacePath, 'config', 'remote.origin.bob-head',
+                                         check=False)
+                if currentCommit != refCommit:
+                    status.add(ScmTaint.switched,
+                        f"> rev '{self.__ref}': expected: '{refCommit}', actual: {currentCommit}")
 
             # Check for modifications wrt. checked out commit
             output = self.callGit(workspacePath, 'status', '--porcelain', '--ignore-submodules=all')
@@ -947,8 +1009,10 @@ async def predictLiveBuildId(self, step):
             if self.__tag:
                 # Annotated tags are objects themselves. We need the commit object!
                 refs = ["refs/tags/" + self.__tag + '^{}', "refs/tags/" + self.__tag]
-            else:
+            elif self.__branch:
                 refs = ["refs/heads/" + self.__branch]
+            else:
+                refs = [self.__ref]
             cmdLine = ['git', 'ls-remote', self.__url] + refs
             try:
                 stdout = await check_output(cmdLine, stderr=subprocess.DEVNULL,