Merge pull request #477 from jkloetzke/consistent-unicode-replace

Consistently use 'replace' unicode error handler

pym/bob/develop/version.py

@@ -24,7 +24,8 @@ def getVersion():
         import subprocess
         try:
             version = subprocess.check_output("git describe --tags --dirty".split(" "),
-                cwd=root, universal_newlines=True, stderr=subprocess.DEVNULL)
+                cwd=root, universal_newlines=True, stderr=subprocess.DEVNULL,
+                errors='replace')
         except (subprocess.CalledProcessError, OSError):
             pass
 

pym/bob/invoker.py

@@ -174,7 +174,7 @@ def __getSandboxHelperPath(self):
 
     async def __runCommand(self, args, cwd, stdout=None, stderr=None,
                            check=False, env=None, universal_newlines=True,
-                           specEnv=True, **kwargs):
+                           errors='replace', specEnv=True, **kwargs):
         cmd = " ".join(quote(a) for a in args)
         self.trace(cmd)
 
@@ -249,7 +249,7 @@ async def __runCommand(self, args, cwd, stdout=None, stderr=None,
         if stdout == True:
             if universal_newlines:
                 stdoutStream.seek(0)
-                stdoutBuf = io.TextIOWrapper(stdoutStream).read()
+                stdoutBuf = io.TextIOWrapper(stdoutStream, errors=errors).read()
             else:
                 stdoutBuf = stdoutStream.getvalue()
         else:
@@ -258,7 +258,7 @@ async def __runCommand(self, args, cwd, stdout=None, stderr=None,
         if stderr == True:
             if universal_newlines:
                 stderrStream.seek(0)
-                stderrBuf = io.TextIOWrapper(stderrStream).read()
+                stderrBuf = io.TextIOWrapper(stderrStream, errors=errors).read()
             else:
                 stderrBuf = stderrStream.getvalue()
         else:
@@ -591,7 +591,7 @@ def joinPath(self, *paths):
 
     def getStdio(self):
         return self.__stdioBuffer.getvalue().decode(
-            locale.getpreferredencoding(), 'surrogateescape')
+            locale.getpreferredencoding(), 'replace')
 
     def setMakeParameters(self, fds, jobs):
         self.__makeFds = fds

pym/bob/scm/cvs.py

@@ -137,6 +137,7 @@ def status(self, workspacePath):
                 output = subprocess.check_output(commandLine,
                                                  cwd=workDir,
                                                  universal_newlines=True,
+                                                 errors='replace',
                                                  env=environment,
                                                  stderr=subprocess.DEVNULL,
                                                  stdin=subprocess.DEVNULL)

pym/bob/scm/git.py

@@ -774,7 +774,7 @@ async def predictLiveBuildId(self, step):
             cmdLine = ['git', 'ls-remote', self.__url] + refs
             try:
                 stdout = await check_output(cmdLine, stderr=subprocess.DEVNULL,
-                    universal_newlines=True)
+                    universal_newlines=True, errors='replace')
                 output = stdout.strip()
             except subprocess.CalledProcessError as e:
                 a.fail("exit {}".format(e.returncode), WARNING)
@@ -824,7 +824,7 @@ def getLiveBuildIdSpec(self, workspacePath):
     def processLiveBuildIdSpec(dir):
         try:
             return subprocess.check_output(["git", "rev-parse", "HEAD"],
-                cwd=dir, universal_newlines=True).strip()
+                cwd=dir, universal_newlines=True, errors='replace').strip()
         except subprocess.CalledProcessError as e:
             raise BuildError("Git audit failed: " + str(e))
         except OSError as e:
@@ -850,15 +850,15 @@ async def _scanDir(self, workspace, dir, extra):
         dir = os.path.join(workspace, dir)
         try:
             remotes = (await check_output(["git", "remote", "-v"],
-                cwd=dir, universal_newlines=True)).split("\n")
+                cwd=dir, universal_newlines=True, errors='replace')).split("\n")
             remotes = (r[:-8].split("\t") for r in remotes if r.endswith("(fetch)"))
             self.__remotes = { remote:url for (remote,url) in remotes }
 
             self.__commit = (await check_output(["git", "rev-parse", "HEAD"],
-                cwd=dir, universal_newlines=True)).strip()
+                cwd=dir, universal_newlines=True, errors='replace')).strip()
             self.__description = (await check_output(
                 ["git", "describe", "--always", "--dirty=-dirty"],
-                cwd=dir, universal_newlines=True)).strip()
+                cwd=dir, universal_newlines=True, errors='replace')).strip()
             subDirty = await self.__scanSubmodules(dir, self.__submodules)
             self.__dirty = subDirty or self.__description.endswith("-dirty")
         except subprocess.CalledProcessError as e:
@@ -875,7 +875,7 @@ async def __scanSubmodules(self, dir, shouldExist, base = "."):
         try:
             allPaths = await check_output(["git", "-C", base, "config", "-f",
                 ".gitmodules", "-z", "--get-regexp", "path"], cwd=dir,
-                universal_newlines=True)
+                universal_newlines=True, errors='replace')
         except subprocess.CalledProcessError:
             allPaths = "" # No key found in file. Probably empty
         allPaths = [ p.split("\n")[1] for p in allPaths.split("\0") if p ]
@@ -884,7 +884,7 @@ async def __scanSubmodules(self, dir, shouldExist, base = "."):
 
         # Fetch the respecive commits as per git ls-tree
         allPaths = await check_output(["git", "-C", base, "ls-tree", "-z",
-            "HEAD"] + allPaths, cwd=dir, universal_newlines=True)
+            "HEAD"] + allPaths, cwd=dir, universal_newlines=True, errors='replace')
         allPaths = {
             path : attribs.split(' ')[2]
                 for attribs, path
@@ -918,7 +918,7 @@ async def __scanSubmodules(self, dir, shouldExist, base = "."):
                 return True
 
             realCommit = (await check_output(["git", "-C", subPath, "rev-parse", "HEAD"],
-                cwd=dir, universal_newlines=True)).strip()
+                cwd=dir, universal_newlines=True, errors='replace')).strip()
             if commit != realCommit:
                 return True # different commit checked out
 

pym/bob/scm/svn.py

@@ -119,7 +119,7 @@ def callSubversion(self, workspacePath, *args):
         cwd = os.path.join(workspacePath, self.__dir)
         try:
             output = subprocess.check_output(cmdLine, cwd=cwd,
-                universal_newlines=True, stderr=subprocess.DEVNULL)
+                universal_newlines=True, errors='replace', stderr=subprocess.DEVNULL)
         except subprocess.CalledProcessError as e:
             raise BuildError("svn error:\n Directory: '{}'\n Command: '{}'\n'{}'".format(
                 cwd, " ".join(cmdLine), e.output.rstrip()))
@@ -175,14 +175,14 @@ async def _scanDir(self, workspace, dir, extra):
         try:
             info = ElementTree.fromstring(await check_output(
                 ["svn", "info", "--xml", dir],
-                cwd=workspace, universal_newlines=True))
+                cwd=workspace, universal_newlines=True, errors='replace'))
             self.__url = info.find('entry/url').text
             self.__revision = int(info.find('entry').get('revision'))
             self.__repoRoot = info.find('entry/repository/root').text
             self.__repoUuid = info.find('entry/repository/uuid').text
 
             status = await check_output(["svn", "status", dir],
-                cwd=workspace, universal_newlines=True)
+                cwd=workspace, universal_newlines=True, errors='replace')
             self.__dirty = status != ""
         except subprocess.CalledProcessError as e:
             raise BuildError("Svn audit failed: " + str(e))

pym/bob/utils.py

@@ -731,7 +731,7 @@ def __exit__(self, exc_type, exc_value, traceback):
         asyncio.set_event_loop(None)
 
 
-async def run(args, universal_newlines=False, check=False, shell=False, **kwargs):
+async def run(args, universal_newlines=False, errors=None, check=False, shell=False, **kwargs):
     """Provide the subprocess.run() function as asyncio corouting.
 
     This takes care of the missing 'universal_newlines' and 'check' options.
@@ -750,9 +750,9 @@ async def run(args, universal_newlines=False, check=False, shell=False, **kwargs
     stdout, stderr = await proc.communicate()
 
     if universal_newlines and (stdout is not None):
-        stdout = io.TextIOWrapper(io.BytesIO(stdout)).read()
+        stdout = io.TextIOWrapper(io.BytesIO(stdout), errors=errors).read()
     if universal_newlines and (stderr is not None):
-        stderr = io.TextIOWrapper(io.BytesIO(stderr)).read()
+        stderr = io.TextIOWrapper(io.BytesIO(stderr), errors=errors).read()
 
     if check and (proc.returncode != 0):
         raise subprocess.CalledProcessError(proc.returncode, args,