Skip to content

Commit

Permalink
test: add unit tests for URL SCM mirrors
Browse files Browse the repository at this point in the history
  • Loading branch information
@jkloetzke
jkloetzke committed Jan 1, 2024
1 parent 081db92 commit d0fc33a
Showing 1 changed file with 323 additions and 11 deletions.
334 changes: 323 additions & 11 deletions test/unit/test_input_urlscm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,17 @@
from bob.errors import ParseError
from bob.utils import asHexStr, runInEventLoop, getProcessPoolExecutor

INVALID_FILE = "C:\\does\\not\\exist" if sys.platform == "win32" else "/does/not/exist/"

def makeFileUrl(fn):
if sys.platform == "win32":
return "file:///" + fn.replace("\\", "/")
else:
return "file://" + fn

def escapeMirrorFileName(fn):
return os.path.abspath(fn).replace('\\', '\\\\')

class DummyPackage:
def getName(self):
return "dummy"
Expand All @@ -37,26 +48,24 @@ class UrlScmTest:
def setUpClass(cls):
cls.__repodir = tempfile.TemporaryDirectory()
cls.dir = cls.__repodir.name
fn = os.path.join(cls.__repodir.name, "test.txt")
if sys.platform == "win32":
cls.url = "file:///" + fn
else:
cls.url = "file://" + fn
cls.fn = "test.txt"
cls.path = os.path.join(cls.__repodir.name, cls.fn)
cls.url = makeFileUrl(cls.path)

with open(fn, "w") as f:
with open(cls.path, "w") as f:
f.write("Hello world!")

with open(fn, "rb") as f:
with open(cls.path, "rb") as f:
d = hashlib.sha1()
d.update(f.read())
cls.urlSha1 = asHexStr(d.digest())

with open(fn, "rb") as f:
with open(cls.path, "rb") as f:
d = hashlib.sha256()
d.update(f.read())
cls.urlSha256 = asHexStr(d.digest())

with open(fn, "rb") as f:
with open(cls.path, "rb") as f:
d = hashlib.sha512()
d.update(f.read())
cls.urlSha512 = asHexStr(d.digest())
Expand All @@ -75,15 +84,15 @@ def invokeScm(self, workspace, scm):
finally:
executor.shutdown()

def createUrlScm(self, spec = {}):
def createUrlScm(self, spec = {}, preMirrors=[], fallbackMirrors=[]):
s = {
'scm' : 'url',
'url' : self.url,
'recipe' : "foo.yaml#0",
'__source' : "Recipe foo",
}
s.update(spec)
return UrlScm(s)
return UrlScm(s, preMirrors=preMirrors, fallbackMirrors=fallbackMirrors)

def assertContent(self, fn):
with open(fn, "rb") as f:
Expand Down Expand Up @@ -413,3 +422,306 @@ def testGzStripComponentsNotSupported(self):
with tempfile.TemporaryDirectory() as workspace:
with self.assertRaises(InvocationError):
self.invokeScm(workspace, scm)


class TestMirrors(UrlScmTest, TestCase):

def assertExists(self, fn):
self.assertTrue(os.path.exists(fn), "file "+fn+" does not exist")

def assertNotExists(self, fn):
self.assertFalse(os.path.exists(fn), "file "+fn+" does exist")

def testFileMirrorFailed(self):
"""A missing file in a local mirror is tolerated"""
scm = self.createUrlScm(
{ "digestSHA1" : self.urlSha1 },
preMirrors=[ { 'scm' : 'url',
'url' : r".+",
'mirror' : escapeMirrorFileName("/does/not/exist"),
}
])

with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)

def testHttpMirrorFailed(self):
"""A missing file in an HTTP mirror is tolerated"""
with tempfile.TemporaryDirectory() as mirror:
with HttpServerMock(mirror) as srv:
scm = self.createUrlScm(
{ "digestSHA1" : self.urlSha1 },
preMirrors = [{
"scm" : "url",
"url" : r".*/(.*)",
"mirror" : r"http://localhost:{}/\1".format(srv.port),
}]
)
with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)

self.assertEqual(0, srv.headRequests)
self.assertEqual(1, srv.getRequests)
self.assertEqual(0, srv.putRequests)

def testNoMirrorsIfIndeterministic(self):
"""Mirrors are only consulted for deterministic SCMs"""
with tempfile.TemporaryDirectory() as mirror:
with HttpServerMock(mirror) as srv:
scm = self.createUrlScm(
preMirrors = [{
"scm" : "url",
"url" : r".*/(.*)",
"mirror" : r"http://localhost:{}/\1".format(srv.port),
"upload" : True,
}]
)
with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)

self.assertEqual(0, srv.getRequests)
self.assertEqual(0, srv.putRequests)

def testPreMirrorFileUsed(self):
"""Test that pre-mirror is used before looking at primary URL"""
with tempfile.TemporaryDirectory() as mirror:
mirrorFile = os.path.join(mirror, "mirror.txt")
shutil.copy(self.path, mirrorFile)

# Make sure to fail if primary URL is used
rogueFile = os.path.join(mirror, "evil.txt")
with open(rogueFile, "w") as f:
f.write("bad")

scm = self.createUrlScm(
{ "url" : makeFileUrl(rogueFile),
"digestSHA1" : self.urlSha1 },
preMirrors=[ { 'scm' : 'url',
'url' : r".+",
'mirror' : escapeMirrorFileName(mirrorFile),
}
])

with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)
self.assertExists(os.path.join(workspace, "evil.txt"))

def testFallbackMirrorFileUsed(self):
"""Test that fallback mirror is used in case primary URL is unavailable."""
with tempfile.TemporaryDirectory() as mirror:
shutil.copy(self.path, os.path.join(mirror, self.fn))

scm = self.createUrlScm(
{ "url" : makeFileUrl(os.path.join(INVALID_FILE, self.fn)),
"digestSHA1" : self.urlSha1 },
fallbackMirrors=[ { 'scm' : 'url',
'url' : r".*/(.*)",
'mirror' : escapeMirrorFileName(mirror) + r"/\1",
}
])

with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)
self.assertExists(os.path.join(workspace, self.fn))

def testHttpMirrorUsed(self):
"""Test HTTP mirror"""
with tempfile.TemporaryDirectory() as mirror:
shutil.copy(self.path, os.path.join(mirror, self.fn))
with HttpServerMock(mirror) as srv:
scm = self.createUrlScm(
{ "url" : makeFileUrl(os.path.join(INVALID_FILE, self.fn)),
"digestSHA1" : self.urlSha1, },
preMirrors = [{
"scm" : "url",
"url" : r".*/(.*)",
"mirror" : r"http://localhost:{}/\1".format(srv.port),
}]
)
with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)

def testGracefulMirrorFallback(self):
"""A failing mirror is ignored and the next mirror is used"""
with tempfile.TemporaryDirectory() as m:
firstMirror = os.path.join(m, "first")
firstMirrorPath = os.path.join(firstMirror, self.fn)
os.makedirs(firstMirror)
secondMirror = os.path.join(m, "second")
secondMirrorPath = os.path.join(secondMirror, self.fn)
os.makedirs(secondMirror)

with HttpServerMock(firstMirror, noResponse=True) as m1:
with HttpServerMock(secondMirror) as m2:
scm = self.createUrlScm(
{ "digestSHA1" : self.urlSha1 },
preMirrors=[
{
'scm' : 'url',
'url' : r".*/(.*)",
'mirror' : r"http://localhost:{}/\1".format(m1.port),
},
{
'scm' : 'url',
'url' : r".*/(.*)",
'mirror' : r"http://localhost:{}/\1".format(m2.port),
'upload' : True,
}
],
)

with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)

self.assertEqual(1, m1.getRequests)
self.assertEqual(0, m1.putRequests)
self.assertEqual(1, m2.getRequests)
self.assertEqual(1, m2.putRequests)

self.assertNotExists(firstMirrorPath)
self.assertExists(secondMirrorPath)

def testHttpMirrorUpload(self):
"""If requrested, a HTTP mirror is filled"""
with tempfile.TemporaryDirectory() as mirror:
mirrorPath = os.path.join(mirror, self.fn)
self.assertNotExists(mirrorPath)

with HttpServerMock(mirror) as srv:
scm = self.createUrlScm(
{ "digestSHA1" : self.urlSha1 },
preMirrors = [{
"scm" : "url",
"url" : r".*/(.*)",
"mirror" : r"http://localhost:{}/\1".format(srv.port),
"upload" : True,
}]
)
with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)

self.assertExists(mirrorPath)
self.assertContent(mirrorPath)

def testHttpMirrorUploadRetry(self):
"""A HTTP mirror upload is retried if configured"""
with tempfile.TemporaryDirectory() as mirror:
mirrorPath = os.path.join(mirror, self.fn)
self.assertNotExists(mirrorPath)

with HttpServerMock(mirror, retries=1) as srv:
scm = self.createUrlScm(
{
"digestSHA1" : self.urlSha1,
"retries" : 2,
},
fallbackMirrors = [{
"scm" : "url",
"url" : r".*/(.*)",
"mirror" : r"http://localhost:{}/\1".format(srv.port),
"upload" : True,
}]
)
with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)

self.assertEqual(2, srv.putRequests)

self.assertExists(mirrorPath)
self.assertContent(mirrorPath)

def testHttpMirrorNoReplaceExisting(self):
"""Existing files on an HTTP mirror are not replaced"""
with tempfile.TemporaryDirectory() as mirror:
mirrorPath = os.path.join(mirror, self.fn)
shutil.copy(self.path, mirrorPath)
with HttpServerMock(mirror) as srv:
scm = self.createUrlScm(
{ "digestSHA1" : self.urlSha1 },
fallbackMirrors = [{
"scm" : "url",
"url" : r".*/(.*)",
"mirror" : r"http://localhost:{}/\1".format(srv.port),
"upload" : True,
}]
)
with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)

self.assertEqual(1, srv.headRequests)
self.assertEqual(0, srv.getRequests)
self.assertEqual(0, srv.putRequests)

self.assertContent(mirrorPath)

def testUploadIfDownloadedFromMirror(self):
"""Mirrors are uploaded if downloaded from another mirror"""
with tempfile.TemporaryDirectory() as m:
firstMirror = os.path.join(m, "first")
firstMirrorPath = os.path.join(firstMirror, self.fn)
os.makedirs(firstMirror)
secondMirror = os.path.join(m, "second")
secondMirrorPath = os.path.join(secondMirror, self.fn)
os.makedirs(secondMirror)

shutil.copy(self.path, firstMirrorPath)
self.assertExists(firstMirrorPath)
self.assertNotExists(secondMirrorPath)

scm = self.createUrlScm(
{ "digestSHA1" : self.urlSha1 },
preMirrors=[{
'scm' : 'url',
'url' : r".*/(.*)",
'mirror' : escapeMirrorFileName(firstMirror) + r"/\1",
'upload' : True,
}],
fallbackMirrors=[{
'scm' : 'url',
'url' : r".*/(.*)",
'mirror' : escapeMirrorFileName(secondMirror) + r"/\1",
'upload' : True,
}],
)

with tempfile.TemporaryDirectory() as workspace:
self.invokeScm(workspace, scm)

self.assertExists(secondMirrorPath)

def testRogueMirrorFails(self):
"""Broken files on mirrors fail the build"""
with tempfile.TemporaryDirectory() as mirror:
mirrorPath = os.path.join(mirror, self.fn)
with open(mirrorPath, "w") as f:
f.write("bad")
scm = self.createUrlScm(
{ "digestSHA1" : self.urlSha1 },
preMirrors=[ { 'scm' : 'url',
'url' : r".*/(.*)",
'mirror' : escapeMirrorFileName(mirror) + r"/\1",
}
])

with tempfile.TemporaryDirectory() as workspace:
with self.assertRaises(InvocationError):
self.invokeScm(workspace, scm)

def testNoUploadBroken(self):
"""Broken artifacts are not uploaded"""
with tempfile.TemporaryDirectory() as mirror:
mirrorPath = os.path.join(mirror, self.fn)
scm = self.createUrlScm(
{ "digestSHA1" : "0"*40 },
preMirrors=[ { 'scm' : 'url',
'url' : r".*/(.*)",
'mirror' : escapeMirrorFileName(mirror) + r"/\1",
'upload' : True
}
])

with tempfile.TemporaryDirectory() as workspace:
with self.assertRaises(InvocationError):
self.invokeScm(workspace, scm)

self.assertNotExists(mirrorPath)

0 comments on commit d0fc33a

Please sign in to comment.