diff --git a/.github/workflows/cargo-package.yml b/.github/workflows/cargo-package.yml index d5beb0f89..d97f011d4 100644 --- a/.github/workflows/cargo-package.yml +++ b/.github/workflows/cargo-package.yml @@ -18,10 +18,10 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - toolchain: [1.80.0] + toolchain: [1.81.0] include: - os: windows-latest - toolchain: 1.80.0-x86_64-pc-windows-gnu + toolchain: 1.81.0-x86_64-pc-windows-gnu runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 0c75531b4..8f5185c89 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -9,16 +9,16 @@ on: - '**.py' - '.github/workflows/python.yml' jobs: - flake8: + ruff: strategy: fail-fast: false runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: chartboost/ruff-action@v1 with: - python-version: '3.9' - - name: flake8 - run: | - pip install flake8==4.0.1 - flake8 --ignore E402,F405,W504,E741 + version: 0.6.5 + - uses: chartboost/ruff-action@v1 + with: + version: 0.6.5 + args: 'format --diff' diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d0707e624..8c97e2b56 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -6,10 +6,10 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - toolchain: [1.80.0] + toolchain: [1.81.0] include: - os: windows-latest - toolchain: 1.80.0-x86_64-pc-windows-gnu + toolchain: 1.81.0-x86_64-pc-windows-gnu runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -59,7 +59,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: - toolchain: 1.80.0 + toolchain: 1.81.0 components: rustfmt - run: cargo fmt -- --check @@ -68,10 +68,10 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - toolchain: [1.80.0, 1.74.0] + toolchain: [1.81.0, 1.74.0] include: - os: windows-latest - toolchain: 1.80.0-x86_64-pc-windows-gnu + toolchain: 1.81.0-x86_64-pc-windows-gnu - os: windows-latest toolchain: 1.74.0-x86_64-pc-windows-gnu runs-on: ${{ matrix.os }} diff --git a/CI/decision.py b/CI/decision.py index 652dc40c1..1cd7634bd 100644 --- a/CI/decision.py +++ b/CI/decision.py @@ -8,25 +8,24 @@ import subprocess import sys - BASE_DIR = os.path.dirname(__file__) sys.path.append(BASE_DIR) -sys.path.append(os.path.join(BASE_DIR, '..')) +sys.path.append(os.path.join(BASE_DIR, "..")) from itertools import chain import osx # noqa: F401 from tasks import ( - action, - parse_version, Task, TaskEnvironment, Tool, + action, + parse_version, ) from tools import ( + ALL_MERCURIAL_VERSIONS, GIT_VERSION, MERCURIAL_VERSION, - ALL_MERCURIAL_VERSIONS, SOME_MERCURIAL_VERSIONS, Build, Git, @@ -38,8 +37,8 @@ def git_rev_parse(committish): return subprocess.check_output( - ['git', 'rev-parse', committish], text=True, - cwd=os.path.join(BASE_DIR, '..')).strip() + ["git", "rev-parse", committish], text=True, cwd=os.path.join(BASE_DIR, "..") + ).strip() def is_old_hg(version): @@ -47,7 +46,7 @@ def is_old_hg(version): if len(version) == 40: return False try: - version = [int(x) for x in version.split('.')] + version = [int(x) for x in version.split(".")] except ValueError: # Assume that an invalid version per the conversion above is # newer. @@ -62,108 +61,113 @@ class TestTask(Task): coverage = [] def __init__(self, **kwargs): - git = kwargs.pop('git', GIT_VERSION) - hg = kwargs.pop('hg', MERCURIAL_VERSION) - hg_clone = kwargs.pop('hg_clone', None) - commit = kwargs.pop('commit', None) - task_env = kwargs.pop('task_env', 'linux') - variant = kwargs.pop('variant', None) - build = kwargs.pop('build', None) - clone = kwargs.pop('clone', TC_COMMIT) - desc = kwargs.pop('description', None) - short_desc = kwargs.pop('short_desc', 'test') - extra_desc = kwargs.pop('extra_desc', None) - pre_command = kwargs.pop('pre_command', None) + git = kwargs.pop("git", GIT_VERSION) + hg = kwargs.pop("hg", MERCURIAL_VERSION) + hg_clone = kwargs.pop("hg_clone", None) + commit = kwargs.pop("commit", None) + task_env = kwargs.pop("task_env", "linux") + variant = kwargs.pop("variant", None) + build = kwargs.pop("build", None) + clone = kwargs.pop("clone", TC_COMMIT) + desc = kwargs.pop("description", None) + short_desc = kwargs.pop("short_desc", "test") + extra_desc = kwargs.pop("extra_desc", None) + pre_command = kwargs.pop("pre_command", None) if build is None: - build = '{}.{}'.format(task_env, variant) if variant else task_env + build = "{}.{}".format(task_env, variant) if variant else task_env build = Build.by_name(build) - kwargs.setdefault('mounts', []).append(build.mount()) + kwargs.setdefault("mounts", []).append(build.mount()) build = build.install() if variant: - kwargs.setdefault('env', {})['VARIANT'] = variant - env = TaskEnvironment.by_name('{}.test'.format(task_env)) + kwargs.setdefault("env", {})["VARIANT"] = variant + env = TaskEnvironment.by_name("{}.test".format(task_env)) command = [] if pre_command: command.extend(pre_command) if hg: - hg_task = Hg.by_name('{}.{}'.format(task_env, hg)) - kwargs.setdefault('mounts', []).append(hg_task.mount()) + hg_task = Hg.by_name("{}.{}".format(task_env, hg)) + kwargs.setdefault("mounts", []).append(hg_task.mount()) command.extend(hg_task.install()) - command.append('hg --version') + command.append("hg --version") if is_old_hg(hg): - kwargs.setdefault('env', {})['NO_CLONEBUNDLES'] = '1' + kwargs.setdefault("env", {})["NO_CLONEBUNDLES"] = "1" if git: - git_task = Git.by_name('{}.{}'.format(task_env, git)) - kwargs.setdefault('mounts', []).append(git_task.mount()) + git_task = Git.by_name("{}.{}".format(task_env, git)) + kwargs.setdefault("mounts", []).append(git_task.mount()) command.extend(git_task.install()) - command.append('git --version') + command.append("git --version") command.extend(Task.checkout(commit=commit)) command.extend(build) if clone: - kwargs.setdefault('mounts', []).append( - {'file:bundle.git': Clone.by_name(clone)}) - command.extend([ - 'git init repo/hg.old.git', - 'git -C repo/hg.old.git fetch ../../bundle.git refs/*:refs/*', - 'git -C repo/hg.old.git remote add origin hg:${REPO#https:}', - 'git -C repo/hg.old.git symbolic-ref HEAD' - ' refs/heads/branches/default/tip', - ]) - kwargs.setdefault('env', {})['REPO'] = REPO - command.extend(( - 'repo/git-cinnabar --version', - )) - if 'command' not in kwargs or hg_clone: + kwargs.setdefault("mounts", []).append( + {"file:bundle.git": Clone.by_name(clone)} + ) + command.extend( + [ + "git init repo/hg.old.git", + "git -C repo/hg.old.git fetch ../../bundle.git refs/*:refs/*", + "git -C repo/hg.old.git remote add origin hg:${REPO#https:}", + "git -C repo/hg.old.git symbolic-ref HEAD" + " refs/heads/branches/default/tip", + ] + ) + kwargs.setdefault("env", {})["REPO"] = REPO + command.extend(("repo/git-cinnabar --version",)) + if "command" not in kwargs or hg_clone: command += [ - 'hg init repo/hg.pure.hg', - 'hg -R repo/hg.pure.hg unbundle bundle.hg', + "hg init repo/hg.pure.hg", + "hg -R repo/hg.pure.hg unbundle bundle.hg", ] - kwargs.setdefault('mounts', []).append( - {'file:bundle.hg': HgClone.by_name(MERCURIAL_VERSION)}) - if 'command' in kwargs: - kwargs['command'] = command + kwargs['command'] + kwargs.setdefault("mounts", []).append( + {"file:bundle.hg": HgClone.by_name(MERCURIAL_VERSION)} + ) + if "command" in kwargs: + kwargs["command"] = command + kwargs["command"] else: if commit: # Always use the current CI scripts command.append( - 'git -C repo -c core.autocrlf=input checkout {} CI' - .format(TC_COMMIT)) - output_sync = ' --output-sync=target' - if env.os == 'macos': - output_sync = '' - kwargs['command'] = command + [ - 'make -C repo -f CI/tests.mk -j$({}){}' - .format(nproc(env), output_sync), + "git -C repo -c core.autocrlf=input checkout {} CI".format( + TC_COMMIT + ) + ) + output_sync = " --output-sync=target" + if env.os == "macos": + output_sync = "" + kwargs["command"] = command + [ + "make -C repo -f CI/tests.mk -j$({}){}".format(nproc(env), output_sync), ] - if variant == 'coverage': - kwargs['command'].extend([ - 'shopt -s nullglob', - 'cd repo', - 'zip $ARTIFACTS/coverage.zip' - ' $(find . -name "*.gcda")', - 'cd ..', - 'shopt -u nullglob', - ]) - artifact = kwargs.pop('artifact', None) - artifacts = kwargs.setdefault('artifacts', []) - assert not(artifacts and artifact) + if variant == "coverage": + kwargs["command"].extend( + [ + "shopt -s nullglob", + "cd repo", + "zip $ARTIFACTS/coverage.zip" ' $(find . -name "*.gcda")', + "cd ..", + "shopt -u nullglob", + ] + ) + artifact = kwargs.pop("artifact", None) + artifacts = kwargs.setdefault("artifacts", []) + assert not (artifacts and artifact) if artifact: artifacts.push(artifact) - artifacts.append('coverage.zip') + artifacts.append("coverage.zip") self.coverage.append(self) - elif variant == 'asan' and task_env == 'linux': - kwargs['caps'] = ['SYS_PTRACE'] + elif variant == "asan" and task_env == "linux": + kwargs["caps"] = ["SYS_PTRACE"] if not desc: - desc = '{} w/ git-{} hg-{}'.format( - short_desc, git, 'r' + hg if len(hg) == 40 else hg) - if variant and variant != 'coverage': - desc = ' '.join((desc, variant)) + desc = "{} w/ git-{} hg-{}".format( + short_desc, git, "r" + hg if len(hg) == 40 else hg + ) + if variant and variant != "coverage": + desc = " ".join((desc, variant)) if extra_desc: - desc = ' '.join((desc, extra_desc)) - if task_env != 'linux': - desc = ' '.join((desc, env.os, env.cpu)) - kwargs['description'] = desc + desc = " ".join((desc, extra_desc)) + if task_env != "linux": + desc = " ".join((desc, env.os, env.cpu)) + kwargs["description"] = desc Task.__init__(self, task_env=env, **kwargs) @@ -172,44 +176,44 @@ class Clone(TestTask, metaclass=Tool): def __init__(self, version): sha1 = git_rev_parse(version) - expireIn = '26 weeks' + expireIn = "26 weeks" kwargs = {} if version == TC_COMMIT or len(version) == 40: if version == TC_COMMIT: - build = Build.by_name('linux') + build = Build.by_name("linux") else: - build = Build.by_name('linux.old:{}'.format(version)) - kwargs.setdefault('mounts', []).append(build.mount()) + build = Build.by_name("linux.old:{}".format(version)) + kwargs.setdefault("mounts", []).append(build.mount()) download = build.install() - expireIn = '26 weeks' - elif parse_version(version) < parse_version('0.6.0'): - download = ['repo/git-cinnabar download'] - if parse_version(version) < parse_version('0.5.7'): - kwargs['git'] = '2.30.2' + expireIn = "26 weeks" + elif parse_version(version) < parse_version("0.6.0"): + download = ["repo/git-cinnabar download"] + if parse_version(version) < parse_version("0.5.7"): + kwargs["git"] = "2.30.2" else: - download = ['repo/download.py'] + download = ["repo/download.py"] if REPO == DEFAULT_REPO: - index = 'bundle.{}'.format(sha1) + index = "bundle.{}".format(sha1) else: - index = 'bundle.{}.{}'.format(hashlib.sha1(REPO).hexdigest(), sha1) + index = "bundle.{}.{}".format(hashlib.sha1(REPO).hexdigest(), sha1) TestTask.__init__( self, hg=MERCURIAL_VERSION, hg_clone=True, - description='clone w/ {}'.format(version), + description="clone w/ {}".format(version), index=index, expireIn=expireIn, build=download, commit=sha1, clone=False, command=[ - 'PATH=$PWD/repo:$PATH' - ' git -c fetch.prune=true clone -n hg::$PWD/repo/hg.pure.hg' - ' hg.old.git', - 'git -C hg.old.git bundle create $ARTIFACTS/bundle.git --all', + "PATH=$PWD/repo:$PATH" + " git -c fetch.prune=true clone -n hg::$PWD/repo/hg.pure.hg" + " hg.old.git", + "git -C hg.old.git bundle create $ARTIFACTS/bundle.git --all", ], - artifact='bundle.git', - priority='high', + artifact="bundle.git", + priority="high", **kwargs, ) @@ -219,146 +223,132 @@ class HgClone(Task, metaclass=Tool): def __init__(self, version): if REPO == DEFAULT_REPO: - index = 'hgclone.{}'.format(version) + index = "hgclone.{}".format(version) else: - index = 'hgclone.{}.{}'.format( - hashlib.sha1(REPO).hexdigest(), version) - hg_task = Hg.by_name(f'linux.{version}') + index = "hgclone.{}.{}".format(hashlib.sha1(REPO).hexdigest(), version) + hg_task = Hg.by_name(f"linux.{version}") Task.__init__( self, - task_env=TaskEnvironment.by_name('linux.test'), - description=f'hg clone w/ {version}', + task_env=TaskEnvironment.by_name("linux.test"), + description=f"hg clone w/ {version}", index=index, - expireIn='26 weeks', - command=hg_task.install() + [ - 'hg clone -U --stream $REPO repo', - 'hg -R repo bundle -t none-v1 -a $ARTIFACTS/bundle.hg', + expireIn="26 weeks", + command=hg_task.install() + + [ + "hg clone -U --stream $REPO repo", + "hg -R repo bundle -t none-v1 -a $ARTIFACTS/bundle.hg", ], mounts=[hg_task.mount()], - artifact='bundle.hg', + artifact="bundle.hg", env={ - 'REPO': REPO, + "REPO": REPO, }, - priority='high', + priority="high", ) -@action('decision') +@action("decision") def decision(): - for env in ('linux', 'mingw64', 'osx', 'arm64-osx'): + for env in ("linux", "mingw64", "osx", "arm64-osx"): # Can't spawn osx workers from pull requests. - if env.endswith('osx') and not TC_IS_PUSH: + if env.endswith("osx") and not TC_IS_PUSH: continue TestTask( task_env=env, - variant='coverage' if env == 'linux' else None, + variant="coverage" if env == "linux" else None, ) - task_env = TaskEnvironment.by_name('{}.test'.format(env)) - if TC_IS_PUSH and TC_BRANCH != "try": - hg = Hg.by_name('{}.{}'.format(env, MERCURIAL_VERSION)) - git = Git.by_name('{}.{}'.format(env, GIT_VERSION)) - Task( - task_env=task_env, - description='download build {} {}'.format(task_env.os, - task_env.cpu), - command=list(chain( + task_env = TaskEnvironment.by_name("{}.test".format(env)) + git = Git.by_name("{}.{}".format(env, GIT_VERSION)) + build = Build.by_name(env) + bin = os.path.basename(build.artifacts[0]) + Task( + task_env=task_env, + description="download build {} {}".format(task_env.os, task_env.cpu), + command=list( + chain( git.install(), - hg.install(), Task.checkout(), + build.install(), [ - '(cd repo ; ./download.py)', - 'PATH=$PWD/repo:$PATH git cinnabar --version', - 'cp repo/download.py .', - './download.py', - 'PATH=$PWD:$PATH', - 'git cinnabar --version', - 'git cinnabar self-update', - 'git cinnabar --version', - 'git cinnabar self-update --branch {}'.format( - TC_BRANCH or "master"), - 'git cinnabar --version', + f"python3 repo/CI/test_download.py repo/{bin}", ], - )), - dependencies=[ - Build.by_name(env), - ], - mounts=[ - hg.mount(), - git.mount(), - ], - ) + ) + ), + mounts=[ + git.mount(), + build.mount(), + ], + ) # Because nothing is using the arm64 linux build, we need to manually # touch it. - Build.by_name('arm64-linux') + Build.by_name("arm64-linux") for upgrade in UPGRADE_FROM: TestTask( - short_desc='upgrade tests', - extra_desc='from-{}'.format(upgrade), - variant='coverage', + short_desc="upgrade tests", + extra_desc="from-{}".format(upgrade), + variant="coverage", clone=upgrade, env={ - 'UPGRADE_FROM': upgrade, + "UPGRADE_FROM": upgrade, }, - hg='5.4.2', + hg="5.4.2", ) - for git in ('1.8.5', '2.7.4'): - TestTask( - git=git, - env={'GIT_OLD_VERSION': '1'} - ) + for git in ("1.8.5", "2.7.4"): + TestTask(git=git, env={"GIT_OLD_VERSION": "1"}) for hg in SOME_MERCURIAL_VERSIONS: if hg != MERCURIAL_VERSION: do_hg_version(hg) TestTask( - task_env='linux', - variant='asan', + task_env="linux", + variant="asan", ) - for env in ('linux', 'mingw64', 'osx', 'arm64-osx'): + for env in ("linux", "mingw64", "osx", "arm64-osx"): # Can't spawn osx workers from pull requests. - if env.endswith('osx') and not TC_IS_PUSH: + if env.endswith("osx") and not TC_IS_PUSH: continue TestTask( task_env=env, - variant='coverage' if env == 'linux' else None, - short_desc='graft tests', + variant="coverage" if env == "linux" else None, + short_desc="graft tests", env={ - 'GRAFT': '1', + "GRAFT": "1", }, ) for env, variant in ( - ('linux', 'coverage'), - ('linux', 'asan'), - ('osx', None), - ('arm64-osx', None), + ("linux", "coverage"), + ("linux", "asan"), + ("osx", None), + ("arm64-osx", None), ): # Can't spawn osx workers from pull requests. - if env.endswith('osx') and not TC_IS_PUSH: + if env.endswith("osx") and not TC_IS_PUSH: continue pre_command = [] - if env != 'linux': - pre_command.append('pip install cram==0.7') + if env != "linux": + pre_command.append("pip install cram==0.7") TestTask( task_env=env, variant=variant, - short_desc='cram', + short_desc="cram", clone=False, - command=pre_command + [ - 'cram --verbose repo/tests', + command=pre_command + + [ + "cram --verbose repo/tests", ], env={ - 'GIT_CINNABAR_CHECK': 'no-version-check', + "GIT_CINNABAR_CHECK": "no-version-check", }, ) @@ -370,48 +360,53 @@ def do_hg_version(hg): # wrt bookmarks. if not is_old_hg(hg): TestTask( - short_desc='cram', + short_desc="cram", clone=False, hg=hg, command=[ - 'cram --verbose repo/tests', + "cram --verbose repo/tests", ], env={ - 'GIT_CINNABAR_CHECK': 'no-version-check', + "GIT_CINNABAR_CHECK": "no-version-check", }, ) -@action('more-hg-versions', - title='More hg versions', - description='Trigger tests against more mercurial versions') +@action( + "more-hg-versions", + title="More hg versions", + description="Trigger tests against more mercurial versions", +) def more_hg_versions(): for hg in ALL_MERCURIAL_VERSIONS: if hg != MERCURIAL_VERSION and hg not in SOME_MERCURIAL_VERSIONS: do_hg_version(hg) -@action('hg-trunk', - title='Test w/ hg trunk', - description='Trigger tests against current mercurial trunk') +@action( + "hg-trunk", + title="Test w/ hg trunk", + description="Trigger tests against current mercurial trunk", +) def hg_trunk(): import requests - r = requests.get('https://www.mercurial-scm.org/repo/hg/?cmd=branchmap') + + r = requests.get("https://www.mercurial-scm.org/repo/hg/?cmd=branchmap") trunk = None for l in r.text.splitlines(): fields = l.split() - if fields[0] == 'default': + if fields[0] == "default": trunk = fields[-1] if not trunk: - raise Exception('Cannot find mercurial trunk changeset') + raise Exception("Cannot find mercurial trunk changeset") do_hg_version(trunk) def main(): try: - func = action.by_name[TC_ACTION or 'decision'].func + func = action.by_name[TC_ACTION or "decision"].func except AttributeError: - raise Exception('Unsupported action: %s', TC_ACTION or 'decision') + raise Exception("Unsupported action: %s", TC_ACTION or "decision") func() @@ -419,73 +414,84 @@ def main(): if TestTask.coverage and TC_IS_PUSH and TC_BRANCH: coverage_mounts = [ - {f'file:cov-{task.id}.zip': task} for task in TestTask.coverage + {f"file:cov-{task.id}.zip": task} for task in TestTask.coverage ] - task = Build.by_name('linux.coverage') - coverage_mounts.append({'file:gcno-build.zip': { - 'artifact': task.artifacts[1], - 'taskId': task.id, - }}) - - merge_coverage.extend([ - 'grcov -s repo -t lcov -o repo/coverage.lcov gcno-build.zip ' + - ' '.join( - f'cov-{task.id}.zip' - for task in TestTask.coverage), - ]) + task = Build.by_name("linux.coverage") + coverage_mounts.append( + { + "file:gcno-build.zip": { + "artifact": task.artifacts[1], + "taskId": task.id, + } + } + ) + + merge_coverage.extend( + [ + "grcov -s repo -t lcov -o repo/coverage.lcov gcno-build.zip " + + " ".join(f"cov-{task.id}.zip" for task in TestTask.coverage), + ] + ) if merge_coverage: Task( - task_env=TaskEnvironment.by_name('linux.codecov'), - description='upload coverage', - scopes=['secrets:get:project/git-cinnabar/codecov'], + task_env=TaskEnvironment.by_name("linux.codecov"), + description="upload coverage", + scopes=["secrets:get:project/git-cinnabar/codecov"], mounts=coverage_mounts, - command=list(chain( - Task.checkout(), - [ - 'set +x', - ('export CODECOV_TOKEN=$(curl -sL ' - f'{PROXY_URL}/api/secrets/v1/secret/project/git-cinnabar' - '/codecov | python3' - ' -c "import json, sys; print(json.load(sys.stdin)' - '[\\"secret\\"][\\"token\\"])")'), - 'set -x', - ], - merge_coverage, - [ - 'cd repo', - 'codecov -Z --name "taskcluster" -C {} -B {}' - .format(TC_COMMIT, TC_BRANCH), - ], - )), + command=list( + chain( + Task.checkout(), + [ + "set +x", + ( + "export CODECOV_TOKEN=$(curl -sL " + f"{PROXY_URL}/api/secrets/v1/secret/project/git-cinnabar" + "/codecov | python3" + ' -c "import json, sys; print(json.load(sys.stdin)' + '[\\"secret\\"][\\"token\\"])")' + ), + "set -x", + ], + merge_coverage, + [ + "cd repo", + 'codecov -Z --name "taskcluster" -C {} -B {}'.format( + TC_COMMIT, TC_BRANCH + ), + ], + ) + ), ) for t in Task.by_id.values(): t.submit() - if not TC_ACTION and 'TC_GROUP_ID' in os.environ: + if not TC_ACTION and "TC_GROUP_ID" in os.environ: actions = { - 'version': 1, - 'actions': [], - 'variables': { - 'e': dict(TC_DATA, decision_id=''), - 'tasks_for': 'action', + "version": 1, + "actions": [], + "variables": { + "e": dict(TC_DATA, decision_id=""), + "tasks_for": "action", }, } for name, a in action.by_name.items(): - if name != 'decision': - actions['actions'].append({ - 'kind': 'task', - 'name': a.name, - 'title': a.title, - 'description': a.description, - 'context': [], - 'task': a.task, - }) - - with open('actions.json', 'w') as out: + if name != "decision": + actions["actions"].append( + { + "kind": "task", + "name": a.name, + "title": a.title, + "description": a.description, + "context": [], + "task": a.task, + } + ) + + with open("actions.json", "w") as out: out.write(json.dumps(actions, indent=True)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/CI/docker.py b/CI/docker.py index e7203d3fb..f0f97d5fc 100644 --- a/CI/docker.py +++ b/CI/docker.py @@ -17,12 +17,12 @@ def sources_list(snapshot, sections): for idx, (archive, dist) in enumerate(sections): if not snapshot: - yield 'deb http://archive.debian.org/{} {} main'.format( + yield "deb http://archive.debian.org/{} {} main".format( archive, dist, ) continue - yield 'deb http://snapshot.debian.org/archive/{}/{} {} main'.format( + yield "deb http://snapshot.debian.org/archive/{}/{} {} main".format( archive, snapshot, dist, @@ -30,180 +30,199 @@ def sources_list(snapshot, sections): LLVM_REPO = ( - 'echo' - ' deb [signed-by=/usr/share/keyrings/llvm.gpg]' - ' https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-18 main' - ' > /etc/apt/sources.list.d/llvm.list' + "echo" + " deb [signed-by=/usr/share/keyrings/llvm.gpg]" + " https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-18 main" + " > /etc/apt/sources.list.d/llvm.list" ) DOCKER_IMAGES = { - 'base': { - 'from': 'debian:bullseye-20220801', - 'commands': [ - '({}) > /etc/apt/sources.list'.format('; '.join( - 'echo ' + l for l in sources_list('20220801T205040Z', ( - ('debian', 'bullseye'), - ('debian', 'bullseye-updates'), - ('debian-security', 'bullseye-security'), - )))), - 'apt-get update -o Acquire::Check-Valid-Until=false', - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'apt-transport-https', - 'bzip2', - 'ca-certificates', - 'curl', - 'gnupg2', - 'libcurl3-gnutls', - 'python-setuptools', - 'python3-setuptools', - 'python3-pip', - 'unzip', - 'xz-utils', - 'zip', - 'zstd', - ])), - 'apt-get clean', - 'curl -sO https://apt.llvm.org/llvm-snapshot.gpg.key', - 'gpg --no-default-keyring --keyring /usr/share/keyrings/llvm.gpg' - ' --import llvm-snapshot.gpg.key', - 'rm llvm-snapshot.gpg.key', - 'curl -sO http://snapshot.debian.org/archive/debian' - '/20220326T025251Z/pool/main/p/python2-pip' - '/python-pip_20.3.4%2Bdfsg-4_all.deb', - 'dpkg-deb -x python-pip*.deb /', - 'python2.7 -m pip install pip==20.3.4 wheel==0.37.1' - ' --upgrade --ignore-installed', - 'python3 -m pip install pip==20.3.4 wheel==0.37.1' - ' --upgrade --ignore-installed', + "base": { + "from": "debian:bullseye-20220801", + "commands": [ + "({}) > /etc/apt/sources.list".format( + "; ".join( + "echo " + l + for l in sources_list( + "20220801T205040Z", + ( + ("debian", "bullseye"), + ("debian", "bullseye-updates"), + ("debian-security", "bullseye-security"), + ), + ) + ) + ), + "apt-get update -o Acquire::Check-Valid-Until=false", + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "apt-transport-https", + "bzip2", + "ca-certificates", + "curl", + "gnupg2", + "libcurl3-gnutls", + "python-setuptools", + "python3-setuptools", + "python3-pip", + "unzip", + "xz-utils", + "zip", + "zstd", + ] + ) + ), + "apt-get clean", + "curl -sO https://apt.llvm.org/llvm-snapshot.gpg.key", + "gpg --no-default-keyring --keyring /usr/share/keyrings/llvm.gpg" + " --import llvm-snapshot.gpg.key", + "rm llvm-snapshot.gpg.key", + "curl -sO http://snapshot.debian.org/archive/debian" + "/20220326T025251Z/pool/main/p/python2-pip" + "/python-pip_20.3.4%2Bdfsg-4_all.deb", + "dpkg-deb -x python-pip*.deb /", + "python2.7 -m pip install pip==20.3.4 wheel==0.37.1" + " --upgrade --ignore-installed", + "python3 -m pip install pip==20.3.4 wheel==0.37.1" + " --upgrade --ignore-installed", ], }, - - 'build': { - 'from': 'base', - 'commands': [ + "build": { + "from": "base", + "commands": [ LLVM_REPO, - 'apt-get update -o Acquire::Check-Valid-Until=false', - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'clang-18', - 'lld-18', - 'git', - 'make', - 'patch', - 'pkg-config', - 'mmdebstrap', - 'debian-archive-keyring', - 'symlinks', - 'fakechroot', - 'gcc-mingw-w64-x86-64-win32', - ])), - 'for arch in amd64 arm64; do' - ' mmdebstrap -d' - ' --architecture=$arch' - ' --mode=chrootless' - ' --variant=extract' - ' --include=libc6-dev,libcurl4-gnutls-dev,zlib1g-dev,libgcc-6-dev' - ' stretch sysroot-$arch' - ' http://archive.debian.org/debian/ ;' - ' LD_PRELOAD=libfakechroot.so FAKECHROOT_BASE=$PWD/sysroot-$arch' - ' symlinks -crv /;' - 'done', - 'apt-get clean', + "apt-get update -o Acquire::Check-Valid-Until=false", + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "clang-18", + "lld-18", + "git", + "make", + "patch", + "pkg-config", + "mmdebstrap", + "debian-archive-keyring", + "symlinks", + "fakechroot", + "gcc-mingw-w64-x86-64-win32", + ] + ) + ), + "for arch in amd64 arm64; do" + " mmdebstrap -d" + " --architecture=$arch" + " --mode=chrootless" + " --variant=extract" + " --include=libc6-dev,libcurl4-gnutls-dev,zlib1g-dev,libgcc-6-dev" + " stretch sysroot-$arch" + " http://archive.debian.org/debian/ ;" + " LD_PRELOAD=libfakechroot.so FAKECHROOT_BASE=$PWD/sysroot-$arch" + " symlinks -crv /;" + "done", + "apt-get clean", ], }, - - 'build-tools': { - 'from': 'base', - 'commands': [ - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'gcc', - 'git', - 'libc6-dev', - 'libcurl4-gnutls-dev', - 'make', - 'patch', - 'python-dev', - 'python3-dev', - 'zlib1g-dev', - ])), - 'apt-get clean', + "build-tools": { + "from": "base", + "commands": [ + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "gcc", + "git", + "libc6-dev", + "libcurl4-gnutls-dev", + "make", + "patch", + "python-dev", + "python3-dev", + "zlib1g-dev", + ] + ) + ), + "apt-get clean", ], }, - - 'codecov': { - 'from': 'base', - 'commands': [ - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'gcc', - 'git', - 'python3-coverage', - ])), - 'apt-get clean', - 'ln -s /usr/bin/python3-coverage /usr/local/bin/coverage', - 'curl -o /usr/local/bin/codecov -sL {}'.format( - 'https://github.com/codecov/uploader/releases/download' - '/v0.1.0_9779/codecov-linux' + "codecov": { + "from": "base", + "commands": [ + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "gcc", + "git", + "python3-coverage", + ] + ) + ), + "apt-get clean", + "ln -s /usr/bin/python3-coverage /usr/local/bin/coverage", + "curl -o /usr/local/bin/codecov -sL {}".format( + "https://github.com/codecov/uploader/releases/download" + "/v0.1.0_9779/codecov-linux" ), - 'chmod +x /usr/local/bin/codecov', - 'curl -sL {} | tar -C /usr/local/bin -jxf -'.format( - 'https://github.com/mozilla/grcov/releases/download/v0.8.7' - '/grcov-x86_64-unknown-linux-gnu.tar.bz2' + "chmod +x /usr/local/bin/codecov", + "curl -sL {} | tar -C /usr/local/bin -jxf -".format( + "https://github.com/mozilla/grcov/releases/download/v0.8.7" + "/grcov-x86_64-unknown-linux-gnu.tar.bz2" ), ], }, - - 'test': { - 'from': 'base', - 'commands': [ + "test": { + "from": "base", + "commands": [ LLVM_REPO, - 'apt-get update -o Acquire::Check-Valid-Until=false', - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'llvm-18', - 'make', - ])), - 'apt-get clean', - 'pip3 install cram==0.7', - 'ln -s llvm-symbolizer-18 /usr/bin/llvm-symbolizer' + "apt-get update -o Acquire::Check-Valid-Until=false", + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "llvm-18", + "make", + ] + ) + ), + "apt-get clean", + "pip3 install cram==0.7", + "ln -s llvm-symbolizer-18 /usr/bin/llvm-symbolizer", ], }, } class DockerImage(Task, metaclass=TaskEnvironment): - PREFIX = 'linux' - cpu = 'x86_64' - os = 'linux' + PREFIX = "linux" + cpu = "x86_64" + os = "linux" def __init__(self, name): defn = DOCKER_IMAGES[name] - base = defn['from'] + base = defn["from"] self.name = name - if ':' not in base: + if ":" not in base: base = DockerImage.by_name(base) self.base = base - self.definition = defn['commands'] + self.definition = defn["commands"] Task.__init__( self, task_env=self, - description='docker image: {}'.format(name), + description="docker image: {}".format(name), index=self.index, - expireIn='26 weeks', - workerType='linux', + expireIn="26 weeks", + workerType="linux", image=base, dockerSave=True, command=self.definition, ) def __str__(self): - return '{}/{}:{}'.format( - TC_REPO_NAME, - self.name, - self.hexdigest - ) + return "{}/{}:{}".format(TC_REPO_NAME, self.name, self.hexdigest) @property def index(self): - return '.'.join((self.PREFIX, self.name, self.hexdigest)) + return ".".join((self.PREFIX, self.name, self.hexdigest)) @property def hexdigest(self): @@ -214,48 +233,49 @@ def hexdigest(self): def prepare_params(self, params): commands = ["mkdir artifacts"] - image = params.pop('image', self) + image = params.pop("image", self) volumes = [ - kind.split(':', 1)[1] - for mount in params.get('mounts', []) + kind.split(":", 1)[1] + for mount in params.get("mounts", []) for kind in mount - if ':' in kind + if ":" in kind ] if isinstance(image, DockerImage): - params.setdefault('mounts', []).append({'file:dockerimage': image}) - image = 'docker-archive:dockerimage' + params.setdefault("mounts", []).append({"file:dockerimage": image}) + image = "docker-archive:dockerimage" run_cmd = [ - 'podman', - 'run', - '--name=taskcontainer', - '--volume=./artifacts:/artifacts', - '--env=ARTIFACTS=/artifacts', + "podman", + "run", + "--name=taskcontainer", + "--volume=./artifacts:/artifacts", + "--env=ARTIFACTS=/artifacts", ] - if any(s.startswith('secrets:') for s in params.get('scopes', [])): + if any(s.startswith("secrets:") for s in params.get("scopes", [])): # There's probably a better way, but it's simpler. - run_cmd.append('--network=host') + run_cmd.append("--network=host") for v in volumes: - run_cmd.append(f'--volume=./{v}:/{v}') - for k, v in params.pop('env', {}).items(): - run_cmd.append(f'--env={k}={v}') - for cap in params.pop('caps', []): - run_cmd.append(f'--cap-add={cap}') + run_cmd.append(f"--volume=./{v}:/{v}") + for k, v in params.pop("env", {}).items(): + run_cmd.append(f"--env={k}={v}") + for cap in params.pop("caps", []): + run_cmd.append(f"--cap-add={cap}") run_cmd.append(image) - run_cmd.extend(bash_command(*params['command'])) + run_cmd.extend(bash_command(*params["command"])) commands.append(join_command(*run_cmd)) - if params.pop('dockerSave', False): - commands.extend([ - 'exit_code=$?', - 'podman commit taskcontainer taskcontainer', - 'podman save taskcontainer' - ' | zstd > artifacts/dockerImage.tar.zst', - 'podman rm taskcontainer', - 'exit $exit_code', - ]) - params['artifacts'] = ["dockerImage.tar.zst"] - params['command'] = bash_command(*commands) + if params.pop("dockerSave", False): + commands.extend( + [ + "exit_code=$?", + "podman commit taskcontainer taskcontainer", + "podman save taskcontainer" + " | zstd > artifacts/dockerImage.tar.zst", + "podman rm taskcontainer", + "exit $exit_code", + ] + ) + params["artifacts"] = ["dockerImage.tar.zst"] + params["command"] = bash_command(*commands) - if 'artifacts' in params: - params['artifacts'] = [f'artifacts/{a}' - for a in params['artifacts']] + if "artifacts" in params: + params["artifacts"] = [f"artifacts/{a}" for a in params["artifacts"]] return params diff --git a/CI/hg-serve-exec.py b/CI/hg-serve-exec.py index b6e26c3b7..ab547907c 100644 --- a/CI/hg-serve-exec.py +++ b/CI/hg-serve-exec.py @@ -3,31 +3,35 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import print_function + import base64 import os import shutil import subprocess + try: from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer from SimpleHTTPServer import SimpleHTTPRequestHandler except ImportError: - from http.server import BaseHTTPRequestHandler, HTTPServer - from http.server import SimpleHTTPRequestHandler + from http.server import BaseHTTPRequestHandler, HTTPServer, SimpleHTTPRequestHandler from threading import Thread try: from mercurial import cmdutil + cmdutil.command except AttributeError: from mercurial import registrar as cmdutil from mercurial import hgweb from mercurial.hgweb import common from mercurial.hgweb.server import openlog + try: httpservice = hgweb.httpservice except AttributeError: try: from mercurial import commands + httpservice = commands.httpservice except AttributeError: from mercurial import util @@ -45,33 +49,35 @@ def init(self): util.setsignalhandler() self.httpd = server.create_server(self.ui, self.app) - if self.opts['port'] and not self.ui.verbose: + if self.opts["port"] and not self.ui.verbose: return if self.httpd.prefix: - prefix = self.httpd.prefix.strip('/') + '/' + prefix = self.httpd.prefix.strip("/") + "/" else: - prefix = '' + prefix = "" - port = ':%d' % self.httpd.port - if port == ':80': - port = '' + port = ":%d" % self.httpd.port + if port == ":80": + port = "" bindaddr = self.httpd.addr - if bindaddr == '0.0.0.0': - bindaddr = '*' - elif ':' in bindaddr: # IPv6 - bindaddr = '[%s]' % bindaddr + if bindaddr == "0.0.0.0": + bindaddr = "*" + elif ":" in bindaddr: # IPv6 + bindaddr = "[%s]" % bindaddr fqaddr = self.httpd.fqaddr - if ':' in fqaddr: - fqaddr = '[%s]' % fqaddr - if self.opts['port']: + if ":" in fqaddr: + fqaddr = "[%s]" % fqaddr + if self.opts["port"]: write = self.ui.status else: write = self.ui.write - write('listening at http://%s%s/%s (bound to %s:%d)\n' % - (fqaddr, port, prefix, bindaddr, self.httpd.port)) + write( + "listening at http://%s%s/%s (bound to %s:%d)\n" + % (fqaddr, port, prefix, bindaddr, self.httpd.port) + ) def run(self): self.httpd.serve_forever() @@ -82,18 +88,20 @@ def run(self): def perform_authentication(hgweb, req, op): - if hasattr(req, 'env'): + if hasattr(req, "env"): env = req.env else: env = req.rawenv - if env.get('REQUEST_METHOD') == 'POST': - auth = env.get('HTTP_AUTHORIZATION') + if env.get("REQUEST_METHOD") == "POST": + auth = env.get("HTTP_AUTHORIZATION") if not auth: raise common.ErrorResponse( - common.HTTP_UNAUTHORIZED, 'who', - [('WWW-Authenticate', 'Basic Realm="mercurial"')]) - if base64.b64decode(auth.split()[1]).split(':', 1) != ['foo', 'bar']: - raise common.ErrorResponse(common.HTTP_FORBIDDEN, 'no') + common.HTTP_UNAUTHORIZED, + "who", + [("WWW-Authenticate", 'Basic Realm="mercurial"')], + ) + if base64.b64decode(auth.split()[1]).split(":", 1) != ["foo", "bar"]: + raise common.ErrorResponse(common.HTTP_FORBIDDEN, "no") def extsetup(ui): @@ -104,11 +112,13 @@ def HgLogging(cls): class Logging(cls): # Copied from mercurial's hgweb/server.py. def _log_any(self, fp, format, *args): - message = '%s - - [%s] %s' % (self.client_address[0], - self.log_date_time_string(), - format % args) + '\n' + message = ( + "%s - - [%s] %s" + % (self.client_address[0], self.log_date_time_string(), format % args) + + "\n" + ) if not isinstance(message, bytes): - message = message.encode('utf-8') + message = message.encode("utf-8") fp.write(message) fp.flush() @@ -130,19 +140,19 @@ def do_POST(self): def git_http_backend(self): env = dict(os.environ) - env['REQUEST_METHOD'] = self.command - env['GIT_HTTP_EXPORT_ALL'] = '1' - env['GIT_PROJECT_ROOT'] = os.path.abspath(os.curdir) - path, _, query = self.path.partition('?') - env['PATH_INFO'] = path - env['QUERY_STRING'] = query + env["REQUEST_METHOD"] = self.command + env["GIT_HTTP_EXPORT_ALL"] = "1" + env["GIT_PROJECT_ROOT"] = os.path.abspath(os.curdir) + path, _, query = self.path.partition("?") + env["PATH_INFO"] = path + env["QUERY_STRING"] = query self.send_response(200, "Script output follows") - if hasattr(self, 'flush_headers'): + if hasattr(self, "flush_headers"): self.flush_headers() - if self.command == 'POST': - length = self.headers.get('Content-Length') - env['CONTENT_LENGTH'] = length - env['CONTENT_TYPE'] = self.headers.get('Content-Type') + if self.command == "POST": + length = self.headers.get("Content-Length") + env["CONTENT_LENGTH"] = length + env["CONTENT_TYPE"] = self.headers.get("Content-Type") try: length = int(length) except (TypeError, ValueError): @@ -152,8 +162,9 @@ def git_http_backend(self): else: stdin = None - p = subprocess.Popen(['git', 'http-backend'], stdin=stdin, - stdout=subprocess.PIPE, env=env) + p = subprocess.Popen( + ["git", "http-backend"], stdin=stdin, stdout=subprocess.PIPE, env=env + ) if stdin: p.stdin.write(data) p.stdin.close() @@ -163,46 +174,43 @@ def git_http_backend(self): class OtherServer(object): def __init__(self, typ, ui): - if typ == b'git': + if typ == b"git": cls = GitServer - elif typ == b'http': + elif typ == b"http": cls = HgLogging(SimpleHTTPRequestHandler) else: assert False self.httpd = HTTPServer( - ('', ui.configint(b'serve', b'otherport', 8080)), + ("", ui.configint(b"serve", b"otherport", 8080)), cls, ) - self.httpd.accesslog = openlog( - ui.config(b'web', b'accesslog'), ui.fout) - self.httpd.errorlog = openlog(ui.config(b'web', b'errorlog'), ui.ferr) + self.httpd.accesslog = openlog(ui.config(b"web", b"accesslog"), ui.fout) + self.httpd.errorlog = openlog(ui.config(b"web", b"errorlog"), ui.ferr) def run(self): self.httpd.serve_forever() -@command(b'serve-and-exec', ()) +@command(b"serve-and-exec", ()) def serve_and_exec(ui, repo, *command): - other_server = ui.config(b'serve', b'other', None) + other_server = ui.config(b"serve", b"other", None) if other_server: other_server = OtherServer(other_server, ui) other_server_thread = Thread(target=other_server.run) other_server_thread.start() - ui.setconfig(b'web', b'push_ssl', False, b'hgweb') - ui.setconfig(b'web', b'allow_push', b'*', b'hgweb') + ui.setconfig(b"web", b"push_ssl", False, b"hgweb") + ui.setconfig(b"web", b"allow_push", b"*", b"hgweb") # For older versions of mercurial - repo.baseui.setconfig(b'web', b'push_ssl', False, b'hgweb') - repo.baseui.setconfig(b'web', b'allow_push', b'*', b'hgweb') + repo.baseui.setconfig(b"web", b"push_ssl", False, b"hgweb") + repo.baseui.setconfig(b"web", b"allow_push", b"*", b"hgweb") app = hgweb.hgweb(repo, baseui=ui) - service = httpservice(ui, app, { - b'port': ui.configint(b'web', b'port', 8000), - b'print_url': False - }) + service = httpservice( + ui, app, {b"port": ui.configint(b"web", b"port", 8000), b"print_url": False} + ) service.init() service_thread = Thread(target=service.run) service_thread.start() - ret = subprocess.call( - [getattr(os, "fsdecode", lambda a: a)(a) for a in command]) + ret = subprocess.call([getattr(os, "fsdecode", lambda a: a)(a) for a in command]) service.httpd.shutdown() service_thread.join() if other_server: diff --git a/CI/msys.py b/CI/msys.py index 11e270811..22984630d 100644 --- a/CI/msys.py +++ b/CI/msys.py @@ -4,6 +4,7 @@ import hashlib +from docker import DockerImage from tasks import ( Task, TaskEnvironment, @@ -11,24 +12,22 @@ bash_command, join_command, ) -from docker import DockerImage - -CPUS = ('x86_64',) +CPUS = ("x86_64",) MSYS_VERSION = { - 'x86_64': '20230318', + "x86_64": "20240727", } def mingw(cpu): return { - 'x86_64': 'MINGW64', + "x86_64": "MINGW64", }.get(cpu) def msys(cpu): return { - 'x86_64': 'msys64', + "x86_64": "msys64", }.get(cpu) @@ -38,41 +37,48 @@ def msys_cpu(cpu): def bits(cpu): return { - 'x86_64': '64', + "x86_64": "64", }.get(cpu) class MsysCommon(object): - os = 'windows' + os = "windows" def prepare_params(self, params): - assert 'workerType' not in params - params['workerType'] = 'win2012r2' - params.setdefault('mounts', []).append({'directory': self}) - params.setdefault('env', {})['MSYSTEM'] = mingw(self.cpu) + assert "workerType" not in params + params["workerType"] = "windows" + params.setdefault("mounts", []).append({"directory": self}) + params.setdefault("env", {})["MSYSTEM"] = mingw(self.cpu) command = [] - command.append('set HOME=%CD%') - command.append('set ARTIFACTS=%CD%') - for path in (mingw(self.cpu), 'usr'): - command.append('set PATH=%CD%\\{}\\{}\\bin;%PATH%' - .format(msys(self.cpu), path)) - command.append( - 'set PATH=%CD%\\git\\{}\\bin;%PATH%'.format(mingw(self.cpu))) - if self.PREFIX != 'msys': - command.append('bash -c -x "{}"'.format('; '.join(( - 'for postinst in /etc/post-install/*.post', - 'do test -e $postinst && . $postinst', - 'done', - )))) + command.append("set HOME=%CD%") + command.append("set ARTIFACTS=%CD%") + for path in (mingw(self.cpu), "usr"): + command.append( + "set PATH=%CD%\\{}\\{}\\bin;%PATH%".format(msys(self.cpu), path) + ) + command.append("set PATH=%CD%\\git\\{}\\bin;%PATH%".format(mingw(self.cpu))) + if self.PREFIX != "msys": + command.append( + 'bash -c -x "{}"'.format( + "; ".join( + ( + "for postinst in /etc/post-install/*.post", + "do test -e $postinst && . $postinst", + "done", + ) + ) + ) + ) command.append( - join_command(*bash_command(*params['command']), for_windows=True)) - params['command'] = command + join_command(*bash_command(*params["command"]), for_windows=True) + ) + params["command"] = command return params @property def index(self): - return '.'.join(('env', self.PREFIX, self.cpu, self.hexdigest)) + return ".".join(("env", self.PREFIX, self.cpu, self.hexdigest)) class MsysBase(MsysCommon, Task, metaclass=Tool): @@ -81,10 +87,11 @@ class MsysBase(MsysCommon, Task, metaclass=Tool): def __init__(self, cpu): assert cpu in CPUS _create_command = ( - 'curl -L http://repo.msys2.org/distrib/{cpu}' - '/msys2-base-{cpu}-{version}.tar.xz | xz -cd | zstd -c' - ' > $ARTIFACTS/msys2.tar.zst'.format( - cpu=msys_cpu(cpu), version=MSYS_VERSION[cpu]) + "curl -L http://repo.msys2.org/distrib/{cpu}" + "/msys2-base-{cpu}-{version}.tar.xz | xz -cd | zstd -c" + " > $ARTIFACTS/msys2.tar.zst".format( + cpu=msys_cpu(cpu), version=MSYS_VERSION[cpu] + ) ) h = hashlib.sha1(_create_command.encode()) self.hexdigest = h.hexdigest() @@ -92,12 +99,12 @@ def __init__(self, cpu): Task.__init__( self, - task_env=DockerImage.by_name('base'), - description='msys2 image: base {}'.format(cpu), + task_env=DockerImage.by_name("base"), + description="msys2 image: base {}".format(cpu), index=self.index, - expireIn='26 weeks', + expireIn="26 weeks", command=[_create_command], - artifact='msys2.tar.zst', + artifact="msys2.tar.zst", ) @@ -105,65 +112,70 @@ class MsysEnvironment(MsysCommon): def __init__(self, name): cpu = self.cpu create_commands = [ - 'pacman-key --init', - 'pacman-key --populate msys2', - 'pacman-key --refresh', - 'pacman --noconfirm -Sy procps tar {}'.format( - ' '.join(self.packages(name))), - 'pkill gpg-agent', - 'pkill dirmngr', - 'rm -rf /var/cache/pacman/pkg', - 'python3 -m pip install pip==22.2.2 wheel==0.37.1 --upgrade', - 'mv {}/{}/bin/{{mingw32-,}}make.exe'.format(msys(cpu), mingw(cpu)), - 'tar -c --hard-dereference {} | zstd -c > msys2.tar.zst'.format( - msys(cpu)), + "pacman-key --init", + "pacman-key --populate msys2", + "pacman-key --refresh", + "pacman --noconfirm -Sy procps tar {}".format( + " ".join(self.packages(name)) + ), + "pkill gpg-agent", + "pkill dirmngr", + "rm -rf /var/cache/pacman/pkg", + "python3 -m pip install pip==22.2.2 wheel==0.37.1 --upgrade", + "mv {}/{}/bin/{{mingw32-,}}make.exe".format(msys(cpu), mingw(cpu)), + "tar -c --hard-dereference {} | zstd -c > msys2.tar.zst".format(msys(cpu)), ] env = MsysBase.by_name(cpu) h = hashlib.sha1(env.hexdigest.encode()) - h.update(';'.join(create_commands).encode()) + h.update(";".join(create_commands).encode()) self.hexdigest = h.hexdigest() Task.__init__( self, task_env=env, - description='msys2 image: {} {}'.format(name, cpu), + description="msys2 image: {} {}".format(name, cpu), index=self.index, - expireIn='26 weeks', + expireIn="26 weeks", command=create_commands, - artifact='msys2.tar.zst', + artifact="msys2.tar.zst", ) def packages(self, name): def mingw_packages(pkgs): - return [ - 'mingw-w64-{}-{}'.format(msys_cpu(self.cpu), pkg) - for pkg in pkgs + return ["mingw-w64-{}-{}".format(msys_cpu(self.cpu), pkg) for pkg in pkgs] + + packages = mingw_packages( + [ + "curl", + "make", + "python3", + "python3-pip", ] + ) - packages = mingw_packages([ - 'curl', - 'make', - 'python3', - 'python3-pip', - ]) - - if name == 'build': - return packages + mingw_packages([ - 'gcc', - ]) + [ - 'patch', - ] - elif name == 'test': + if name == "build": + return ( + packages + + mingw_packages( + [ + "gcc", + ] + ) + + [ + "patch", + ] + ) + elif name == "test": return packages + [ - 'diffutils', - 'git', + "diffutils", + "git", ] - raise Exception('Unknown name: {}'.format(name)) + raise Exception("Unknown name: {}".format(name)) class Msys64Environment(MsysEnvironment, Task, metaclass=TaskEnvironment): - PREFIX = 'mingw64' - cpu = 'x86_64' + PREFIX = "mingw64" + cpu = "x86_64" __init__ = MsysEnvironment.__init__ diff --git a/CI/osx.py b/CI/osx.py index bebf3e40e..05c8d9add 100644 --- a/CI/osx.py +++ b/CI/osx.py @@ -11,57 +11,56 @@ class OsxCommon(object): - os = 'macos' - cpu = 'x86_64' + os = "macos" + cpu = "x86_64" def __init__(self, name): - self.hexdigest = hashlib.sha1( - self.ITERATION.encode('utf-8')).hexdigest() + self.hexdigest = hashlib.sha1(self.ITERATION.encode("utf-8")).hexdigest() self.name = name def prepare_params(self, params): - assert 'workerType' not in params - params['provisionerId'] = 'proj-git-cinnabar' - params['workerType'] = self.WORKER_TYPE + assert "workerType" not in params + params["provisionerId"] = "proj-git-cinnabar" + params["workerType"] = self.WORKER_TYPE command = [] - command.append('export PWD=$(pwd)') - command.append('export ARTIFACTS=$PWD') - command.extend(params['command']) - params['command'] = bash_command(*command) - env = params.setdefault('env', {}) + command.append("export PWD=$(pwd)") + command.append("export ARTIFACTS=$PWD") + command.extend(params["command"]) + params["command"] = bash_command(*command) + env = params.setdefault("env", {}) dev = env.setdefault( - 'DEVELOPER_DIR', - '/Applications/Xcode_13.2.1.app/Contents/Developer') + "DEVELOPER_DIR", "/Applications/Xcode_13.2.1.app/Contents/Developer" + ) env.setdefault( - 'SDKROOT', - '{}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.1.sdk' - .format(dev)) + "SDKROOT", + "{}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.1.sdk".format(dev), + ) return params class Osx(OsxCommon, metaclass=TaskEnvironment): - ITERATION = '4' - PREFIX = 'osx' - WORKER_TYPE = 'osx' - os_version = '10.15' + ITERATION = "4" + PREFIX = "osx" + WORKER_TYPE = "osx" + os_version = "10.15" class OsxArm64(OsxCommon, metaclass=TaskEnvironment): - cpu = 'arm64' - ITERATION = '2' - PREFIX = 'arm64-osx' - WORKER_TYPE = 'macos' - os_version = '11.0' + cpu = "arm64" + ITERATION = "2" + PREFIX = "arm64-osx" + WORKER_TYPE = "macos" + os_version = "11.0" def prepare_params(self, params): - env = params.setdefault('env', {}) + env = params.setdefault("env", {}) dev = env.setdefault( - 'DEVELOPER_DIR', - '/Applications/Xcode_15.0.1.app/Contents/Developer') + "DEVELOPER_DIR", "/Applications/Xcode_15.0.1.app/Contents/Developer" + ) env.setdefault( - 'SDKROOT', - '{}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.0.sdk' - .format(dev)) - env.setdefault('PIP_DISABLE_PIP_VERSION_CHECK', '1') - params['command'].insert(0, 'export PATH=$PATH:/opt/homebrew/bin') + "SDKROOT", + "{}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.0.sdk".format(dev), + ) + env.setdefault("PIP_DISABLE_PIP_VERSION_CHECK", "1") + params["command"].insert(0, "export PATH=$PATH:/opt/homebrew/bin") return super(OsxArm64, self).prepare_params(params) diff --git a/CI/package.mk b/CI/package.mk deleted file mode 100644 index f5c64e69b..000000000 --- a/CI/package.mk +++ /dev/null @@ -1,53 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -TOPDIR = $(abspath $(or $(dir $(firstword $(MAKEFILE_LIST))),$(CURDIR))/..) - -ifeq (a,$(firstword a$(subst /, ,$(abspath .)))) -PATHSEP = : -else -PATHSEP = ; -endif - -export PATH := $(TOPDIR)$(PATHSEP)$(PATH) -export PYTHONDONTWRITEBYTECODE := 1 - -ifeq ($(MAKECMDGOALS),package) -LOWER = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) - -system := $(call LOWER,$(SYSTEM)) -machine := $(call LOWER,$(MACHINE)) -PACKAGE_FLAGS = $(addprefix --system ,$(SYSTEM)) $(addprefix --machine ,$(MACHINE)) -PACKAGE_EXT = $(if $(filter windows,$(system)),zip,tar.xz) -PACKAGE = git-cinnabar.$(system).$(machine).$(PACKAGE_EXT) -EXT = $(if $(filter windows,$(system)),.exe) - -export XZ_OPT=-9 -export GZIP=-9 - -$(PACKAGE): - @mkdir -p tmp - @rm -rf tmp/git-cinnabar - @mkdir -p tmp/git-cinnabar - @$(TOPDIR)/download.py -o tmp/git-cinnabar/git-cinnabar$(EXT) $(PACKAGE_FLAGS) -ifeq (,$(filter zip,$(PACKAGE_EXT))) - tar --owner cinnabar:1000 --group cinnabar:1000 -C tmp --remove-files --sort=name -acvf $@ git-cinnabar -else - @rm -f $@ - cd tmp && find git-cinnabar | sort | zip -9 $(if $(filter windows,$(system)),,--symlinks )--move $(CURDIR)/$@ -@ -endif - rm -rf tmp - -.PHONY: $(PACKAGE) - -package: $(PACKAGE) -endif - -define CR - - -endef - -packages: - $(foreach c,$(shell $(TOPDIR)/download.py --list),$(MAKE) -f $(firstword $(MAKEFILE_LIST)) package SYSTEM=$(firstword $(subst /, ,$(c))) MACHINE=$(word 2,$(subst /, ,$(c)))$(CR)) diff --git a/CI/package.py b/CI/package.py new file mode 100644 index 000000000..2aab05199 --- /dev/null +++ b/CI/package.py @@ -0,0 +1,184 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import concurrent.futures +import os +import shutil +import struct +import subprocess +import sys +import tarfile +import zipfile +from io import BytesIO +from pathlib import Path +from urllib.request import urlopen + + +def main(): + ret = 0 + args = sys.argv[1:] + if args and args[0] == "--download": + download_py = Path(__file__).parent.parent / "download.py" + args = ( + subprocess.check_output( + [sys.executable, download_py, "--url", "--list"] + args[1:] + ) + .decode() + .splitlines() + ) + else: + for arg in args: + if arg.startswith("-"): + print(f"{arg} is not supported.") + return 1 + if len(args) > 1: + executor = concurrent.futures.ProcessPoolExecutor() + map_ = executor.map + else: + map_ = map + for path, pkg in zip(args, map_(package_from, args)): + if pkg: + print(f"Created {pkg} from {path}", file=sys.stderr) + else: + print(f"Can't determine platform type for {path}", file=sys.stderr) + ret = 1 + return ret + + +def package_from(path): + if path.startswith(("http:", "https:")): + fh = urlopen(path) + size = fh.length + else: + fh = open(path, "rb") + size = os.stat(path).st_size + with fh: + fh = RewindOnce(fh) + system, machine = detect_platform(fh) + if not system or not machine: + return + + fh.rewind() + if size is None: + fh = BytesIO(fh.read()) + size = len(fh.getbuffer()) + return package(fh, size, system, machine) + + +def package(fh, size, system, machine): + stem = f"git-cinnabar.{system.lower()}.{machine.lower()}" + if system == "Windows": + pkg = f"{stem}.zip" + zip = zipfile.ZipFile( + pkg, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=9 + ) + # Manually do zip.mkdir("git-cinnabar") until we can depend on python 3.11 + zinfo = zipfile.ZipInfo("git-cinnabar/") + zinfo.external_attr = (0o40777 << 16) | 0x10 + zip.writestr(zinfo, "") + fh = RewindOnce(fh) + with zip.open("git-cinnabar/git-cinnabar.exe", mode="w") as zipped: + shutil.copyfileobj(fh, zipped) + fh.rewind() + with zip.open("git-cinnabar/git-remote-hg.exe", mode="w") as zipped: + shutil.copyfileobj(fh, zipped) + else: + pkg = f"{stem}.tar.xz" + tar = tarfile.open(pkg, mode="w:xz", preset=9) + info = tarinfo("git-cinnabar/") + info.mode = 0o755 + info.type = tarfile.DIRTYPE + tar.addfile(info) + + info = tarinfo("git-cinnabar/git-cinnabar") + info.mode = 0o700 + info.size = size + info.type = tarfile.REGTYPE + tar.addfile(info, fh) + + info = tarinfo("git-cinnabar/git-remote-hg") + info.mode = 0o777 + info.type = tarfile.SYMTYPE + info.linkname = "git-cinnabar" + tar.addfile(info) + return pkg + + +def tarinfo(name): + info = tarfile.TarInfo(name) + info.uid = 1000 + info.gid = 1000 + info.uname = "cinnabar" + info.gname = "cinnabar" + return info + + +class RewindOnce: + def __init__(self, fh): + self.buf = b"" + self.off = 0 + self.rewound = False + self.fh = fh + + def read(self, length=None): + if self.rewound: + if length is None: + return self.buf[self.off :] + self.fh.read() + ret = self.buf[self.off :][:length] + self.off += len(ret) + missing = length - len(ret) + if not missing: + return ret + return ret + self.fh.read(missing) + + ret = self.fh.read(length) + self.buf += ret + return ret + + def rewind(self): + assert not self.rewound + self.rewound = True + + +def detect_platform(executable): + system, machine = None, None + head = executable.read(4) + if head[:2] == b"MZ": + # Seek to 0x3c + executable.read(0x3C - 4) + (pe_offset,) = struct.unpack("L", executable.read(4)) + # 64-bits little-endian Linux (in theory, System-V) + if ident == 0x02010100: + system = "Linux" + # Seek to 0x12 + executable.read(10) + (machine_type,) = struct.unpack("= 0xd0: - rawBytes[0] = rawBytes[0] & 0x7f + if rawBytes[0] >= 0xD0: + rawBytes[0] = rawBytes[0] & 0x7F result = base64.urlsafe_b64encode(rawBytes)[:-2] # Drop '==' padding return result.decode() @@ -43,10 +41,10 @@ def slugid(): class datetime(datetime.datetime): def format(self, no_usec=True): if no_usec: - return self.replace(microsecond=0).isoformat() + 'Z' + return self.replace(microsecond=0).isoformat() + "Z" if self.microsecond == 0: - return self.isoformat() + '.000000Z' - return self.isoformat() + 'Z' + return self.isoformat() + ".000000Z" + return self.isoformat() + "Z" def __add__(self, other): if isinstance(other, numbers.Number): @@ -55,24 +53,20 @@ def __add__(self, other): return self.combine(d.date(), d.timetz()) -task_group_id = (os.environ.get('TC_GROUP_ID') or - os.environ.get('TASK_ID') or slugid()) -if os.environ.get('DETERMINISTIC'): +task_group_id = os.environ.get("TC_GROUP_ID") or os.environ.get("TASK_ID") or slugid() +if os.environ.get("DETERMINISTIC"): now = datetime.fromtimestamp(0) else: now = datetime.utcnow() def index_env(idx): - return 'INDEX_{}'.format( - idx.replace('.', '_').replace('-', '_').upper() - ) + return "INDEX_{}".format(idx.replace(".", "_").replace("-", "_").upper()) def expires_soon(expires): try: - expires = datetime.strptime( - expires.rstrip('Z'), '%Y-%m-%dT%H:%M:%S.%f') + expires = datetime.strptime(expires.rstrip("Z"), "%Y-%m-%dT%H:%M:%S.%f") return expires < now + 86400 except (KeyError, ValueError): return True @@ -94,7 +88,7 @@ class Existing(str): def __init__(self, requests=requests): super(Index, self).__init__() - self.requests = None if os.environ.get('NO_INDEX') else requests + self.requests = None if os.environ.get("NO_INDEX") else requests def __missing__(self, key): result = None @@ -104,7 +98,7 @@ def __missing__(self, key): elif hint is not None: # empty environment variable pass else: - result = self._try_key('project.git-cinnabar.{}'.format(key)) + result = self._try_key("project.git-cinnabar.{}".format(key)) if not result: result = slugid() self[key] = result @@ -114,8 +108,8 @@ def _try_key(self, key, create=False): if not self.requests: return data = http_get(self.requests, PROXY_INDEX_URL.format(key)) - if data and not expires_soon(data['expires']): - result = data.get('taskId') + if data and not expires_soon(data["expires"]): + result = data.get("taskId") print('Found task "{}" for "{}"'.format(result, key)) return self.Existing(result) @@ -136,13 +130,13 @@ class TaskNamespace(type): def by_name(cls, fqdn): env = cls._namespace.get(fqdn) if not env: - n = fqdn.split('.') + n = fqdn.split(".") prefix = n[:-1] name = n[-1:] while prefix: - kls = cls._namespace.get('.'.join(prefix)) + kls = cls._namespace.get(".".join(prefix)) if isinstance(kls, type): - cls._namespace[fqdn] = env = kls('.'.join(name)) + cls._namespace[fqdn] = env = kls(".".join(name)) break name.insert(0, prefix.pop()) return env @@ -150,11 +144,12 @@ def by_name(cls, fqdn): def __new__(cls, name, bases, dic): @classmethod def by_name(kls, name): - return cls.by_name('.'.join((kls.PREFIX, name))) - dic['by_name'] = by_name + return cls.by_name(".".join((kls.PREFIX, name))) + + dic["by_name"] = by_name kls = super(TaskNamespace, cls).__new__(cls, name, bases, dic) - cls._namespace[dic['PREFIX']] = kls + cls._namespace[dic["PREFIX"]] = kls return kls @@ -173,134 +168,133 @@ class Task(object): @staticmethod def normalize_params(params): try: - artifact = params.pop('artifact') - assert 'artifacts' not in params - params['artifacts'] = [artifact] + artifact = params.pop("artifact") + assert "artifacts" not in params + params["artifacts"] = [artifact] except KeyError: pass return params try: - mount = params.pop('mount') - assert 'mounts' not in params - params['mounts'] = [mount] + mount = params.pop("mount") + assert "mounts" not in params + params["mounts"] = [mount] except KeyError: pass @staticmethod - def checkout(repo=None, commit=None, dest='repo'): + def checkout(repo=None, commit=None, dest="repo"): repo = repo or TC_REPO_URL commit = commit or TC_COMMIT return [ - 'git clone -n {} {}'.format(repo, dest), - 'git -c core.autocrlf=input -c advice.detachedHead=false' - ' -C {} checkout {}'.format(dest, commit), + "git clone -n {} {}".format(repo, dest), + "git -c core.autocrlf=input -c advice.detachedHead=false" + " -C {} checkout {}".format(dest, commit), ] def __init__(self, **kwargs): - task_env = kwargs.pop('task_env', None) + task_env = kwargs.pop("task_env", None) kwargs = self.normalize_params(kwargs) if task_env: kwargs = task_env.prepare_params(kwargs) - maxRunTime = kwargs.pop('maxRunTime', 1800) + maxRunTime = kwargs.pop("maxRunTime", 1800) task = { - 'created': now.format(), - 'deadline': (now + maxRunTime * 5 + 1800).format(), - 'retries': 5, - 'provisionerId': 'proj-git-cinnabar', - 'workerType': 'linux', - 'schedulerId': 'taskcluster-github', - 'taskGroupId': task_group_id, - 'metadata': { - 'owner': '{}@users.noreply.github.com'.format(TC_LOGIN), - 'source': TC_REPO_URL, + "created": now.format(), + "deadline": (now + maxRunTime * 5 + 1800).format(), + "retries": 5, + "provisionerId": "proj-git-cinnabar", + "workerType": "linux", + "schedulerId": "taskcluster-github", + "taskGroupId": task_group_id, + "metadata": { + "owner": "{}@users.noreply.github.com".format(TC_LOGIN), + "source": TC_REPO_URL, }, - 'payload': { - 'maxRunTime': maxRunTime, + "payload": { + "maxRunTime": maxRunTime, }, } - kwargs.setdefault('expireIn', '4 weeks') - dependencies = [os.environ.get('TASK_ID') or task_group_id] + kwargs.setdefault("expireIn", "4 weeks") + dependencies = [os.environ.get("TASK_ID") or task_group_id] artifact_paths = [] for k, v in kwargs.items(): - if k in ('provisionerId', 'workerType', 'priority'): + if k in ("provisionerId", "workerType", "priority"): task[k] = v - elif k == 'description': - task['metadata'][k] = task['metadata']['name'] = v - elif k == 'index': + elif k == "description": + task["metadata"][k] = task["metadata"]["name"] = v + elif k == "index": if TC_IS_PUSH and TC_BRANCH != "try": - task['routes'] = [ - 'index.project.git-cinnabar.{}'.format(v)] - elif k == 'expireIn': + task["routes"] = ["index.project.git-cinnabar.{}".format(v)] + elif k == "expireIn": value = v.split() if len(value) == 1: value, multiplier = value, 1 elif len(value) == 2: value, unit = value value = int(value) - unit = unit.rstrip('s') + unit = unit.rstrip("s") multiplier = 1 - if unit == 'year': + if unit == "year": multiplier *= 365 - unit = 'day' - if unit == 'week': + unit = "day" + if unit == "week": multiplier *= 7 - unit = 'day' - if unit == 'day': + unit = "day" + if unit == "day": multiplier *= 24 - unit = 'hour' - if unit == 'hour': + unit = "hour" + if unit == "hour": multiplier *= 60 - unit = 'minute' - if unit == 'minute': + unit = "minute" + if unit == "minute": multiplier *= 60 - unit = 'second' - if unit == 'second': - unit = '' + unit = "second" + if unit == "second": + unit = "" if unit: - raise Exception( - "Don't know how to handle {}".format(unit)) + raise Exception("Don't know how to handle {}".format(unit)) else: raise Exception("Don't know how to handle {}".format(v)) if not TC_IS_PUSH or TC_BRANCH == "try": if value * multiplier > 4 * 7 * 24 * 60 * 60: value = 4 multiplier = 7 * 24 * 60 * 60 # weeks - task['expires'] = (now + value * multiplier).format() - elif k == 'command': - task['payload']['command'] = v - if not kwargs.get('workerType', '').startswith('win'): - task['payload']['command'] = [task['payload']['command']] + task["expires"] = (now + value * multiplier).format() + elif k == "command": + task["payload"]["command"] = v + if not kwargs.get("workerType", "").startswith("win"): + task["payload"]["command"] = [task["payload"]["command"]] - elif k == 'artifacts': + elif k == "artifacts": artifacts = [ { - 'name': 'public/{}'.format(os.path.basename(a)), - 'path': a, - 'type': 'file', - } for a in v + "name": "public/{}".format(os.path.basename(a)), + "path": a, + "type": "file", + } + for a in v ] - artifact_paths.extend(a['name'] for a in artifacts) - task['payload']['artifacts'] = artifacts - elif k == 'env': - task['payload'].setdefault('env', {}).update(v) - elif k == 'scopes': + artifact_paths.extend(a["name"] for a in artifacts) + task["payload"]["artifacts"] = artifacts + elif k == "env": + task["payload"].setdefault("env", {}).update(v) + elif k == "scopes": task[k] = v for s in v: - if s.startswith('secrets:'): - features = task['payload'].setdefault('features', {}) - features['taskclusterProxy'] = True - elif k == 'mounts': + if s.startswith("secrets:"): + features = task["payload"].setdefault("features", {}) + features["taskclusterProxy"] = True + elif k == "mounts": + def file_format(url): - for ext in ('rar', 'tar.zst', 'tar.bz2', 'tar.gz', 'zip'): - if url.endswith('.{}'.format(ext)): + for ext in ("rar", "tar.zst", "tar.bz2", "tar.gz", "zip"): + if url.endswith(".{}".format(ext)): return ext - raise Exception( - 'Unsupported/unknown format for {}'.format(url)) + raise Exception("Unsupported/unknown format for {}".format(url)) - mounts = task['payload']['mounts'] = [] + mounts = task["payload"]["mounts"] = [] for m in v: assert isinstance(m, dict) m = list(m.items()) @@ -308,57 +302,59 @@ def file_format(url): kind, m = m[0] if isinstance(m, Task): content = { - 'artifact': m.artifacts[0], - 'taskId': m.id, + "artifact": m.artifacts[0], + "taskId": m.id, } dependencies.append(m.id) elif isinstance(m, dict): content = m - dependencies.append(m['taskId']) + dependencies.append(m["taskId"]) else: content = { - 'url': m, + "url": m, } - artifact = content.get('artifact') or content['url'] + artifact = content.get("artifact") or content["url"] if kind == "file" or kind.startswith("file:"): mount = { - 'content': content, - 'file': kind[5:] or os.path.basename(artifact), + "content": content, + "file": kind[5:] or os.path.basename(artifact), } - if kind[5:] == 'dockerimage': - mount['format'] = os.path.splitext( - content['artifact'])[-1].replace('.', '') + if kind[5:] == "dockerimage": + mount["format"] = os.path.splitext(content["artifact"])[ + -1 + ].replace(".", "") mounts.append(mount) elif kind == "directory" or kind.startswith("directory:"): - mounts.append({ - 'content': content, - 'directory': os.path.dirname(kind[10:]) or '.', - 'format': file_format(artifact), - }) - elif k == 'dependencies': + mounts.append( + { + "content": content, + "directory": os.path.dirname(kind[10:]) or ".", + "format": file_format(artifact), + } + ) + elif k == "dependencies": for t in v: dependencies.append(t.id) else: raise Exception("Don't know how to handle {}".format(k)) - task['dependencies'] = sorted(dependencies) - index = kwargs.get('index') + task["dependencies"] = sorted(dependencies) + index = kwargs.get("index") id = None - if index and all(isinstance(d, Index.Existing) - for d in dependencies[1:]): + if index and all(isinstance(d, Index.Existing) for d in dependencies[1:]): id = Task.by_index[index] if isinstance(id, Index.Existing): - data = http_get( - session, ARTIFACT_URL.format(id, '').rstrip('/')) or {} + data = http_get(session, ARTIFACT_URL.format(id, "").rstrip("/")) or {} artifacts_expire = [ - expires_soon(a.get('expires')) - for a in data.get('artifacts', []) - if a.get('name') in artifact_paths + expires_soon(a.get("expires")) + for a in data.get("artifacts", []) + if a.get("name") in artifact_paths ] - if len(artifact_paths) != len(artifacts_expire) \ - or any(artifacts_expire): + if len(artifact_paths) != len(artifacts_expire) or any(artifacts_expire): print( - 'Ignore task "{}" because of missing or expiring artifacts' - .format(id)) + 'Ignore task "{}" because of missing or expiring artifacts'.format( + id + ) + ) id = None self.id = id or slugid() @@ -380,40 +376,40 @@ def submit(self): return print('Submitting task "{}":'.format(self.id)) print(json.dumps(self.task, indent=4, sort_keys=True)) - if 'TC_PROXY' not in os.environ: + if "TC_PROXY" not in os.environ: return - url = f'{PROXY_URL}/api/queue/v1/task/{self.id}' + url = f"{PROXY_URL}/api/queue/v1/task/{self.id}" res = session.put(url, json=self.task) try: res.raise_for_status() except Exception: print(res.headers) try: - print(res.json()['message']) + print(res.json()["message"]) except Exception: print(res.content) raise print(res.json()) -SHELL_QUOTE_RE = re.compile(r'[\\\t\r\n \'\"#<>&|`~(){}$;\*\?]') +SHELL_QUOTE_RE = re.compile(r"[\\\t\r\n \'\"#<>&|`~(){}$;\*\?]") def _quote(s, for_windows=False): if s and not SHELL_QUOTE_RE.search(s): return s if for_windows: - for c in '^&\\<>|': - s = s.replace(c, '^' + c) + for c in "^&\\<>|": + s = s.replace(c, "^" + c) return "'{}'".format(s.replace("'", "'\\''")) def join_command(*command, for_windows=False): - return ' '.join(_quote(a, for_windows) for a in command) + return " ".join(_quote(a, for_windows) for a in command) def bash_command(*commands): - return ['bash', '-c', '-x', '-e', '; '.join(commands)] + return ["bash", "-c", "-x", "-e", "; ".join(commands)] class action(object): @@ -430,25 +426,31 @@ def __init__(self, name, title=None, description=None): if self.template is None: import yaml - with open(os.path.join(os.path.dirname(__file__), '..', - '.taskcluster.yml')) as fh: + + with open( + os.path.join(os.path.dirname(__file__), "..", ".taskcluster.yml") + ) as fh: contents = yaml.safe_load(fh) - task = contents['tasks'][0]['then']['in'] - del task['taskId'] + task = contents["tasks"][0]["then"]["in"] + del task["taskId"] self.__class__.template = task def adjust(s): - return s.replace('decision', 'action') + ' ({})'.format(title) + return s.replace("decision", "action") + " ({})".format(title) - metadata = self.template['metadata'] + metadata = self.template["metadata"] self.task = dict( self.template, - payload=dict(self.template['payload'], - env=dict(self.template['payload']['env'], - TC_ACTION=name)), - metadata=dict(metadata, - name=adjust(metadata['name']), - description=adjust(metadata['description']))) + payload=dict( + self.template["payload"], + env=dict(self.template["payload"]["env"], TC_ACTION=name), + ), + metadata=dict( + metadata, + name=adjust(metadata["name"]), + description=adjust(metadata["description"]), + ), + ) def __call__(self, func): self.func = func diff --git a/CI/test_download.py b/CI/test_download.py new file mode 100644 index 000000000..01aca99f0 --- /dev/null +++ b/CI/test_download.py @@ -0,0 +1,784 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import http.client +import http.server +import inspect +import itertools +import os +import shutil +import socket +import ssl +import subprocess +import sys +import types +import urllib.error +import urllib.parse +import urllib.request +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import contextmanager +from io import BytesIO +from pathlib import Path +from tempfile import TemporaryDirectory +from threading import Thread + +REPOSITORY = "https://github.com/glandium/git-cinnabar" + +# The versions below are expected to be semantically ordered. +VERSIONS = { + # Tag: Version in Cargo.toml + "0.6.0rc2": "0.6.0-rc2", + "0.6.0": "0.6.0", + "0.6.3": "0.6.3", + "0.7.0beta1": "0.7.0-beta.1", + # Newer versions below. We're bound to what older versions were doing to find the + # right download on self-update. I don't know what went through my head when I + # made that code strip dashes from tag names... + "0.7.0beta2": "0.7.0-beta.2", + "0.7.0rc1": "0.7.0-rc.1", + "0.7.0": "0.7.0", + "0.7.1": "0.7.1", + # Here's a trick to make things happy-ish in the future: older versions don't + # handle tags prefixed with "v", but will still do a self-update to the first + # one it finds. + # Tag: Shortened version + "v0.8.0-beta.1": "0.8.0beta1", + "v0.8.0-rc.1": "0.8.0rc1", + "v0.8.0": "0.8.0", + "v0.8.1": "0.8.1", + "v0.9.0-beta.1": "0.9.0beta1", + "v0.9.0": "0.9.0", + "v0.10.0-beta.1": "0.10.0beta1", + "v0.10.0": "0.10.0", +} +VERSIONS_ORDER = {v: n for n, v in enumerate(VERSIONS)} + + +def version_for_tag(tag): + if tag.startswith("v"): + return tag[1:] + return VERSIONS[tag] + + +def do_test(cwd, worktree, git_cinnabar, download_py, package_py, proxy): + # Avoid extra ls-remote traffic by forcing git to use a local mirror + # created from the original clone. + repo = cwd / "git-cinnabar" + subprocess.check_call( + [ + "git", + "clone", + "--bare", + "--reference", + worktree, + REPOSITORY, + repo, + ] + ) + subprocess.check_call( + ["git", "-C", repo, "fetch", worktree, "refs/tags/*:refs/tags/*"] + ) + env = CommandEnvironment(cwd).derive_with( + GIT_CONFIG_COUNT="1", + GIT_CONFIG_KEY_0=f"url.{repo}.insteadOf", + GIT_CONFIG_VALUE_0=REPOSITORY, + GIT_CINNABAR_CHECK="no-version-check", + HTTPS_PROXY=proxy.url, + GIT_SSL_NO_VERIFY="1", + ) + + def listdir(p): + return sorted(os.listdir(p)) + + executor = ThreadPoolExecutor(max_workers=1) + + def get_pkg(): + pkg_dir = cwd / "pkg" + pkg_dir.mkdir() + env.derive_with(cwd=pkg_dir).check_call( + [ + sys.executable, + package_py, + git_cinnabar, + ], + ) + return pkg_dir / os.listdir(pkg_dir)[0] + + pkg = executor.submit(get_pkg) + standalone_download_py = Path(shutil.copy2(download_py, cwd)) + + worktree_head = env.check_output(["git", "-C", worktree, "rev-parse", "HEAD"]) + head_version = env.get_version([git_cinnabar, "-V"]) + head_full_version = env.get_version([git_cinnabar, "--version"]) + _, _, head = head_full_version.removesuffix("-modified").rpartition("-") + head_branch = "release" + if head_version.endswith(".0-a"): + head_branch = "next" + elif head_version.endswith(("-a", "-b")): + head_branch = "master" + # We may be testing a version that is not the current tip of the + # head_branch. Update our mirror so that it is. + env.check_call(["git", "-C", repo, "update-ref", f"refs/heads/{head_branch}", head]) + + status = Status() + last_known_tag = None + previous_tag = None + future_tags = {} + tags = {} + envs = {head: env} + for t, v in VERSIONS.items(): + try: + rev = env.check_output( + ["git", "-C", repo, "rev-parse", t], stderr=subprocess.DEVNULL + ) + last_known_tag = t + envs[t] = env + except Exception: + assert previous_tag is not None + previous_rev = tags[previous_tag] + rev = env.derive_with( + GIT_AUTHOR_NAME="foobar", + GIT_AUTHOR_EMAIL="foo@bar", + GIT_COMMITTER_NAME="foobar", + GIT_COMMITTER_EMAIL="foo@bar", + ).check_output( + [ + "git", + "-C", + repo, + "commit-tree", + f"{previous_rev}^{{tree}}", + "-p", + previous_rev, + "-m", + t, + ] + ) + repo_t = repo.parent / (repo.name + f"-{t}") + envs[previous_tag].check_call( + ["git", "clone", "--mirror", "--reference", repo, REPOSITORY, repo_t] + ) + envs[t] = env.derive_with(GIT_CONFIG_KEY_0=f"url.{repo_t}.insteadOf") + status += assert_eq( + env.check_output(["git", "-C", repo_t, "tag", t, rev]), "" + ) + future_tags[t] = None + if v != t: + assert v not in envs + envs[v] = envs[t] + tags[t] = rev + previous_tag = t + + tag_by_sha1 = {sha1: t for t, sha1 in tags.items()} + + def get_url_with(script, args): + use_env = None + if args[:1] == ["--exact"]: + use_env = envs.get(args[1]) + if not use_env: + tag = tag_by_sha1.get(args[1]) + if tag: + use_env = envs.get(tag) + if not use_env: + use_env = env + if script.suffix == ".py": + cmd = [sys.executable, script] + else: + cmd = [script, "self-update"] + return Result( + use_env.check_output, + cmd + ["--url"] + args, + stderr=subprocess.PIPE, + ) + + BRANCHES = ("release", "master", "next") + results = { + script: { + what: get_url_with(script, args) + for what, args in itertools.chain( + ( + (None, []), + (head, ["--exact", head]), + (worktree_head, ["--exact", worktree_head]), + ), + ((t, ["--exact", t]) for t in tags), + ((v, ["--exact", v]) for v in VERSIONS.values() if v not in tags), + ((sha1, ["--exact", sha1]) for v, sha1 in tags.items()), + ((branch, ["--branch", branch]) for branch in BRANCHES), + ) + } + for script in (standalone_download_py, download_py, git_cinnabar) + } + urls = { + what: result.value + for what, result in results[download_py].items() + if not isinstance(result.value, Exception) + } + + for t, v in VERSIONS.items(): + if t != v: + status += assert_eq( + results[download_py][t], + results[download_py][v], + "download.py should support different types of version strings", + ) + + for k in results[download_py].keys(): + if k: + status += assert_eq( + results[standalone_download_py][k], + results[download_py][k], + "Same url should be used whether run standalone or not", + ) + if k and k != head_branch: + status += assert_eq( + results[download_py][k], + results[git_cinnabar][k], + "git cinnabar self-update should work the same as download.py", + ) + status += assert_eq( + results[standalone_download_py][None], + results[download_py]["release"], + "Standalone download should download from release by default", + ) + status += assert_eq( + results[standalone_download_py][worktree_head], + results[download_py][None], + "Download from a worktree should download for the exact commit", + ) + for branch in BRANCHES: + if branch in urls and branch != "release": + status += assert_startswith( + urls[branch], + "https://community-tc.services.mozilla.com/", + f"Url from --branch {branch} should be on taskcluster", + ) + if "release" in urls: + status += assert_startswith( + urls["release"], + REPOSITORY, + "Url from --branch release should be on github", + ) + status += assert_eq( + urls["release"], + urls[last_known_tag], + f"Url from --branch release should be the same as --exact {last_known_tag}", + ) + for t, sha1 in tags.items(): + status += assert_eq( + urls[t], + urls[sha1], + f"Url from --exact {t} should be the same as --exact {sha1}", + ) + # Now that we've established that, we change the sha1 urls for vX.Y.Z tags + # to point to TC. That's what versions prior to 0.7.0-beta.2 would use. + if t.startswith("v"): + urls[sha1] = get_url_with(download_py, ["--url", "--exact", sha1]).value + + full_versions = {t: f"{version_for_tag(t)}-{sha1}" for t, sha1 in tags.items()} + mappings = { + urls[h]: pkg if urls[h].startswith(REPOSITORY) else git_cinnabar + for h in (head, None, "master", "next") + } + for h, url in urls.items(): + if h in (head, "master", "next"): + full_versions[h] = head_full_version + if url not in mappings: + if h in future_tags: + mappings[url] = pkg + elif tag_by_sha1.get(h) in future_tags: + mappings[url] = git_cinnabar + else: + mappings[url] = urllib.request.urlopen(url).read() + for url, content in mappings.items(): + proxy.map(url, content) + + for t, v in itertools.chain([(head, head_version)], VERSIONS.items()): + git_cinnabar_v = cwd / v / git_cinnabar.name + envs[t].run( + [sys.executable, download_py, "-o", git_cinnabar_v, "--exact", t], + ) + status += assert_eq( + Result(listdir, cwd / v), + sorted((git_cinnabar.name, f"git-remote-hg{git_cinnabar.suffix}")), + ) + status += assert_eq( + Result(env.get_version, [git_cinnabar_v, "-V"]), + head_version if t in future_tags else v, + ) + + for upgrade_to in itertools.chain([last_known_tag], future_tags): + for t, v in itertools.chain( + [(head, head_version)] if head_branch != "release" else [], VERSIONS.items() + ): + upgrade_env = envs[upgrade_to] + if t in future_tags: + upgrade_env = upgrade_env.derive_with( + GIT_CINNABAR_EXPERIMENTS="test", + GIT_CINNABAR_VERSION=v, + GIT_CINNABAR_MODIFIED="", + GIT_CINNABAR_COMMIT=tags[t], + ) + git_cinnabar_v = cwd / v / git_cinnabar.name + version = Result( + upgrade_env.derive_with(GIT_CINNABAR_CHECK="").get_version, + [git_cinnabar_v, "--version"], + stderr=subprocess.STDOUT, + ) + new_version_warning = "" + # Starting with version 0.7.0beta1, a warning is shown when there is a + # new version available. Unfortunately, 0.7.0beta1 has a bug that makes + # it believe there's always an update even if it's the last version. + # It also, like older versions doesn't support tags prefixed with "v", + # and in that case, doesn't show the warning. + if ( + "." in t + and tuple(int(x) for x in t.replace("v", "").split(".")[:2]) >= (0, 7) + and ( + VERSIONS_ORDER[upgrade_to] > VERSIONS_ORDER[t] or t == "0.7.0beta1" + ) + and (not upgrade_to.startswith("v") or t != "0.7.0beta1") + ): + new_version = version_for_tag(upgrade_to) + current_version = "" + if t == "0.7.0beta1": + new_version = upgrade_to.replace("b", "-b").replace("rc", "-rc") + current_version = f" (current version: {v})" + new_version_warning = ( + f"\n\nWARNING New git-cinnabar version available: {new_version}{current_version}" + "\n\nWARNING You may run `git cinnabar self-update` to update." + ) + + version_status = assert_eq( + version, + full_versions[t] + new_version_warning, + ) + + status += version_status + version_status = True + if not version_status or VERSIONS_ORDER[upgrade_to] <= VERSIONS_ORDER.get( + t, -1 + ): + continue + + for branch in (None,) + (BRANCHES if upgrade_to == last_known_tag else ()): + update_dir = cwd / "update" / v + if branch: + update_dir = update_dir / branch + + shutil.copytree(cwd / v, update_dir, symlinks=True, dirs_exist_ok=True) + git_cinnabar_v = update_dir / git_cinnabar.name + extra_args = [] + if branch: + extra_args += ["--branch", branch] + if ( + branch in (None, "release") + and VERSIONS_ORDER[upgrade_to] < VERSIONS_ORDER.get(t, -1) + or branch in (None, head_branch) + and t == head + ): + update = None + elif branch in (None, "release"): + if ( + upgrade_to.startswith("v") + and VERSIONS_ORDER[t] <= VERSIONS_ORDER["0.7.0beta1"] + ): + # The mishandling of version parsing error in these versions + # makes it so that the first tag starting with "v" in alphanumeric + # order wins. + # Which, interestingly, means older versions will self-update to + # 0.8.0-beta.1, but no subsequent versions until 0.8.0. + # And jump to 0.10.0-beta.1 then 0.10.0. Of course, that only + # happens for versions that haven't been updated all that time. + mishandled_versions = [ + v + for v in VERSIONS + if v.startswith("v") + and VERSIONS_ORDER[v] <= VERSIONS_ORDER[upgrade_to] + ] + update = tags[ + min(mishandled_versions) + if mishandled_versions + else upgrade_to + ] + else: + update = upgrade_to + else: + update = branch + with proxy.capture_log() as log: + status += assert_eq( + Result( + upgrade_env.check_output, + [git_cinnabar_v, "self-update"] + extra_args, + stderr=subprocess.STDOUT, + ), + f"Installing update from {urls[update]}" + if update + else "WARNING Did not find an update to install.", + ) + if update: + status += assert_eq(log, [urls[update]]) + else: + status += assert_eq(log, []) + status += assert_eq( + Result(env.get_version, [git_cinnabar_v, "--version"]), + full_versions[head if upgrade_to in future_tags else (update or t)], + ) + shutil.rmtree(update_dir) + + return status.as_return_code() + + +def main(): + assert len(sys.argv) <= 2 + if len(sys.argv) == 2: + git_cinnabar = Path(sys.argv[1]).resolve() + if not git_cinnabar.exists(): + print(f"{sys.argv[1]} not found") + return 1 + if not git_cinnabar.is_file(): + print(f"{sys.argv[1]} not a file") + return 1 + else: + git_cinnabar = shutil.which("git-cinnabar") + if not git_cinnabar: + print("A git-cinnabar executable couldn't be found in $PATH") + return 1 + git_cinnabar = Path(git_cinnabar) + worktree = Path(__file__).parent.parent.absolute() + git = worktree / ".git" + download_py = worktree / "download.py" + package_py = worktree / "CI" / "package.py" + for f in (git, download_py, package_py): + if not f.exists(): + print(f"{f} doesn't exist.") + return 1 + with TemporaryDirectory() as d: + proxy = ProxyServer() + try: + return do_test( + Path(d), worktree, git_cinnabar, download_py, package_py, proxy + ) + finally: + proxy.shutdown() + + +class CommandEnvironment: + def __init__(self, cwd, environ=os.environ): + self.cwd = cwd + self.env = environ.copy() + + class CalledProcessError(subprocess.CalledProcessError): + def __repr__(self): + return ( + super().__repr__().removesuffix(")") + + f", stdout={self.stdout!r}, stderr={self.stderr!r})" + ) + + def check_call(self, x, **kwargs): + return self.subprocess_func(subprocess.check_call, x, **kwargs) + + def check_output(self, x, **kwargs): + return self.subprocess_func(subprocess.check_output, x, **kwargs) + + def run(self, x, **kwargs): + return self.subprocess_func(subprocess.run, x, **kwargs) + + def subprocess_func(self, func, x, **kwargs): + try: + result = func(x, env=self.env, cwd=self.cwd, text=True, **kwargs) + except Exception as e: + if isinstance(e, subprocess.CalledProcessError): + e.__class__ = self.CalledProcessError + raise e + if isinstance(result, str): + return result.strip() + return result + + def get_version(self, x, **kwargs): + return self.check_output(x, **kwargs).removeprefix("git-cinnabar ") + + def derive_with(self, cwd=None, **kwargs): + env = self.env.copy() + for k, v in kwargs.items(): + if v is None: + env.pop(k, None) + else: + env[k] = v + return CommandEnvironment(self.cwd if cwd is None else cwd, env) + + def __repr__(self): + return f"{self.__class__.__name__}(cwd={str(self.cwd)!r})" + + +class Status: + def __init__(self): + self.success = True + + def __iadd__(self, other): + self.success = bool(other) and self.success + return self + + def __bool__(self): + return self.success + + def as_return_code(self): + return 0 if self.success else 1 + + +class Func: + def __init__(self, func): + self.func = func + + def __repr__(self): + func = self.func + name = func.__name__ + if code := getattr(func, "__code__", None): + name = f"{name}@{code.co_filename}:{code.co_firstlineno}" + if isinstance(func, types.MethodType): + name = f"{func.__self__}.{name}" + return name + + def __eq__(self, other): + return other == self.func + + +class Args: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def __repr__(self): + args = (repr(a) for a in self.args) + kwargs = (f"{k}={v!r}" for k, v in self.kwargs.items()) + return ", ".join(itertools.chain(args, kwargs)) + + +class Result: + def __init__(self, func, *args, **kwargs): + self.func = Func(func) + self.args = Args(*args, **kwargs) + try: + self.value = func(*args, **kwargs) + except Exception as e: + self.value = e + + def __repr__(self): + return repr(self.value) + + def __eq__(self, other): + return other == self.value + + def __getattr__(self, name): + return getattr(self.value, name) + + +def assertion_message(assertion, msg): + msg = f": {msg}" if msg else "" + for frame in inspect.stack(): + info = inspect.getframeinfo(frame.frame) + if not info.function.startswith("assert"): + return ( + f"assertion `{assertion}` failed at {info.filename}:{info.lineno}{msg}" + ) + # Just in case, but this shouldn't happen + return f"assertion `{assertion}` failed{msg}" + + +def assert_op(op_msg, op, a, b, msg=None): + if op(a, b): + return True + + show = {} + if isinstance(a, Result): + show["f"] = a.func + left = "f(a)" + show["a"] = a.args + else: + left = "a" + if isinstance(b, Result): + f = show.get("f") + if f: + if b.func == f: + right = "f(b)" + else: + show["g"] = b.func + right = "g(b)" + show["b"] = b.args + else: + show["f"] = b.func + right = "f(b)" + else: + right = "b" + show[left] = a + show[right] = b + print( + assertion_message(op_msg.format(left=left, right=right), msg), file=sys.stderr + ) + for k, v in show.items(): + if "f" in show: + print(f"{k:>5}: {v!r}", file=sys.stderr) + else: + print(f" {k}: {v!r}", file=sys.stderr) + return False + + +def assert_eq(a, b, msg=None): + return assert_op("{left} == {right}", (lambda a, b: a == b), a, b, msg) + + +def assert_startswith(a, b, msg=None): + def startswith(a, b): + try: + return a.startswith(b) + except Exception: + return False + + return assert_op("{left}.startswith({right})", startswith, a, b, msg) + + +class ProxyServer(http.server.ThreadingHTTPServer): + def __init__(self): + super().__init__(("localhost", 0), ProxyHTTPRequestHandler) + self.log = None + self.mappings = {} + self.url = f"http://localhost:{self.server_port}" + self.thread = Thread(target=self.serve_forever) + self.thread.start() + + this_script = Path(__file__) + self.context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # Created with `openssl req -x509 -newkey rsa:2048 -keyout selfsigned.key + # -out selfsigned.crt -days 36524 -nodes -subj "/CN=localhost"` + self.context.load_cert_chain( + this_script.with_name("selfsigned.crt"), + this_script.with_name("selfsigned.key"), + ) + + @contextmanager + def capture_log(self): + assert self.log is None + self.log = [] + yield self.log + self.log = None + + def log_url(self, url_elements): + if self.log is not None: + host, port, path = url_elements + url = f"https://{host}" + if port != 443: + url += f":{port}" + url += path + self.log.append(url) + + @staticmethod + def urlsplit(url): + u = urllib.parse.urlsplit(url) + assert u.scheme == "https" + path = u.path + if u.query: + path = f"{path}?{u.query}" + return (u.hostname, u.port or 443, path) + + def map(self, url, content): + host, port, path = self.urlsplit(url) + self.mappings.setdefault((host, port), {})[path] = content + + +class ProxyHTTPRequestHandler(http.server.BaseHTTPRequestHandler): + def do_CONNECT(self): + host, _, port = self.path.partition(":") + port = int(port) + self.send_response_only(200) + self.end_headers() + + mappings = self.server.mappings.get((host, port), {}) + if mappings or self.server.log is not None: + self.handle_locally(mappings, host, port) + else: + self.pass_through(host, port) + + def handle_locally(self, mappings, host, port): + with self.server.context.wrap_socket(self.connection, server_side=True) as sock: + HTTPRequestHandler( + sock, self.client_address, (self.server, mappings, host, port) + ) + + def pass_through(self, host, port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.connect((host, port)) + + def relay(src, dest): + try: + shutil.copyfileobj( + src.makefile("rb", buffering=False), + dest.makefile("wb", buffering=False), + ) + except ConnectionResetError: + pass + + t1 = Thread(target=relay, args=(self.connection, s)) + t1.start() + t2 = Thread(target=relay, args=(s, self.connection)) + t2.start() + t1.join() + t2.join() + + +class HTTPRequestHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + server, mappings, host, port = self.server + content = mappings.get(self.path) + server.log_url((host, port, self.path)) + if content: + self.send_content(content) + else: + self.pass_through(host, port) + + def log_request(self, code="-", size="-"): + pass + + def send_content(self, content): + self.send_response(200) + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + if isinstance(content, Future): + content = content.result() + if isinstance(content, str): + content = content.encode() + if isinstance(content, bytes): + content = BytesIO(content) + elif isinstance(content, Path): + content = content.open("rb") + else: + raise RuntimeError("mapped content is neither bytes, str nor Path") + out = Chunker(self.wfile) + shutil.copyfileobj(content, out) + out.write(b"") + + def pass_through(self, host, port): + conn = http.client.HTTPSConnection(host, port) + conn.request("GET", self.path, headers=self.headers) + response = conn.getresponse() + self.send_response(response.status) + for k, v in response.getheaders(): + self.send_header(k, v) + self.end_headers() + shutil.copyfileobj(response, self.wfile) + + +class Chunker: + def __init__(self, out): + self.out = out + + def write(self, data): + self.out.write(f"{len(data):x}\r\n".encode()) + if data: + self.out.write(data) + self.out.write(b"\r\n") + self.out.flush() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/CI/tools.py b/CI/tools.py index 81434a8e2..b51222cae 100644 --- a/CI/tools.py +++ b/CI/tools.py @@ -5,6 +5,8 @@ import hashlib import os +import msys +from docker import DockerImage from tasks import ( Task, TaskEnvironment, @@ -12,28 +14,68 @@ parse_version, ) from util import build_commit -from docker import DockerImage -import msys - +from variables import TC_BRANCH, TC_IS_PUSH -MERCURIAL_VERSION = '6.8' -# Not using 2.46.0 because of -# https://lore.kernel.org/git/20240727191917.p64ul4jybpm2a7hm@glandium.org/ -GIT_VERSION = '2.45.2' +MERCURIAL_VERSION = "6.8" +GIT_VERSION = "2.46.2" ALL_MERCURIAL_VERSIONS = ( - '1.9.3', '2.0.2', '2.1.2', '2.2.3', '2.3.2', '2.4.2', '2.5.4', - '2.6.3', '2.7.2', '2.8.2', '2.9.1', '3.0.1', '3.1.2', '3.2.4', - '3.3.3', '3.4.2', '3.5.2', '3.6.3', '3.7.3', '3.8.4', '3.9.2', - '4.0.2', '4.1.3', '4.2.2', '4.3.3', '4.4.2', '4.5.3', '4.6.2', - '4.7.2', '4.8.2', '4.9.1', '5.0.2', '5.1.2', '5.2.2', '5.3.2', - '5.4.2', '5.5.2', '5.6.1', '5.7.1', '5.8.1', '5.9.3', '6.0.3', - '6.1.4', '6.2.3', '6.3.3', '6.4.2', '6.5.3', '6.6.3', '6.7.4', - '6.8', + "1.9.3", + "2.0.2", + "2.1.2", + "2.2.3", + "2.3.2", + "2.4.2", + "2.5.4", + "2.6.3", + "2.7.2", + "2.8.2", + "2.9.1", + "3.0.1", + "3.1.2", + "3.2.4", + "3.3.3", + "3.4.2", + "3.5.2", + "3.6.3", + "3.7.3", + "3.8.4", + "3.9.2", + "4.0.2", + "4.1.3", + "4.2.2", + "4.3.3", + "4.4.2", + "4.5.3", + "4.6.2", + "4.7.2", + "4.8.2", + "4.9.1", + "5.0.2", + "5.1.2", + "5.2.2", + "5.3.2", + "5.4.2", + "5.5.2", + "5.6.1", + "5.7.1", + "5.8.1", + "5.9.3", + "6.0.3", + "6.1.4", + "6.2.3", + "6.3.3", + "6.4.2", + "6.5.3", + "6.6.3", + "6.7.4", + "6.8", ) SOME_MERCURIAL_VERSIONS = ( - '1.9.3', '2.5.4', '3.4.2', + "1.9.3", + "2.5.4", + "3.4.2", ) assert MERCURIAL_VERSION in ALL_MERCURIAL_VERSIONS @@ -41,97 +83,106 @@ def nproc(env): - if env.os == 'macos': - return 'sysctl -n hw.physicalcpu' - return 'nproc --all' + if env.os == "macos": + return "sysctl -n hw.physicalcpu" + return "nproc --all" class Git(Task, metaclass=Tool): PREFIX = "git" def __init__(self, os_and_version): - (os, version) = os_and_version.split('.', 1) + (os, version) = os_and_version.split(".", 1) self.os = os - if os.endswith('osx'): - build_image = TaskEnvironment.by_name('{}.build'.format(os)) + if os.endswith("osx"): + build_image = TaskEnvironment.by_name("{}.build".format(os)) else: - build_image = DockerImage.by_name('build-tools') - if os == 'linux' or os.endswith('osx'): + build_image = DockerImage.by_name("build-tools") + if os == "linux" or os.endswith("osx"): h = hashlib.sha1(build_image.hexdigest.encode()) - h.update(b'v4' if version == GIT_VERSION else b'v3') - if os == 'linux': - description = 'git v{}'.format(version) + h.update(b"v4" if version == GIT_VERSION else b"v3") + if os == "linux": + description = "git v{}".format(version) else: env = build_image - description = 'git v{} {} {}'.format(version, env.os, env.cpu) + description = "git v{} {} {}".format(version, env.os, env.cpu) Task.__init__( self, task_env=build_image, description=description, - index='{}.git.v{}'.format(h.hexdigest(), version), - expireIn='26 weeks', + index="{}.git.v{}".format(h.hexdigest(), version), + expireIn="26 weeks", command=Task.checkout( - 'git://git.kernel.org/pub/scm/git/git.git', - 'v{}'.format(version), - dest='git', - ) + Task.checkout() + ([ - 'patch -d git -p1 < repo/CI/git-transport-disconnect.diff', - ] if version == GIT_VERSION else []) + [ - 'make -C git -j$({}) install prefix=/ NO_GETTEXT=1' - ' NO_OPENSSL=1 NO_TCLTK=1 NO_UNCOMPRESS2=1' - ' DESTDIR=$PWD/git'.format( - nproc(build_image)), - 'tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst' - .format(version), + "git://git.kernel.org/pub/scm/git/git.git", + "v{}".format(version), + dest="git", + ) + + Task.checkout() + + ( + [ + "patch -d git -p1 < repo/CI/git-transport-disconnect.diff", + ] + if version == GIT_VERSION + else [] + ) + + [ + "make -C git -j$({}) install prefix=/ NO_GETTEXT=1" + " NO_OPENSSL=1 NO_TCLTK=1 NO_UNCOMPRESS2=1" + " DESTDIR=$PWD/git".format(nproc(build_image)), + "tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst".format(version), ], - artifact='git-{}.tar.zst'.format(version), + artifact="git-{}.tar.zst".format(version), ) else: - env = TaskEnvironment.by_name('{}.build'.format(os)) + env = TaskEnvironment.by_name("{}.build".format(os)) raw_version = version - if 'windows' not in version: + if "windows" not in version: version = { - version: version + '.windows.1', - '2.17.1': '2.17.1.windows.2', + version: version + ".windows.1", + "2.17.1": "2.17.1.windows.2", }.get(version) - if version.endswith('.windows.1'): - min_ver = version[:-len('.windows.1')] + if version.endswith(".windows.1"): + min_ver = version[: -len(".windows.1")] else: - min_ver = version.replace('windows.', '') + min_ver = version.replace("windows.", "") h = hashlib.sha1(env.hexdigest.encode()) - h.update(b'v1') + h.update(b"v1") Task.__init__( self, task_env=build_image, - description='git v{} {} {}'.format(version, env.os, env.cpu), - index='{}.git.v{}'.format(h.hexdigest(), raw_version), - expireIn='26 weeks', + description="git v{} {} {}".format(version, env.os, env.cpu), + index="{}.git.v{}".format(h.hexdigest(), raw_version), + expireIn="26 weeks", command=[ - 'curl -L https://github.com/git-for-windows/git/releases/' - 'download/v{}/MinGit-{}-{}-bit.zip' - ' -o git.zip'.format(version, min_ver, msys.bits(env.cpu)), - 'unzip -d git git.zip', - 'curl -L https://github.com/git-for-windows/git/releases/' - 'download/v{}/Git-{}-{}-bit.tar.bz2 | ' - 'tar -C git --no-same-owner -jx ' - '{}/libexec/git-core/git-http-backend.exe' - .format(version, min_ver, msys.bits(env.cpu), - msys.mingw(env.cpu).lower()), - 'tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst'.format( - raw_version), + "curl -L https://github.com/git-for-windows/git/releases/" + "download/v{}/MinGit-{}-{}-bit.zip" + " -o git.zip".format(version, min_ver, msys.bits(env.cpu)), + "unzip -d git git.zip", + "curl -L https://github.com/git-for-windows/git/releases/" + "download/v{}/Git-{}-{}-bit.tar.bz2 | " + "tar -C git --no-same-owner -jx " + "{}/libexec/git-core/git-http-backend.exe".format( + version, + min_ver, + msys.bits(env.cpu), + msys.mingw(env.cpu).lower(), + ), + "tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst".format( + raw_version + ), ], - artifact='git-{}.tar.zst'.format(raw_version), + artifact="git-{}.tar.zst".format(raw_version), ) def mount(self): - return {'directory:git': self} + return {"directory:git": self} def install(self): - if self.os.endswith(('linux', 'osx')): + if self.os.endswith(("linux", "osx")): return [ - 'export PATH=$PWD/git/bin:$PATH', - 'export GIT_EXEC_PATH=$PWD/git/libexec/git-core', - 'export GIT_TEMPLATE_DIR=$PWD/git/share/git-core/templates', + "export PATH=$PWD/git/bin:$PATH", + "export GIT_EXEC_PATH=$PWD/git/libexec/git-core", + "export GIT_TEMPLATE_DIR=$PWD/git/share/git-core/templates", ] else: return [] @@ -141,63 +192,66 @@ class Hg(Task, metaclass=Tool): PREFIX = "hg" def __init__(self, os_and_version): - (os, version) = os_and_version.split('.', 1) - (version, suffix, _) = version.partition('.py3') - if suffix or len(version) == 40 or \ - parse_version(version) >= parse_version('6.2'): - python = 'python3' + (os, version) = os_and_version.split(".", 1) + (version, suffix, _) = version.partition(".py3") + if ( + suffix + or len(version) == 40 + or parse_version(version) >= parse_version("6.2") + ): + python = "python3" else: - python = 'python2.7' - if os == 'linux': - env = TaskEnvironment.by_name('{}.build-tools'.format(os)) + python = "python2.7" + if os == "linux": + env = TaskEnvironment.by_name("{}.build-tools".format(os)) else: - env = TaskEnvironment.by_name('{}.build'.format(os)) + env = TaskEnvironment.by_name("{}.build".format(os)) kwargs = {} if len(version) == 40: # Assume it's a sha1 - pretty_version = 'r{}{}'.format(version, suffix) - artifact_version = '99.0' - expire = '2 weeks' + pretty_version = "r{}{}".format(version, suffix) + artifact_version = "99.0" + expire = "2 weeks" else: - pretty_version = 'v{}{}'.format(version, suffix) + pretty_version = "v{}{}".format(version, suffix) artifact_version = version - expire = '26 weeks' - desc = 'hg {}'.format(pretty_version) - if os == 'linux': - platform_tag = 'linux_x86_64' - if python == 'python3': - python_tag = 'cp39' - abi_tag = 'cp39' + expire = "26 weeks" + desc = "hg {}".format(pretty_version) + if os == "linux": + platform_tag = "linux_x86_64" + if python == "python3": + python_tag = "cp39" + abi_tag = "cp39" else: - python_tag = 'cp27' - abi_tag = 'cp27mu' + python_tag = "cp27" + abi_tag = "cp27mu" else: - desc = '{} {} {}'.format(desc, env.os, env.cpu) - if os.endswith('osx'): - py_host_plat = 'macosx-{}-{}'.format(env.os_version, env.cpu) - platform_tag = py_host_plat.replace('.', '_').replace('-', '_') - if python == 'python3': - python_tag = 'cp311' if os == 'arm64-osx' else 'cp39' + desc = "{} {} {}".format(desc, env.os, env.cpu) + if os.endswith("osx"): + py_host_plat = "macosx-{}-{}".format(env.os_version, env.cpu) + platform_tag = py_host_plat.replace(".", "_").replace("-", "_") + if python == "python3": + python_tag = "cp311" if os == "arm64-osx" else "cp39" abi_tag = python_tag else: - python_tag = 'cp27' - abi_tag = 'cp27m' - env_ = kwargs.setdefault('env', {}) - env_.setdefault('MACOSX_DEPLOYMENT_TARGET', env.os_version) - env_.setdefault('ARCHFLAGS', '-arch {}'.format(env.cpu)) - env_.setdefault('_PYTHON_HOST_PLATFORM', py_host_plat) + python_tag = "cp27" + abi_tag = "cp27m" + env_ = kwargs.setdefault("env", {}) + env_.setdefault("MACOSX_DEPLOYMENT_TARGET", env.os_version) + env_.setdefault("ARCHFLAGS", "-arch {}".format(env.cpu)) + env_.setdefault("_PYTHON_HOST_PLATFORM", py_host_plat) else: - if python == 'python3': - platform_tag = 'mingw_x86_64' - python_tag = 'cp310' - abi_tag = 'cp310' + if python == "python3": + platform_tag = "mingw_x86_64" + python_tag = "cp311" + abi_tag = "cp311" else: - platform_tag = 'mingw' - python_tag = 'cp27' - abi_tag = 'cp27m' + platform_tag = "mingw" + python_tag = "cp27" + abi_tag = "cp27m" - artifact = 'mercurial-{{}}-{}-{}-{}.whl'.format( + artifact = "mercurial-{{}}-{}-{}-{}.whl".format( python_tag, abi_tag, platform_tag, @@ -205,126 +259,126 @@ def __init__(self, os_and_version): pre_command = [] if len(version) == 40: - hg = self.by_name('{}.{}'.format(os, MERCURIAL_VERSION)) - kwargs.setdefault('mounts', []).append(hg.mount()) + hg = self.by_name("{}.{}".format(os, MERCURIAL_VERSION)) + kwargs.setdefault("mounts", []).append(hg.mount()) pre_command.extend(hg.install()) - pre_command.extend([ - 'hg clone https://www.mercurial-scm.org/repo/hg' - ' -r {} mercurial-{}'.format(version, version), - 'rm -rf mercurial-{}/.hg'.format(version), - 'echo tag: {} > mercurial-{}/.hg_archival.txt' - .format(artifact_version, version), - ]) + pre_command.extend( + [ + "hg clone https://www.mercurial-scm.org/repo/hg" + " -r {} mercurial-{}".format(version, version), + "rm -rf mercurial-{}/.hg".format(version), + "echo tag: {} > mercurial-{}/.hg_archival.txt".format( + artifact_version, version + ), + ] + ) # 2.6.2 is the first version available on pypi - elif parse_version('2.6.2') <= parse_version(version) and \ - parse_version(version) < parse_version('6.2'): + elif parse_version("2.6.2") <= parse_version(version) and parse_version( + version + ) < parse_version("6.2"): # pip download does more than download, and while it runs setup.py # for version 6.2, a DistutilsPlatformError exception is thrown on # Windows. pre_command.append( - '{} -m pip download --no-binary mercurial --no-deps' - ' --progress-bar off mercurial=={}'.format(python, version)) + "{} -m pip download --no-binary mercurial --no-deps" + " --progress-bar off mercurial=={}".format(python, version) + ) else: - url = 'https://mercurial-scm.org/release/mercurial-{}.tar.gz' - pre_command.append( - 'curl -sLO {}'.format(url.format(version))) + url = "https://mercurial-scm.org/release/mercurial-{}.tar.gz" + pre_command.append("curl -sLO {}".format(url.format(version))) if len(version) != 40: - pre_command.append( - 'tar -zxf mercurial-{}.tar.gz'.format(version)) + pre_command.append("tar -zxf mercurial-{}.tar.gz".format(version)) - if os.startswith('mingw'): + if os.startswith("mingw"): # Work around https://bz.mercurial-scm.org/show_bug.cgi?id=6654 # and https://bz.mercurial-scm.org/show_bug.cgi?id=6757 pre_command.append( 'sed -i "s/, output_dir=self.build_temp/' - ', output_dir=self.build_temp, extra_postargs=[$EXTRA_FLAGS]/;' - '/self.addlongpathsmanifest/d;' - '" mercurial-{}/setup.py' - .format(version)) - if python == 'python3': - kwargs.setdefault('env', {}).setdefault( - 'EXTRA_FLAGS', '"-municode"') + ", output_dir=self.build_temp, extra_postargs=[$EXTRA_FLAGS]/;" + "/self.addlongpathsmanifest/d;" + '" mercurial-{}/setup.py'.format(version) + ) + if python == "python3": + kwargs.setdefault("env", {}).setdefault("EXTRA_FLAGS", '"-municode"') pre_command.append( 'sed -i "s/ifdef __GNUC__/if 0/"' - ' mercurial-{}/mercurial/exewrapper.c' - .format(version)) + " mercurial-{}/mercurial/exewrapper.c".format(version) + ) h = hashlib.sha1(env.hexdigest.encode()) h.update(artifact.encode()) - if os.endswith('osx'): - h.update(b'v2') - elif os.startswith('mingw'): - h.update(b'v4') + if os.endswith("osx"): + h.update(b"v2") + elif os.startswith("mingw"): + h.update(b"v4") else: - h.update(b'v1') + h.update(b"v1") Task.__init__( self, task_env=env, description=desc, - index='{}.hg.{}'.format(h.hexdigest(), pretty_version), + index="{}.hg.{}".format(h.hexdigest(), pretty_version), expireIn=expire, - command=pre_command + [ + command=pre_command + + [ # pyproject.toml enables PEP 517, which can't be disabled. # pip wheel doesn't accept --build-option when PEP 517 is # enabled. --build-option is necessary on msys2 because # of problems with the bdist-dir otherwise. - 'rm -f mercurial-{}/pyproject.toml'.format(version), - '{} -m pip wheel -v --build-option -b --build-option' - ' $PWD/wheel -w $ARTIFACTS ./mercurial-{}'.format( - python, - version), + "rm -f mercurial-{}/pyproject.toml".format(version), + "{} -m pip wheel -v --build-option -b --build-option" + " $PWD/wheel -w $ARTIFACTS ./mercurial-{}".format(python, version), ], artifact=artifact.format(artifact_version), - **kwargs + **kwargs, ) def mount(self): - return {f'file:{os.path.basename(self.artifacts[0])}': self} + return {f"file:{os.path.basename(self.artifacts[0])}": self} def install(self): filename = os.path.basename(self.artifacts[0]) - if 'cp3' in filename: - python = 'python3' + if "cp3" in filename: + python = "python3" else: - python = 'python2.7' - return [ - '{} -m pip install {}'.format(python, filename) - ] + python = "python2.7" + return ["{} -m pip install {}".format(python, filename)] -def install_rust(version='1.80.0', target='x86_64-unknown-linux-gnu'): - rustup_opts = '-y --default-toolchain none' - cargo_dir = '$HOME/.cargo/bin/' - rustup = cargo_dir + 'rustup' +def install_rust(version="1.81.0", target="x86_64-unknown-linux-gnu"): + rustup_opts = "-y --default-toolchain none" + cargo_dir = "$HOME/.cargo/bin/" + rustup = cargo_dir + "rustup" rust_install = [ - 'curl -o rustup.sh https://sh.rustup.rs', - 'sh rustup.sh {rustup_opts}', - '{rustup} install {version} --profile minimal', - '{rustup} default {version}', - 'PATH={cargo_dir}:$PATH', - '{rustup} target add {target}', + "curl -o rustup.sh https://sh.rustup.rs", + "sh rustup.sh {rustup_opts}", + "{rustup} install {version} --profile minimal", + "{rustup} default {version}", + "PATH={cargo_dir}:$PATH", + "{rustup} target add {target}", ] loc = locals() return [r.format(**loc) for r in rust_install] class Build(Task, metaclass=Tool): - PREFIX = 'build' + PREFIX = "build" def __init__(self, os_and_variant): - os, variant = (os_and_variant.split('.', 1) + [''])[:2] + os, variant = (os_and_variant.split(".", 1) + [""])[:2] env = TaskEnvironment.by_name( - '{}.build'.format(os.replace('arm64-linux', 'linux'))) - if os.startswith('mingw'): - build_env = TaskEnvironment.by_name('linux.build') + "{}.build".format(os.replace("arm64-linux", "linux")) + ) + if os.startswith("mingw"): + build_env = TaskEnvironment.by_name("linux.build") else: build_env = env - artifact = 'git-cinnabar' - if os.startswith('mingw'): - artifact += '.exe' + artifact = "git-cinnabar" + if os.startswith("mingw"): + artifact += ".exe" artifacts = [artifact] def prefix(p, s): @@ -335,156 +389,168 @@ def prefix(p, s): desc_variant = variant extra_commands = [] environ = { - 'WARNINGS_AS_ERRORS': '1', + "WARNINGS_AS_ERRORS": "1", } - cargo_flags = ['-vv', '--release'] - cargo_features = ['self-update', 'gitdev'] + cargo_flags = ["-vv", "--release"] + cargo_features = ["self-update", "gitdev", "xz2/static", "bzip2/static"] rust_version = None - if variant == 'asan': - if os.endswith('osx'): - opt = '-O2' + if variant == "asan": + if os.endswith("osx"): + opt = "-O2" else: - opt = '-Og' - environ['TARGET_CFLAGS'] = ' '.join([ - opt, - '-g', - '-fsanitize=address', - '-fno-omit-frame-pointer', - '-fPIC', - ]) - environ['RUSTFLAGS'] = ' '.join([ - '-Zsanitizer=address', - '-Copt-level=1', - '-Cdebuginfo=full', - '-Cforce-frame-pointers=yes', - ]) - elif variant == 'coverage': - environ['TARGET_CFLAGS'] = ' '.join([ - '-coverage', - '-fPIC', - ]) - artifacts += ['coverage.zip'] + opt = "-Og" + environ["TARGET_CFLAGS"] = " ".join( + [ + opt, + "-g", + "-fsanitize=address", + "-fno-omit-frame-pointer", + "-fPIC", + ] + ) + environ["RUSTFLAGS"] = " ".join( + [ + "-Zsanitizer=address", + "-Copt-level=1", + "-Cdebuginfo=full", + "-Cforce-frame-pointers=yes", + ] + ) + elif variant == "coverage": + environ["TARGET_CFLAGS"] = " ".join( + [ + "-coverage", + "-fPIC", + ] + ) + artifacts += ["coverage.zip"] extra_commands = [ - '(cd repo && zip $ARTIFACTS/coverage.zip' + "(cd repo && zip $ARTIFACTS/coverage.zip" ' $(find . -name "*.gcno" -not -name "build_script*"))', ] - environ['RUSTFLAGS'] = ' '.join([ - '-Zprofile', - '-Ccodegen-units=1', - '-Cinline-threshold=0', - ]) + environ["RUSTFLAGS"] = " ".join( + [ + "-Zprofile", + "-Ccodegen-units=1", + "-Cinline-threshold=0", + ] + ) # Build without --release - cargo_flags.remove('--release') - environ['CARGO_INCREMENTAL'] = '0' - elif variant.startswith('old:'): + cargo_flags.remove("--release") + environ["CARGO_INCREMENTAL"] = "0" + elif variant.startswith("old:"): head = variant[4:] hash = build_commit(head) - variant = '' - elif variant.startswith('rust-'): + variant = "" + elif variant.startswith("rust-"): rust_version = variant[5:] elif variant: - raise Exception('Unknown variant: {}'.format(variant)) + raise Exception("Unknown variant: {}".format(variant)) - if 'osx' not in os: - environ['CC'] = 'clang-18' - if os in ('linux', 'arm64-linux'): - cargo_features.append('curl-compat') + if "osx" not in os: + environ["CC"] = "clang-18" + if os in ("linux", "arm64-linux"): + cargo_features.append("curl-compat") - if os.startswith('mingw'): + if os.startswith("mingw"): cpu = msys.msys_cpu(env.cpu) rust_target = "{}-pc-windows-gnu".format(cpu) - elif os.startswith('osx'): - rust_target = 'x86_64-apple-darwin' - elif os.startswith('arm64-osx'): - rust_target = 'aarch64-apple-darwin' - elif os == 'linux': - rust_target = 'x86_64-unknown-linux-gnu' - elif os == 'arm64-linux': - rust_target = 'aarch64-unknown-linux-gnu' - if 'osx' not in os: + elif os.startswith("osx"): + rust_target = "x86_64-apple-darwin" + elif os.startswith("arm64-osx"): + rust_target = "aarch64-apple-darwin" + elif os == "linux": + rust_target = "x86_64-unknown-linux-gnu" + elif os == "arm64-linux": + rust_target = "aarch64-unknown-linux-gnu" + if "osx" not in os: for target in dict.fromkeys( - ["x86_64-unknown-linux-gnu", rust_target]).keys(): + ["x86_64-unknown-linux-gnu", rust_target] + ).keys(): arch = { - 'x86_64': 'amd64', - 'aarch64': 'arm64', - }[target.partition('-')[0]] - multiarch = target.replace('unknown-', '') - TARGET = target.replace('-', '_').upper() - environ[f'CARGO_TARGET_{TARGET}_LINKER'] = environ['CC'] - if 'linux' in os: - extra_link_arg = f'--sysroot=/sysroot-{arch}' - if os.startswith('mingw'): - extra_link_arg = \ - f'-L/usr/lib/gcc/{cpu}-w64-mingw32/10-win32' - environ[f'CARGO_TARGET_{TARGET}_RUSTFLAGS'] = \ - f'-C link-arg=--target={target} ' + \ - f'-C link-arg={extra_link_arg} ' + \ - '-C link-arg=-fuse-ld=lld-18' - rustflags = environ.pop('RUSTFLAGS', None) + "x86_64": "amd64", + "aarch64": "arm64", + }[target.partition("-")[0]] + multiarch = target.replace("unknown-", "") + TARGET = target.replace("-", "_").upper() + environ[f"CARGO_TARGET_{TARGET}_LINKER"] = environ["CC"] + if "linux" in os: + extra_link_arg = f"--sysroot=/sysroot-{arch}" + if os.startswith("mingw"): + extra_link_arg = f"-L/usr/lib/gcc/{cpu}-w64-mingw32/10-win32" + environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] = ( + f"-C link-arg=--target={target} " + + f"-C link-arg={extra_link_arg} " + + "-C link-arg=-fuse-ld=lld-18" + ) + rustflags = environ.pop("RUSTFLAGS", None) if rustflags: - environ[f'CARGO_TARGET_{TARGET}_RUSTFLAGS'] += \ - f' {rustflags}' - if 'linux' in os: - environ[f'CFLAGS_{target}'] = f'--sysroot=/sysroot-{arch}' - if 'linux' in os: - environ['PKG_CONFIG_PATH'] = '' - environ['PKG_CONFIG_SYSROOT_DIR'] = f'/sysroot-{arch}' - environ['PKG_CONFIG_LIBDIR'] = ':'.join(( - f'/sysroot-{arch}/usr/lib/pkgconfig', - f'/sysroot-{arch}/usr/lib/{multiarch}/pkgconfig', - f'/sysroot-{arch}/usr/share/pkgconfig', - )) - if variant in ('coverage', 'asan'): - environ['RUSTC_BOOTSTRAP'] = '1' + environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] += f" {rustflags}" + if "linux" in os: + environ[f"CFLAGS_{target}"] = f"--sysroot=/sysroot-{arch}" + if "linux" in os: + environ["PKG_CONFIG_PATH"] = "" + environ["PKG_CONFIG_SYSROOT_DIR"] = f"/sysroot-{arch}" + environ["PKG_CONFIG_LIBDIR"] = ":".join( + ( + f"/sysroot-{arch}/usr/lib/pkgconfig", + f"/sysroot-{arch}/usr/lib/{multiarch}/pkgconfig", + f"/sysroot-{arch}/usr/share/pkgconfig", + ) + ) + if variant in ("coverage", "asan"): + environ["RUSTC_BOOTSTRAP"] = "1" if rust_version: rust_install = install_rust(rust_version, target=rust_target) else: rust_install = install_rust(target=rust_target) - cargo_flags.extend(['--target', rust_target]) + cargo_flags.extend(["--target", rust_target]) if cargo_features: - cargo_flags.extend(['--features', ','.join(cargo_features)]) + cargo_flags.extend(["--features", ",".join(cargo_features)]) for key, value in list(environ.items()): # RUSTFLAGS values in the environment override builds.rustflags # from .cargo/config.toml. - if 'RUSTFLAGS' in key: - environ[key] = value + ' -Cforce-unwind-tables=yes' + if "RUSTFLAGS" in key: + environ[key] = value + " -Cforce-unwind-tables=yes" hash = hash or build_commit() - if os.startswith('osx'): - environ.setdefault( - 'MACOSX_DEPLOYMENT_TARGET', '10.7') - if os.startswith('arm64-osx'): - environ.setdefault( - 'MACOSX_DEPLOYMENT_TARGET', '11.0') + if os.startswith("osx"): + environ.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.7") + if os.startswith("arm64-osx"): + environ.setdefault("MACOSX_DEPLOYMENT_TARGET", "11.0") - cpu = 'arm64' if os == 'arm64-linux' else env.cpu + cpu = "arm64" if os == "arm64-linux" else env.cpu Task.__init__( self, task_env=build_env, - description='build {} {}{}'.format( - env.os, cpu, prefix(' ', desc_variant)), - index='build.{}.{}.{}{}'.format( - hash, env.os, cpu, prefix('.', variant)), - expireIn='26 weeks', - command=Task.checkout(commit=head) + rust_install + [ - '(cd repo ; cargo build {})'.format(' '.join(cargo_flags)), - 'mv repo/target/{}/{}/{} $ARTIFACTS/'.format( + description="build {} {}{}".format(env.os, cpu, prefix(" ", desc_variant)), + index="build.{}.{}.{}{}".format(hash, env.os, cpu, prefix(".", variant)), + expireIn="100 years" + if TC_IS_PUSH and TC_BRANCH == "release" and not variant + else "26 weeks", + command=Task.checkout(commit=head) + + rust_install + + [ + "(cd repo ; cargo build {})".format(" ".join(cargo_flags)), + "mv repo/target/{}/{}/{} $ARTIFACTS/".format( rust_target, - 'release' if '--release' in cargo_flags else 'debug', - artifact), - ] + extra_commands, + "release" if "--release" in cargo_flags else "debug", + artifact, + ), + ] + + extra_commands, artifacts=artifacts, env=environ, ) def mount(self): - return {f'file:{os.path.basename(self.artifacts[0])}': self} + return {f"file:{os.path.basename(self.artifacts[0])}": self} def install(self): filename = os.path.basename(self.artifacts[0]) return [ - f'cp {filename} repo/', - 'chmod +x repo/{}'.format(filename), - '$PWD/repo/{} setup'.format(filename), + f"cp {filename} repo/", + "chmod +x repo/{}".format(filename), + "$PWD/repo/{} setup".format(filename), ] diff --git a/CI/util.py b/CI/util.py index 2d714e408..14ac9bd9f 100644 --- a/CI/util.py +++ b/CI/util.py @@ -6,8 +6,16 @@ import subprocess -def build_commit(head='HEAD'): +def build_commit(head="HEAD"): return subprocess.check_output( - ['git', '-C', os.path.join(os.path.dirname(__file__), '..'), - 'rev-parse', '--verify', head], text=True, - stderr=open(os.devnull, 'wb')).strip() + [ + "git", + "-C", + os.path.join(os.path.dirname(__file__), ".."), + "rev-parse", + "--verify", + head, + ], + text=True, + stderr=open(os.devnull, "wb"), + ).strip() diff --git a/CI/variables.py b/CI/variables.py index 6ba8f195f..350e4ccbf 100644 --- a/CI/variables.py +++ b/CI/variables.py @@ -6,46 +6,47 @@ import os rootUrl = os.environ.get( - 'TASKCLUSTER_ROOT_URL', - 'https://community-tc.services.mozilla.com') + "TASKCLUSTER_ROOT_URL", "https://community-tc.services.mozilla.com" +) -if 'TC_PROXY' in os.environ: - PROXY_URL = os.environ.get('TASKCLUSTER_PROXY_URL', 'http://taskcluster') +if "TC_PROXY" in os.environ: + PROXY_URL = os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster") else: PROXY_URL = rootUrl -PROXY_INDEX_URL = PROXY_URL + '/api/index/v1/task/{}' -ARTIFACT_URL = rootUrl + '/api/queue/v1/task/{}/artifacts/{}' +PROXY_INDEX_URL = PROXY_URL + "/api/index/v1/task/{}" +ARTIFACT_URL = rootUrl + "/api/queue/v1/task/{}/artifacts/{}" DEFAULT_DATA = { - 'repo_name': 'git-cinnabar', - 'login': 'glandium', - 'commit': 'HEAD', - 'branch': '', - 'decision_id': '', + "repo_name": "git-cinnabar", + "login": "glandium", + "commit": "HEAD", + "branch": "", + "decision_id": "", } -DEFAULT_DATA['repo_url'] = 'https://github.com/{}/{}'.format( - DEFAULT_DATA['login'], DEFAULT_DATA['repo_name']) -for k in ('repo_name', 'login'): - DEFAULT_DATA['base_{}'.format(k)] = DEFAULT_DATA[k] +DEFAULT_DATA["repo_url"] = "https://github.com/{}/{}".format( + DEFAULT_DATA["login"], DEFAULT_DATA["repo_name"] +) +for k in ("repo_name", "login"): + DEFAULT_DATA["base_{}".format(k)] = DEFAULT_DATA[k] -TC_DATA = json.loads(os.environ.get('TC_DATA', json.dumps(DEFAULT_DATA))) +TC_DATA = json.loads(os.environ.get("TC_DATA", json.dumps(DEFAULT_DATA))) def get(k): return TC_DATA.get(k, DEFAULT_DATA[k]) -TC_LOGIN = get('login') -TC_REPO_NAME = get('repo_name') -TC_REPO_URL = get('repo_url') -TC_COMMIT = get('commit') -TC_BRANCH = get('branch') -TC_BASE_LOGIN = get('base_login') -TC_BASE_REPO_NAME = get('base_repo_name') +TC_LOGIN = get("login") +TC_REPO_NAME = get("repo_name") +TC_REPO_URL = get("repo_url") +TC_COMMIT = get("commit") +TC_BRANCH = get("branch") +TC_BASE_LOGIN = get("base_login") +TC_BASE_REPO_NAME = get("base_repo_name") -TC_ACTION = os.environ.get('TC_ACTION') -TC_IS_PUSH = os.environ.get('TC_IS_PUSH') == '1' +TC_ACTION = os.environ.get("TC_ACTION") +TC_IS_PUSH = os.environ.get("TC_IS_PUSH") == "1" -DEFAULT_REPO = 'https://hg.mozilla.org/users/mh_glandium.org/jqplot' -REPO = os.environ.get('REPO', DEFAULT_REPO) +DEFAULT_REPO = "https://hg.mozilla.org/users/mh_glandium.org/jqplot" +REPO = os.environ.get("REPO", DEFAULT_REPO) diff --git a/Cargo.lock b/Cargo.lock index a6592e146..7a0b876d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aho-corasick" @@ -62,7 +62,7 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -72,7 +72,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -92,17 +92,17 @@ checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -111,12 +111,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.6.0" @@ -134,9 +128,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" dependencies = [ "memchr", "serde", @@ -171,12 +165,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.6" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "3bbb537bb4a30b90362caddba8f360c0a56bc13d3a5570028e7197204cb54a17" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -187,9 +182,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.11" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -197,9 +192,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.11" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -209,14 +204,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.11" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -231,11 +226,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +[[package]] +name = "concat_const" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "105f2bacffc2751fc3f4f2b69b9c2d6f793d8807f03ee733e89a7baa8c9f8c42" + [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -277,9 +278,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.73+curl-8.8.0" +version = "0.4.77+curl-8.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "450ab250ecf17227c39afb9a2dd9261dc0035cb80f2612472fc0c4aac2dcb84d" +checksum = "f469e8a5991f277a208224f6c7ad72ecb5f986e36d09ae1f2c1bb9259478a480" dependencies = [ "cc", "libc", @@ -287,7 +288,7 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -309,7 +310,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -320,7 +321,7 @@ checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -341,7 +342,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -363,32 +364,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "filetime" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", - "redox_syscall", - "windows-sys", + "libredox", + "windows-sys 0.59.0", ] [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", "libz-sys", @@ -427,36 +428,37 @@ dependencies = [ [[package]] name = "getset" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" +checksum = "f636605b743120a8d32ed92fc27b6cde1a769f8f936c065151eb66f88ded513c" dependencies = [ - "proc-macro-error", + "proc-macro-error2", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.79", ] [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "git-cinnabar" -version = "0.7.0-beta.1" +version = "0.7.0-beta.2" dependencies = [ "array-init", "backtrace", "bit-vec", - "bitflags 2.6.0", + "bitflags", "bstr", "byteorder", "bzip2", "cc", "cfg-if", "clap", + "concat_const", "cstr", "curl-sys", "derivative", @@ -483,14 +485,14 @@ dependencies = [ "semver", "sha-1", "shared_child", - "syn 2.0.72", + "syn 2.0.79", "tar", "target", "tee", "tempfile", "typenum", "url", - "windows-sys", + "windows-sys 0.52.0", "xz2", "zip", "zstd", @@ -513,7 +515,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -552,9 +554,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown", @@ -586,15 +588,26 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags", + "libc", + "redox_syscall", +] [[package]] name = "libz-sys" -version = "1.1.18" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "libc", @@ -616,9 +629,9 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" [[package]] name = "lzma-sys" @@ -645,27 +658,30 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] name = "object" -version = "0.36.2" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "openssl-sys" @@ -687,38 +703,45 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] -name = "proc-macro-error" -version = "1.0.4" +name = "proc-macro-error-attr2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" dependencies = [ - "proc-macro-error-attr", "proc-macro2", "quote", - "syn 1.0.109", - "version_check", ] [[package]] -name = "proc-macro-error-attr" -version = "1.0.4" +name = "proc-macro-error2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" dependencies = [ + "proc-macro-error-attr2", "proc-macro2", "quote", - "version_check", + "syn 2.0.79", ] [[package]] @@ -732,9 +755,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -771,18 +794,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ - "bitflags 1.3.2", + "bitflags", ] [[package]] name = "regex" -version = "1.10.5" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", @@ -792,9 +815,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", @@ -803,9 +826,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rustc-demangle" @@ -821,15 +844,15 @@ checksum = "d7fc92159fb50a431c5da366f7627751fe7263cf867f8a30f27fa6063ba02ac0" [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ - "bitflags 2.6.0", + "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -840,22 +863,22 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -871,14 +894,20 @@ dependencies = [ [[package]] name = "shared_child" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d94659ad3c2137fef23ae75b03d5241d633f8acded53d672decfa0e6e0caef" +checksum = "09fa9338aed9a1df411814a5b2252f7cd206c55ae9bf2fa763f8de84603aa60c" dependencies = [ "libc", - "winapi", + "windows-sys 0.59.0", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "strsim" version = "0.11.1" @@ -898,9 +927,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -909,9 +938,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" dependencies = [ "filetime", "libc", @@ -932,34 +961,35 @@ checksum = "37c12559dba7383625faaff75be24becf35bfc885044375bcab931111799a3da" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -991,15 +1021,15 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -1029,9 +1059,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wasi" @@ -1040,32 +1070,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "winapi" -version = "0.3.9" +name = "windows-sys" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "windows-targets", ] -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-sys" -version = "0.52.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ "windows-targets", ] @@ -1154,11 +1171,32 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "zip" -version = "2.1.5" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b895748a3ebcb69b9d38dcfdf21760859a4b0d0b0015277640c2ef4c69640e6f" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" dependencies = [ "arbitrary", "crc32fast", @@ -1181,18 +1219,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 8a0657542..944356572 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "git-cinnabar" -version = "0.7.0-beta.1" +version = "0.7.0-beta.2" description = "git remote helper to interact with mercurial repositories" authors = ["Mike Hommey "] edition = "2021" @@ -13,6 +13,7 @@ repository = "https://github.com/glandium/git-cinnabar" rust-version = "1.74.0" include = [ "/src", + "/MPL-2.0", "/build.rs", "/.cargo", "/git-core/COPYING", @@ -34,6 +35,7 @@ bitflags = "2" bzip2 = "0.4" byteorder = "1" cfg-if = "1" +concat_const = "0.1" cstr = "0.2.10" derivative = "2" digest = "0.10" @@ -48,6 +50,7 @@ libc = "0.2" once_cell = "1.13" percent-encoding = "2" rand = "0.8" +semver = "1.0" sha-1 = "0.10" tee = "0.1" tempfile = "3" @@ -75,7 +78,7 @@ default-features = false [dependencies.derive_more] version = "0.99" default-features = false -features = ["deref", "from", "try_into"] +features = ["deref", "display", "from", "try_into"] [dependencies.flate2] version = "1" @@ -102,10 +105,6 @@ version = "1" default-features = false features = ["std"] -[dependencies.semver] -version = "1.0" -optional = true - [dependencies.shared_child] version = "1.0" optional = true @@ -158,9 +157,9 @@ default = ["version-check"] # libcurl.so compatibility (Linux only). curl-compat = ["rustflags"] # Check and report when a new version is available. -version-check = ["semver", "shared_child"] +version-check = ["shared_child"] # Download and apply new versions. -self-update = ["semver", "shared_child", "dep:tar", "dep:xz2", "dep:zip", "windows-sys/Win32_System_Threading"] +self-update = ["shared_child", "dep:tar", "dep:xz2", "dep:zip", "windows-sys/Win32_System_Threading"] # Development features @@ -168,7 +167,3 @@ self-update = ["semver", "shared_child", "dep:tar", "dep:xz2", "dep:zip", "windo compile_commands = [] # Enable libgit development options. gitdev = [] - -[lints.rust.unexpected_cfgs] -level = "warn" -check-cfg = ['cfg(version_check_branch)'] diff --git a/README.md b/README.md index d44d030b0..8d88a6b00 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Setup: - Assuming a prebuilt binary is available for your system, get the [download.py script](https://raw.githubusercontent.com/glandium/git-cinnabar/master/download.py) - and run it (requires python 3.6 or newer) with: + and run it (requires python 3.9 or newer) with: ``` $ ./download.py @@ -196,9 +196,11 @@ preference with one of the following values: - `always` - `never` - `phase` +- `force` `phase` is the default described above. `always` and `never` are -self-explanatory. +self-explanatory. `force` has the same meaning as `always`, but also +forces `git push --dry-run` to store metadata. Cinnabar clone: --------------- diff --git a/build.rs b/build.rs index 25c92640a..e309f8c6a 100644 --- a/build.rs +++ b/build.rs @@ -75,7 +75,7 @@ fn prepare_make(make: &mut Command) -> &mut Command { if chunk.len() == 2 { let name = chunk[0].trim_start().trim_end_matches('='); let value = chunk[1]; - result = result.arg(&format!("{}={}", name, value)); + result = result.arg(format!("{}={}", name, value)); } } result.env_remove("PROFILE") @@ -305,33 +305,4 @@ fn main() { } println!("cargo:rerun-if-env-changed=CINNABAR_MAKE_FLAGS"); - - #[cfg(any(feature = "version-check", feature = "self-update"))] - { - // The expected lifecycle is: - // - 0.x.0a on branch next - // - 0.x.0b on branch master - // - 0.x.0b1 on branch release (optionally) - // - 0.(x+1).0a on branch next (possibly later) - // - 0.x.0rc1 on branch release (optionally) - // - 0.x.0 on branch release - // - 0.x.1a on branch master - // - 0.x.1 on branch release - // - // Releases will only check tags. Others will check branch heads. - let pre = env("CARGO_PKG_VERSION_PRE"); - let patch = env("CARGO_PKG_VERSION_PATCH"); - if let "a" | "b" = &*pre { - println!("cargo:rustc-cfg=version_check_branch"); - println!( - "cargo:rustc-env=VERSION_CHECK_BRANCH={}", - match (&*patch, &*pre) { - ("0", "a") => "next", - ("0", "b") | (_, "a") => "master", - _ => panic!("Unsupported version"), - } - ); - } - println!("cargo:rerun-if-env-changed=CARGO_PKG_VERSION"); - } } diff --git a/deny.toml b/deny.toml index af8307403..39ef14833 100644 --- a/deny.toml +++ b/deny.toml @@ -23,5 +23,10 @@ exceptions = [ [bans] multiple-versions = "deny" skip = [ - { crate = "syn", reason = "ongoing transition from syn 1 to syn 2" } + { crate = "syn", reason = "ongoing transition from syn 1 to syn 2" }, + { crate = "windows-sys", reason = "ongoing transition from windows-sys 0.52 to windows-sys 0.59. The alternative at the moment would be to stay with winapi, which is actually a bigger dependency." }, +] +deny = [ + # Now that we're off winapi, let's stay that way. + "winapi", ] diff --git a/download.py b/download.py index 330725b79..2c6eeede7 100755 --- a/download.py +++ b/download.py @@ -3,7 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -''':' +""":" if command -v python3 > /dev/null; then PYTHON=python3 else @@ -12,35 +12,37 @@ fi exec $PYTHON -B $0 "$@" exit 1 -''' +""" + +import argparse +import errno import os +import platform import re +import subprocess import sys -import argparse -import platform import tarfile import tempfile import time -import subprocess -import errno from gzip import GzipFile -from shutil import copyfileobj, copyfile -from urllib.request import urlopen +from shutil import copyfile, copyfileobj from urllib.error import HTTPError +from urllib.request import urlopen from zipfile import ZipFile + try: from CI.util import build_commit except ImportError: build_commit = None -REPOSITORY = 'https://github.com/glandium/git-cinnabar' +REPOSITORY = "https://github.com/glandium/git-cinnabar" AVAILABLE = ( - ('Linux', 'x86_64'), - ('Linux', 'arm64'), - ('macOS', 'x86_64'), - ('macOS', 'arm64'), - ('Windows', 'x86_64'), + ("Linux", "x86_64"), + ("Linux", "arm64"), + ("macOS", "x86_64"), + ("macOS", "arm64"), + ("Windows", "x86_64"), ) @@ -53,7 +55,7 @@ def __init__(self, reader, length): self._length = length self._read = 0 self._pos = 0 - self._buf = b'' + self._buf = b"" def read(self, length): if self._pos < self._read: @@ -89,44 +91,52 @@ def seek(self, pos, how=os.SEEK_SET): def get_binary(system): - binary = 'git-cinnabar' - if system == 'Windows': - binary += '.exe' + binary = "git-cinnabar" + if system == "Windows": + binary += ".exe" return binary def get_url(system, machine, variant, sha1): - url = 'https://community-tc.services.mozilla.com/api/index/v1/task/' - url += 'project.git-cinnabar.build.' - url += '{}.{}.{}.{}'.format( - sha1, system.lower(), machine, - variant.lower() if variant else '').rstrip('.') - url += '/artifacts/public/{}'.format(get_binary(system)) + url = "https://community-tc.services.mozilla.com/api/index/v1/task/" + url += "project.git-cinnabar.build." + url += "{}.{}.{}.{}".format( + sha1, system.lower(), machine, variant.lower() if variant else "" + ).rstrip(".") + url += "/artifacts/public/{}".format(get_binary(system)) return url def get_release_url(system, machine, tag): - ext = 'zip' if system == 'Windows' else 'tar.xz' - url = f'{REPOSITORY}/releases/download/{tag}/git-cinnabar' - url += f'.{system.lower()}.{machine}.{ext}' + ext = "zip" if system == "Windows" else "tar.xz" + url = f"{REPOSITORY}/releases/download/{tag}/git-cinnabar" + url += f".{system.lower()}.{machine}.{ext}" return url def download(url, system, binary_path): - print('Downloading from %s...' % url) + print("Downloading from %s..." % url, file=sys.stderr) + if os.environ.get("GIT_SSL_NO_VERIFY"): + import ssl + + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + else: + context = None try: - reader = urlopen(url) + reader = urlopen(url, context=context) except HTTPError: # Try again, just in case try: - reader = urlopen(url) + reader = urlopen(url, context=context) except HTTPError as e: - print('Download failed with status code %d\n' % e.code, - file=sys.stderr) + print("Download failed with status code %d\n" % e.code, file=sys.stderr) print( - 'Error body was:\n\n%s' % e.read().decode('utf-8', 'replace'), - file=sys.stderr) + "Error body was:\n\n%s" % e.read().decode("utf-8", "replace"), + file=sys.stderr, + ) return 1 class ReaderProgress(object): @@ -138,9 +148,9 @@ def __init__(self, reader, length=None): def show_progress(self): if self._length: - count = f'\r {self._read * 100 // self._length}%' + count = f"\r {self._read * 100 // self._length}%" else: - count = f'\r {self._read} bytes' + count = f"\r {self._read} bytes" sys.stderr.write(count) def read(self, length): @@ -154,18 +164,19 @@ def read(self, length): def finish(self): self.show_progress() - sys.stderr.write('\n') + sys.stderr.write("\n") sys.stderr.flush() - encoding = reader.headers.get('Content-Encoding', 'identity') + encoding = reader.headers.get("Content-Encoding", "identity") progress = ReaderProgress(reader, reader.length) binary_content = Seekable(progress, reader.length) - if encoding == 'gzip': - binary_content = GzipFile(mode='rb', fileobj=binary_content) + if encoding == "gzip": + binary_content = GzipFile(mode="rb", fileobj=binary_content) (dirname, filename) = os.path.split(binary_path) + os.makedirs(dirname, exist_ok=True) fd, path = tempfile.mkstemp(prefix=filename, dir=dirname) - fh = os.fdopen(fd, 'wb') + fh = os.fdopen(fd, "wb") success = False try: @@ -175,33 +186,35 @@ def finish(self): progress.finish() fh.close() if success: - if url.endswith(('.zip', '.tar.xz')): + if url.endswith((".zip", ".tar.xz")): binary_name = get_binary(system) binary_content = None size = 0 archive_path = path - if url.endswith('.zip'): + if url.endswith(".zip"): archive = zip = ZipFile(path) for info in zip.infolist(): if os.path.basename(info.filename) == binary_name: size = info.file_size binary_content = zip.open(info) break - elif url.endswith('tar.xz'): - archive = tar = tarfile.open(path, 'r:*') + elif url.endswith("tar.xz"): + archive = tar = tarfile.open(path, "r:*") while True: member = tar.next() if member is None: break - if (member.isfile() and - os.path.basename(member.name) == binary_name): + if ( + member.isfile() + and os.path.basename(member.name) == binary_name + ): size = member.size binary_content = tar.extractfile(member) break fd, path = tempfile.mkstemp(prefix=filename, dir=dirname) - fh = os.fdopen(fd, 'wb') + fh = os.fdopen(fd, "wb") try: - print('Extracting %s...' % binary_name) + print("Extracting %s..." % binary_name, file=sys.stderr) progress = ReaderProgress(binary_content, size) copyfileobj(progress, fh) finally: @@ -249,39 +262,102 @@ def maybe_int(s): def split_version(s): - s = s.decode('ascii') - version = s.replace('-', '').split('.') - version[-1:] = [x for x in re.split(r'([0-9]+)', version[-1]) if x] - version = [maybe_int(x) for x in version] + s = s.decode("ascii").removeprefix("v") + version = [x.replace("-", "").replace(".", "") for x in re.split(r"([0-9]+)", s)] + version = [maybe_int(x) for x in version if x] if isinstance(version[-1], int): - version += ['z'] + version += ["z"] return version -def main(args): - if args.list: - for system, machine in AVAILABLE: - print("%s/%s" % (system, machine)) - return 0 +def normalize_platform(system, machine): + if system: + s = system.lower() + if s.startswith("msys_nt") or s == "windows": + system = "Windows" + elif s in ("darwin", "macos"): + system = "macOS" + elif s == "linux": + system = "Linux" - system = args.system - machine = args.machine + if machine: + m = machine.lower() + if m in ("x86_64", "amd64"): + machine = "x86_64" + elif m in ("aarch64", "arm64"): + machine = "arm64" - if system.startswith('MSYS_NT'): - system = 'Windows' + return system, machine - if system == 'Darwin': - system = 'macOS' - elif system == 'Windows': - if machine == 'AMD64': - machine = 'x86_64' - if machine == 'aarch64': - machine = 'arm64' - if (system, machine) not in AVAILABLE: - print('No download available for %s/%s' % (system, machine), - file=sys.stderr) - return 1 +def find_tag(exact, locally): + if locally: + tags = ( + (sha1, ref) + for sha1, _, ref in ( + l.split(None, 2) + for l in subprocess.check_output( + ["git", "for-each-ref", "refs/tags/"], + cwd=os.path.dirname(__file__), + ).splitlines() + ) + ) + else: + try: + tags = ( + tuple(l.split(None, 1)) + for l in subprocess.check_output( + ["git", "ls-remote", REPOSITORY, "refs/tags/*"] + ).splitlines() + ) + except Exception: + tags = () + + if "." in exact: + version = split_version(exact.encode()) + matches = [ + (sha1, r) + for sha1, r in tags + if split_version(r[len("refs/tags/") :]) == version + ] + if matches: + return ( + matches[0][1].decode("ascii").removeprefix("refs/tags/"), + matches[0][0].decode("ascii"), + ) + else: + tags = [ + ref[len("refs/tags/") :] + for sha1, ref in tags + if sha1.decode("ascii") == exact + ] + tags = sorted(tags, key=lambda x: split_version(x), reverse=True) + if tags: + return (tags[0].decode("ascii"), exact) + + +def main(args): + if args.list: + system, machine = normalize_platform(args.system, args.machine) + platforms = [ + (s, m) + for s, m in AVAILABLE + if (not system or s == system) and (not machine or m == machine) + ] + if not args.url: + for system, machine in platforms: + print("%s/%s" % (system, machine)) + return 0 + else: + system = args.system or platform.system() + machine = args.machine or platform.machine() + ptform = normalize_platform(system, machine) + if ptform == ("Windows", "arm64"): + ptform = ("Windows", "x86_64") + if ptform not in AVAILABLE: + print("No download available for %s/%s" % ptform, file=sys.stderr) + return 1 + platforms = (ptform,) tag = None local_sha1 = None @@ -292,65 +368,82 @@ def main(args): pass exact = args.exact or (not args.branch and local_sha1) - branch = args.branch or 'release' - - if build_commit and exact and not args.variant: - tags = [ - ref[len('refs/tags/'):] - for sha1, _, ref in ( - l.split(None, 2) - for l in subprocess.check_output( - ['git', 'for-each-ref', 'refs/tags/']).splitlines() - ) - if sha1.decode('ascii') == exact - ] - tags = sorted(tags, key=lambda x: split_version(x), reverse=True) - if tags: - tag = tags[0].decode('ascii') + branch = args.branch or "release" + + if exact and not args.variant: + tag = None + if build_commit: + tag = find_tag(exact, True) + if tag is None: + tag = find_tag(exact, False) + if tag: + tag, exact = tag + elif "." in exact: + print(f"Couldn't find a tag for {exact}", file=sys.stderr) + return 1 if exact: sha1 = exact - elif branch == 'release': + elif branch == "release": if args.variant: - print('Cannot use --variant without --branch {master,next}') + print( + "Cannot use --variant without --branch {master,next}", file=sys.stderr + ) return 1 - result = sorted(((sha1, ref[len('refs/tags/'):]) for sha1, ref in [ - l.split(b'\t', 1) for l in subprocess.check_output( - ['git', 'ls-remote', REPOSITORY, 'refs/tags/*'] - ).splitlines() - ]), key=lambda x: split_version(x[1]), reverse=True) + result = sorted( + ( + (sha1, ref[len("refs/tags/") :]) + for sha1, ref in [ + l.split(b"\t", 1) + for l in subprocess.check_output( + ["git", "ls-remote", REPOSITORY, "refs/tags/*"] + ).splitlines() + ] + ), + key=lambda x: split_version(x[1]), + reverse=True, + ) if len(result) == 0: - print('Could not find release tags') + print("Could not find release tags", file=sys.stderr) return 1 sha1, tag = result[0] - sha1 = sha1.decode('ascii') - tag = tag.decode('ascii') + sha1 = sha1.decode("ascii") + tag = tag.decode("ascii") elif branch: - ref = f'refs/heads/{branch}'.encode('utf-8') - result = [sha1 for sha1, ref_ in [ - l.split(b'\t', 1) for l in subprocess.check_output( - ['git', 'ls-remote', REPOSITORY, ref] - ).splitlines()] + ref = f"refs/heads/{branch}".encode("utf-8") + result = [ + sha1 + for sha1, ref_ in [ + l.split(b"\t", 1) + for l in subprocess.check_output( + ["git", "ls-remote", REPOSITORY, ref] + ).splitlines() + ] if ref == ref_ ] if len(result) == 0: - print(f'Could not find branch {branch}') + print(f"Could not find branch {branch}", file=sys.stderr) return 1 - sha1 = result[0].decode('ascii') + sha1 = result[0].decode("ascii") else: sha1 = None if sha1 is None: - print('Cannot find the right binary for git-cinnabar.' - ' Try --exact or --branch.', - file=sys.stderr) + print( + "Cannot find the right binary for git-cinnabar." + " Try --exact or --branch.", + file=sys.stderr, + ) return 1 - if tag: - url = get_release_url(system, machine, tag) - else: - url = get_url(system, machine, args.variant, sha1) + for system, machine in platforms: + if tag: + url = get_release_url(system, machine, tag) + else: + url = get_url(system, machine, args.variant, sha1) + if args.url: + print(url) + if args.url: - print(url) return 0 script_path = os.path.dirname(os.path.abspath(sys.argv[0])) @@ -360,39 +453,48 @@ def main(args): else: d = script_path if not os.access(d, os.W_OK): - d = os.path.join(os.path.expanduser('~'), '.git-cinnabar') + d = os.path.join(os.path.expanduser("~"), ".git-cinnabar") try: os.makedirs(d) except Exception: pass if not os.path.isdir(d): - print('Cannot write to either %s or %s.' % (d, script_path), - file=sys.stderr) + print( + "Cannot write to either %s or %s." % (d, script_path), + file=sys.stderr, + ) return 1 binary_path = os.path.join(d, get_binary(system)) return download(url, system, binary_path) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--url', action='store_true', - help='only print the download url') + parser.add_argument( + "--url", action="store_true", help="only print the download url" + ) pgroup = parser.add_mutually_exclusive_group() - pgroup.add_argument('--branch', metavar='BRANCH', - default=os.environ.get("GIT_CINNABAR_DOWNLOAD_BRANCH"), - help='download a build for the given branch') - pgroup.add_argument('--exact', metavar='EXACT', - default=os.environ.get("GIT_CINNABAR_DOWNLOAD_EXACT"), - help='download a build for the given commit') - parser.add_argument('--variant', metavar='VARIANT', - default=os.environ.get( - "GIT_CINNABAR_DOWNLOAD_VARIANT"), - help='download the given variant') - parser.add_argument('--system', default=platform.system(), - help=argparse.SUPPRESS) - parser.add_argument('--machine', default=platform.machine(), - help=argparse.SUPPRESS) - parser.add_argument('-o', '--output', help=argparse.SUPPRESS) - parser.add_argument('--list', action='store_true', help=argparse.SUPPRESS) + pgroup.add_argument( + "--branch", + metavar="BRANCH", + default=os.environ.get("GIT_CINNABAR_DOWNLOAD_BRANCH"), + help="download a build for the given branch", + ) + pgroup.add_argument( + "--exact", + metavar="EXACT", + default=os.environ.get("GIT_CINNABAR_DOWNLOAD_EXACT"), + help="download a build for the given commit", + ) + parser.add_argument( + "--variant", + metavar="VARIANT", + default=os.environ.get("GIT_CINNABAR_DOWNLOAD_VARIANT"), + help="download the given variant", + ) + parser.add_argument("--system", help=argparse.SUPPRESS) + parser.add_argument("--machine", help=argparse.SUPPRESS) + parser.add_argument("-o", "--output", help=argparse.SUPPRESS) + parser.add_argument("--list", action="store_true", help=argparse.SUPPRESS) sys.exit(main(parser.parse_args())) diff --git a/git-core b/git-core index 2e6a859ff..1e20af0e8 160000 --- a/git-core +++ b/git-core @@ -1 +1 @@ -Subproject commit 2e6a859ffc0471f60f79c1256f766042b0d5d17d +Subproject commit 1e20af0e809e08cca5714727dfa0fb50c0c4664d diff --git a/mercurial/cinnabarclone.py b/mercurial/cinnabarclone.py index b961f9197..ab9b7cc96 100644 --- a/mercurial/cinnabarclone.py +++ b/mercurial/cinnabarclone.py @@ -46,10 +46,10 @@ """ from __future__ import absolute_import, unicode_literals + import errno import os - testedwith = ( "1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 2.8 2.9 " "3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 " @@ -71,10 +71,12 @@ def add_cinnabar_cap(repo, caps): try: exists = vfs.exists except AttributeError: + def exists(path): return os.path.exists(os.path.join(vfs.base, path)) - if exists(b'cinnabar.manifest'): - caps.append(b'cinnabarclone') + + if exists(b"cinnabar.manifest"): + caps.append(b"cinnabarclone") def _capabilities(orig, repo, proto): @@ -86,20 +88,20 @@ def _capabilities(orig, repo, proto): def capabilities(orig, repo, proto): caps = orig(repo, proto).split() add_cinnabar_cap(repo, caps) - return b' '.join(caps) + return b" ".join(caps) def cinnabar(repo, proto): vfs = get_vfs(repo) try: - return vfs.tryread(b'cinnabar.manifest') + return vfs.tryread(b"cinnabar.manifest") except AttributeError: try: - return vfs.read(b'cinnabar.manifest') + return vfs.read(b"cinnabar.manifest") except IOError as e: if e.errno != errno.ENOENT: raise - return b'' + return b"" def extsetup(ui): @@ -110,17 +112,16 @@ def extsetup(ui): from mercurial import extensions try: - extensions.wrapfunction(wireproto, '_capabilities', _capabilities) + extensions.wrapfunction(wireproto, "_capabilities", _capabilities) except AttributeError: - extensions.wrapcommand( - wireproto.commands, 'capabilities', capabilities) + extensions.wrapcommand(wireproto.commands, "capabilities", capabilities) - def wireprotocommand(name, args=b'', permission=b'push'): - if hasattr(wireproto, 'wireprotocommand'): + def wireprotocommand(name, args=b"", permission=b"push"): + if hasattr(wireproto, "wireprotocommand"): try: return wireproto.wireprotocommand(name, args, permission) except TypeError: - if hasattr(wireproto, 'permissions'): + if hasattr(wireproto, "permissions"): wireproto.permissions[name] = permission return wireproto.wireprotocommand(name, args) @@ -131,4 +132,4 @@ def register(func): return register - wireprotocommand(b'cinnabarclone', permission=b'pull')(cinnabar) + wireprotocommand(b"cinnabarclone", permission=b"pull")(cinnabar) diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..b47877ec3 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,3 @@ +[lint] +ignore = ["E402", "F405", "E741"] +extend-select = ["I"] diff --git a/src/csum-file.c.patch b/src/csum-file.c.patch index 1b484158d..f987e4f91 100644 --- a/src/csum-file.c.patch +++ b/src/csum-file.c.patch @@ -1,9 +1,9 @@ diff --git a/csum-file.c b/csum-file.c -index 59ef3398ca..46f753d012 100644 +index 2131ee6b12..f31d551613 100644 --- a/csum-file.c +++ b/csum-file.c -@@ -92,7 +92,8 @@ int finalize_hashfile(struct hashfile *f, unsigned char *result, - return fd; +@@ -111,7 +111,8 @@ void discard_hashfile(struct hashfile *f) + free_hashfile(f); } -void hashwrite(struct hashfile *f, const void *buf, unsigned int count) diff --git a/src/main.rs b/src/main.rs index 00a46bbe3..0d704a254 100644 --- a/src/main.rs +++ b/src/main.rs @@ -57,6 +57,7 @@ mod progress; pub mod store; pub mod tree_util; mod util; +mod version; mod xdiff; pub(crate) mod hg_bundle; @@ -94,12 +95,11 @@ use cinnabar::{ GitChangesetId, GitFileMetadataId, GitManifestId, GitManifestTree, GitManifestTreeId, }; use clap::error::ErrorKind; -use clap::{crate_version, CommandFactory, FromArgMatches, Parser}; +use clap::{CommandFactory, FromArgMatches, Parser}; use cstr::cstr; use digest::OutputSizeUser; use either::Either; use git::{BlobId, CommitId, GitObjectId, RawBlob, RawCommit, RawTree, RecursedTreeEntry, TreeIsh}; -use git_version::git_version; use graft::{graft_finish, grafted, init_graft}; use hg::{HgChangesetId, HgFileId, HgManifestId, ManifestEntry}; use hg_bundle::{create_bundle, create_chunk_data, read_rev_chunk, BundleSpec, RevChunkIter}; @@ -143,6 +143,7 @@ use crate::progress::set_progress; use crate::store::{clear_manifest_heads, do_set_replace, set_changeset_heads, Dag}; use crate::tree_util::{Empty, ParseTree, WithPath}; use crate::util::{FromBytes, ToBoxed}; +use crate::version::{FULL_VERSION, SHORT_VERSION}; #[cfg(any(feature = "version-check", feature = "self-update"))] mod version_check; @@ -161,20 +162,11 @@ impl VersionChecker { } #[cfg(feature = "self-update")] -use version_check::{VersionInfo, VersionRequest}; +use version::Version; +#[cfg(feature = "self-update")] +use version_check::{find_version, VersionInfo, VersionRequest}; pub const CARGO_PKG_REPOSITORY: &str = env!("CARGO_PKG_REPOSITORY"); -pub const FULL_VERSION: &str = git_version!( - args = [ - "--always", - "--match=nothing/", - "--abbrev=40", - "--dirty=-modified" - ], - prefix = concat!(crate_version!(), "-"), - cargo_prefix = "", - fallback = crate_version!(), -); #[allow(improper_ctypes)] extern "C" { @@ -1870,16 +1862,18 @@ cfg_if::cfg_if! { } } +#[cfg(all(feature = "self-update", windows))] +const BINARY: &str = "git-cinnabar.exe"; +#[cfg(all(feature = "self-update", not(windows)))] +const BINARY: &str = "git-cinnabar"; + #[cfg(feature = "self-update")] -fn do_self_update(branch: Option, exact: Option) -> Result<(), String> { +fn do_self_update( + branch: Option, + exact: Option, + url: bool, +) -> Result<(), String> { assert!(!(branch.is_some() && exact.is_some())); - cfg_if::cfg_if! { - if #[cfg(windows)] { - const BINARY: &str = "git-cinnabar.exe"; - } else { - const BINARY: &str = "git-cinnabar"; - } - }; #[cfg(windows)] const FINISH_SELF_UPDATE: &str = "GIT_CINNABAR_FINISH_SELF_UPDATE"; #[cfg(windows)] @@ -1902,17 +1896,32 @@ fn do_self_update(branch: Option, exact: Option) -> Result<(), let exe = std::env::current_exe().map_err(|e| e.to_string())?; let exe_dir = exe.parent().unwrap(); - if let Some(version) = exact.map(VersionInfo::Commit).or_else(|| { + let version = if let Some(exact) = exact { + let v0_6 = Version::parse("0.6.0-rc.1").unwrap(); + find_version(exact.clone()) + .or_else(|| matches!(exact, VersionInfo::Commit(_)).then_some(exact)) + .filter(|info| match info { + VersionInfo::Tagged(v, _) => v >= &v0_6, + _ => true, + }) + } else { version_check::check_new_version( branch .as_ref() .map_or_else(VersionRequest::default, |b| VersionRequest::from(&**b)), ) - }) { + }; + if url { + if let Some(version) = version { + println!("{}", GitCinnabarBuild::url(version)); + } + return Ok(()); + } + if let Some(version) = version { let mut tmpbuilder = tempfile::Builder::new(); tmpbuilder.prefix(BINARY); let mut tmpfile = tmpbuilder.tempfile_in(exe_dir).map_err(|e| e.to_string())?; - download_build(version, &mut tmpfile, BINARY)?; + download_build(version, &mut tmpfile)?; tmpfile.flush().map_err(|e| e.to_string())?; let old_exe = tmpbuilder.tempfile_in(exe_dir).map_err(|e| e.to_string())?; let old_exe_path = old_exe.path().to_path_buf(); @@ -1969,35 +1978,65 @@ fn do_self_update(branch: Option, exact: Option) -> Result<(), } #[cfg(feature = "self-update")] -fn download_build( - version: VersionInfo, - tmpfile: &mut impl Write, - binary: &str, -) -> Result<(), String> { - use crate::hg_connect_http::HttpRequest; +enum GitCinnabarBuild { + Executable(String), + Archive(String), +} - cfg_if::cfg_if! { - if #[cfg(all(target_arch = "x86_64", target_os = "linux"))] { - const SYSTEM_MACHINE: &str = "linux.x86_64"; - } else if #[cfg(all(target_arch = "aarch64", target_os = "linux"))] { - const SYSTEM_MACHINE: &str = "linux.arm64"; - } else if #[cfg(all(target_arch = "x86_64", target_os = "macos"))] { - const SYSTEM_MACHINE: &str = "macos.x86_64"; - } else if #[cfg(all(target_arch = "aarch64", target_os = "macos"))] { - const SYSTEM_MACHINE: &str = "macos.arm64"; - } else if #[cfg(all(target_arch = "x86_64", target_os = "windows"))] { - const SYSTEM_MACHINE: &str = "windows.x86_64"; - } else { - compile_error!("self-update is not supported on this platform"); +#[cfg(feature = "self-update")] +impl GitCinnabarBuild { + fn new(version: VersionInfo) -> Self { + use concat_const::concat as cat; + cfg_if::cfg_if! { + if #[cfg(all(target_arch = "x86_64", target_os = "linux"))] { + const SYSTEM_MACHINE: &str = "linux.x86_64"; + } else if #[cfg(all(target_arch = "aarch64", target_os = "linux"))] { + const SYSTEM_MACHINE: &str = "linux.arm64"; + } else if #[cfg(all(target_arch = "x86_64", target_os = "macos"))] { + const SYSTEM_MACHINE: &str = "macos.x86_64"; + } else if #[cfg(all(target_arch = "aarch64", target_os = "macos"))] { + const SYSTEM_MACHINE: &str = "macos.arm64"; + } else if #[cfg(all(target_arch = "x86_64", target_os = "windows"))] { + const SYSTEM_MACHINE: &str = "windows.x86_64"; + } else { + compile_error!("self-update is not supported on this platform"); + } + } + cfg_if::cfg_if! { + if #[cfg(windows)] { + const ARCHIVE_EXT: &str = "zip"; + } else { + const ARCHIVE_EXT: &str = "tar.xz"; + } + } + + match version { + VersionInfo::Commit(cid) => { + const URL_BASE: &str = + "https://community-tc.services.mozilla.com/api/index/v1/task/project.git-cinnabar.build."; + const URL_TAIL: &str = cat!(".", SYSTEM_MACHINE, "/artifacts/public/", BINARY); + GitCinnabarBuild::Executable(format!("{URL_BASE}{cid}{URL_TAIL}")) + } + VersionInfo::Tagged(version, _) => { + let tag = version.as_tag(); + const URL_BASE: &str = cat!(CARGO_PKG_REPOSITORY, "/releases/download/"); + const URL_TAIL: &str = cat!("/git-cinnabar.", SYSTEM_MACHINE, ".", ARCHIVE_EXT); + GitCinnabarBuild::Archive(format!("{URL_BASE}{tag}{URL_TAIL}")) + } } } - cfg_if::cfg_if! { - if #[cfg(windows)] { - const ARCHIVE_EXT: &str = "zip"; - } else { - const ARCHIVE_EXT: &str = "tar.xz"; + + fn url(version: VersionInfo) -> String { + match Self::new(version) { + GitCinnabarBuild::Archive(url) => url, + GitCinnabarBuild::Executable(url) => url, } } +} + +#[cfg(feature = "self-update")] +fn download_build(version: VersionInfo, tmpfile: &mut impl Write) -> Result<(), String> { + use crate::hg_connect_http::HttpRequest; let request = |url: &str| { eprintln!("Installing update from {url}"); @@ -2007,19 +2046,14 @@ fn download_build( req.execute() }; - match version { - VersionInfo::Commit(cid) => { - const URL_BASE: &str = - "https://community-tc.services.mozilla.com/api/index/v1/task/project.git-cinnabar.build"; - let url = format!("{URL_BASE}.{cid}.{SYSTEM_MACHINE}/artifacts/public/{binary}"); + match GitCinnabarBuild::new(version) { + GitCinnabarBuild::Executable(url) => { let mut response = request(&url)?; std::io::copy(&mut response, tmpfile) .map(|_| ()) .map_err(|e| e.to_string()) } - VersionInfo::Tagged(tag, _) => { - let tag = tag.to_string().replace('-', ""); - let url = format!("{CARGO_PKG_REPOSITORY}/releases/download/{tag}/git-cinnabar.{SYSTEM_MACHINE}.{ARCHIVE_EXT}"); + GitCinnabarBuild::Archive(url) => { let mut extracted = false; #[cfg(windows)] { @@ -2030,7 +2064,7 @@ fn download_build( if file.is_file() && file .enclosed_name() - .map_or(false, |p| p.file_name().unwrap() == binary) + .map_or(false, |p| p.file_name().unwrap() == BINARY) { std::io::copy(file, tmpfile).map_err(|e| e.to_string())?; extracted = true; @@ -2050,7 +2084,7 @@ fn download_build( .map_err(|e| e.to_string())? .file_name() .unwrap() - == binary + == BINARY { std::io::copy(&mut file, tmpfile).map_err(|e| e.to_string())?; extracted = true; @@ -2061,9 +2095,13 @@ fn download_build( if extracted { Ok(()) } else { - Err(format!( - "Could not find the {binary} executable in the downloaded archive." - )) + use concat_const::concat as cat; + Err(cat!( + "Could not find the ", + BINARY, + " executable in the downloaded archive." + ) + .to_string()) } } } @@ -3708,8 +3746,8 @@ impl FromStr for AbbrevSize { #[derive(Parser)] #[command( name = "git-cinnabar", - version=crate_version!(), - long_version=FULL_VERSION, + version=SHORT_VERSION.as_ref(), + long_version=FULL_VERSION.as_ref(), arg_required_else_help = true, dont_collapse_args_in_usage = true, subcommand_required = true, @@ -3850,14 +3888,25 @@ enum CinnabarCommand { #[arg(long)] branch: Option, /// Exact commit to get a version from - #[arg(long, value_parser, conflicts_with = "branch")] - exact: Option, + #[arg(long, value_parser = self_update_exact, conflicts_with = "branch")] + exact: Option, + /// Show where the self-update would be downloaded from + #[arg(long, hide = true)] + url: bool, }, /// Setup git-cinnabar #[command(name = "setup", hide = true)] Setup, } +#[cfg(feature = "self-update")] +fn self_update_exact(arg: &str) -> Result { + CommitId::from_str(arg) + .map(VersionInfo::Commit) + .or_else(|_| Version::parse(arg).map(|v| VersionInfo::Tagged(v, CommitId::NULL))) + .map_err(|e| e.to_string()) +} + /// Create, list, delete tags #[derive(Parser)] #[command(name = "tag")] @@ -3939,9 +3988,14 @@ fn git_cinnabar(args: Option<&[&OsStr]>) -> Result { e.print().unwrap(); #[cfg(feature = "version-check")] if e.kind() == ErrorKind::DisplayVersion - && format!("{}", e.render()) != concat!("git-cinnabar ", crate_version!(), "\n") + && e.render() + .to_string() + .strip_suffix('\n') + .and_then(|s| s.strip_prefix("git-cinnabar ")) + .unwrap_or("") + != *SHORT_VERSION { - if let Some(mut checker) = VersionChecker::force_now() { + if let Some(mut checker) = VersionChecker::for_dashdash_version() { checker.wait(Duration::from_secs(1)); } } @@ -3949,8 +4003,8 @@ fn git_cinnabar(args: Option<&[&OsStr]>) -> Result { } }; #[cfg(feature = "self-update")] - if let SelfUpdate { branch, exact } = command { - return do_self_update(branch, exact).map(|()| 0); + if let SelfUpdate { branch, exact, url } = command { + return do_self_update(branch, exact, url).map(|()| 0); } if let Setup = command { return do_setup().map(|()| 0); @@ -4483,6 +4537,39 @@ fn check_graft_refs() { } } +#[derive(PartialEq, Eq)] +enum RecordMetadata { + Never, + Phase, + Always, + // Like Always, but also applies to dry-run + Force, +} + +impl TryFrom<&OsStr> for RecordMetadata { + type Error = String; + + fn try_from(value: &OsStr) -> Result { + value + .to_str() + .and_then(|s| { + Some(match s { + "never" => RecordMetadata::Never, + "phase" => RecordMetadata::Phase, + "always" => RecordMetadata::Always, + "force" => RecordMetadata::Force, + _ => return None, + }) + }) + .ok_or_else(|| { + format!( + "`{}` is not one of `never`, `phase`, `always` or `force`", + value.as_bytes().as_bstr() + ) + }) + } +} + fn remote_helper_push( store: &mut Store, conn: &mut dyn HgRepo, @@ -4535,6 +4622,11 @@ fn remote_helper_push( } }); + let data = get_config_remote("data", remote) + .filter(|d| !d.is_empty()) + .as_deref() + .map_or(Ok(RecordMetadata::Phase), RecordMetadata::try_from)?; + let mut pushed = ChangesetHeads::new(); let result = (|| { let push_commits = push_refs.iter().filter_map(|(c, _, _)| *c).collect_vec(); @@ -4697,7 +4789,7 @@ fn remote_helper_push( } let mut result = None; - if !push_commits.is_empty() && !dry_run { + if !push_commits.is_empty() && (!dry_run || data == RecordMetadata::Force) { conn.require_capability(b"unbundle"); let b2caps = conn @@ -4723,6 +4815,7 @@ fn remote_helper_push( .unwrap_or(1); (BundleSpec::V2None, version) }; + // TODO: Ideally, for dry-run, we wouldn't even create a temporary file. let tempfile = tempfile::Builder::new() .prefix("hg-bundle-") .suffix(".hg") @@ -4739,40 +4832,43 @@ fn remote_helper_push( version == 2, )?; drop(file); - let file = File::open(path).unwrap(); - let empty_heads = [HgChangesetId::NULL]; - let heads = if force { - None - } else if no_topological_heads { - Some(&empty_heads[..]) - } else { - Some(&info.topological_heads[..]) - }; - let response = conn.unbundle(heads, file); - match response { - UnbundleResponse::Bundlev2(data) => { - let mut bundle = BundleReader::new(data).unwrap(); - while let Some(part) = bundle.next_part().unwrap() { - match part.part_type.as_bytes() { - b"reply:changegroup" => { - // TODO: should check in-reply-to param. - let response = part.get_param("return").unwrap(); - result = u32::from_str(response).ok(); - } - b"error:abort" => { - let mut message = part.get_param("message").unwrap().to_string(); - if let Some(hint) = part.get_param("hint") { - message.push_str("\n\n"); - message.push_str(hint); + if !dry_run { + let file = File::open(path).unwrap(); + let empty_heads = [HgChangesetId::NULL]; + let heads = if force { + None + } else if no_topological_heads { + Some(&empty_heads[..]) + } else { + Some(&info.topological_heads[..]) + }; + let response = conn.unbundle(heads, file); + match response { + UnbundleResponse::Bundlev2(data) => { + let mut bundle = BundleReader::new(data).unwrap(); + while let Some(part) = bundle.next_part().unwrap() { + match part.part_type.as_bytes() { + b"reply:changegroup" => { + // TODO: should check in-reply-to param. + let response = part.get_param("return").unwrap(); + result = u32::from_str(response).ok(); + } + b"error:abort" => { + let mut message = + part.get_param("message").unwrap().to_string(); + if let Some(hint) = part.get_param("hint") { + message.push_str("\n\n"); + message.push_str(hint); + } + error!(target: "root", "{}", message); } - error!(target: "root", "{}", message); + _ => {} } - _ => {} } } - } - UnbundleResponse::Raw(response) => { - result = u32::from_bytes(&response).ok(); + UnbundleResponse::Raw(response) => { + result = u32::from_bytes(&response).ok(); + } } } } @@ -4821,7 +4917,7 @@ fn remote_helper_push( .map(|n| n > 0) .unwrap_or_default()) } else if source_cid.is_some() { - Ok(!pushed.is_empty()) + Ok(!pushed.is_empty() || dry_run) } else { Err("Deleting remote branches is unsupported") }; @@ -4856,76 +4952,60 @@ fn remote_helper_push( stdout.flush().unwrap(); } - let data = get_config_remote("data", remote); - let data = data - .as_deref() - .and_then(|d| (!d.is_empty()).then_some(d)) - .unwrap_or_else(|| OsStr::new("phase")); - let valid = [ - OsStr::new("never"), - OsStr::new("phase"), - OsStr::new("always"), - ]; - if !valid.contains(&data) { - die!( - "`{}` is not one of `never`, `phase` or `always`", - data.as_bytes().as_bstr() - ); - } - let rollback = if status.is_empty() || pushed.is_empty() || dry_run { - true - } else { - match data.to_str().unwrap() { - "always" => false, - "never" => true, - "phase" => { - let phases = conn.phases(); - let phases = ByteSlice::lines(&*phases) - .filter_map(|l| { - l.splitn_exact(b'\t') - .map(|[k, v]| (k.as_bstr(), v.as_bstr())) - }) - .collect::>(); - let drafts = (!phases.contains_key(b"publishing".as_bstr())) - .then(|| { - phases - .into_iter() - .filter_map(|(phase, is_draft)| { - u32::from_bytes(is_draft).ok().and_then(|is_draft| { - (is_draft > 0).then(|| HgChangesetId::from_bytes(phase)) + let rollback = + if status.is_empty() || pushed.is_empty() || (dry_run && data != RecordMetadata::Force) { + true + } else { + match data { + RecordMetadata::Force | RecordMetadata::Always => false, + RecordMetadata::Never => true, + RecordMetadata::Phase => { + let phases = conn.phases(); + let phases = ByteSlice::lines(&*phases) + .filter_map(|l| { + l.splitn_exact(b'\t') + .map(|[k, v]| (k.as_bstr(), v.as_bstr())) + }) + .collect::>(); + let drafts = (!phases.contains_key(b"publishing".as_bstr())) + .then(|| { + phases + .into_iter() + .filter_map(|(phase, is_draft)| { + u32::from_bytes(is_draft).ok().and_then(|is_draft| { + (is_draft > 0).then(|| HgChangesetId::from_bytes(phase)) + }) }) - }) - .collect::, _>>() - }) - .transpose() - .unwrap() - .unwrap_or_default(); - if drafts.is_empty() { - false - } else { - // Theoretically, we could have commits with no - // metadata that the remote declares are public, while - // the rest of our push is in a draft state. That is - // however so unlikely that it's not worth the effort - // to support partial metadata storage. - !reachable_subset( - pushed - .heads() - .copied() - .filter_map(|h| h.to_git(store)) - .map(Into::into), - drafts - .iter() - .copied() - .filter_map(|h| h.to_git(store)) - .map(Into::into), - ) - .is_empty() + .collect::, _>>() + }) + .transpose() + .unwrap() + .unwrap_or_default(); + if drafts.is_empty() { + false + } else { + // Theoretically, we could have commits with no + // metadata that the remote declares are public, while + // the rest of our push is in a draft state. That is + // however so unlikely that it's not worth the effort + // to support partial metadata storage. + !reachable_subset( + pushed + .heads() + .copied() + .filter_map(|h| h.to_git(store)) + .map(Into::into), + drafts + .iter() + .copied() + .filter_map(|h| h.to_git(store)) + .map(Into::into), + ) + .is_empty() + } } } - _ => unreachable!(), - } - }; + }; if rollback { unsafe { do_cleanup(1); @@ -5170,6 +5250,30 @@ pub fn get_config(name: &str) -> Option { get_config_remote(name, None) } +pub trait ConfigType: ToOwned { + fn from_os_string(value: OsString) -> Option; +} + +impl ConfigType for str { + fn from_os_string(value: OsString) -> Option { + value.into_string().ok() + } +} + +impl ConfigType for bool { + fn from_os_string(value: OsString) -> Option { + match value.as_bytes() { + b"1" | b"true" => Some(true), + b"" | b"0" | b"false" => Some(false), + _ => None, + } + } +} + +pub fn get_typed_config(name: &str) -> Option { + get_config(name).and_then(T::from_os_string) +} + pub fn get_config_remote(name: &str, remote: Option<&str>) -> Option { const PREFIX: &str = "GIT_CINNABAR_"; let mut env_key = String::with_capacity(name.len() + PREFIX.len()); @@ -5286,6 +5390,7 @@ bitflags! { const GIT_COMMIT = 0x2; const TAG = 0x4; const BRANCH = 0x8; + const TEST = 0x100; } } pub struct AllExperiments { @@ -5314,6 +5419,9 @@ static EXPERIMENTS: Lazy = Lazy::new(|| { b"branch" => { flags |= Experiments::BRANCH; } + b"test" => { + flags |= Experiments::TEST; + } s if s.starts_with(b"similarity") => { if let Some(value) = s[b"similarity".len()..].strip_prefix(b"=") { match u8::from_bytes(value) { diff --git a/src/version.rs b/src/version.rs new file mode 100644 index 000000000..1b15c336b --- /dev/null +++ b/src/version.rs @@ -0,0 +1,318 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::borrow::Cow; +use std::cmp::Ordering; +use std::str::FromStr; + +use derive_more::Display; +use once_cell::sync::Lazy; + +use crate::git::CommitId; +use crate::{experiment, get_typed_config, ConfigType, Experiments}; + +#[derive(Clone, Display, Debug)] +#[display(fmt = "{version}")] +pub struct Version { + version: semver::Version, + tag_has_v_prefix: bool, +} + +impl PartialEq for Version { + fn eq(&self, other: &Version) -> bool { + self.version == other.version + } +} + +impl PartialOrd for Version { + fn partial_cmp(&self, other: &Version) -> Option { + self.version.partial_cmp(&other.version) + } +} + +impl Version { + pub fn parse(v: &str) -> Result { + let (v, tag_has_v_prefix) = v.strip_prefix("v").map_or((v, false), |v| (v, true)); + let mut version = semver::Version::parse(v).or_else(|e| { + // If the version didn't parse, try again by separating + // x.y.z from everything that follows with a dash. + v.find(|c: char| !c.is_ascii_digit() && c != '.') + .map(|pos| { + let (digits, rest) = v.split_at(pos); + format!("{}-{}", digits, rest) + }) + .as_deref() + .and_then(|v| semver::Version::parse(v).ok()) + .ok_or(e) + })?; + if !version.pre.is_empty() { + if let Some(i) = version.pre.find(|c: char| c.is_ascii_digit()) { + if version.pre.chars().nth(i - 1) != Some('.') { + let (a, b) = version.pre.split_at(i); + version.pre = semver::Prerelease::new(&format!("{a}.{b}"))?; + } + } + } + Ok(Version { + version, + tag_has_v_prefix, + }) + } + + #[cfg(any(feature = "self-update", test))] + pub fn as_tag(&self) -> String { + if self.tag_has_v_prefix { + format!("v{}", self.version) + } else { + let v = &self.version; + let mut tag = semver::Version::new(v.major, v.minor, v.patch).to_string(); + if !v.pre.is_empty() { + let mut pre = v.pre.chars(); + tag.extend(pre.by_ref().take_while(|&c| c != '.')); + tag.extend(pre); + } + + tag + } + } +} + +#[test] +fn test_version() { + use crate::util::{assert_gt, assert_lt}; + // Because earlier versions of git-cinnabar removed the dash from x.y.z-beta* + // versions, we have to continue not putting a dash in tags. + // But because (arbitrarily) I don't like the beta.1 form without the + // dash, we also remove the dot there. + assert_eq!( + Version::parse("0.7.0-beta.1").unwrap().as_tag(), + "0.7.0beta1".to_owned() + ); + // There most probably won't be versions like this, but just in case, + // ensure we deal with them properly. + assert_eq!( + Version::parse("0.7.0-beta.1.1").unwrap().as_tag(), + "0.7.0beta1.1".to_owned() + ); + + // semver doesn't handle beta.x and betax the same way, and the ordering + // of the latter is unfortunate. So internally, we normalize the latter + // to the former. + assert_lt!( + semver::Version::parse("0.7.0-beta11").unwrap(), + semver::Version::parse("0.7.0-beta2").unwrap() + ); + assert_ne!( + semver::Version::parse("0.7.0-beta2").unwrap(), + semver::Version::parse("0.7.0-beta.2").unwrap() + ); + assert!(semver::Version::parse("0.7.0beta2").is_err()); + assert_gt!( + Version::parse("0.7.0-beta11").unwrap(), + Version::parse("0.7.0-beta2").unwrap() + ); + assert_eq!( + Version::parse("0.7.0-beta2").unwrap(), + Version::parse("0.7.0-beta.2").unwrap() + ); + assert_eq!( + Version::parse("0.7.0beta2").unwrap(), + Version::parse("0.7.0-beta.2").unwrap() + ); + + assert_eq!( + Version::parse("0.7.0-beta.2").unwrap().as_tag(), + "0.7.0beta2".to_owned() + ); + assert_eq!( + Version::parse("0.7.0beta2").unwrap().to_string(), + "0.7.0-beta.2".to_owned() + ); + + assert_eq!( + Version::parse("v0.7.0-beta.2").unwrap(), + Version::parse("0.7.0beta2").unwrap(), + ); + assert_eq!( + Version::parse("v0.7.0-beta.2").unwrap().as_tag(), + "v0.7.0-beta.2".to_owned() + ); + assert_eq!( + Version::parse("v0.7.0-beta.2").unwrap().to_string(), + "0.7.0-beta.2".to_owned() + ); +} + +macro_rules! join { + ($s:expr) => { + Cow::Borrowed($s) + }; + ($($s:expr),+) => { + Cow::Owned(itertools::join(&[$($s),+], "")) + } +} + +macro_rules! full_version { + ($short_version:expr, $build_commit:expr, $modified:expr, $macro:ident) => { + if $build_commit.is_empty() { + $macro!($short_version) + } else { + $macro!( + $short_version, + "-", + $build_commit, + if $modified { "-modified" } else { "" } + ) + } + }; +} + +mod static_ { + use clap::crate_version; + // Work around https://github.com/rust-lang/rust-analyzer/issues/8828 with `as cat`. + use concat_const::concat as cat; + use git_version::git_version; + + #[cfg(any(feature = "version-check", feature = "self-update"))] + use super::BuildBranch; + + pub const SHORT_VERSION: &str = crate_version!(); + const GIT_VERSION: &str = git_version!( + args = ["--always", "--match=nothing/", "--abbrev=40", "--dirty=m"], + fallback = "", + ); + pub const MODIFIED: bool = matches!(GIT_VERSION.as_bytes().last(), Some(b'm')); + pub const BUILD_COMMIT: &str = unsafe { + // Subslicing is not supported in const yet. + std::str::from_utf8_unchecked(std::slice::from_raw_parts( + GIT_VERSION.as_ptr(), + GIT_VERSION.len() - if MODIFIED { 1 } else { 0 }, + )) + }; + + #[cfg(any(feature = "version-check", feature = "self-update"))] + pub const BUILD_BRANCH: BuildBranch = BuildBranch::from_version(SHORT_VERSION); + + // #[allow(clippy::const_is_empty)] + pub const FULL_VERSION: &str = full_version!(SHORT_VERSION, BUILD_COMMIT, MODIFIED, cat); +} + +fn value<'a, T: 'a + ConfigType + ?Sized, F: FnOnce(T::Owned) -> Option>( + config: &str, + static_value: &'a T, + filter: F, +) -> Cow<'a, T> { + if let Some(value) = experiment(Experiments::TEST) + .then(|| get_typed_config::(config)) + .flatten() + .and_then(filter) + { + Cow::Owned(value) + } else { + Cow::Borrowed(static_value) + } +} + +macro_rules! value { + ($config:expr, $static_value:expr) => { + value!($config, $static_value, |x| Some(x)) + }; + ($config:expr, $static_value:expr, $filter:expr) => { + Lazy::new(|| value($config, $static_value, $filter)) + }; +} + +type Value = Lazy>; + +fn is_overridden(value: &Value) -> bool { + matches!(**value, Cow::Owned(_)) +} + +pub static SHORT_VERSION: Value = + value!("version", static_::SHORT_VERSION, |v| Version::parse(&v) + .ok() + .map(|v| v.to_string())); +pub static BUILD_COMMIT: Value = value!("commit", static_::BUILD_COMMIT, |c| (c.is_empty() + || CommitId::from_str(&c).is_ok()) +.then_some(c)); +static MODIFIED: Value = value!("modified", &static_::MODIFIED); +#[cfg(any(feature = "version-check", feature = "self-update"))] +pub static BUILD_BRANCH: Lazy = Lazy::new(|| { + if is_overridden(&SHORT_VERSION) { + BuildBranch::from_version(&SHORT_VERSION) + } else { + static_::BUILD_BRANCH + } +}); +pub static FULL_VERSION: Value = Lazy::new(|| { + if is_overridden(&SHORT_VERSION) || is_overridden(&BUILD_COMMIT) || is_overridden(&MODIFIED) { + full_version!( + SHORT_VERSION.as_ref(), + BUILD_COMMIT.as_ref(), + *MODIFIED.as_ref(), + join + ) + } else { + Cow::Borrowed(static_::FULL_VERSION) + } +}); + +#[cfg(any(feature = "version-check", feature = "self-update"))] +#[derive(PartialEq, Eq, Debug)] +pub enum BuildBranch { + Release, + Master, + Next, +} + +#[cfg(any(feature = "version-check", feature = "self-update"))] +impl BuildBranch { + pub const fn from_version(version: &str) -> BuildBranch { + let version = version.as_bytes(); + // TODO: Use version.last_chunk when MSRV >= 1.80 + // or version.split_last_chunk when MSRV >= 1.77 + const fn last_chunk(b: &[u8]) -> Option<&[u8]> { + if b.len() >= N { + Some(b.split_at(b.len() - N).1) + } else { + None + } + } + if matches!(last_chunk::<4>(version), Some(b".0-a")) { + BuildBranch::Next + } else if matches!(last_chunk::<2>(version), Some(b"-a" | b"-b")) { + BuildBranch::Master + } else { + BuildBranch::Release + } + } + + pub const fn as_str(&self) -> &'static str { + match self { + BuildBranch::Release => "release", + BuildBranch::Master => "master", + BuildBranch::Next => "next", + } + } +} + +#[cfg(any(feature = "version-check", feature = "self-update"))] +#[test] +fn test_build_branch() { + // The following tests outline the expected lifecycle. + let from_version = |v| { + assert!(Version::parse(v).is_ok()); + BuildBranch::from_version(v) + }; + assert_eq!(from_version("0.2.0-a"), BuildBranch::Next); + assert_eq!(from_version("0.2.0-b"), BuildBranch::Master); + assert_eq!(from_version("0.2.0-b1"), BuildBranch::Release); // optionally + assert_eq!(from_version("0.2.0-beta1"), BuildBranch::Release); // alternative form + assert_eq!(from_version("0.2.0-beta1"), BuildBranch::Release); // alternative form + assert_eq!(from_version("0.3.0-a"), BuildBranch::Next); // possibly later + assert_eq!(from_version("0.2.0-rc1"), BuildBranch::Release); // optionally + assert_eq!(from_version("0.2.0"), BuildBranch::Release); // optionally + assert_eq!(from_version("0.2.1-a"), BuildBranch::Master); + assert_eq!(from_version("0.2.1"), BuildBranch::Release); +} diff --git a/src/version_check.rs b/src/version_check.rs index fc04bda3f..5f2d1b014 100644 --- a/src/version_check.rs +++ b/src/version_check.rs @@ -2,8 +2,8 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +use std::ops::Deref; use std::process::{Command, Stdio}; -#[cfg(feature = "version-check")] use std::str::FromStr; #[cfg(feature = "version-check")] use std::sync::Arc; @@ -13,25 +13,19 @@ use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use bstr::ByteSlice; -use clap::crate_version; use itertools::Itertools; -use semver::Version; use shared_child::SharedChild; use crate::git::CommitId; #[cfg(feature = "version-check")] use crate::util::DurationExt; use crate::util::{FromBytes, OsStrExt, ReadExt, SliceExt}; -use crate::FULL_VERSION; +use crate::version::BuildBranch::*; +use crate::version::{Version, BUILD_BRANCH, BUILD_COMMIT, SHORT_VERSION}; #[cfg(feature = "version-check")] use crate::{check_enabled, get_config, Checks}; const ALL_TAG_REFS: &str = "refs/tags/*"; -const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); -#[cfg(version_check_branch)] -const VERSION_CHECK_REF: &str = env!("VERSION_CHECK_BRANCH"); -#[cfg(not(version_check_branch))] -const VERSION_CHECK_REF: &str = ALL_TAG_REFS; #[cfg(feature = "version-check")] const VERSION_CHECK_CONFIG: &str = "cinnabar.version-check"; @@ -51,15 +45,73 @@ impl<'a> From<&'a str> for VersionRequest<'a> { impl<'a> Default for VersionRequest<'a> { fn default() -> Self { - if VERSION_CHECK_REF == ALL_TAG_REFS { + if *BUILD_BRANCH == Release { VersionRequest::Tagged } else { - VersionRequest::Branch(VERSION_CHECK_REF) + VersionRequest::Branch(BUILD_BRANCH.as_str()) + } + } +} + +struct VersionRequestChild<'a> { + child: SharedChild, + request: VersionRequest<'a>, +} + +impl<'a> VersionRequestChild<'a> { + fn new(request: VersionRequest<'a>) -> Option> { + let mut cmd = Command::new("git"); + cmd.args(["ls-remote", crate::CARGO_PKG_REPOSITORY]); + match request { + VersionRequest::Tagged => cmd.arg(ALL_TAG_REFS), + VersionRequest::Branch(branch) => cmd.arg(format!("refs/heads/{branch}")), + }; + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::null()); + + debug!(target: "version-check", "Running git {}", cmd.get_args().map(|arg| arg.as_bytes().as_bstr()).join(" ")); + + SharedChild::spawn(&mut cmd) + .ok() + .map(|child| VersionRequestChild { child, request }) + } + + fn fold B>(&self, init: B, f: F) -> Result { + let output = self.child.take_stdout().unwrap().read_all().map_err(|_| ()); + self.child.wait().map_err(|_| ())?; + let output = output?; + if output.is_empty() { + return Err(()); } + Ok(output + .lines() + .filter_map(|line| { + let [sha1, r] = line.splitn_exact(u8::is_ascii_whitespace)?; + let cid = CommitId::from_bytes(sha1).ok()?; + debug!(target: "version-check", "Found {}@{}", r.as_bstr(), cid); + if matches!(self.request, VersionRequest::Tagged) { + std::str::from_utf8(r) + .ok() + .and_then(|tag| tag.strip_prefix("refs/tags/")) + .and_then(|tag| Version::parse(tag).ok()) + .map(|v| VersionInfo::Tagged(v, cid)) + } else { + Some(VersionInfo::Commit(cid)) + } + }) + .fold(init, f)) + } +} + +impl<'a> Deref for VersionRequestChild<'a> { + type Target = SharedChild; + fn deref(&self) -> &SharedChild { + &self.child } } #[allow(unused)] +#[derive(Clone)] pub enum VersionInfo { Tagged(Version, CommitId), Commit(CommitId), @@ -67,14 +119,15 @@ pub enum VersionInfo { #[cfg(feature = "version-check")] pub struct VersionChecker { - child: Option>, + child: Option>>, thread: Option, ()>>>, when: Option, + show_current: bool, } #[cfg(feature = "version-check")] impl VersionChecker { - fn new_inner(force_now: bool) -> Option { + fn new_inner(force_now: bool, show_current: bool) -> Option { if !check_enabled(Checks::VERSION) { debug!(target: "version-check", "Version check is disabled"); return None; @@ -102,7 +155,7 @@ impl VersionChecker { return None; } - let child = create_child(VersionRequest::default()).map(Arc::new); + let child = VersionRequestChild::new(VersionRequest::default()).map(Arc::new); let thread = child.clone().and_then(|child| { thread::Builder::new() .name("version-check".into()) @@ -113,15 +166,16 @@ impl VersionChecker { child, thread, when: Some(now), + show_current, }) } pub fn new() -> Option { - Self::new_inner(false) + Self::new_inner(false, true) } - pub fn force_now() -> Option { - Self::new_inner(true) + pub fn for_dashdash_version() -> Option { + Self::new_inner(true, false) } pub fn wait(&mut self, timeout: Duration) { @@ -159,12 +213,20 @@ impl VersionChecker { impl Drop for VersionChecker { fn drop(&mut self) { match self.take_result() { - Some(VersionInfo::Tagged(version, _)) if VERSION_CHECK_REF == ALL_TAG_REFS => { - warn!( - target: "root", - "New git-cinnabar version available: {} (current version: {})", - version, CARGO_PKG_VERSION - ); + Some(VersionInfo::Tagged(version, _)) if *BUILD_BRANCH == Release => { + if self.show_current { + warn!( + target: "root", + "New git-cinnabar version available: {} (current version: {})", + version, *SHORT_VERSION + ); + } else { + warn!( + target: "root", + "New git-cinnabar version available: {}", + version, + ); + } if cfg!(feature = "self-update") { warn!( target: "root", @@ -172,11 +234,11 @@ impl Drop for VersionChecker { ); } } - Some(VersionInfo::Commit(_)) if VERSION_CHECK_REF != ALL_TAG_REFS => { + Some(VersionInfo::Commit(_)) if *BUILD_BRANCH != Release => { warn!( target: "root", "The {} branch of git-cinnabar was updated. {}", - VERSION_CHECK_REF, + BUILD_BRANCH.as_str(), if cfg!(feature = "self-update") { "You may run `git cinnabar self-update` to update." } else { @@ -214,87 +276,52 @@ impl Drop for VersionChecker { #[cfg(feature = "self-update")] pub fn check_new_version(req: VersionRequest) -> Option { - create_child(req) + VersionRequestChild::new(req) .as_ref() .and_then(|child| get_version(child).ok().flatten()) } -fn create_child(req: VersionRequest) -> Option { - let mut cmd = Command::new("git"); - cmd.args(["ls-remote", crate::CARGO_PKG_REPOSITORY]); - match req { - VersionRequest::Tagged => cmd.arg(ALL_TAG_REFS), - VersionRequest::Branch(branch) => cmd.arg(&format!("refs/heads/{branch}")), - }; - cmd.stdout(Stdio::piped()); - cmd.stderr(Stdio::null()); - - debug!(target: "version-check", "Running git {}", cmd.get_args().map(|arg| arg.as_bytes().as_bstr()).join(" ")); - - SharedChild::spawn(&mut cmd).ok() +#[cfg(feature = "self-update")] +pub fn find_version(what: VersionInfo) -> Option { + VersionRequestChild::new(VersionRequest::Tagged) + .as_ref() + .and_then(|child| { + child + .fold(None, |result, info| match &info { + VersionInfo::Tagged(v, c) => match &what { + VersionInfo::Tagged(version, _) if v == version => Some(info), + VersionInfo::Commit(cid) if c == cid => Some(info), + _ => result, + }, + _ => unreachable!(), + }) + .ok() + }) + .flatten() } -fn get_version(child: &SharedChild) -> Result, ()> { - let build_commit = FULL_VERSION - .strip_suffix("-modified") - .unwrap_or(FULL_VERSION) - .strip_prefix(concat!(crate_version!(), "-")) - .unwrap_or(""); - let output = child.take_stdout().unwrap().read_all().map_err(|_| ()); - child.wait().map_err(|_| ())?; - let output = output?; - if output.is_empty() { - return Err(()); - } - let current_version = Version::parse(CARGO_PKG_VERSION).unwrap(); - let mut newest_version = None; - for [sha1, r] in output - .lines() - .filter_map(|line| line.splitn_exact(u8::is_ascii_whitespace)) - { - debug!(target: "version-check", "Found {}@{}", r.as_bstr(), sha1.as_bstr()); - let cid = if let Ok(cid) = CommitId::from_bytes(sha1) { - cid - } else { - continue; - }; - if let Some(version) = r - .strip_prefix(b"refs/tags/") - .and_then(|tag| std::str::from_utf8(tag).ok()) - .and_then(parse_version) - { - if version > current_version - && newest_version - .as_ref() - .map_or(true, |(n_v, _)| &version > n_v) - { - newest_version = Some((version, cid)); - } - } else if sha1 != build_commit.as_bytes() { - debug!(target: "version-check", "Current version ({}) is different", build_commit); - return Ok(Some(VersionInfo::Commit(cid))); +fn get_version(child: &VersionRequestChild) -> Result, ()> { + let current_version = Version::parse(&SHORT_VERSION).unwrap(); + let build_commit = CommitId::from_str(&BUILD_COMMIT).ok(); + let result = child.fold(None, |result, info| match &info { + VersionInfo::Commit(cid) => build_commit.is_some_and(|c| &c != cid).then_some(info), + VersionInfo::Tagged(version, _) if version <= ¤t_version => result, + VersionInfo::Tagged(version, _) => match result { + Some(VersionInfo::Tagged(ref result_ver, _)) if result_ver < version => Some(info), + None => Some(info), + _ => result, + }, + })?; + match &result { + Some(VersionInfo::Tagged(v, _)) => { + debug!(target: "version-check", "Newest version found: {}", v); + } + Some(VersionInfo::Commit(_)) => { + debug!(target: "version-check", "Current version ({}) is different", *BUILD_COMMIT); + } + None => { + debug!(target: "version-check", "No version is newer than current ({})", current_version); } } - if let Some((v, cid)) = newest_version { - debug!(target: "version-check", "Newest version found: {}", v); - Ok(Some(VersionInfo::Tagged(v, cid))) - } else { - debug!(target: "version-check", "No version is newer than current ({})", current_version); - Ok(None) - } -} - -fn parse_version(v: &str) -> Option { - Version::parse(v).ok().or_else(|| { - // If the version didn't parse, try again by separating - // x.y.z from everything that follows, and try parsing - // again with a dash in between. - v.find(|c: char| !c.is_ascii_digit() && c != '.') - .map(|pos| { - let (digits, rest) = v.split_at(pos); - format!("{}-{}", digits, rest) - }) - .as_deref() - .and_then(|v| Version::parse(v).ok()) - }) + Ok(result) } diff --git a/tests/push.t b/tests/push.t index f21ceebb4..12c1cf0cf 100755 --- a/tests/push.t +++ b/tests/push.t @@ -81,6 +81,9 @@ Create an empty mercurial repository where we are going to push. Pushing from a repo with cinnabar metadata to an empty mercurial repo works + $ git -C abc-git push --dry-run origin d04f6df4abe2870ceb759263ee6aaa9241c4f93c:refs/heads/branches/default/tip + To hg::.*/push.t/repo (re) + * [new branch] d04f6df4abe2870ceb759263ee6aaa9241c4f93c -> branches/default/tip $ git -C abc-git push origin d04f6df4abe2870ceb759263ee6aaa9241c4f93c:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -94,6 +97,9 @@ Pushing from a repo without cinnabar metadata to an empty mercurial repo works $ git -C abc-git cinnabar rollback 0000000000000000000000000000000000000000 $ rm -rf $REPO/.hg $ hg init $REPO + $ git -C abc-git push --dry-run origin d04f6df4abe2870ceb759263ee6aaa9241c4f93c:refs/heads/branches/default/tip + To hg::.*/push.t/repo (re) + * [new branch] d04f6df4abe2870ceb759263ee6aaa9241c4f93c -> branches/default/tip $ git -C abc-git push origin d04f6df4abe2870ceb759263ee6aaa9241c4f93c:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -106,6 +112,11 @@ Pushing from a repo without cinnabar metadata to a non-empty mercurial repo requires pulling first. $ git -C abc-git cinnabar rollback 0000000000000000000000000000000000000000 + $ git -C abc-git push --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + \r (no-eol) (esc) + ERROR Cannot push to this remote without pulling/updating first. + error: failed to push some refs to 'hg::.*/push.t/repo' (re) + [1] $ git -C abc-git push origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip \r (no-eol) (esc) ERROR Cannot push to this remote without pulling/updating first. @@ -114,6 +125,11 @@ requires pulling first. Same, even when forced. + $ git -C abc-git push -f --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + \r (no-eol) (esc) + ERROR Cannot push to this remote without pulling/updating first. + error: failed to push some refs to 'hg::.*/push.t/repo' (re) + [1] $ git -C abc-git push -f origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip \r (no-eol) (esc) ERROR Cannot push to this remote without pulling/updating first. @@ -125,6 +141,9 @@ However, after pulling, we have a shared root, and we can push $ git -c fetch.prune=true -C abc-git remote update origin Fetching origin + $ git -C abc-git push --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + To hg::.*/push.t/repo (re) + d04f6df..687e015 687e015f9f646bb19797d991f2f53087297fbe14 -> branches/default/tip $ git -C abc-git push origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -137,6 +156,11 @@ Pushing from a repo without cinnabar metadata to a non-empty mercurial repo with different contents requires pulling first. $ git -C def-git cinnabar rollback 0000000000000000000000000000000000000000 + $ git -C def-git push --dry-run origin 62326f34fea5b80510f57599da9fd6e5997c0ca4:refs/heads/branches/default/tip + \r (no-eol) (esc) + ERROR Cannot push to this remote without pulling/updating first. + error: failed to push some refs to 'hg::.*/push.t/repo' (re) + [1] $ git -C def-git push origin 62326f34fea5b80510f57599da9fd6e5997c0ca4:refs/heads/branches/default/tip \r (no-eol) (esc) ERROR Cannot push to this remote without pulling/updating first. @@ -145,6 +169,11 @@ with different contents requires pulling first. Same, even when forced. + $ git -C def-git push -f --dry-run origin 62326f34fea5b80510f57599da9fd6e5997c0ca4:refs/heads/branches/default/tip + \r (no-eol) (esc) + ERROR Cannot push to this remote without pulling/updating first. + error: failed to push some refs to 'hg::.*/push.t/repo' (re) + [1] $ git -C def-git push -f origin 62326f34fea5b80510f57599da9fd6e5997c0ca4:refs/heads/branches/default/tip \r (no-eol) (esc) ERROR Cannot push to this remote without pulling/updating first. @@ -159,6 +188,15 @@ and can't push, but that's caught by git itself. From hg::.*/push.t/repo (re) + 62326f3...687e015 branches/default/tip -> origin/branches/default/tip (forced update) + $ git -c advice.pushnonffcurrent=true -c advice.pushupdaterejected=true -C def-git push --dry-run origin 62326f34fea5b80510f57599da9fd6e5997c0ca4:refs/heads/branches/default/tip + To hg::.*/push.t/repo (re) + ! [rejected] 62326f34fea5b80510f57599da9fd6e5997c0ca4 -> branches/default/tip (non-fast-forward) + error: failed to push some refs to 'hg::.*/push.t/repo' (re) + hint: Updates were rejected because the tip of your current branch is behind + hint: its remote counterpart.* (re) + hint: .*before pushing again. (re) + hint: See the 'Note about fast-forwards' in 'git push --help' for details. + [1] $ git -c advice.pushnonffcurrent=true -c advice.pushupdaterejected=true -C def-git push origin 62326f34fea5b80510f57599da9fd6e5997c0ca4:refs/heads/branches/default/tip To hg::.*/push.t/repo (re) ! [rejected] 62326f34fea5b80510f57599da9fd6e5997c0ca4 -> branches/default/tip (non-fast-forward) @@ -171,6 +209,11 @@ and can't push, but that's caught by git itself. This time, forced push is allowed. + $ git -c advice.pushnonffcurrent=true -c advice.pushupdaterejected=true -C def-git push origin -f --dry-run 62326f34fea5b80510f57599da9fd6e5997c0ca4:refs/heads/branches/default/tip + \r (no-eol) (esc) + WARNING Pushing a new root + To hg::.*/push.t/repo (re) + + 687e015...62326f3 62326f34fea5b80510f57599da9fd6e5997c0ca4 -> branches/default/tip (forced update) $ git -c advice.pushnonffcurrent=true -c advice.pushupdaterejected=true -C def-git push origin -f 62326f34fea5b80510f57599da9fd6e5997c0ca4:refs/heads/branches/default/tip \r (no-eol) (esc) WARNING Pushing a new root @@ -188,11 +231,21 @@ Similarly, when pushing from a shallow git repository. $ rm -rf $REPO/.hg $ hg init $REPO $ git -C abc-shallow remote set-url origin hg::$REPO + $ git -C abc-shallow push --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + \r (no-eol) (esc) + ERROR Pushing git shallow clones is not supported. + error: failed to push some refs to 'hg::.*/push.t/repo' (re) + [1] $ git -C abc-shallow push origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip \r (no-eol) (esc) ERROR Pushing git shallow clones is not supported. error: failed to push some refs to 'hg::.*/push.t/repo' (re) [1] + $ git -C abc-shallow push -f --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + \r (no-eol) (esc) + ERROR Pushing git shallow clones is not supported. + error: failed to push some refs to 'hg::.*/push.t/repo' (re) + [1] $ git -C abc-shallow push -f origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip \r (no-eol) (esc) ERROR Pushing git shallow clones is not supported. @@ -204,6 +257,11 @@ After pulling from a different repo, we still recognize we have a shallow clone. $ git -C abc-shallow cinnabar fetch hg::$DEF 90f6163d2820561ebe0f6c28e87d766ef619e43c From hg::.*/push.t/def (re) * branch hg/revs/90f6163d2820561ebe0f6c28e87d766ef619e43c -> FETCH_HEAD + $ git -C abc-shallow push -f --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + \r (no-eol) (esc) + ERROR Pushing git shallow clones is not supported. + error: failed to push some refs to 'hg::.*/push.t/repo' (re) + [1] $ git -C abc-shallow push -f origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip \r (no-eol) (esc) ERROR Pushing git shallow clones is not supported. @@ -216,6 +274,9 @@ supported and makes it unshallowed. $ git -C abc-shallow cinnabar fetch hg::$ABC bd623dea939349b06a47d5dce064255e5f1d9ec1 From hg::.*/push.t/abc (re) * branch hg/revs/bd623dea939349b06a47d5dce064255e5f1d9ec1 -> FETCH_HEAD + $ git -C abc-shallow push -f --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + To hg::.*/push.t/repo (re) + * [new branch] 687e015f9f646bb19797d991f2f53087297fbe14 -> branches/default/tip $ git -C abc-shallow push -f origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -234,6 +295,9 @@ Phase and cinnabar.data tests. From ../def-git * branch 62326f34fea5b80510f57599da9fd6e5997c0ca4 -> FETCH_HEAD + $ git -c cinnabar.data=never -C xyz-git push --dry-run origin 8b86a58578d5270969543e287634e3a2f122a338:refs/heads/branches/default/tip + To hg::.*/push.t/xyz (re) + * [new branch] 8b86a58578d5270969543e287634e3a2f122a338 -> branches/default/tip $ git -c cinnabar.data=never -C xyz-git push origin 8b86a58578d5270969543e287634e3a2f122a338:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -249,6 +313,9 @@ Phase and cinnabar.data tests. $ git -C xyz-git cinnabar rollback --candidates 2836e453f32b1ecccd3acca412f75b07c88176bf (current) + $ git -c cinnabar.data=phase -C xyz-git push --dry-run origin d04f6df4abe2870ceb759263ee6aaa9241c4f93c:refs/heads/branches/default/tip + To hg::.*/push.t/xyz (re) + 8b86a58..d04f6df d04f6df4abe2870ceb759263ee6aaa9241c4f93c -> branches/default/tip $ git -c cinnabar.data=phase -C xyz-git push origin d04f6df4abe2870ceb759263ee6aaa9241c4f93c:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -268,6 +335,9 @@ Server is publishing, so metadata was stored. > publish = False > EOF + $ git -c cinnabar.data=phase -C xyz-git push --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + To hg::.*/push.t/xyz (re) + d04f6df..687e015 687e015f9f646bb19797d991f2f53087297fbe14 -> branches/default/tip $ git -c cinnabar.data=phase -C xyz-git push origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -289,6 +359,37 @@ Server is now non-publishing, so metadata is unchanged. 8b8194eefb69ec89edc35dafb965311fe48c49d0 2836e453f32b1ecccd3acca412f75b07c88176bf + $ git -c cinnabar.data=always -C xyz-git push -f --dry-run origin 7ca6a3c32ec0dbcbcd155b2be6e2f4505012c273:refs/heads/branches/default/tip + \r (no-eol) (esc) + WARNING Pushing a new root + To hg::.*/push.t/xyz (re) + + 687e015...7ca6a3c 7ca6a3c32ec0dbcbcd155b2be6e2f4505012c273 -> branches/default/tip (forced update) + +Whatever happens, --dry-run doesn't store metadata + + $ git -C xyz-git cinnabar rollback --candidates + a2341d430e5acddf9481eabcad901fda12d023d3 (current) + 8b8194eefb69ec89edc35dafb965311fe48c49d0 + 2836e453f32b1ecccd3acca412f75b07c88176bf + +There is an extra mode for cinnabar.data that forces --dry-run to store metadata + + $ git -c cinnabar.data=force -C xyz-git push -f --dry-run origin 7ca6a3c32ec0dbcbcd155b2be6e2f4505012c273:refs/heads/branches/default/tip + \r (no-eol) (esc) + WARNING Pushing a new root + To hg::.*/push.t/xyz (re) + + 687e015...7ca6a3c 7ca6a3c32ec0dbcbcd155b2be6e2f4505012c273 -> branches/default/tip (forced update) + $ git -C xyz-git cinnabar rollback --candidates + 4305cef3fa610b3370f64ce10d2b50693a904278 (current) + a2341d430e5acddf9481eabcad901fda12d023d3 + 8b8194eefb69ec89edc35dafb965311fe48c49d0 + 2836e453f32b1ecccd3acca412f75b07c88176bf + $ git -C xyz-git cinnabar rollback + $ git -C xyz-git cinnabar rollback --candidates + a2341d430e5acddf9481eabcad901fda12d023d3 (current) + 8b8194eefb69ec89edc35dafb965311fe48c49d0 + 2836e453f32b1ecccd3acca412f75b07c88176bf + $ git -c cinnabar.data=always -C xyz-git push -f origin 7ca6a3c32ec0dbcbcd155b2be6e2f4505012c273:refs/heads/branches/default/tip \r (no-eol) (esc) WARNING Pushing a new root @@ -318,6 +419,9 @@ Pushing a root to a new non-publishing repo should work. From ../xyz-git * branch 687e015f9f646bb19797d991f2f53087297fbe14 -> FETCH_HEAD + $ git -c cinnabar.data=phase -C uvw-git push --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + To hg::.*/push.t/uvw (re) + * [new branch] 687e015f9f646bb19797d991f2f53087297fbe14 -> branches/default/tip $ git -c cinnabar.data=phase -C uvw-git push origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -352,6 +456,9 @@ We won't be able to push without first pulling something, so fetch the first com 2: draft $ hg --config extensions.strip= -R $UVW strip -r 2 --no-backup + $ git -c cinnabar.data=phase -C uvw-git push --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip + To hg::.*/push.t/uvw (re) + * [new branch] 687e015f9f646bb19797d991f2f53087297fbe14 -> branches/default/tip $ git -c cinnabar.data=phase -C uvw-git push origin 687e015f9f646bb19797d991f2f53087297fbe14:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -371,6 +478,9 @@ Test the git_commit experimental feature. $ rm -rf $REPO/.hg $ hg init $REPO $ git -C abc-git cinnabar rollback 0000000000000000000000000000000000000000 + $ git -C abc-git push --dry-run origin 8b86a58578d5270969543e287634e3a2f122a338:refs/heads/branches/default/tip + To hg::.*/push.t/repo (re) + * [new branch] 8b86a58578d5270969543e287634e3a2f122a338 -> branches/default/tip $ git -C abc-git push origin 8b86a58578d5270969543e287634e3a2f122a338:refs/heads/branches/default/tip remote: adding changesets remote: adding manifests @@ -379,6 +489,9 @@ Test the git_commit experimental feature. To hg::.*/push.t/repo (re) * [new branch] 8b86a58578d5270969543e287634e3a2f122a338 -> branches/default/tip + $ git -c cinnabar.experiments=git_commit -C abc-git push --dry-run origin 687e015f9f646bb19797d991f2f53087297fbe14:branches/default/tip + To hg::.*/push.t/repo (re) + 8b86a58..687e015 687e015f9f646bb19797d991f2f53087297fbe14 -> branches/default/tip $ git -c cinnabar.experiments=git_commit -C abc-git push origin 687e015f9f646bb19797d991f2f53087297fbe14:branches/default/tip remote: adding changesets remote: adding manifests