diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index b87041d24..87254aa98 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -18,3 +18,7 @@ jobs: - uses: chartboost/ruff-action@v1 with: version: 0.5.5 + - uses: chartboost/ruff-action@v1 + with: + version: 0.5.5 + args: 'format --diff' diff --git a/CI/decision.py b/CI/decision.py index 652dc40c1..32181a36c 100644 --- a/CI/decision.py +++ b/CI/decision.py @@ -11,7 +11,7 @@ BASE_DIR = os.path.dirname(__file__) sys.path.append(BASE_DIR) -sys.path.append(os.path.join(BASE_DIR, '..')) +sys.path.append(os.path.join(BASE_DIR, "..")) from itertools import chain @@ -38,8 +38,8 @@ def git_rev_parse(committish): return subprocess.check_output( - ['git', 'rev-parse', committish], text=True, - cwd=os.path.join(BASE_DIR, '..')).strip() + ["git", "rev-parse", committish], text=True, cwd=os.path.join(BASE_DIR, "..") + ).strip() def is_old_hg(version): @@ -47,7 +47,7 @@ def is_old_hg(version): if len(version) == 40: return False try: - version = [int(x) for x in version.split('.')] + version = [int(x) for x in version.split(".")] except ValueError: # Assume that an invalid version per the conversion above is # newer. @@ -62,108 +62,113 @@ class TestTask(Task): coverage = [] def __init__(self, **kwargs): - git = kwargs.pop('git', GIT_VERSION) - hg = kwargs.pop('hg', MERCURIAL_VERSION) - hg_clone = kwargs.pop('hg_clone', None) - commit = kwargs.pop('commit', None) - task_env = kwargs.pop('task_env', 'linux') - variant = kwargs.pop('variant', None) - build = kwargs.pop('build', None) - clone = kwargs.pop('clone', TC_COMMIT) - desc = kwargs.pop('description', None) - short_desc = kwargs.pop('short_desc', 'test') - extra_desc = kwargs.pop('extra_desc', None) - pre_command = kwargs.pop('pre_command', None) + git = kwargs.pop("git", GIT_VERSION) + hg = kwargs.pop("hg", MERCURIAL_VERSION) + hg_clone = kwargs.pop("hg_clone", None) + commit = kwargs.pop("commit", None) + task_env = kwargs.pop("task_env", "linux") + variant = kwargs.pop("variant", None) + build = kwargs.pop("build", None) + clone = kwargs.pop("clone", TC_COMMIT) + desc = kwargs.pop("description", None) + short_desc = kwargs.pop("short_desc", "test") + extra_desc = kwargs.pop("extra_desc", None) + pre_command = kwargs.pop("pre_command", None) if build is None: - build = '{}.{}'.format(task_env, variant) if variant else task_env + build = "{}.{}".format(task_env, variant) if variant else task_env build = Build.by_name(build) - kwargs.setdefault('mounts', []).append(build.mount()) + kwargs.setdefault("mounts", []).append(build.mount()) build = build.install() if variant: - kwargs.setdefault('env', {})['VARIANT'] = variant - env = TaskEnvironment.by_name('{}.test'.format(task_env)) + kwargs.setdefault("env", {})["VARIANT"] = variant + env = TaskEnvironment.by_name("{}.test".format(task_env)) command = [] if pre_command: command.extend(pre_command) if hg: - hg_task = Hg.by_name('{}.{}'.format(task_env, hg)) - kwargs.setdefault('mounts', []).append(hg_task.mount()) + hg_task = Hg.by_name("{}.{}".format(task_env, hg)) + kwargs.setdefault("mounts", []).append(hg_task.mount()) command.extend(hg_task.install()) - command.append('hg --version') + command.append("hg --version") if is_old_hg(hg): - kwargs.setdefault('env', {})['NO_CLONEBUNDLES'] = '1' + kwargs.setdefault("env", {})["NO_CLONEBUNDLES"] = "1" if git: - git_task = Git.by_name('{}.{}'.format(task_env, git)) - kwargs.setdefault('mounts', []).append(git_task.mount()) + git_task = Git.by_name("{}.{}".format(task_env, git)) + kwargs.setdefault("mounts", []).append(git_task.mount()) command.extend(git_task.install()) - command.append('git --version') + command.append("git --version") command.extend(Task.checkout(commit=commit)) command.extend(build) if clone: - kwargs.setdefault('mounts', []).append( - {'file:bundle.git': Clone.by_name(clone)}) - command.extend([ - 'git init repo/hg.old.git', - 'git -C repo/hg.old.git fetch ../../bundle.git refs/*:refs/*', - 'git -C repo/hg.old.git remote add origin hg:${REPO#https:}', - 'git -C repo/hg.old.git symbolic-ref HEAD' - ' refs/heads/branches/default/tip', - ]) - kwargs.setdefault('env', {})['REPO'] = REPO - command.extend(( - 'repo/git-cinnabar --version', - )) - if 'command' not in kwargs or hg_clone: + kwargs.setdefault("mounts", []).append( + {"file:bundle.git": Clone.by_name(clone)} + ) + command.extend( + [ + "git init repo/hg.old.git", + "git -C repo/hg.old.git fetch ../../bundle.git refs/*:refs/*", + "git -C repo/hg.old.git remote add origin hg:${REPO#https:}", + "git -C repo/hg.old.git symbolic-ref HEAD" + " refs/heads/branches/default/tip", + ] + ) + kwargs.setdefault("env", {})["REPO"] = REPO + command.extend(("repo/git-cinnabar --version",)) + if "command" not in kwargs or hg_clone: command += [ - 'hg init repo/hg.pure.hg', - 'hg -R repo/hg.pure.hg unbundle bundle.hg', + "hg init repo/hg.pure.hg", + "hg -R repo/hg.pure.hg unbundle bundle.hg", ] - kwargs.setdefault('mounts', []).append( - {'file:bundle.hg': HgClone.by_name(MERCURIAL_VERSION)}) - if 'command' in kwargs: - kwargs['command'] = command + kwargs['command'] + kwargs.setdefault("mounts", []).append( + {"file:bundle.hg": HgClone.by_name(MERCURIAL_VERSION)} + ) + if "command" in kwargs: + kwargs["command"] = command + kwargs["command"] else: if commit: # Always use the current CI scripts command.append( - 'git -C repo -c core.autocrlf=input checkout {} CI' - .format(TC_COMMIT)) - output_sync = ' --output-sync=target' - if env.os == 'macos': - output_sync = '' - kwargs['command'] = command + [ - 'make -C repo -f CI/tests.mk -j$({}){}' - .format(nproc(env), output_sync), + "git -C repo -c core.autocrlf=input checkout {} CI".format( + TC_COMMIT + ) + ) + output_sync = " --output-sync=target" + if env.os == "macos": + output_sync = "" + kwargs["command"] = command + [ + "make -C repo -f CI/tests.mk -j$({}){}".format(nproc(env), output_sync), ] - if variant == 'coverage': - kwargs['command'].extend([ - 'shopt -s nullglob', - 'cd repo', - 'zip $ARTIFACTS/coverage.zip' - ' $(find . -name "*.gcda")', - 'cd ..', - 'shopt -u nullglob', - ]) - artifact = kwargs.pop('artifact', None) - artifacts = kwargs.setdefault('artifacts', []) - assert not(artifacts and artifact) + if variant == "coverage": + kwargs["command"].extend( + [ + "shopt -s nullglob", + "cd repo", + "zip $ARTIFACTS/coverage.zip" ' $(find . -name "*.gcda")', + "cd ..", + "shopt -u nullglob", + ] + ) + artifact = kwargs.pop("artifact", None) + artifacts = kwargs.setdefault("artifacts", []) + assert not (artifacts and artifact) if artifact: artifacts.push(artifact) - artifacts.append('coverage.zip') + artifacts.append("coverage.zip") self.coverage.append(self) - elif variant == 'asan' and task_env == 'linux': - kwargs['caps'] = ['SYS_PTRACE'] + elif variant == "asan" and task_env == "linux": + kwargs["caps"] = ["SYS_PTRACE"] if not desc: - desc = '{} w/ git-{} hg-{}'.format( - short_desc, git, 'r' + hg if len(hg) == 40 else hg) - if variant and variant != 'coverage': - desc = ' '.join((desc, variant)) + desc = "{} w/ git-{} hg-{}".format( + short_desc, git, "r" + hg if len(hg) == 40 else hg + ) + if variant and variant != "coverage": + desc = " ".join((desc, variant)) if extra_desc: - desc = ' '.join((desc, extra_desc)) - if task_env != 'linux': - desc = ' '.join((desc, env.os, env.cpu)) - kwargs['description'] = desc + desc = " ".join((desc, extra_desc)) + if task_env != "linux": + desc = " ".join((desc, env.os, env.cpu)) + kwargs["description"] = desc Task.__init__(self, task_env=env, **kwargs) @@ -172,44 +177,44 @@ class Clone(TestTask, metaclass=Tool): def __init__(self, version): sha1 = git_rev_parse(version) - expireIn = '26 weeks' + expireIn = "26 weeks" kwargs = {} if version == TC_COMMIT or len(version) == 40: if version == TC_COMMIT: - build = Build.by_name('linux') + build = Build.by_name("linux") else: - build = Build.by_name('linux.old:{}'.format(version)) - kwargs.setdefault('mounts', []).append(build.mount()) + build = Build.by_name("linux.old:{}".format(version)) + kwargs.setdefault("mounts", []).append(build.mount()) download = build.install() - expireIn = '26 weeks' - elif parse_version(version) < parse_version('0.6.0'): - download = ['repo/git-cinnabar download'] - if parse_version(version) < parse_version('0.5.7'): - kwargs['git'] = '2.30.2' + expireIn = "26 weeks" + elif parse_version(version) < parse_version("0.6.0"): + download = ["repo/git-cinnabar download"] + if parse_version(version) < parse_version("0.5.7"): + kwargs["git"] = "2.30.2" else: - download = ['repo/download.py'] + download = ["repo/download.py"] if REPO == DEFAULT_REPO: - index = 'bundle.{}'.format(sha1) + index = "bundle.{}".format(sha1) else: - index = 'bundle.{}.{}'.format(hashlib.sha1(REPO).hexdigest(), sha1) + index = "bundle.{}.{}".format(hashlib.sha1(REPO).hexdigest(), sha1) TestTask.__init__( self, hg=MERCURIAL_VERSION, hg_clone=True, - description='clone w/ {}'.format(version), + description="clone w/ {}".format(version), index=index, expireIn=expireIn, build=download, commit=sha1, clone=False, command=[ - 'PATH=$PWD/repo:$PATH' - ' git -c fetch.prune=true clone -n hg::$PWD/repo/hg.pure.hg' - ' hg.old.git', - 'git -C hg.old.git bundle create $ARTIFACTS/bundle.git --all', + "PATH=$PWD/repo:$PATH" + " git -c fetch.prune=true clone -n hg::$PWD/repo/hg.pure.hg" + " hg.old.git", + "git -C hg.old.git bundle create $ARTIFACTS/bundle.git --all", ], - artifact='bundle.git', - priority='high', + artifact="bundle.git", + priority="high", **kwargs, ) @@ -219,68 +224,70 @@ class HgClone(Task, metaclass=Tool): def __init__(self, version): if REPO == DEFAULT_REPO: - index = 'hgclone.{}'.format(version) + index = "hgclone.{}".format(version) else: - index = 'hgclone.{}.{}'.format( - hashlib.sha1(REPO).hexdigest(), version) - hg_task = Hg.by_name(f'linux.{version}') + index = "hgclone.{}.{}".format(hashlib.sha1(REPO).hexdigest(), version) + hg_task = Hg.by_name(f"linux.{version}") Task.__init__( self, - task_env=TaskEnvironment.by_name('linux.test'), - description=f'hg clone w/ {version}', + task_env=TaskEnvironment.by_name("linux.test"), + description=f"hg clone w/ {version}", index=index, - expireIn='26 weeks', - command=hg_task.install() + [ - 'hg clone -U --stream $REPO repo', - 'hg -R repo bundle -t none-v1 -a $ARTIFACTS/bundle.hg', + expireIn="26 weeks", + command=hg_task.install() + + [ + "hg clone -U --stream $REPO repo", + "hg -R repo bundle -t none-v1 -a $ARTIFACTS/bundle.hg", ], mounts=[hg_task.mount()], - artifact='bundle.hg', + artifact="bundle.hg", env={ - 'REPO': REPO, + "REPO": REPO, }, - priority='high', + priority="high", ) -@action('decision') +@action("decision") def decision(): - for env in ('linux', 'mingw64', 'osx', 'arm64-osx'): + for env in ("linux", "mingw64", "osx", "arm64-osx"): # Can't spawn osx workers from pull requests. - if env.endswith('osx') and not TC_IS_PUSH: + if env.endswith("osx") and not TC_IS_PUSH: continue TestTask( task_env=env, - variant='coverage' if env == 'linux' else None, + variant="coverage" if env == "linux" else None, ) - task_env = TaskEnvironment.by_name('{}.test'.format(env)) + task_env = TaskEnvironment.by_name("{}.test".format(env)) if TC_IS_PUSH and TC_BRANCH != "try": - hg = Hg.by_name('{}.{}'.format(env, MERCURIAL_VERSION)) - git = Git.by_name('{}.{}'.format(env, GIT_VERSION)) + hg = Hg.by_name("{}.{}".format(env, MERCURIAL_VERSION)) + git = Git.by_name("{}.{}".format(env, GIT_VERSION)) Task( task_env=task_env, - description='download build {} {}'.format(task_env.os, - task_env.cpu), - command=list(chain( - git.install(), - hg.install(), - Task.checkout(), - [ - '(cd repo ; ./download.py)', - 'PATH=$PWD/repo:$PATH git cinnabar --version', - 'cp repo/download.py .', - './download.py', - 'PATH=$PWD:$PATH', - 'git cinnabar --version', - 'git cinnabar self-update', - 'git cinnabar --version', - 'git cinnabar self-update --branch {}'.format( - TC_BRANCH or "master"), - 'git cinnabar --version', - ], - )), + description="download build {} {}".format(task_env.os, task_env.cpu), + command=list( + chain( + git.install(), + hg.install(), + Task.checkout(), + [ + "(cd repo ; ./download.py)", + "PATH=$PWD/repo:$PATH git cinnabar --version", + "cp repo/download.py .", + "./download.py", + "PATH=$PWD:$PATH", + "git cinnabar --version", + "git cinnabar self-update", + "git cinnabar --version", + "git cinnabar self-update --branch {}".format( + TC_BRANCH or "master" + ), + "git cinnabar --version", + ], + ) + ), dependencies=[ Build.by_name(env), ], @@ -292,73 +299,71 @@ def decision(): # Because nothing is using the arm64 linux build, we need to manually # touch it. - Build.by_name('arm64-linux') + Build.by_name("arm64-linux") for upgrade in UPGRADE_FROM: TestTask( - short_desc='upgrade tests', - extra_desc='from-{}'.format(upgrade), - variant='coverage', + short_desc="upgrade tests", + extra_desc="from-{}".format(upgrade), + variant="coverage", clone=upgrade, env={ - 'UPGRADE_FROM': upgrade, + "UPGRADE_FROM": upgrade, }, - hg='5.4.2', + hg="5.4.2", ) - for git in ('1.8.5', '2.7.4'): - TestTask( - git=git, - env={'GIT_OLD_VERSION': '1'} - ) + for git in ("1.8.5", "2.7.4"): + TestTask(git=git, env={"GIT_OLD_VERSION": "1"}) for hg in SOME_MERCURIAL_VERSIONS: if hg != MERCURIAL_VERSION: do_hg_version(hg) TestTask( - task_env='linux', - variant='asan', + task_env="linux", + variant="asan", ) - for env in ('linux', 'mingw64', 'osx', 'arm64-osx'): + for env in ("linux", "mingw64", "osx", "arm64-osx"): # Can't spawn osx workers from pull requests. - if env.endswith('osx') and not TC_IS_PUSH: + if env.endswith("osx") and not TC_IS_PUSH: continue TestTask( task_env=env, - variant='coverage' if env == 'linux' else None, - short_desc='graft tests', + variant="coverage" if env == "linux" else None, + short_desc="graft tests", env={ - 'GRAFT': '1', + "GRAFT": "1", }, ) for env, variant in ( - ('linux', 'coverage'), - ('linux', 'asan'), - ('osx', None), - ('arm64-osx', None), + ("linux", "coverage"), + ("linux", "asan"), + ("osx", None), + ("arm64-osx", None), ): # Can't spawn osx workers from pull requests. - if env.endswith('osx') and not TC_IS_PUSH: + if env.endswith("osx") and not TC_IS_PUSH: continue pre_command = [] - if env != 'linux': - pre_command.append('pip install cram==0.7') + if env != "linux": + pre_command.append("pip install cram==0.7") TestTask( task_env=env, variant=variant, - short_desc='cram', + short_desc="cram", clone=False, - command=pre_command + [ - 'cram --verbose repo/tests', + command=pre_command + + [ + "cram --verbose repo/tests", ], env={ - 'GIT_CINNABAR_CHECK': 'no-version-check', + "GIT_CINNABAR_CHECK": "no-version-check", }, ) @@ -370,48 +375,53 @@ def do_hg_version(hg): # wrt bookmarks. if not is_old_hg(hg): TestTask( - short_desc='cram', + short_desc="cram", clone=False, hg=hg, command=[ - 'cram --verbose repo/tests', + "cram --verbose repo/tests", ], env={ - 'GIT_CINNABAR_CHECK': 'no-version-check', + "GIT_CINNABAR_CHECK": "no-version-check", }, ) -@action('more-hg-versions', - title='More hg versions', - description='Trigger tests against more mercurial versions') +@action( + "more-hg-versions", + title="More hg versions", + description="Trigger tests against more mercurial versions", +) def more_hg_versions(): for hg in ALL_MERCURIAL_VERSIONS: if hg != MERCURIAL_VERSION and hg not in SOME_MERCURIAL_VERSIONS: do_hg_version(hg) -@action('hg-trunk', - title='Test w/ hg trunk', - description='Trigger tests against current mercurial trunk') +@action( + "hg-trunk", + title="Test w/ hg trunk", + description="Trigger tests against current mercurial trunk", +) def hg_trunk(): import requests - r = requests.get('https://www.mercurial-scm.org/repo/hg/?cmd=branchmap') + + r = requests.get("https://www.mercurial-scm.org/repo/hg/?cmd=branchmap") trunk = None for l in r.text.splitlines(): fields = l.split() - if fields[0] == 'default': + if fields[0] == "default": trunk = fields[-1] if not trunk: - raise Exception('Cannot find mercurial trunk changeset') + raise Exception("Cannot find mercurial trunk changeset") do_hg_version(trunk) def main(): try: - func = action.by_name[TC_ACTION or 'decision'].func + func = action.by_name[TC_ACTION or "decision"].func except AttributeError: - raise Exception('Unsupported action: %s', TC_ACTION or 'decision') + raise Exception("Unsupported action: %s", TC_ACTION or "decision") func() @@ -419,73 +429,84 @@ def main(): if TestTask.coverage and TC_IS_PUSH and TC_BRANCH: coverage_mounts = [ - {f'file:cov-{task.id}.zip': task} for task in TestTask.coverage + {f"file:cov-{task.id}.zip": task} for task in TestTask.coverage ] - task = Build.by_name('linux.coverage') - coverage_mounts.append({'file:gcno-build.zip': { - 'artifact': task.artifacts[1], - 'taskId': task.id, - }}) - - merge_coverage.extend([ - 'grcov -s repo -t lcov -o repo/coverage.lcov gcno-build.zip ' + - ' '.join( - f'cov-{task.id}.zip' - for task in TestTask.coverage), - ]) + task = Build.by_name("linux.coverage") + coverage_mounts.append( + { + "file:gcno-build.zip": { + "artifact": task.artifacts[1], + "taskId": task.id, + } + } + ) + + merge_coverage.extend( + [ + "grcov -s repo -t lcov -o repo/coverage.lcov gcno-build.zip " + + " ".join(f"cov-{task.id}.zip" for task in TestTask.coverage), + ] + ) if merge_coverage: Task( - task_env=TaskEnvironment.by_name('linux.codecov'), - description='upload coverage', - scopes=['secrets:get:project/git-cinnabar/codecov'], + task_env=TaskEnvironment.by_name("linux.codecov"), + description="upload coverage", + scopes=["secrets:get:project/git-cinnabar/codecov"], mounts=coverage_mounts, - command=list(chain( - Task.checkout(), - [ - 'set +x', - ('export CODECOV_TOKEN=$(curl -sL ' - f'{PROXY_URL}/api/secrets/v1/secret/project/git-cinnabar' - '/codecov | python3' - ' -c "import json, sys; print(json.load(sys.stdin)' - '[\\"secret\\"][\\"token\\"])")'), - 'set -x', - ], - merge_coverage, - [ - 'cd repo', - 'codecov -Z --name "taskcluster" -C {} -B {}' - .format(TC_COMMIT, TC_BRANCH), - ], - )), + command=list( + chain( + Task.checkout(), + [ + "set +x", + ( + "export CODECOV_TOKEN=$(curl -sL " + f"{PROXY_URL}/api/secrets/v1/secret/project/git-cinnabar" + "/codecov | python3" + ' -c "import json, sys; print(json.load(sys.stdin)' + '[\\"secret\\"][\\"token\\"])")' + ), + "set -x", + ], + merge_coverage, + [ + "cd repo", + 'codecov -Z --name "taskcluster" -C {} -B {}'.format( + TC_COMMIT, TC_BRANCH + ), + ], + ) + ), ) for t in Task.by_id.values(): t.submit() - if not TC_ACTION and 'TC_GROUP_ID' in os.environ: + if not TC_ACTION and "TC_GROUP_ID" in os.environ: actions = { - 'version': 1, - 'actions': [], - 'variables': { - 'e': dict(TC_DATA, decision_id=''), - 'tasks_for': 'action', + "version": 1, + "actions": [], + "variables": { + "e": dict(TC_DATA, decision_id=""), + "tasks_for": "action", }, } for name, a in action.by_name.items(): - if name != 'decision': - actions['actions'].append({ - 'kind': 'task', - 'name': a.name, - 'title': a.title, - 'description': a.description, - 'context': [], - 'task': a.task, - }) - - with open('actions.json', 'w') as out: + if name != "decision": + actions["actions"].append( + { + "kind": "task", + "name": a.name, + "title": a.title, + "description": a.description, + "context": [], + "task": a.task, + } + ) + + with open("actions.json", "w") as out: out.write(json.dumps(actions, indent=True)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/CI/docker.py b/CI/docker.py index e7203d3fb..f0f97d5fc 100644 --- a/CI/docker.py +++ b/CI/docker.py @@ -17,12 +17,12 @@ def sources_list(snapshot, sections): for idx, (archive, dist) in enumerate(sections): if not snapshot: - yield 'deb http://archive.debian.org/{} {} main'.format( + yield "deb http://archive.debian.org/{} {} main".format( archive, dist, ) continue - yield 'deb http://snapshot.debian.org/archive/{}/{} {} main'.format( + yield "deb http://snapshot.debian.org/archive/{}/{} {} main".format( archive, snapshot, dist, @@ -30,180 +30,199 @@ def sources_list(snapshot, sections): LLVM_REPO = ( - 'echo' - ' deb [signed-by=/usr/share/keyrings/llvm.gpg]' - ' https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-18 main' - ' > /etc/apt/sources.list.d/llvm.list' + "echo" + " deb [signed-by=/usr/share/keyrings/llvm.gpg]" + " https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-18 main" + " > /etc/apt/sources.list.d/llvm.list" ) DOCKER_IMAGES = { - 'base': { - 'from': 'debian:bullseye-20220801', - 'commands': [ - '({}) > /etc/apt/sources.list'.format('; '.join( - 'echo ' + l for l in sources_list('20220801T205040Z', ( - ('debian', 'bullseye'), - ('debian', 'bullseye-updates'), - ('debian-security', 'bullseye-security'), - )))), - 'apt-get update -o Acquire::Check-Valid-Until=false', - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'apt-transport-https', - 'bzip2', - 'ca-certificates', - 'curl', - 'gnupg2', - 'libcurl3-gnutls', - 'python-setuptools', - 'python3-setuptools', - 'python3-pip', - 'unzip', - 'xz-utils', - 'zip', - 'zstd', - ])), - 'apt-get clean', - 'curl -sO https://apt.llvm.org/llvm-snapshot.gpg.key', - 'gpg --no-default-keyring --keyring /usr/share/keyrings/llvm.gpg' - ' --import llvm-snapshot.gpg.key', - 'rm llvm-snapshot.gpg.key', - 'curl -sO http://snapshot.debian.org/archive/debian' - '/20220326T025251Z/pool/main/p/python2-pip' - '/python-pip_20.3.4%2Bdfsg-4_all.deb', - 'dpkg-deb -x python-pip*.deb /', - 'python2.7 -m pip install pip==20.3.4 wheel==0.37.1' - ' --upgrade --ignore-installed', - 'python3 -m pip install pip==20.3.4 wheel==0.37.1' - ' --upgrade --ignore-installed', + "base": { + "from": "debian:bullseye-20220801", + "commands": [ + "({}) > /etc/apt/sources.list".format( + "; ".join( + "echo " + l + for l in sources_list( + "20220801T205040Z", + ( + ("debian", "bullseye"), + ("debian", "bullseye-updates"), + ("debian-security", "bullseye-security"), + ), + ) + ) + ), + "apt-get update -o Acquire::Check-Valid-Until=false", + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "apt-transport-https", + "bzip2", + "ca-certificates", + "curl", + "gnupg2", + "libcurl3-gnutls", + "python-setuptools", + "python3-setuptools", + "python3-pip", + "unzip", + "xz-utils", + "zip", + "zstd", + ] + ) + ), + "apt-get clean", + "curl -sO https://apt.llvm.org/llvm-snapshot.gpg.key", + "gpg --no-default-keyring --keyring /usr/share/keyrings/llvm.gpg" + " --import llvm-snapshot.gpg.key", + "rm llvm-snapshot.gpg.key", + "curl -sO http://snapshot.debian.org/archive/debian" + "/20220326T025251Z/pool/main/p/python2-pip" + "/python-pip_20.3.4%2Bdfsg-4_all.deb", + "dpkg-deb -x python-pip*.deb /", + "python2.7 -m pip install pip==20.3.4 wheel==0.37.1" + " --upgrade --ignore-installed", + "python3 -m pip install pip==20.3.4 wheel==0.37.1" + " --upgrade --ignore-installed", ], }, - - 'build': { - 'from': 'base', - 'commands': [ + "build": { + "from": "base", + "commands": [ LLVM_REPO, - 'apt-get update -o Acquire::Check-Valid-Until=false', - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'clang-18', - 'lld-18', - 'git', - 'make', - 'patch', - 'pkg-config', - 'mmdebstrap', - 'debian-archive-keyring', - 'symlinks', - 'fakechroot', - 'gcc-mingw-w64-x86-64-win32', - ])), - 'for arch in amd64 arm64; do' - ' mmdebstrap -d' - ' --architecture=$arch' - ' --mode=chrootless' - ' --variant=extract' - ' --include=libc6-dev,libcurl4-gnutls-dev,zlib1g-dev,libgcc-6-dev' - ' stretch sysroot-$arch' - ' http://archive.debian.org/debian/ ;' - ' LD_PRELOAD=libfakechroot.so FAKECHROOT_BASE=$PWD/sysroot-$arch' - ' symlinks -crv /;' - 'done', - 'apt-get clean', + "apt-get update -o Acquire::Check-Valid-Until=false", + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "clang-18", + "lld-18", + "git", + "make", + "patch", + "pkg-config", + "mmdebstrap", + "debian-archive-keyring", + "symlinks", + "fakechroot", + "gcc-mingw-w64-x86-64-win32", + ] + ) + ), + "for arch in amd64 arm64; do" + " mmdebstrap -d" + " --architecture=$arch" + " --mode=chrootless" + " --variant=extract" + " --include=libc6-dev,libcurl4-gnutls-dev,zlib1g-dev,libgcc-6-dev" + " stretch sysroot-$arch" + " http://archive.debian.org/debian/ ;" + " LD_PRELOAD=libfakechroot.so FAKECHROOT_BASE=$PWD/sysroot-$arch" + " symlinks -crv /;" + "done", + "apt-get clean", ], }, - - 'build-tools': { - 'from': 'base', - 'commands': [ - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'gcc', - 'git', - 'libc6-dev', - 'libcurl4-gnutls-dev', - 'make', - 'patch', - 'python-dev', - 'python3-dev', - 'zlib1g-dev', - ])), - 'apt-get clean', + "build-tools": { + "from": "base", + "commands": [ + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "gcc", + "git", + "libc6-dev", + "libcurl4-gnutls-dev", + "make", + "patch", + "python-dev", + "python3-dev", + "zlib1g-dev", + ] + ) + ), + "apt-get clean", ], }, - - 'codecov': { - 'from': 'base', - 'commands': [ - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'gcc', - 'git', - 'python3-coverage', - ])), - 'apt-get clean', - 'ln -s /usr/bin/python3-coverage /usr/local/bin/coverage', - 'curl -o /usr/local/bin/codecov -sL {}'.format( - 'https://github.com/codecov/uploader/releases/download' - '/v0.1.0_9779/codecov-linux' + "codecov": { + "from": "base", + "commands": [ + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "gcc", + "git", + "python3-coverage", + ] + ) + ), + "apt-get clean", + "ln -s /usr/bin/python3-coverage /usr/local/bin/coverage", + "curl -o /usr/local/bin/codecov -sL {}".format( + "https://github.com/codecov/uploader/releases/download" + "/v0.1.0_9779/codecov-linux" ), - 'chmod +x /usr/local/bin/codecov', - 'curl -sL {} | tar -C /usr/local/bin -jxf -'.format( - 'https://github.com/mozilla/grcov/releases/download/v0.8.7' - '/grcov-x86_64-unknown-linux-gnu.tar.bz2' + "chmod +x /usr/local/bin/codecov", + "curl -sL {} | tar -C /usr/local/bin -jxf -".format( + "https://github.com/mozilla/grcov/releases/download/v0.8.7" + "/grcov-x86_64-unknown-linux-gnu.tar.bz2" ), ], }, - - 'test': { - 'from': 'base', - 'commands': [ + "test": { + "from": "base", + "commands": [ LLVM_REPO, - 'apt-get update -o Acquire::Check-Valid-Until=false', - 'apt-get install -y --no-install-recommends {}'.format(' '.join([ - 'llvm-18', - 'make', - ])), - 'apt-get clean', - 'pip3 install cram==0.7', - 'ln -s llvm-symbolizer-18 /usr/bin/llvm-symbolizer' + "apt-get update -o Acquire::Check-Valid-Until=false", + "apt-get install -y --no-install-recommends {}".format( + " ".join( + [ + "llvm-18", + "make", + ] + ) + ), + "apt-get clean", + "pip3 install cram==0.7", + "ln -s llvm-symbolizer-18 /usr/bin/llvm-symbolizer", ], }, } class DockerImage(Task, metaclass=TaskEnvironment): - PREFIX = 'linux' - cpu = 'x86_64' - os = 'linux' + PREFIX = "linux" + cpu = "x86_64" + os = "linux" def __init__(self, name): defn = DOCKER_IMAGES[name] - base = defn['from'] + base = defn["from"] self.name = name - if ':' not in base: + if ":" not in base: base = DockerImage.by_name(base) self.base = base - self.definition = defn['commands'] + self.definition = defn["commands"] Task.__init__( self, task_env=self, - description='docker image: {}'.format(name), + description="docker image: {}".format(name), index=self.index, - expireIn='26 weeks', - workerType='linux', + expireIn="26 weeks", + workerType="linux", image=base, dockerSave=True, command=self.definition, ) def __str__(self): - return '{}/{}:{}'.format( - TC_REPO_NAME, - self.name, - self.hexdigest - ) + return "{}/{}:{}".format(TC_REPO_NAME, self.name, self.hexdigest) @property def index(self): - return '.'.join((self.PREFIX, self.name, self.hexdigest)) + return ".".join((self.PREFIX, self.name, self.hexdigest)) @property def hexdigest(self): @@ -214,48 +233,49 @@ def hexdigest(self): def prepare_params(self, params): commands = ["mkdir artifacts"] - image = params.pop('image', self) + image = params.pop("image", self) volumes = [ - kind.split(':', 1)[1] - for mount in params.get('mounts', []) + kind.split(":", 1)[1] + for mount in params.get("mounts", []) for kind in mount - if ':' in kind + if ":" in kind ] if isinstance(image, DockerImage): - params.setdefault('mounts', []).append({'file:dockerimage': image}) - image = 'docker-archive:dockerimage' + params.setdefault("mounts", []).append({"file:dockerimage": image}) + image = "docker-archive:dockerimage" run_cmd = [ - 'podman', - 'run', - '--name=taskcontainer', - '--volume=./artifacts:/artifacts', - '--env=ARTIFACTS=/artifacts', + "podman", + "run", + "--name=taskcontainer", + "--volume=./artifacts:/artifacts", + "--env=ARTIFACTS=/artifacts", ] - if any(s.startswith('secrets:') for s in params.get('scopes', [])): + if any(s.startswith("secrets:") for s in params.get("scopes", [])): # There's probably a better way, but it's simpler. - run_cmd.append('--network=host') + run_cmd.append("--network=host") for v in volumes: - run_cmd.append(f'--volume=./{v}:/{v}') - for k, v in params.pop('env', {}).items(): - run_cmd.append(f'--env={k}={v}') - for cap in params.pop('caps', []): - run_cmd.append(f'--cap-add={cap}') + run_cmd.append(f"--volume=./{v}:/{v}") + for k, v in params.pop("env", {}).items(): + run_cmd.append(f"--env={k}={v}") + for cap in params.pop("caps", []): + run_cmd.append(f"--cap-add={cap}") run_cmd.append(image) - run_cmd.extend(bash_command(*params['command'])) + run_cmd.extend(bash_command(*params["command"])) commands.append(join_command(*run_cmd)) - if params.pop('dockerSave', False): - commands.extend([ - 'exit_code=$?', - 'podman commit taskcontainer taskcontainer', - 'podman save taskcontainer' - ' | zstd > artifacts/dockerImage.tar.zst', - 'podman rm taskcontainer', - 'exit $exit_code', - ]) - params['artifacts'] = ["dockerImage.tar.zst"] - params['command'] = bash_command(*commands) + if params.pop("dockerSave", False): + commands.extend( + [ + "exit_code=$?", + "podman commit taskcontainer taskcontainer", + "podman save taskcontainer" + " | zstd > artifacts/dockerImage.tar.zst", + "podman rm taskcontainer", + "exit $exit_code", + ] + ) + params["artifacts"] = ["dockerImage.tar.zst"] + params["command"] = bash_command(*commands) - if 'artifacts' in params: - params['artifacts'] = [f'artifacts/{a}' - for a in params['artifacts']] + if "artifacts" in params: + params["artifacts"] = [f"artifacts/{a}" for a in params["artifacts"]] return params diff --git a/CI/hg-serve-exec.py b/CI/hg-serve-exec.py index b6e26c3b7..47e333c66 100644 --- a/CI/hg-serve-exec.py +++ b/CI/hg-serve-exec.py @@ -7,6 +7,7 @@ import os import shutil import subprocess + try: from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer from SimpleHTTPServer import SimpleHTTPRequestHandler @@ -17,17 +18,20 @@ try: from mercurial import cmdutil + cmdutil.command except AttributeError: from mercurial import registrar as cmdutil from mercurial import hgweb from mercurial.hgweb import common from mercurial.hgweb.server import openlog + try: httpservice = hgweb.httpservice except AttributeError: try: from mercurial import commands + httpservice = commands.httpservice except AttributeError: from mercurial import util @@ -45,33 +49,35 @@ def init(self): util.setsignalhandler() self.httpd = server.create_server(self.ui, self.app) - if self.opts['port'] and not self.ui.verbose: + if self.opts["port"] and not self.ui.verbose: return if self.httpd.prefix: - prefix = self.httpd.prefix.strip('/') + '/' + prefix = self.httpd.prefix.strip("/") + "/" else: - prefix = '' + prefix = "" - port = ':%d' % self.httpd.port - if port == ':80': - port = '' + port = ":%d" % self.httpd.port + if port == ":80": + port = "" bindaddr = self.httpd.addr - if bindaddr == '0.0.0.0': - bindaddr = '*' - elif ':' in bindaddr: # IPv6 - bindaddr = '[%s]' % bindaddr + if bindaddr == "0.0.0.0": + bindaddr = "*" + elif ":" in bindaddr: # IPv6 + bindaddr = "[%s]" % bindaddr fqaddr = self.httpd.fqaddr - if ':' in fqaddr: - fqaddr = '[%s]' % fqaddr - if self.opts['port']: + if ":" in fqaddr: + fqaddr = "[%s]" % fqaddr + if self.opts["port"]: write = self.ui.status else: write = self.ui.write - write('listening at http://%s%s/%s (bound to %s:%d)\n' % - (fqaddr, port, prefix, bindaddr, self.httpd.port)) + write( + "listening at http://%s%s/%s (bound to %s:%d)\n" + % (fqaddr, port, prefix, bindaddr, self.httpd.port) + ) def run(self): self.httpd.serve_forever() @@ -82,18 +88,20 @@ def run(self): def perform_authentication(hgweb, req, op): - if hasattr(req, 'env'): + if hasattr(req, "env"): env = req.env else: env = req.rawenv - if env.get('REQUEST_METHOD') == 'POST': - auth = env.get('HTTP_AUTHORIZATION') + if env.get("REQUEST_METHOD") == "POST": + auth = env.get("HTTP_AUTHORIZATION") if not auth: raise common.ErrorResponse( - common.HTTP_UNAUTHORIZED, 'who', - [('WWW-Authenticate', 'Basic Realm="mercurial"')]) - if base64.b64decode(auth.split()[1]).split(':', 1) != ['foo', 'bar']: - raise common.ErrorResponse(common.HTTP_FORBIDDEN, 'no') + common.HTTP_UNAUTHORIZED, + "who", + [("WWW-Authenticate", 'Basic Realm="mercurial"')], + ) + if base64.b64decode(auth.split()[1]).split(":", 1) != ["foo", "bar"]: + raise common.ErrorResponse(common.HTTP_FORBIDDEN, "no") def extsetup(ui): @@ -104,11 +112,13 @@ def HgLogging(cls): class Logging(cls): # Copied from mercurial's hgweb/server.py. def _log_any(self, fp, format, *args): - message = '%s - - [%s] %s' % (self.client_address[0], - self.log_date_time_string(), - format % args) + '\n' + message = ( + "%s - - [%s] %s" + % (self.client_address[0], self.log_date_time_string(), format % args) + + "\n" + ) if not isinstance(message, bytes): - message = message.encode('utf-8') + message = message.encode("utf-8") fp.write(message) fp.flush() @@ -130,19 +140,19 @@ def do_POST(self): def git_http_backend(self): env = dict(os.environ) - env['REQUEST_METHOD'] = self.command - env['GIT_HTTP_EXPORT_ALL'] = '1' - env['GIT_PROJECT_ROOT'] = os.path.abspath(os.curdir) - path, _, query = self.path.partition('?') - env['PATH_INFO'] = path - env['QUERY_STRING'] = query + env["REQUEST_METHOD"] = self.command + env["GIT_HTTP_EXPORT_ALL"] = "1" + env["GIT_PROJECT_ROOT"] = os.path.abspath(os.curdir) + path, _, query = self.path.partition("?") + env["PATH_INFO"] = path + env["QUERY_STRING"] = query self.send_response(200, "Script output follows") - if hasattr(self, 'flush_headers'): + if hasattr(self, "flush_headers"): self.flush_headers() - if self.command == 'POST': - length = self.headers.get('Content-Length') - env['CONTENT_LENGTH'] = length - env['CONTENT_TYPE'] = self.headers.get('Content-Type') + if self.command == "POST": + length = self.headers.get("Content-Length") + env["CONTENT_LENGTH"] = length + env["CONTENT_TYPE"] = self.headers.get("Content-Type") try: length = int(length) except (TypeError, ValueError): @@ -152,8 +162,9 @@ def git_http_backend(self): else: stdin = None - p = subprocess.Popen(['git', 'http-backend'], stdin=stdin, - stdout=subprocess.PIPE, env=env) + p = subprocess.Popen( + ["git", "http-backend"], stdin=stdin, stdout=subprocess.PIPE, env=env + ) if stdin: p.stdin.write(data) p.stdin.close() @@ -163,46 +174,43 @@ def git_http_backend(self): class OtherServer(object): def __init__(self, typ, ui): - if typ == b'git': + if typ == b"git": cls = GitServer - elif typ == b'http': + elif typ == b"http": cls = HgLogging(SimpleHTTPRequestHandler) else: assert False self.httpd = HTTPServer( - ('', ui.configint(b'serve', b'otherport', 8080)), + ("", ui.configint(b"serve", b"otherport", 8080)), cls, ) - self.httpd.accesslog = openlog( - ui.config(b'web', b'accesslog'), ui.fout) - self.httpd.errorlog = openlog(ui.config(b'web', b'errorlog'), ui.ferr) + self.httpd.accesslog = openlog(ui.config(b"web", b"accesslog"), ui.fout) + self.httpd.errorlog = openlog(ui.config(b"web", b"errorlog"), ui.ferr) def run(self): self.httpd.serve_forever() -@command(b'serve-and-exec', ()) +@command(b"serve-and-exec", ()) def serve_and_exec(ui, repo, *command): - other_server = ui.config(b'serve', b'other', None) + other_server = ui.config(b"serve", b"other", None) if other_server: other_server = OtherServer(other_server, ui) other_server_thread = Thread(target=other_server.run) other_server_thread.start() - ui.setconfig(b'web', b'push_ssl', False, b'hgweb') - ui.setconfig(b'web', b'allow_push', b'*', b'hgweb') + ui.setconfig(b"web", b"push_ssl", False, b"hgweb") + ui.setconfig(b"web", b"allow_push", b"*", b"hgweb") # For older versions of mercurial - repo.baseui.setconfig(b'web', b'push_ssl', False, b'hgweb') - repo.baseui.setconfig(b'web', b'allow_push', b'*', b'hgweb') + repo.baseui.setconfig(b"web", b"push_ssl", False, b"hgweb") + repo.baseui.setconfig(b"web", b"allow_push", b"*", b"hgweb") app = hgweb.hgweb(repo, baseui=ui) - service = httpservice(ui, app, { - b'port': ui.configint(b'web', b'port', 8000), - b'print_url': False - }) + service = httpservice( + ui, app, {b"port": ui.configint(b"web", b"port", 8000), b"print_url": False} + ) service.init() service_thread = Thread(target=service.run) service_thread.start() - ret = subprocess.call( - [getattr(os, "fsdecode", lambda a: a)(a) for a in command]) + ret = subprocess.call([getattr(os, "fsdecode", lambda a: a)(a) for a in command]) service.httpd.shutdown() service_thread.join() if other_server: diff --git a/CI/msys.py b/CI/msys.py index 11e270811..2110ff9d4 100644 --- a/CI/msys.py +++ b/CI/msys.py @@ -14,21 +14,21 @@ from docker import DockerImage -CPUS = ('x86_64',) +CPUS = ("x86_64",) MSYS_VERSION = { - 'x86_64': '20230318', + "x86_64": "20230318", } def mingw(cpu): return { - 'x86_64': 'MINGW64', + "x86_64": "MINGW64", }.get(cpu) def msys(cpu): return { - 'x86_64': 'msys64', + "x86_64": "msys64", }.get(cpu) @@ -38,41 +38,48 @@ def msys_cpu(cpu): def bits(cpu): return { - 'x86_64': '64', + "x86_64": "64", }.get(cpu) class MsysCommon(object): - os = 'windows' + os = "windows" def prepare_params(self, params): - assert 'workerType' not in params - params['workerType'] = 'win2012r2' - params.setdefault('mounts', []).append({'directory': self}) - params.setdefault('env', {})['MSYSTEM'] = mingw(self.cpu) + assert "workerType" not in params + params["workerType"] = "win2012r2" + params.setdefault("mounts", []).append({"directory": self}) + params.setdefault("env", {})["MSYSTEM"] = mingw(self.cpu) command = [] - command.append('set HOME=%CD%') - command.append('set ARTIFACTS=%CD%') - for path in (mingw(self.cpu), 'usr'): - command.append('set PATH=%CD%\\{}\\{}\\bin;%PATH%' - .format(msys(self.cpu), path)) + command.append("set HOME=%CD%") + command.append("set ARTIFACTS=%CD%") + for path in (mingw(self.cpu), "usr"): + command.append( + "set PATH=%CD%\\{}\\{}\\bin;%PATH%".format(msys(self.cpu), path) + ) + command.append("set PATH=%CD%\\git\\{}\\bin;%PATH%".format(mingw(self.cpu))) + if self.PREFIX != "msys": + command.append( + 'bash -c -x "{}"'.format( + "; ".join( + ( + "for postinst in /etc/post-install/*.post", + "do test -e $postinst && . $postinst", + "done", + ) + ) + ) + ) command.append( - 'set PATH=%CD%\\git\\{}\\bin;%PATH%'.format(mingw(self.cpu))) - if self.PREFIX != 'msys': - command.append('bash -c -x "{}"'.format('; '.join(( - 'for postinst in /etc/post-install/*.post', - 'do test -e $postinst && . $postinst', - 'done', - )))) - command.append( - join_command(*bash_command(*params['command']), for_windows=True)) - params['command'] = command + join_command(*bash_command(*params["command"]), for_windows=True) + ) + params["command"] = command return params @property def index(self): - return '.'.join(('env', self.PREFIX, self.cpu, self.hexdigest)) + return ".".join(("env", self.PREFIX, self.cpu, self.hexdigest)) class MsysBase(MsysCommon, Task, metaclass=Tool): @@ -81,10 +88,11 @@ class MsysBase(MsysCommon, Task, metaclass=Tool): def __init__(self, cpu): assert cpu in CPUS _create_command = ( - 'curl -L http://repo.msys2.org/distrib/{cpu}' - '/msys2-base-{cpu}-{version}.tar.xz | xz -cd | zstd -c' - ' > $ARTIFACTS/msys2.tar.zst'.format( - cpu=msys_cpu(cpu), version=MSYS_VERSION[cpu]) + "curl -L http://repo.msys2.org/distrib/{cpu}" + "/msys2-base-{cpu}-{version}.tar.xz | xz -cd | zstd -c" + " > $ARTIFACTS/msys2.tar.zst".format( + cpu=msys_cpu(cpu), version=MSYS_VERSION[cpu] + ) ) h = hashlib.sha1(_create_command.encode()) self.hexdigest = h.hexdigest() @@ -92,12 +100,12 @@ def __init__(self, cpu): Task.__init__( self, - task_env=DockerImage.by_name('base'), - description='msys2 image: base {}'.format(cpu), + task_env=DockerImage.by_name("base"), + description="msys2 image: base {}".format(cpu), index=self.index, - expireIn='26 weeks', + expireIn="26 weeks", command=[_create_command], - artifact='msys2.tar.zst', + artifact="msys2.tar.zst", ) @@ -105,65 +113,70 @@ class MsysEnvironment(MsysCommon): def __init__(self, name): cpu = self.cpu create_commands = [ - 'pacman-key --init', - 'pacman-key --populate msys2', - 'pacman-key --refresh', - 'pacman --noconfirm -Sy procps tar {}'.format( - ' '.join(self.packages(name))), - 'pkill gpg-agent', - 'pkill dirmngr', - 'rm -rf /var/cache/pacman/pkg', - 'python3 -m pip install pip==22.2.2 wheel==0.37.1 --upgrade', - 'mv {}/{}/bin/{{mingw32-,}}make.exe'.format(msys(cpu), mingw(cpu)), - 'tar -c --hard-dereference {} | zstd -c > msys2.tar.zst'.format( - msys(cpu)), + "pacman-key --init", + "pacman-key --populate msys2", + "pacman-key --refresh", + "pacman --noconfirm -Sy procps tar {}".format( + " ".join(self.packages(name)) + ), + "pkill gpg-agent", + "pkill dirmngr", + "rm -rf /var/cache/pacman/pkg", + "python3 -m pip install pip==22.2.2 wheel==0.37.1 --upgrade", + "mv {}/{}/bin/{{mingw32-,}}make.exe".format(msys(cpu), mingw(cpu)), + "tar -c --hard-dereference {} | zstd -c > msys2.tar.zst".format(msys(cpu)), ] env = MsysBase.by_name(cpu) h = hashlib.sha1(env.hexdigest.encode()) - h.update(';'.join(create_commands).encode()) + h.update(";".join(create_commands).encode()) self.hexdigest = h.hexdigest() Task.__init__( self, task_env=env, - description='msys2 image: {} {}'.format(name, cpu), + description="msys2 image: {} {}".format(name, cpu), index=self.index, - expireIn='26 weeks', + expireIn="26 weeks", command=create_commands, - artifact='msys2.tar.zst', + artifact="msys2.tar.zst", ) def packages(self, name): def mingw_packages(pkgs): - return [ - 'mingw-w64-{}-{}'.format(msys_cpu(self.cpu), pkg) - for pkg in pkgs + return ["mingw-w64-{}-{}".format(msys_cpu(self.cpu), pkg) for pkg in pkgs] + + packages = mingw_packages( + [ + "curl", + "make", + "python3", + "python3-pip", ] + ) - packages = mingw_packages([ - 'curl', - 'make', - 'python3', - 'python3-pip', - ]) - - if name == 'build': - return packages + mingw_packages([ - 'gcc', - ]) + [ - 'patch', - ] - elif name == 'test': + if name == "build": + return ( + packages + + mingw_packages( + [ + "gcc", + ] + ) + + [ + "patch", + ] + ) + elif name == "test": return packages + [ - 'diffutils', - 'git', + "diffutils", + "git", ] - raise Exception('Unknown name: {}'.format(name)) + raise Exception("Unknown name: {}".format(name)) class Msys64Environment(MsysEnvironment, Task, metaclass=TaskEnvironment): - PREFIX = 'mingw64' - cpu = 'x86_64' + PREFIX = "mingw64" + cpu = "x86_64" __init__ = MsysEnvironment.__init__ diff --git a/CI/osx.py b/CI/osx.py index bebf3e40e..05c8d9add 100644 --- a/CI/osx.py +++ b/CI/osx.py @@ -11,57 +11,56 @@ class OsxCommon(object): - os = 'macos' - cpu = 'x86_64' + os = "macos" + cpu = "x86_64" def __init__(self, name): - self.hexdigest = hashlib.sha1( - self.ITERATION.encode('utf-8')).hexdigest() + self.hexdigest = hashlib.sha1(self.ITERATION.encode("utf-8")).hexdigest() self.name = name def prepare_params(self, params): - assert 'workerType' not in params - params['provisionerId'] = 'proj-git-cinnabar' - params['workerType'] = self.WORKER_TYPE + assert "workerType" not in params + params["provisionerId"] = "proj-git-cinnabar" + params["workerType"] = self.WORKER_TYPE command = [] - command.append('export PWD=$(pwd)') - command.append('export ARTIFACTS=$PWD') - command.extend(params['command']) - params['command'] = bash_command(*command) - env = params.setdefault('env', {}) + command.append("export PWD=$(pwd)") + command.append("export ARTIFACTS=$PWD") + command.extend(params["command"]) + params["command"] = bash_command(*command) + env = params.setdefault("env", {}) dev = env.setdefault( - 'DEVELOPER_DIR', - '/Applications/Xcode_13.2.1.app/Contents/Developer') + "DEVELOPER_DIR", "/Applications/Xcode_13.2.1.app/Contents/Developer" + ) env.setdefault( - 'SDKROOT', - '{}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.1.sdk' - .format(dev)) + "SDKROOT", + "{}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.1.sdk".format(dev), + ) return params class Osx(OsxCommon, metaclass=TaskEnvironment): - ITERATION = '4' - PREFIX = 'osx' - WORKER_TYPE = 'osx' - os_version = '10.15' + ITERATION = "4" + PREFIX = "osx" + WORKER_TYPE = "osx" + os_version = "10.15" class OsxArm64(OsxCommon, metaclass=TaskEnvironment): - cpu = 'arm64' - ITERATION = '2' - PREFIX = 'arm64-osx' - WORKER_TYPE = 'macos' - os_version = '11.0' + cpu = "arm64" + ITERATION = "2" + PREFIX = "arm64-osx" + WORKER_TYPE = "macos" + os_version = "11.0" def prepare_params(self, params): - env = params.setdefault('env', {}) + env = params.setdefault("env", {}) dev = env.setdefault( - 'DEVELOPER_DIR', - '/Applications/Xcode_15.0.1.app/Contents/Developer') + "DEVELOPER_DIR", "/Applications/Xcode_15.0.1.app/Contents/Developer" + ) env.setdefault( - 'SDKROOT', - '{}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.0.sdk' - .format(dev)) - env.setdefault('PIP_DISABLE_PIP_VERSION_CHECK', '1') - params['command'].insert(0, 'export PATH=$PATH:/opt/homebrew/bin') + "SDKROOT", + "{}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.0.sdk".format(dev), + ) + env.setdefault("PIP_DISABLE_PIP_VERSION_CHECK", "1") + params["command"].insert(0, "export PATH=$PATH:/opt/homebrew/bin") return super(OsxArm64, self).prepare_params(params) diff --git a/CI/tasks.py b/CI/tasks.py index c5431ef9d..812bcd36a 100644 --- a/CI/tasks.py +++ b/CI/tasks.py @@ -17,7 +17,7 @@ from variables import * # noqa: F403 -if os.environ.get('DETERMINISTIC'): +if os.environ.get("DETERMINISTIC"): from uuid import UUID import random @@ -31,8 +31,8 @@ def uuid4(): # noqa: F811 def slugid(): rawBytes = bytearray(uuid4().bytes) # Ensure base64-encoded bytes start with [A-Za-f] - if rawBytes[0] >= 0xd0: - rawBytes[0] = rawBytes[0] & 0x7f + if rawBytes[0] >= 0xD0: + rawBytes[0] = rawBytes[0] & 0x7F result = base64.urlsafe_b64encode(rawBytes)[:-2] # Drop '==' padding return result.decode() @@ -43,10 +43,10 @@ def slugid(): class datetime(datetime.datetime): def format(self, no_usec=True): if no_usec: - return self.replace(microsecond=0).isoformat() + 'Z' + return self.replace(microsecond=0).isoformat() + "Z" if self.microsecond == 0: - return self.isoformat() + '.000000Z' - return self.isoformat() + 'Z' + return self.isoformat() + ".000000Z" + return self.isoformat() + "Z" def __add__(self, other): if isinstance(other, numbers.Number): @@ -55,24 +55,20 @@ def __add__(self, other): return self.combine(d.date(), d.timetz()) -task_group_id = (os.environ.get('TC_GROUP_ID') or - os.environ.get('TASK_ID') or slugid()) -if os.environ.get('DETERMINISTIC'): +task_group_id = os.environ.get("TC_GROUP_ID") or os.environ.get("TASK_ID") or slugid() +if os.environ.get("DETERMINISTIC"): now = datetime.fromtimestamp(0) else: now = datetime.utcnow() def index_env(idx): - return 'INDEX_{}'.format( - idx.replace('.', '_').replace('-', '_').upper() - ) + return "INDEX_{}".format(idx.replace(".", "_").replace("-", "_").upper()) def expires_soon(expires): try: - expires = datetime.strptime( - expires.rstrip('Z'), '%Y-%m-%dT%H:%M:%S.%f') + expires = datetime.strptime(expires.rstrip("Z"), "%Y-%m-%dT%H:%M:%S.%f") return expires < now + 86400 except (KeyError, ValueError): return True @@ -94,7 +90,7 @@ class Existing(str): def __init__(self, requests=requests): super(Index, self).__init__() - self.requests = None if os.environ.get('NO_INDEX') else requests + self.requests = None if os.environ.get("NO_INDEX") else requests def __missing__(self, key): result = None @@ -104,7 +100,7 @@ def __missing__(self, key): elif hint is not None: # empty environment variable pass else: - result = self._try_key('project.git-cinnabar.{}'.format(key)) + result = self._try_key("project.git-cinnabar.{}".format(key)) if not result: result = slugid() self[key] = result @@ -114,8 +110,8 @@ def _try_key(self, key, create=False): if not self.requests: return data = http_get(self.requests, PROXY_INDEX_URL.format(key)) - if data and not expires_soon(data['expires']): - result = data.get('taskId') + if data and not expires_soon(data["expires"]): + result = data.get("taskId") print('Found task "{}" for "{}"'.format(result, key)) return self.Existing(result) @@ -136,13 +132,13 @@ class TaskNamespace(type): def by_name(cls, fqdn): env = cls._namespace.get(fqdn) if not env: - n = fqdn.split('.') + n = fqdn.split(".") prefix = n[:-1] name = n[-1:] while prefix: - kls = cls._namespace.get('.'.join(prefix)) + kls = cls._namespace.get(".".join(prefix)) if isinstance(kls, type): - cls._namespace[fqdn] = env = kls('.'.join(name)) + cls._namespace[fqdn] = env = kls(".".join(name)) break name.insert(0, prefix.pop()) return env @@ -150,11 +146,12 @@ def by_name(cls, fqdn): def __new__(cls, name, bases, dic): @classmethod def by_name(kls, name): - return cls.by_name('.'.join((kls.PREFIX, name))) - dic['by_name'] = by_name + return cls.by_name(".".join((kls.PREFIX, name))) + + dic["by_name"] = by_name kls = super(TaskNamespace, cls).__new__(cls, name, bases, dic) - cls._namespace[dic['PREFIX']] = kls + cls._namespace[dic["PREFIX"]] = kls return kls @@ -173,134 +170,133 @@ class Task(object): @staticmethod def normalize_params(params): try: - artifact = params.pop('artifact') - assert 'artifacts' not in params - params['artifacts'] = [artifact] + artifact = params.pop("artifact") + assert "artifacts" not in params + params["artifacts"] = [artifact] except KeyError: pass return params try: - mount = params.pop('mount') - assert 'mounts' not in params - params['mounts'] = [mount] + mount = params.pop("mount") + assert "mounts" not in params + params["mounts"] = [mount] except KeyError: pass @staticmethod - def checkout(repo=None, commit=None, dest='repo'): + def checkout(repo=None, commit=None, dest="repo"): repo = repo or TC_REPO_URL commit = commit or TC_COMMIT return [ - 'git clone -n {} {}'.format(repo, dest), - 'git -c core.autocrlf=input -c advice.detachedHead=false' - ' -C {} checkout {}'.format(dest, commit), + "git clone -n {} {}".format(repo, dest), + "git -c core.autocrlf=input -c advice.detachedHead=false" + " -C {} checkout {}".format(dest, commit), ] def __init__(self, **kwargs): - task_env = kwargs.pop('task_env', None) + task_env = kwargs.pop("task_env", None) kwargs = self.normalize_params(kwargs) if task_env: kwargs = task_env.prepare_params(kwargs) - maxRunTime = kwargs.pop('maxRunTime', 1800) + maxRunTime = kwargs.pop("maxRunTime", 1800) task = { - 'created': now.format(), - 'deadline': (now + maxRunTime * 5 + 1800).format(), - 'retries': 5, - 'provisionerId': 'proj-git-cinnabar', - 'workerType': 'linux', - 'schedulerId': 'taskcluster-github', - 'taskGroupId': task_group_id, - 'metadata': { - 'owner': '{}@users.noreply.github.com'.format(TC_LOGIN), - 'source': TC_REPO_URL, + "created": now.format(), + "deadline": (now + maxRunTime * 5 + 1800).format(), + "retries": 5, + "provisionerId": "proj-git-cinnabar", + "workerType": "linux", + "schedulerId": "taskcluster-github", + "taskGroupId": task_group_id, + "metadata": { + "owner": "{}@users.noreply.github.com".format(TC_LOGIN), + "source": TC_REPO_URL, }, - 'payload': { - 'maxRunTime': maxRunTime, + "payload": { + "maxRunTime": maxRunTime, }, } - kwargs.setdefault('expireIn', '4 weeks') - dependencies = [os.environ.get('TASK_ID') or task_group_id] + kwargs.setdefault("expireIn", "4 weeks") + dependencies = [os.environ.get("TASK_ID") or task_group_id] artifact_paths = [] for k, v in kwargs.items(): - if k in ('provisionerId', 'workerType', 'priority'): + if k in ("provisionerId", "workerType", "priority"): task[k] = v - elif k == 'description': - task['metadata'][k] = task['metadata']['name'] = v - elif k == 'index': + elif k == "description": + task["metadata"][k] = task["metadata"]["name"] = v + elif k == "index": if TC_IS_PUSH and TC_BRANCH != "try": - task['routes'] = [ - 'index.project.git-cinnabar.{}'.format(v)] - elif k == 'expireIn': + task["routes"] = ["index.project.git-cinnabar.{}".format(v)] + elif k == "expireIn": value = v.split() if len(value) == 1: value, multiplier = value, 1 elif len(value) == 2: value, unit = value value = int(value) - unit = unit.rstrip('s') + unit = unit.rstrip("s") multiplier = 1 - if unit == 'year': + if unit == "year": multiplier *= 365 - unit = 'day' - if unit == 'week': + unit = "day" + if unit == "week": multiplier *= 7 - unit = 'day' - if unit == 'day': + unit = "day" + if unit == "day": multiplier *= 24 - unit = 'hour' - if unit == 'hour': + unit = "hour" + if unit == "hour": multiplier *= 60 - unit = 'minute' - if unit == 'minute': + unit = "minute" + if unit == "minute": multiplier *= 60 - unit = 'second' - if unit == 'second': - unit = '' + unit = "second" + if unit == "second": + unit = "" if unit: - raise Exception( - "Don't know how to handle {}".format(unit)) + raise Exception("Don't know how to handle {}".format(unit)) else: raise Exception("Don't know how to handle {}".format(v)) if not TC_IS_PUSH or TC_BRANCH == "try": if value * multiplier > 4 * 7 * 24 * 60 * 60: value = 4 multiplier = 7 * 24 * 60 * 60 # weeks - task['expires'] = (now + value * multiplier).format() - elif k == 'command': - task['payload']['command'] = v - if not kwargs.get('workerType', '').startswith('win'): - task['payload']['command'] = [task['payload']['command']] + task["expires"] = (now + value * multiplier).format() + elif k == "command": + task["payload"]["command"] = v + if not kwargs.get("workerType", "").startswith("win"): + task["payload"]["command"] = [task["payload"]["command"]] - elif k == 'artifacts': + elif k == "artifacts": artifacts = [ { - 'name': 'public/{}'.format(os.path.basename(a)), - 'path': a, - 'type': 'file', - } for a in v + "name": "public/{}".format(os.path.basename(a)), + "path": a, + "type": "file", + } + for a in v ] - artifact_paths.extend(a['name'] for a in artifacts) - task['payload']['artifacts'] = artifacts - elif k == 'env': - task['payload'].setdefault('env', {}).update(v) - elif k == 'scopes': + artifact_paths.extend(a["name"] for a in artifacts) + task["payload"]["artifacts"] = artifacts + elif k == "env": + task["payload"].setdefault("env", {}).update(v) + elif k == "scopes": task[k] = v for s in v: - if s.startswith('secrets:'): - features = task['payload'].setdefault('features', {}) - features['taskclusterProxy'] = True - elif k == 'mounts': + if s.startswith("secrets:"): + features = task["payload"].setdefault("features", {}) + features["taskclusterProxy"] = True + elif k == "mounts": + def file_format(url): - for ext in ('rar', 'tar.zst', 'tar.bz2', 'tar.gz', 'zip'): - if url.endswith('.{}'.format(ext)): + for ext in ("rar", "tar.zst", "tar.bz2", "tar.gz", "zip"): + if url.endswith(".{}".format(ext)): return ext - raise Exception( - 'Unsupported/unknown format for {}'.format(url)) + raise Exception("Unsupported/unknown format for {}".format(url)) - mounts = task['payload']['mounts'] = [] + mounts = task["payload"]["mounts"] = [] for m in v: assert isinstance(m, dict) m = list(m.items()) @@ -308,57 +304,59 @@ def file_format(url): kind, m = m[0] if isinstance(m, Task): content = { - 'artifact': m.artifacts[0], - 'taskId': m.id, + "artifact": m.artifacts[0], + "taskId": m.id, } dependencies.append(m.id) elif isinstance(m, dict): content = m - dependencies.append(m['taskId']) + dependencies.append(m["taskId"]) else: content = { - 'url': m, + "url": m, } - artifact = content.get('artifact') or content['url'] + artifact = content.get("artifact") or content["url"] if kind == "file" or kind.startswith("file:"): mount = { - 'content': content, - 'file': kind[5:] or os.path.basename(artifact), + "content": content, + "file": kind[5:] or os.path.basename(artifact), } - if kind[5:] == 'dockerimage': - mount['format'] = os.path.splitext( - content['artifact'])[-1].replace('.', '') + if kind[5:] == "dockerimage": + mount["format"] = os.path.splitext(content["artifact"])[ + -1 + ].replace(".", "") mounts.append(mount) elif kind == "directory" or kind.startswith("directory:"): - mounts.append({ - 'content': content, - 'directory': os.path.dirname(kind[10:]) or '.', - 'format': file_format(artifact), - }) - elif k == 'dependencies': + mounts.append( + { + "content": content, + "directory": os.path.dirname(kind[10:]) or ".", + "format": file_format(artifact), + } + ) + elif k == "dependencies": for t in v: dependencies.append(t.id) else: raise Exception("Don't know how to handle {}".format(k)) - task['dependencies'] = sorted(dependencies) - index = kwargs.get('index') + task["dependencies"] = sorted(dependencies) + index = kwargs.get("index") id = None - if index and all(isinstance(d, Index.Existing) - for d in dependencies[1:]): + if index and all(isinstance(d, Index.Existing) for d in dependencies[1:]): id = Task.by_index[index] if isinstance(id, Index.Existing): - data = http_get( - session, ARTIFACT_URL.format(id, '').rstrip('/')) or {} + data = http_get(session, ARTIFACT_URL.format(id, "").rstrip("/")) or {} artifacts_expire = [ - expires_soon(a.get('expires')) - for a in data.get('artifacts', []) - if a.get('name') in artifact_paths + expires_soon(a.get("expires")) + for a in data.get("artifacts", []) + if a.get("name") in artifact_paths ] - if len(artifact_paths) != len(artifacts_expire) \ - or any(artifacts_expire): + if len(artifact_paths) != len(artifacts_expire) or any(artifacts_expire): print( - 'Ignore task "{}" because of missing or expiring artifacts' - .format(id)) + 'Ignore task "{}" because of missing or expiring artifacts'.format( + id + ) + ) id = None self.id = id or slugid() @@ -380,40 +378,40 @@ def submit(self): return print('Submitting task "{}":'.format(self.id)) print(json.dumps(self.task, indent=4, sort_keys=True)) - if 'TC_PROXY' not in os.environ: + if "TC_PROXY" not in os.environ: return - url = f'{PROXY_URL}/api/queue/v1/task/{self.id}' + url = f"{PROXY_URL}/api/queue/v1/task/{self.id}" res = session.put(url, json=self.task) try: res.raise_for_status() except Exception: print(res.headers) try: - print(res.json()['message']) + print(res.json()["message"]) except Exception: print(res.content) raise print(res.json()) -SHELL_QUOTE_RE = re.compile(r'[\\\t\r\n \'\"#<>&|`~(){}$;\*\?]') +SHELL_QUOTE_RE = re.compile(r"[\\\t\r\n \'\"#<>&|`~(){}$;\*\?]") def _quote(s, for_windows=False): if s and not SHELL_QUOTE_RE.search(s): return s if for_windows: - for c in '^&\\<>|': - s = s.replace(c, '^' + c) + for c in "^&\\<>|": + s = s.replace(c, "^" + c) return "'{}'".format(s.replace("'", "'\\''")) def join_command(*command, for_windows=False): - return ' '.join(_quote(a, for_windows) for a in command) + return " ".join(_quote(a, for_windows) for a in command) def bash_command(*commands): - return ['bash', '-c', '-x', '-e', '; '.join(commands)] + return ["bash", "-c", "-x", "-e", "; ".join(commands)] class action(object): @@ -430,25 +428,31 @@ def __init__(self, name, title=None, description=None): if self.template is None: import yaml - with open(os.path.join(os.path.dirname(__file__), '..', - '.taskcluster.yml')) as fh: + + with open( + os.path.join(os.path.dirname(__file__), "..", ".taskcluster.yml") + ) as fh: contents = yaml.safe_load(fh) - task = contents['tasks'][0]['then']['in'] - del task['taskId'] + task = contents["tasks"][0]["then"]["in"] + del task["taskId"] self.__class__.template = task def adjust(s): - return s.replace('decision', 'action') + ' ({})'.format(title) + return s.replace("decision", "action") + " ({})".format(title) - metadata = self.template['metadata'] + metadata = self.template["metadata"] self.task = dict( self.template, - payload=dict(self.template['payload'], - env=dict(self.template['payload']['env'], - TC_ACTION=name)), - metadata=dict(metadata, - name=adjust(metadata['name']), - description=adjust(metadata['description']))) + payload=dict( + self.template["payload"], + env=dict(self.template["payload"]["env"], TC_ACTION=name), + ), + metadata=dict( + metadata, + name=adjust(metadata["name"]), + description=adjust(metadata["description"]), + ), + ) def __call__(self, func): self.func = func diff --git a/CI/tools.py b/CI/tools.py index 81434a8e2..b3ac2c5a1 100644 --- a/CI/tools.py +++ b/CI/tools.py @@ -16,24 +16,68 @@ import msys -MERCURIAL_VERSION = '6.8' +MERCURIAL_VERSION = "6.8" # Not using 2.46.0 because of # https://lore.kernel.org/git/20240727191917.p64ul4jybpm2a7hm@glandium.org/ -GIT_VERSION = '2.45.2' +GIT_VERSION = "2.45.2" ALL_MERCURIAL_VERSIONS = ( - '1.9.3', '2.0.2', '2.1.2', '2.2.3', '2.3.2', '2.4.2', '2.5.4', - '2.6.3', '2.7.2', '2.8.2', '2.9.1', '3.0.1', '3.1.2', '3.2.4', - '3.3.3', '3.4.2', '3.5.2', '3.6.3', '3.7.3', '3.8.4', '3.9.2', - '4.0.2', '4.1.3', '4.2.2', '4.3.3', '4.4.2', '4.5.3', '4.6.2', - '4.7.2', '4.8.2', '4.9.1', '5.0.2', '5.1.2', '5.2.2', '5.3.2', - '5.4.2', '5.5.2', '5.6.1', '5.7.1', '5.8.1', '5.9.3', '6.0.3', - '6.1.4', '6.2.3', '6.3.3', '6.4.2', '6.5.3', '6.6.3', '6.7.4', - '6.8', + "1.9.3", + "2.0.2", + "2.1.2", + "2.2.3", + "2.3.2", + "2.4.2", + "2.5.4", + "2.6.3", + "2.7.2", + "2.8.2", + "2.9.1", + "3.0.1", + "3.1.2", + "3.2.4", + "3.3.3", + "3.4.2", + "3.5.2", + "3.6.3", + "3.7.3", + "3.8.4", + "3.9.2", + "4.0.2", + "4.1.3", + "4.2.2", + "4.3.3", + "4.4.2", + "4.5.3", + "4.6.2", + "4.7.2", + "4.8.2", + "4.9.1", + "5.0.2", + "5.1.2", + "5.2.2", + "5.3.2", + "5.4.2", + "5.5.2", + "5.6.1", + "5.7.1", + "5.8.1", + "5.9.3", + "6.0.3", + "6.1.4", + "6.2.3", + "6.3.3", + "6.4.2", + "6.5.3", + "6.6.3", + "6.7.4", + "6.8", ) SOME_MERCURIAL_VERSIONS = ( - '1.9.3', '2.5.4', '3.4.2', + "1.9.3", + "2.5.4", + "3.4.2", ) assert MERCURIAL_VERSION in ALL_MERCURIAL_VERSIONS @@ -41,97 +85,106 @@ def nproc(env): - if env.os == 'macos': - return 'sysctl -n hw.physicalcpu' - return 'nproc --all' + if env.os == "macos": + return "sysctl -n hw.physicalcpu" + return "nproc --all" class Git(Task, metaclass=Tool): PREFIX = "git" def __init__(self, os_and_version): - (os, version) = os_and_version.split('.', 1) + (os, version) = os_and_version.split(".", 1) self.os = os - if os.endswith('osx'): - build_image = TaskEnvironment.by_name('{}.build'.format(os)) + if os.endswith("osx"): + build_image = TaskEnvironment.by_name("{}.build".format(os)) else: - build_image = DockerImage.by_name('build-tools') - if os == 'linux' or os.endswith('osx'): + build_image = DockerImage.by_name("build-tools") + if os == "linux" or os.endswith("osx"): h = hashlib.sha1(build_image.hexdigest.encode()) - h.update(b'v4' if version == GIT_VERSION else b'v3') - if os == 'linux': - description = 'git v{}'.format(version) + h.update(b"v4" if version == GIT_VERSION else b"v3") + if os == "linux": + description = "git v{}".format(version) else: env = build_image - description = 'git v{} {} {}'.format(version, env.os, env.cpu) + description = "git v{} {} {}".format(version, env.os, env.cpu) Task.__init__( self, task_env=build_image, description=description, - index='{}.git.v{}'.format(h.hexdigest(), version), - expireIn='26 weeks', + index="{}.git.v{}".format(h.hexdigest(), version), + expireIn="26 weeks", command=Task.checkout( - 'git://git.kernel.org/pub/scm/git/git.git', - 'v{}'.format(version), - dest='git', - ) + Task.checkout() + ([ - 'patch -d git -p1 < repo/CI/git-transport-disconnect.diff', - ] if version == GIT_VERSION else []) + [ - 'make -C git -j$({}) install prefix=/ NO_GETTEXT=1' - ' NO_OPENSSL=1 NO_TCLTK=1 NO_UNCOMPRESS2=1' - ' DESTDIR=$PWD/git'.format( - nproc(build_image)), - 'tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst' - .format(version), + "git://git.kernel.org/pub/scm/git/git.git", + "v{}".format(version), + dest="git", + ) + + Task.checkout() + + ( + [ + "patch -d git -p1 < repo/CI/git-transport-disconnect.diff", + ] + if version == GIT_VERSION + else [] + ) + + [ + "make -C git -j$({}) install prefix=/ NO_GETTEXT=1" + " NO_OPENSSL=1 NO_TCLTK=1 NO_UNCOMPRESS2=1" + " DESTDIR=$PWD/git".format(nproc(build_image)), + "tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst".format(version), ], - artifact='git-{}.tar.zst'.format(version), + artifact="git-{}.tar.zst".format(version), ) else: - env = TaskEnvironment.by_name('{}.build'.format(os)) + env = TaskEnvironment.by_name("{}.build".format(os)) raw_version = version - if 'windows' not in version: + if "windows" not in version: version = { - version: version + '.windows.1', - '2.17.1': '2.17.1.windows.2', + version: version + ".windows.1", + "2.17.1": "2.17.1.windows.2", }.get(version) - if version.endswith('.windows.1'): - min_ver = version[:-len('.windows.1')] + if version.endswith(".windows.1"): + min_ver = version[: -len(".windows.1")] else: - min_ver = version.replace('windows.', '') + min_ver = version.replace("windows.", "") h = hashlib.sha1(env.hexdigest.encode()) - h.update(b'v1') + h.update(b"v1") Task.__init__( self, task_env=build_image, - description='git v{} {} {}'.format(version, env.os, env.cpu), - index='{}.git.v{}'.format(h.hexdigest(), raw_version), - expireIn='26 weeks', + description="git v{} {} {}".format(version, env.os, env.cpu), + index="{}.git.v{}".format(h.hexdigest(), raw_version), + expireIn="26 weeks", command=[ - 'curl -L https://github.com/git-for-windows/git/releases/' - 'download/v{}/MinGit-{}-{}-bit.zip' - ' -o git.zip'.format(version, min_ver, msys.bits(env.cpu)), - 'unzip -d git git.zip', - 'curl -L https://github.com/git-for-windows/git/releases/' - 'download/v{}/Git-{}-{}-bit.tar.bz2 | ' - 'tar -C git --no-same-owner -jx ' - '{}/libexec/git-core/git-http-backend.exe' - .format(version, min_ver, msys.bits(env.cpu), - msys.mingw(env.cpu).lower()), - 'tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst'.format( - raw_version), + "curl -L https://github.com/git-for-windows/git/releases/" + "download/v{}/MinGit-{}-{}-bit.zip" + " -o git.zip".format(version, min_ver, msys.bits(env.cpu)), + "unzip -d git git.zip", + "curl -L https://github.com/git-for-windows/git/releases/" + "download/v{}/Git-{}-{}-bit.tar.bz2 | " + "tar -C git --no-same-owner -jx " + "{}/libexec/git-core/git-http-backend.exe".format( + version, + min_ver, + msys.bits(env.cpu), + msys.mingw(env.cpu).lower(), + ), + "tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst".format( + raw_version + ), ], - artifact='git-{}.tar.zst'.format(raw_version), + artifact="git-{}.tar.zst".format(raw_version), ) def mount(self): - return {'directory:git': self} + return {"directory:git": self} def install(self): - if self.os.endswith(('linux', 'osx')): + if self.os.endswith(("linux", "osx")): return [ - 'export PATH=$PWD/git/bin:$PATH', - 'export GIT_EXEC_PATH=$PWD/git/libexec/git-core', - 'export GIT_TEMPLATE_DIR=$PWD/git/share/git-core/templates', + "export PATH=$PWD/git/bin:$PATH", + "export GIT_EXEC_PATH=$PWD/git/libexec/git-core", + "export GIT_TEMPLATE_DIR=$PWD/git/share/git-core/templates", ] else: return [] @@ -141,63 +194,66 @@ class Hg(Task, metaclass=Tool): PREFIX = "hg" def __init__(self, os_and_version): - (os, version) = os_and_version.split('.', 1) - (version, suffix, _) = version.partition('.py3') - if suffix or len(version) == 40 or \ - parse_version(version) >= parse_version('6.2'): - python = 'python3' + (os, version) = os_and_version.split(".", 1) + (version, suffix, _) = version.partition(".py3") + if ( + suffix + or len(version) == 40 + or parse_version(version) >= parse_version("6.2") + ): + python = "python3" else: - python = 'python2.7' - if os == 'linux': - env = TaskEnvironment.by_name('{}.build-tools'.format(os)) + python = "python2.7" + if os == "linux": + env = TaskEnvironment.by_name("{}.build-tools".format(os)) else: - env = TaskEnvironment.by_name('{}.build'.format(os)) + env = TaskEnvironment.by_name("{}.build".format(os)) kwargs = {} if len(version) == 40: # Assume it's a sha1 - pretty_version = 'r{}{}'.format(version, suffix) - artifact_version = '99.0' - expire = '2 weeks' + pretty_version = "r{}{}".format(version, suffix) + artifact_version = "99.0" + expire = "2 weeks" else: - pretty_version = 'v{}{}'.format(version, suffix) + pretty_version = "v{}{}".format(version, suffix) artifact_version = version - expire = '26 weeks' - desc = 'hg {}'.format(pretty_version) - if os == 'linux': - platform_tag = 'linux_x86_64' - if python == 'python3': - python_tag = 'cp39' - abi_tag = 'cp39' + expire = "26 weeks" + desc = "hg {}".format(pretty_version) + if os == "linux": + platform_tag = "linux_x86_64" + if python == "python3": + python_tag = "cp39" + abi_tag = "cp39" else: - python_tag = 'cp27' - abi_tag = 'cp27mu' + python_tag = "cp27" + abi_tag = "cp27mu" else: - desc = '{} {} {}'.format(desc, env.os, env.cpu) - if os.endswith('osx'): - py_host_plat = 'macosx-{}-{}'.format(env.os_version, env.cpu) - platform_tag = py_host_plat.replace('.', '_').replace('-', '_') - if python == 'python3': - python_tag = 'cp311' if os == 'arm64-osx' else 'cp39' + desc = "{} {} {}".format(desc, env.os, env.cpu) + if os.endswith("osx"): + py_host_plat = "macosx-{}-{}".format(env.os_version, env.cpu) + platform_tag = py_host_plat.replace(".", "_").replace("-", "_") + if python == "python3": + python_tag = "cp311" if os == "arm64-osx" else "cp39" abi_tag = python_tag else: - python_tag = 'cp27' - abi_tag = 'cp27m' - env_ = kwargs.setdefault('env', {}) - env_.setdefault('MACOSX_DEPLOYMENT_TARGET', env.os_version) - env_.setdefault('ARCHFLAGS', '-arch {}'.format(env.cpu)) - env_.setdefault('_PYTHON_HOST_PLATFORM', py_host_plat) + python_tag = "cp27" + abi_tag = "cp27m" + env_ = kwargs.setdefault("env", {}) + env_.setdefault("MACOSX_DEPLOYMENT_TARGET", env.os_version) + env_.setdefault("ARCHFLAGS", "-arch {}".format(env.cpu)) + env_.setdefault("_PYTHON_HOST_PLATFORM", py_host_plat) else: - if python == 'python3': - platform_tag = 'mingw_x86_64' - python_tag = 'cp310' - abi_tag = 'cp310' + if python == "python3": + platform_tag = "mingw_x86_64" + python_tag = "cp310" + abi_tag = "cp310" else: - platform_tag = 'mingw' - python_tag = 'cp27' - abi_tag = 'cp27m' + platform_tag = "mingw" + python_tag = "cp27" + abi_tag = "cp27m" - artifact = 'mercurial-{{}}-{}-{}-{}.whl'.format( + artifact = "mercurial-{{}}-{}-{}-{}.whl".format( python_tag, abi_tag, platform_tag, @@ -205,126 +261,126 @@ def __init__(self, os_and_version): pre_command = [] if len(version) == 40: - hg = self.by_name('{}.{}'.format(os, MERCURIAL_VERSION)) - kwargs.setdefault('mounts', []).append(hg.mount()) + hg = self.by_name("{}.{}".format(os, MERCURIAL_VERSION)) + kwargs.setdefault("mounts", []).append(hg.mount()) pre_command.extend(hg.install()) - pre_command.extend([ - 'hg clone https://www.mercurial-scm.org/repo/hg' - ' -r {} mercurial-{}'.format(version, version), - 'rm -rf mercurial-{}/.hg'.format(version), - 'echo tag: {} > mercurial-{}/.hg_archival.txt' - .format(artifact_version, version), - ]) + pre_command.extend( + [ + "hg clone https://www.mercurial-scm.org/repo/hg" + " -r {} mercurial-{}".format(version, version), + "rm -rf mercurial-{}/.hg".format(version), + "echo tag: {} > mercurial-{}/.hg_archival.txt".format( + artifact_version, version + ), + ] + ) # 2.6.2 is the first version available on pypi - elif parse_version('2.6.2') <= parse_version(version) and \ - parse_version(version) < parse_version('6.2'): + elif parse_version("2.6.2") <= parse_version(version) and parse_version( + version + ) < parse_version("6.2"): # pip download does more than download, and while it runs setup.py # for version 6.2, a DistutilsPlatformError exception is thrown on # Windows. pre_command.append( - '{} -m pip download --no-binary mercurial --no-deps' - ' --progress-bar off mercurial=={}'.format(python, version)) + "{} -m pip download --no-binary mercurial --no-deps" + " --progress-bar off mercurial=={}".format(python, version) + ) else: - url = 'https://mercurial-scm.org/release/mercurial-{}.tar.gz' - pre_command.append( - 'curl -sLO {}'.format(url.format(version))) + url = "https://mercurial-scm.org/release/mercurial-{}.tar.gz" + pre_command.append("curl -sLO {}".format(url.format(version))) if len(version) != 40: - pre_command.append( - 'tar -zxf mercurial-{}.tar.gz'.format(version)) + pre_command.append("tar -zxf mercurial-{}.tar.gz".format(version)) - if os.startswith('mingw'): + if os.startswith("mingw"): # Work around https://bz.mercurial-scm.org/show_bug.cgi?id=6654 # and https://bz.mercurial-scm.org/show_bug.cgi?id=6757 pre_command.append( 'sed -i "s/, output_dir=self.build_temp/' - ', output_dir=self.build_temp, extra_postargs=[$EXTRA_FLAGS]/;' - '/self.addlongpathsmanifest/d;' - '" mercurial-{}/setup.py' - .format(version)) - if python == 'python3': - kwargs.setdefault('env', {}).setdefault( - 'EXTRA_FLAGS', '"-municode"') + ", output_dir=self.build_temp, extra_postargs=[$EXTRA_FLAGS]/;" + "/self.addlongpathsmanifest/d;" + '" mercurial-{}/setup.py'.format(version) + ) + if python == "python3": + kwargs.setdefault("env", {}).setdefault("EXTRA_FLAGS", '"-municode"') pre_command.append( 'sed -i "s/ifdef __GNUC__/if 0/"' - ' mercurial-{}/mercurial/exewrapper.c' - .format(version)) + " mercurial-{}/mercurial/exewrapper.c".format(version) + ) h = hashlib.sha1(env.hexdigest.encode()) h.update(artifact.encode()) - if os.endswith('osx'): - h.update(b'v2') - elif os.startswith('mingw'): - h.update(b'v4') + if os.endswith("osx"): + h.update(b"v2") + elif os.startswith("mingw"): + h.update(b"v4") else: - h.update(b'v1') + h.update(b"v1") Task.__init__( self, task_env=env, description=desc, - index='{}.hg.{}'.format(h.hexdigest(), pretty_version), + index="{}.hg.{}".format(h.hexdigest(), pretty_version), expireIn=expire, - command=pre_command + [ + command=pre_command + + [ # pyproject.toml enables PEP 517, which can't be disabled. # pip wheel doesn't accept --build-option when PEP 517 is # enabled. --build-option is necessary on msys2 because # of problems with the bdist-dir otherwise. - 'rm -f mercurial-{}/pyproject.toml'.format(version), - '{} -m pip wheel -v --build-option -b --build-option' - ' $PWD/wheel -w $ARTIFACTS ./mercurial-{}'.format( - python, - version), + "rm -f mercurial-{}/pyproject.toml".format(version), + "{} -m pip wheel -v --build-option -b --build-option" + " $PWD/wheel -w $ARTIFACTS ./mercurial-{}".format(python, version), ], artifact=artifact.format(artifact_version), - **kwargs + **kwargs, ) def mount(self): - return {f'file:{os.path.basename(self.artifacts[0])}': self} + return {f"file:{os.path.basename(self.artifacts[0])}": self} def install(self): filename = os.path.basename(self.artifacts[0]) - if 'cp3' in filename: - python = 'python3' + if "cp3" in filename: + python = "python3" else: - python = 'python2.7' - return [ - '{} -m pip install {}'.format(python, filename) - ] + python = "python2.7" + return ["{} -m pip install {}".format(python, filename)] -def install_rust(version='1.80.0', target='x86_64-unknown-linux-gnu'): - rustup_opts = '-y --default-toolchain none' - cargo_dir = '$HOME/.cargo/bin/' - rustup = cargo_dir + 'rustup' +def install_rust(version="1.80.0", target="x86_64-unknown-linux-gnu"): + rustup_opts = "-y --default-toolchain none" + cargo_dir = "$HOME/.cargo/bin/" + rustup = cargo_dir + "rustup" rust_install = [ - 'curl -o rustup.sh https://sh.rustup.rs', - 'sh rustup.sh {rustup_opts}', - '{rustup} install {version} --profile minimal', - '{rustup} default {version}', - 'PATH={cargo_dir}:$PATH', - '{rustup} target add {target}', + "curl -o rustup.sh https://sh.rustup.rs", + "sh rustup.sh {rustup_opts}", + "{rustup} install {version} --profile minimal", + "{rustup} default {version}", + "PATH={cargo_dir}:$PATH", + "{rustup} target add {target}", ] loc = locals() return [r.format(**loc) for r in rust_install] class Build(Task, metaclass=Tool): - PREFIX = 'build' + PREFIX = "build" def __init__(self, os_and_variant): - os, variant = (os_and_variant.split('.', 1) + [''])[:2] + os, variant = (os_and_variant.split(".", 1) + [""])[:2] env = TaskEnvironment.by_name( - '{}.build'.format(os.replace('arm64-linux', 'linux'))) - if os.startswith('mingw'): - build_env = TaskEnvironment.by_name('linux.build') + "{}.build".format(os.replace("arm64-linux", "linux")) + ) + if os.startswith("mingw"): + build_env = TaskEnvironment.by_name("linux.build") else: build_env = env - artifact = 'git-cinnabar' - if os.startswith('mingw'): - artifact += '.exe' + artifact = "git-cinnabar" + if os.startswith("mingw"): + artifact += ".exe" artifacts = [artifact] def prefix(p, s): @@ -335,156 +391,166 @@ def prefix(p, s): desc_variant = variant extra_commands = [] environ = { - 'WARNINGS_AS_ERRORS': '1', + "WARNINGS_AS_ERRORS": "1", } - cargo_flags = ['-vv', '--release'] - cargo_features = ['self-update', 'gitdev'] + cargo_flags = ["-vv", "--release"] + cargo_features = ["self-update", "gitdev"] rust_version = None - if variant == 'asan': - if os.endswith('osx'): - opt = '-O2' + if variant == "asan": + if os.endswith("osx"): + opt = "-O2" else: - opt = '-Og' - environ['TARGET_CFLAGS'] = ' '.join([ - opt, - '-g', - '-fsanitize=address', - '-fno-omit-frame-pointer', - '-fPIC', - ]) - environ['RUSTFLAGS'] = ' '.join([ - '-Zsanitizer=address', - '-Copt-level=1', - '-Cdebuginfo=full', - '-Cforce-frame-pointers=yes', - ]) - elif variant == 'coverage': - environ['TARGET_CFLAGS'] = ' '.join([ - '-coverage', - '-fPIC', - ]) - artifacts += ['coverage.zip'] + opt = "-Og" + environ["TARGET_CFLAGS"] = " ".join( + [ + opt, + "-g", + "-fsanitize=address", + "-fno-omit-frame-pointer", + "-fPIC", + ] + ) + environ["RUSTFLAGS"] = " ".join( + [ + "-Zsanitizer=address", + "-Copt-level=1", + "-Cdebuginfo=full", + "-Cforce-frame-pointers=yes", + ] + ) + elif variant == "coverage": + environ["TARGET_CFLAGS"] = " ".join( + [ + "-coverage", + "-fPIC", + ] + ) + artifacts += ["coverage.zip"] extra_commands = [ - '(cd repo && zip $ARTIFACTS/coverage.zip' + "(cd repo && zip $ARTIFACTS/coverage.zip" ' $(find . -name "*.gcno" -not -name "build_script*"))', ] - environ['RUSTFLAGS'] = ' '.join([ - '-Zprofile', - '-Ccodegen-units=1', - '-Cinline-threshold=0', - ]) + environ["RUSTFLAGS"] = " ".join( + [ + "-Zprofile", + "-Ccodegen-units=1", + "-Cinline-threshold=0", + ] + ) # Build without --release - cargo_flags.remove('--release') - environ['CARGO_INCREMENTAL'] = '0' - elif variant.startswith('old:'): + cargo_flags.remove("--release") + environ["CARGO_INCREMENTAL"] = "0" + elif variant.startswith("old:"): head = variant[4:] hash = build_commit(head) - variant = '' - elif variant.startswith('rust-'): + variant = "" + elif variant.startswith("rust-"): rust_version = variant[5:] elif variant: - raise Exception('Unknown variant: {}'.format(variant)) + raise Exception("Unknown variant: {}".format(variant)) - if 'osx' not in os: - environ['CC'] = 'clang-18' - if os in ('linux', 'arm64-linux'): - cargo_features.append('curl-compat') + if "osx" not in os: + environ["CC"] = "clang-18" + if os in ("linux", "arm64-linux"): + cargo_features.append("curl-compat") - if os.startswith('mingw'): + if os.startswith("mingw"): cpu = msys.msys_cpu(env.cpu) rust_target = "{}-pc-windows-gnu".format(cpu) - elif os.startswith('osx'): - rust_target = 'x86_64-apple-darwin' - elif os.startswith('arm64-osx'): - rust_target = 'aarch64-apple-darwin' - elif os == 'linux': - rust_target = 'x86_64-unknown-linux-gnu' - elif os == 'arm64-linux': - rust_target = 'aarch64-unknown-linux-gnu' - if 'osx' not in os: + elif os.startswith("osx"): + rust_target = "x86_64-apple-darwin" + elif os.startswith("arm64-osx"): + rust_target = "aarch64-apple-darwin" + elif os == "linux": + rust_target = "x86_64-unknown-linux-gnu" + elif os == "arm64-linux": + rust_target = "aarch64-unknown-linux-gnu" + if "osx" not in os: for target in dict.fromkeys( - ["x86_64-unknown-linux-gnu", rust_target]).keys(): + ["x86_64-unknown-linux-gnu", rust_target] + ).keys(): arch = { - 'x86_64': 'amd64', - 'aarch64': 'arm64', - }[target.partition('-')[0]] - multiarch = target.replace('unknown-', '') - TARGET = target.replace('-', '_').upper() - environ[f'CARGO_TARGET_{TARGET}_LINKER'] = environ['CC'] - if 'linux' in os: - extra_link_arg = f'--sysroot=/sysroot-{arch}' - if os.startswith('mingw'): - extra_link_arg = \ - f'-L/usr/lib/gcc/{cpu}-w64-mingw32/10-win32' - environ[f'CARGO_TARGET_{TARGET}_RUSTFLAGS'] = \ - f'-C link-arg=--target={target} ' + \ - f'-C link-arg={extra_link_arg} ' + \ - '-C link-arg=-fuse-ld=lld-18' - rustflags = environ.pop('RUSTFLAGS', None) + "x86_64": "amd64", + "aarch64": "arm64", + }[target.partition("-")[0]] + multiarch = target.replace("unknown-", "") + TARGET = target.replace("-", "_").upper() + environ[f"CARGO_TARGET_{TARGET}_LINKER"] = environ["CC"] + if "linux" in os: + extra_link_arg = f"--sysroot=/sysroot-{arch}" + if os.startswith("mingw"): + extra_link_arg = f"-L/usr/lib/gcc/{cpu}-w64-mingw32/10-win32" + environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] = ( + f"-C link-arg=--target={target} " + + f"-C link-arg={extra_link_arg} " + + "-C link-arg=-fuse-ld=lld-18" + ) + rustflags = environ.pop("RUSTFLAGS", None) if rustflags: - environ[f'CARGO_TARGET_{TARGET}_RUSTFLAGS'] += \ - f' {rustflags}' - if 'linux' in os: - environ[f'CFLAGS_{target}'] = f'--sysroot=/sysroot-{arch}' - if 'linux' in os: - environ['PKG_CONFIG_PATH'] = '' - environ['PKG_CONFIG_SYSROOT_DIR'] = f'/sysroot-{arch}' - environ['PKG_CONFIG_LIBDIR'] = ':'.join(( - f'/sysroot-{arch}/usr/lib/pkgconfig', - f'/sysroot-{arch}/usr/lib/{multiarch}/pkgconfig', - f'/sysroot-{arch}/usr/share/pkgconfig', - )) - if variant in ('coverage', 'asan'): - environ['RUSTC_BOOTSTRAP'] = '1' + environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] += f" {rustflags}" + if "linux" in os: + environ[f"CFLAGS_{target}"] = f"--sysroot=/sysroot-{arch}" + if "linux" in os: + environ["PKG_CONFIG_PATH"] = "" + environ["PKG_CONFIG_SYSROOT_DIR"] = f"/sysroot-{arch}" + environ["PKG_CONFIG_LIBDIR"] = ":".join( + ( + f"/sysroot-{arch}/usr/lib/pkgconfig", + f"/sysroot-{arch}/usr/lib/{multiarch}/pkgconfig", + f"/sysroot-{arch}/usr/share/pkgconfig", + ) + ) + if variant in ("coverage", "asan"): + environ["RUSTC_BOOTSTRAP"] = "1" if rust_version: rust_install = install_rust(rust_version, target=rust_target) else: rust_install = install_rust(target=rust_target) - cargo_flags.extend(['--target', rust_target]) + cargo_flags.extend(["--target", rust_target]) if cargo_features: - cargo_flags.extend(['--features', ','.join(cargo_features)]) + cargo_flags.extend(["--features", ",".join(cargo_features)]) for key, value in list(environ.items()): # RUSTFLAGS values in the environment override builds.rustflags # from .cargo/config.toml. - if 'RUSTFLAGS' in key: - environ[key] = value + ' -Cforce-unwind-tables=yes' + if "RUSTFLAGS" in key: + environ[key] = value + " -Cforce-unwind-tables=yes" hash = hash or build_commit() - if os.startswith('osx'): - environ.setdefault( - 'MACOSX_DEPLOYMENT_TARGET', '10.7') - if os.startswith('arm64-osx'): - environ.setdefault( - 'MACOSX_DEPLOYMENT_TARGET', '11.0') + if os.startswith("osx"): + environ.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.7") + if os.startswith("arm64-osx"): + environ.setdefault("MACOSX_DEPLOYMENT_TARGET", "11.0") - cpu = 'arm64' if os == 'arm64-linux' else env.cpu + cpu = "arm64" if os == "arm64-linux" else env.cpu Task.__init__( self, task_env=build_env, - description='build {} {}{}'.format( - env.os, cpu, prefix(' ', desc_variant)), - index='build.{}.{}.{}{}'.format( - hash, env.os, cpu, prefix('.', variant)), - expireIn='26 weeks', - command=Task.checkout(commit=head) + rust_install + [ - '(cd repo ; cargo build {})'.format(' '.join(cargo_flags)), - 'mv repo/target/{}/{}/{} $ARTIFACTS/'.format( + description="build {} {}{}".format(env.os, cpu, prefix(" ", desc_variant)), + index="build.{}.{}.{}{}".format(hash, env.os, cpu, prefix(".", variant)), + expireIn="26 weeks", + command=Task.checkout(commit=head) + + rust_install + + [ + "(cd repo ; cargo build {})".format(" ".join(cargo_flags)), + "mv repo/target/{}/{}/{} $ARTIFACTS/".format( rust_target, - 'release' if '--release' in cargo_flags else 'debug', - artifact), - ] + extra_commands, + "release" if "--release" in cargo_flags else "debug", + artifact, + ), + ] + + extra_commands, artifacts=artifacts, env=environ, ) def mount(self): - return {f'file:{os.path.basename(self.artifacts[0])}': self} + return {f"file:{os.path.basename(self.artifacts[0])}": self} def install(self): filename = os.path.basename(self.artifacts[0]) return [ - f'cp {filename} repo/', - 'chmod +x repo/{}'.format(filename), - '$PWD/repo/{} setup'.format(filename), + f"cp {filename} repo/", + "chmod +x repo/{}".format(filename), + "$PWD/repo/{} setup".format(filename), ] diff --git a/CI/util.py b/CI/util.py index 2d714e408..14ac9bd9f 100644 --- a/CI/util.py +++ b/CI/util.py @@ -6,8 +6,16 @@ import subprocess -def build_commit(head='HEAD'): +def build_commit(head="HEAD"): return subprocess.check_output( - ['git', '-C', os.path.join(os.path.dirname(__file__), '..'), - 'rev-parse', '--verify', head], text=True, - stderr=open(os.devnull, 'wb')).strip() + [ + "git", + "-C", + os.path.join(os.path.dirname(__file__), ".."), + "rev-parse", + "--verify", + head, + ], + text=True, + stderr=open(os.devnull, "wb"), + ).strip() diff --git a/CI/variables.py b/CI/variables.py index 6ba8f195f..350e4ccbf 100644 --- a/CI/variables.py +++ b/CI/variables.py @@ -6,46 +6,47 @@ import os rootUrl = os.environ.get( - 'TASKCLUSTER_ROOT_URL', - 'https://community-tc.services.mozilla.com') + "TASKCLUSTER_ROOT_URL", "https://community-tc.services.mozilla.com" +) -if 'TC_PROXY' in os.environ: - PROXY_URL = os.environ.get('TASKCLUSTER_PROXY_URL', 'http://taskcluster') +if "TC_PROXY" in os.environ: + PROXY_URL = os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster") else: PROXY_URL = rootUrl -PROXY_INDEX_URL = PROXY_URL + '/api/index/v1/task/{}' -ARTIFACT_URL = rootUrl + '/api/queue/v1/task/{}/artifacts/{}' +PROXY_INDEX_URL = PROXY_URL + "/api/index/v1/task/{}" +ARTIFACT_URL = rootUrl + "/api/queue/v1/task/{}/artifacts/{}" DEFAULT_DATA = { - 'repo_name': 'git-cinnabar', - 'login': 'glandium', - 'commit': 'HEAD', - 'branch': '', - 'decision_id': '', + "repo_name": "git-cinnabar", + "login": "glandium", + "commit": "HEAD", + "branch": "", + "decision_id": "", } -DEFAULT_DATA['repo_url'] = 'https://github.com/{}/{}'.format( - DEFAULT_DATA['login'], DEFAULT_DATA['repo_name']) -for k in ('repo_name', 'login'): - DEFAULT_DATA['base_{}'.format(k)] = DEFAULT_DATA[k] +DEFAULT_DATA["repo_url"] = "https://github.com/{}/{}".format( + DEFAULT_DATA["login"], DEFAULT_DATA["repo_name"] +) +for k in ("repo_name", "login"): + DEFAULT_DATA["base_{}".format(k)] = DEFAULT_DATA[k] -TC_DATA = json.loads(os.environ.get('TC_DATA', json.dumps(DEFAULT_DATA))) +TC_DATA = json.loads(os.environ.get("TC_DATA", json.dumps(DEFAULT_DATA))) def get(k): return TC_DATA.get(k, DEFAULT_DATA[k]) -TC_LOGIN = get('login') -TC_REPO_NAME = get('repo_name') -TC_REPO_URL = get('repo_url') -TC_COMMIT = get('commit') -TC_BRANCH = get('branch') -TC_BASE_LOGIN = get('base_login') -TC_BASE_REPO_NAME = get('base_repo_name') +TC_LOGIN = get("login") +TC_REPO_NAME = get("repo_name") +TC_REPO_URL = get("repo_url") +TC_COMMIT = get("commit") +TC_BRANCH = get("branch") +TC_BASE_LOGIN = get("base_login") +TC_BASE_REPO_NAME = get("base_repo_name") -TC_ACTION = os.environ.get('TC_ACTION') -TC_IS_PUSH = os.environ.get('TC_IS_PUSH') == '1' +TC_ACTION = os.environ.get("TC_ACTION") +TC_IS_PUSH = os.environ.get("TC_IS_PUSH") == "1" -DEFAULT_REPO = 'https://hg.mozilla.org/users/mh_glandium.org/jqplot' -REPO = os.environ.get('REPO', DEFAULT_REPO) +DEFAULT_REPO = "https://hg.mozilla.org/users/mh_glandium.org/jqplot" +REPO = os.environ.get("REPO", DEFAULT_REPO) diff --git a/download.py b/download.py index 4c65a8295..a80494b5a 100755 --- a/download.py +++ b/download.py @@ -3,7 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -''':' +""":" if command -v python3 > /dev/null; then PYTHON=python3 else @@ -12,7 +12,8 @@ fi exec $PYTHON -B $0 "$@" exit 1 -''' +""" + import os import re import sys @@ -28,19 +29,20 @@ from urllib.request import urlopen from urllib.error import HTTPError from zipfile import ZipFile + try: from CI.util import build_commit except ImportError: build_commit = None -REPOSITORY = 'https://github.com/glandium/git-cinnabar' +REPOSITORY = "https://github.com/glandium/git-cinnabar" AVAILABLE = ( - ('Linux', 'x86_64'), - ('Linux', 'arm64'), - ('macOS', 'x86_64'), - ('macOS', 'arm64'), - ('Windows', 'x86_64'), + ("Linux", "x86_64"), + ("Linux", "arm64"), + ("macOS", "x86_64"), + ("macOS", "arm64"), + ("Windows", "x86_64"), ) @@ -53,7 +55,7 @@ def __init__(self, reader, length): self._length = length self._read = 0 self._pos = 0 - self._buf = b'' + self._buf = b"" def read(self, length): if self._pos < self._read: @@ -89,32 +91,32 @@ def seek(self, pos, how=os.SEEK_SET): def get_binary(system): - binary = 'git-cinnabar' - if system == 'Windows': - binary += '.exe' + binary = "git-cinnabar" + if system == "Windows": + binary += ".exe" return binary def get_url(system, machine, variant, sha1): - url = 'https://community-tc.services.mozilla.com/api/index/v1/task/' - url += 'project.git-cinnabar.build.' - url += '{}.{}.{}.{}'.format( - sha1, system.lower(), machine, - variant.lower() if variant else '').rstrip('.') - url += '/artifacts/public/{}'.format(get_binary(system)) + url = "https://community-tc.services.mozilla.com/api/index/v1/task/" + url += "project.git-cinnabar.build." + url += "{}.{}.{}.{}".format( + sha1, system.lower(), machine, variant.lower() if variant else "" + ).rstrip(".") + url += "/artifacts/public/{}".format(get_binary(system)) return url def get_release_url(system, machine, tag): - ext = 'zip' if system == 'Windows' else 'tar.xz' - url = f'{REPOSITORY}/releases/download/{tag}/git-cinnabar' - url += f'.{system.lower()}.{machine}.{ext}' + ext = "zip" if system == "Windows" else "tar.xz" + url = f"{REPOSITORY}/releases/download/{tag}/git-cinnabar" + url += f".{system.lower()}.{machine}.{ext}" return url def download(url, system, binary_path): - print('Downloading from %s...' % url) + print("Downloading from %s..." % url) try: reader = urlopen(url) except HTTPError: @@ -122,11 +124,11 @@ def download(url, system, binary_path): try: reader = urlopen(url) except HTTPError as e: - print('Download failed with status code %d\n' % e.code, - file=sys.stderr) + print("Download failed with status code %d\n" % e.code, file=sys.stderr) print( - 'Error body was:\n\n%s' % e.read().decode('utf-8', 'replace'), - file=sys.stderr) + "Error body was:\n\n%s" % e.read().decode("utf-8", "replace"), + file=sys.stderr, + ) return 1 class ReaderProgress(object): @@ -138,9 +140,9 @@ def __init__(self, reader, length=None): def show_progress(self): if self._length: - count = f'\r {self._read * 100 // self._length}%' + count = f"\r {self._read * 100 // self._length}%" else: - count = f'\r {self._read} bytes' + count = f"\r {self._read} bytes" sys.stderr.write(count) def read(self, length): @@ -154,18 +156,18 @@ def read(self, length): def finish(self): self.show_progress() - sys.stderr.write('\n') + sys.stderr.write("\n") sys.stderr.flush() - encoding = reader.headers.get('Content-Encoding', 'identity') + encoding = reader.headers.get("Content-Encoding", "identity") progress = ReaderProgress(reader, reader.length) binary_content = Seekable(progress, reader.length) - if encoding == 'gzip': - binary_content = GzipFile(mode='rb', fileobj=binary_content) + if encoding == "gzip": + binary_content = GzipFile(mode="rb", fileobj=binary_content) (dirname, filename) = os.path.split(binary_path) fd, path = tempfile.mkstemp(prefix=filename, dir=dirname) - fh = os.fdopen(fd, 'wb') + fh = os.fdopen(fd, "wb") success = False try: @@ -175,33 +177,35 @@ def finish(self): progress.finish() fh.close() if success: - if url.endswith(('.zip', '.tar.xz')): + if url.endswith((".zip", ".tar.xz")): binary_name = get_binary(system) binary_content = None size = 0 archive_path = path - if url.endswith('.zip'): + if url.endswith(".zip"): archive = zip = ZipFile(path) for info in zip.infolist(): if os.path.basename(info.filename) == binary_name: size = info.file_size binary_content = zip.open(info) break - elif url.endswith('tar.xz'): - archive = tar = tarfile.open(path, 'r:*') + elif url.endswith("tar.xz"): + archive = tar = tarfile.open(path, "r:*") while True: member = tar.next() if member is None: break - if (member.isfile() and - os.path.basename(member.name) == binary_name): + if ( + member.isfile() + and os.path.basename(member.name) == binary_name + ): size = member.size binary_content = tar.extractfile(member) break fd, path = tempfile.mkstemp(prefix=filename, dir=dirname) - fh = os.fdopen(fd, 'wb') + fh = os.fdopen(fd, "wb") try: - print('Extracting %s...' % binary_name) + print("Extracting %s..." % binary_name) progress = ReaderProgress(binary_content, size) copyfileobj(progress, fh) finally: @@ -249,12 +253,12 @@ def maybe_int(s): def split_version(s): - s = s.decode('ascii') - version = s.replace('-', '').split('.') - version[-1:] = [x for x in re.split(r'([0-9]+)', version[-1]) if x] + s = s.decode("ascii") + version = s.replace("-", "").split(".") + version[-1:] = [x for x in re.split(r"([0-9]+)", version[-1]) if x] version = [maybe_int(x) for x in version] if isinstance(version[-1], int): - version += ['z'] + version += ["z"] return version @@ -267,20 +271,19 @@ def main(args): system = args.system machine = args.machine - if system.startswith('MSYS_NT'): - system = 'Windows' + if system.startswith("MSYS_NT"): + system = "Windows" - if system == 'Darwin': - system = 'macOS' - elif system == 'Windows': - if machine == 'AMD64': - machine = 'x86_64' - if machine == 'aarch64': - machine = 'arm64' + if system == "Darwin": + system = "macOS" + elif system == "Windows": + if machine == "AMD64": + machine = "x86_64" + if machine == "aarch64": + machine = "arm64" if (system, machine) not in AVAILABLE: - print('No download available for %s/%s' % (system, machine), - file=sys.stderr) + print("No download available for %s/%s" % (system, machine), file=sys.stderr) return 1 tag = None @@ -292,7 +295,7 @@ def main(args): pass exact = args.exact or (not args.branch and local_sha1) - branch = args.branch or 'release' + branch = args.branch or "release" if exact and not args.variant: if build_commit: @@ -301,7 +304,8 @@ def main(args): for sha1, _, ref in ( l.split(None, 2) for l in subprocess.check_output( - ['git', 'for-each-ref', 'refs/tags/']).splitlines() + ["git", "for-each-ref", "refs/tags/"] + ).splitlines() ) ) else: @@ -309,19 +313,15 @@ def main(args): tags = ( tuple(l.split(None, 1)) for l in subprocess.check_output( - ['git', 'ls-remote', REPOSITORY, 'refs/tags/*']) - .splitlines() + ["git", "ls-remote", REPOSITORY, "refs/tags/*"] + ).splitlines() ) except Exception: tags = () - if '.' in exact: - ref = f'refs/tags/{exact}'.encode() - matches = [ - sha1 - for sha1, r in tags - if r == ref - ] + if "." in exact: + ref = f"refs/tags/{exact}".encode() + matches = [sha1 for sha1, r in tags if r == ref] if not matches: print(f"Couldn't find a tag for {exact}") return 1 @@ -329,49 +329,63 @@ def main(args): exact = matches[0] else: tags = [ - ref[len('refs/tags/'):] + ref[len("refs/tags/") :] for sha1, ref in tags - if sha1.decode('ascii') == exact + if sha1.decode("ascii") == exact ] tags = sorted(tags, key=lambda x: split_version(x), reverse=True) if tags: - tag = tags[0].decode('ascii') + tag = tags[0].decode("ascii") if exact: sha1 = exact - elif branch == 'release': + elif branch == "release": if args.variant: - print('Cannot use --variant without --branch {master,next}') + print("Cannot use --variant without --branch {master,next}") return 1 - result = sorted(((sha1, ref[len('refs/tags/'):]) for sha1, ref in [ - l.split(b'\t', 1) for l in subprocess.check_output( - ['git', 'ls-remote', REPOSITORY, 'refs/tags/*'] - ).splitlines() - ]), key=lambda x: split_version(x[1]), reverse=True) + result = sorted( + ( + (sha1, ref[len("refs/tags/") :]) + for sha1, ref in [ + l.split(b"\t", 1) + for l in subprocess.check_output( + ["git", "ls-remote", REPOSITORY, "refs/tags/*"] + ).splitlines() + ] + ), + key=lambda x: split_version(x[1]), + reverse=True, + ) if len(result) == 0: - print('Could not find release tags') + print("Could not find release tags") return 1 sha1, tag = result[0] - sha1 = sha1.decode('ascii') - tag = tag.decode('ascii') + sha1 = sha1.decode("ascii") + tag = tag.decode("ascii") elif branch: - ref = f'refs/heads/{branch}'.encode('utf-8') - result = [sha1 for sha1, ref_ in [ - l.split(b'\t', 1) for l in subprocess.check_output( - ['git', 'ls-remote', REPOSITORY, ref] - ).splitlines()] + ref = f"refs/heads/{branch}".encode("utf-8") + result = [ + sha1 + for sha1, ref_ in [ + l.split(b"\t", 1) + for l in subprocess.check_output( + ["git", "ls-remote", REPOSITORY, ref] + ).splitlines() + ] if ref == ref_ ] if len(result) == 0: - print(f'Could not find branch {branch}') + print(f"Could not find branch {branch}") return 1 - sha1 = result[0].decode('ascii') + sha1 = result[0].decode("ascii") else: sha1 = None if sha1 is None: - print('Cannot find the right binary for git-cinnabar.' - ' Try --exact or --branch.', - file=sys.stderr) + print( + "Cannot find the right binary for git-cinnabar." + " Try --exact or --branch.", + file=sys.stderr, + ) return 1 if tag: @@ -389,39 +403,48 @@ def main(args): else: d = script_path if not os.access(d, os.W_OK): - d = os.path.join(os.path.expanduser('~'), '.git-cinnabar') + d = os.path.join(os.path.expanduser("~"), ".git-cinnabar") try: os.makedirs(d) except Exception: pass if not os.path.isdir(d): - print('Cannot write to either %s or %s.' % (d, script_path), - file=sys.stderr) + print( + "Cannot write to either %s or %s." % (d, script_path), + file=sys.stderr, + ) return 1 binary_path = os.path.join(d, get_binary(system)) return download(url, system, binary_path) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--url', action='store_true', - help='only print the download url') + parser.add_argument( + "--url", action="store_true", help="only print the download url" + ) pgroup = parser.add_mutually_exclusive_group() - pgroup.add_argument('--branch', metavar='BRANCH', - default=os.environ.get("GIT_CINNABAR_DOWNLOAD_BRANCH"), - help='download a build for the given branch') - pgroup.add_argument('--exact', metavar='EXACT', - default=os.environ.get("GIT_CINNABAR_DOWNLOAD_EXACT"), - help='download a build for the given commit') - parser.add_argument('--variant', metavar='VARIANT', - default=os.environ.get( - "GIT_CINNABAR_DOWNLOAD_VARIANT"), - help='download the given variant') - parser.add_argument('--system', default=platform.system(), - help=argparse.SUPPRESS) - parser.add_argument('--machine', default=platform.machine(), - help=argparse.SUPPRESS) - parser.add_argument('-o', '--output', help=argparse.SUPPRESS) - parser.add_argument('--list', action='store_true', help=argparse.SUPPRESS) + pgroup.add_argument( + "--branch", + metavar="BRANCH", + default=os.environ.get("GIT_CINNABAR_DOWNLOAD_BRANCH"), + help="download a build for the given branch", + ) + pgroup.add_argument( + "--exact", + metavar="EXACT", + default=os.environ.get("GIT_CINNABAR_DOWNLOAD_EXACT"), + help="download a build for the given commit", + ) + parser.add_argument( + "--variant", + metavar="VARIANT", + default=os.environ.get("GIT_CINNABAR_DOWNLOAD_VARIANT"), + help="download the given variant", + ) + parser.add_argument("--system", default=platform.system(), help=argparse.SUPPRESS) + parser.add_argument("--machine", default=platform.machine(), help=argparse.SUPPRESS) + parser.add_argument("-o", "--output", help=argparse.SUPPRESS) + parser.add_argument("--list", action="store_true", help=argparse.SUPPRESS) sys.exit(main(parser.parse_args())) diff --git a/mercurial/cinnabarclone.py b/mercurial/cinnabarclone.py index b961f9197..be24a2bc8 100644 --- a/mercurial/cinnabarclone.py +++ b/mercurial/cinnabarclone.py @@ -71,10 +71,12 @@ def add_cinnabar_cap(repo, caps): try: exists = vfs.exists except AttributeError: + def exists(path): return os.path.exists(os.path.join(vfs.base, path)) - if exists(b'cinnabar.manifest'): - caps.append(b'cinnabarclone') + + if exists(b"cinnabar.manifest"): + caps.append(b"cinnabarclone") def _capabilities(orig, repo, proto): @@ -86,20 +88,20 @@ def _capabilities(orig, repo, proto): def capabilities(orig, repo, proto): caps = orig(repo, proto).split() add_cinnabar_cap(repo, caps) - return b' '.join(caps) + return b" ".join(caps) def cinnabar(repo, proto): vfs = get_vfs(repo) try: - return vfs.tryread(b'cinnabar.manifest') + return vfs.tryread(b"cinnabar.manifest") except AttributeError: try: - return vfs.read(b'cinnabar.manifest') + return vfs.read(b"cinnabar.manifest") except IOError as e: if e.errno != errno.ENOENT: raise - return b'' + return b"" def extsetup(ui): @@ -110,17 +112,16 @@ def extsetup(ui): from mercurial import extensions try: - extensions.wrapfunction(wireproto, '_capabilities', _capabilities) + extensions.wrapfunction(wireproto, "_capabilities", _capabilities) except AttributeError: - extensions.wrapcommand( - wireproto.commands, 'capabilities', capabilities) + extensions.wrapcommand(wireproto.commands, "capabilities", capabilities) - def wireprotocommand(name, args=b'', permission=b'push'): - if hasattr(wireproto, 'wireprotocommand'): + def wireprotocommand(name, args=b"", permission=b"push"): + if hasattr(wireproto, "wireprotocommand"): try: return wireproto.wireprotocommand(name, args, permission) except TypeError: - if hasattr(wireproto, 'permissions'): + if hasattr(wireproto, "permissions"): wireproto.permissions[name] = permission return wireproto.wireprotocommand(name, args) @@ -131,4 +132,4 @@ def register(func): return register - wireprotocommand(b'cinnabarclone', permission=b'pull')(cinnabar) + wireprotocommand(b"cinnabarclone", permission=b"pull")(cinnabar)