diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py new file mode 100644 index 000000000..f393892fc --- /dev/null +++ b/conda_smithy/cirun_utils.py @@ -0,0 +1,82 @@ +""" +See http://py.cirun.io/api.html for cirun client docs +""" +import os +from functools import lru_cache +from typing import List, Dict, Any, Optional + +from cirun import Cirun +from .github import gh_token, Github + + +@lru_cache +def get_cirun_installation_id(owner: str) -> int: + # This ID needs a token with admin: org privileges. + # Hard-code instead for easier use. + if owner == "conda-forge": + return 18453316 + else: + gh = Github(gh_token) + user = gh.get_user() + if user.login == owner: + user_or_org = user + else: + user_or_org = gh.get_organization(owner) + for inst in user_or_org.get_installations: + if inst.raw_data["app_slug"] == "cirun-application": + return inst.app_id + raise ValueError(f"cirun not found for owner {owner}") + + +def enable_cirun_for_project(owner: str, repo: str) -> Dict[str, Any]: + """Enable the cirun.io Github Application for a particular repository.""" + print(f"Enabling cirun for {owner}/{repo} ...") + cirun = _get_cirun_client() + return cirun.set_repo( + f"{owner}/{repo}", installation_id=get_cirun_installation_id(owner) + ) + + +def add_repo_to_cirun_resource( + owner: str, + repo: str, + resource: str, + cirun_policy_args: Optional[List[str]] = None, +) -> Dict[str, Any]: + """Grant access to a cirun resource to a particular repository, with a particular policy.""" + cirun = _get_cirun_client() + policy_args: Optional[Dict[str, Any]] = None + if cirun_policy_args and "pull_request" in cirun_policy_args: + policy_args = {"pull_request": True} + print( + f"Adding repo {owner}/{repo} to resource {resource} with policy_args: {policy_args}" + ) + response = cirun.add_repo_to_resources( + owner, + repo, + resources=[resource], + teams=[repo], + policy_args=policy_args, + ) + print(f"response: {response} | {response.json().keys()}") + return response + + +def remove_repo_from_cirun_resource(owner: str, repo: str, resource: str): + """Revoke access to a cirun resource to a particular repository, with a particular policy.""" + cirun = _get_cirun_client() + print(f"Removing repo {owner}/{repo} from resource {resource}.") + response = cirun.remove_repo_from_resources(owner, repo, [resource]) + print(f"response: {response} | {response.json().keys()}") + return response + + +@lru_cache +def _get_cirun_client() -> Cirun: + try: + return Cirun() + except KeyError: + raise RuntimeError( + "You must have CIRUN_API_KEY defined to do Cirun CI registration. " + "This requirement can be overriden by specifying `--without-cirun`" + ) diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index 477d0fbc2..18b101617 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -12,6 +12,8 @@ import conda # noqa import conda_build.api from conda_build.metadata import MetaData + +import conda_smithy.cirun_utils from conda_smithy.utils import get_feedstock_name_from_meta, merge_dict from ruamel.yaml import YAML @@ -208,7 +210,7 @@ def __init__(self, parser): # conda-smithy register-ci ./ super(RegisterCI, self).__init__( parser, - "Register a feedstock at the CI " "services which do the builds.", + "Register a feedstock at the CI services which do the builds.", ) scp = self.subcommand_parser scp.add_argument( @@ -238,6 +240,7 @@ def __init__(self, parser): "Appveyor", "Drone", "Webservice", + "Cirun", ]: scp.add_argument( "--without-{}".format(ci.lower()), @@ -257,6 +260,23 @@ def __init__(self, parser): action="append", help="drone server URL to register this repo. multiple values allowed", ) + scp.add_argument( + "--cirun-resources", + default=[], + action="append", + help="cirun resources to enable for this repo. multiple values allowed", + ) + scp.add_argument( + "--cirun-policy-args", + action="append", + help="extra arguments for cirun policy to create for this repo. multiple values allowed", + ) + scp.add_argument( + "--remove", + action="store_true", + help="Revoke access to the configured CI services. " + "Only available for Cirun for now", + ) def __call__(self, args): from conda_smithy import ci_register @@ -278,7 +298,19 @@ def __call__(self, args): ) print("CI Summary for {}/{} (can take ~30s):".format(owner, repo)) - + if args.remove and any( + [ + args.azure, + args.circle, + args.appveyor, + args.drone, + args.webservice, + args.anaconda_token, + ] + ): + raise RuntimeError( + "The --remove flag is only supported for Cirun for now" + ) if not args.anaconda_token: print( "Warning: By not registering an Anaconda/Binstar token" @@ -345,6 +377,31 @@ def __call__(self, args): else: print("Drone registration disabled.") + if args.cirun: + print("Cirun Registration") + if args.remove: + if args.cirun_resources: + to_remove = args.cirun_resources + else: + to_remove = ["*"] + + print(f"Cirun Registration: resources to remove: {to_remove}") + for resource in to_remove: + conda_smithy.cirun_utils.remove_repo_from_cirun_resource( + owner, repo, resource + ) + else: + print( + f"Cirun Registration: resources to add to: {owner}/{repo}" + ) + conda_smithy.cirun_utils.enable_cirun_for_project(owner, repo) + for resource in args.cirun_resources: + conda_smithy.cirun_utils.add_repo_to_cirun_resource( + owner, repo, resource, args.cirun_policy_args + ) + else: + print("Cirun registration disabled.") + if args.webservice: ci_register.add_conda_forge_webservice_hooks(owner, repo) else: diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index b13590165..5f49009d0 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -377,6 +377,9 @@ def _collapse_subpackage_variants( if not is_noarch: always_keep_keys.add("target_platform") + if forge_config["github_actions"]["self_hosted"]: + always_keep_keys.add("github_actions_labels") + all_used_vars.update(always_keep_keys) all_used_vars.update(top_level_vars) @@ -695,7 +698,6 @@ def _render_ci_provider( channel_target.startswith("conda-forge ") and provider_name == "github_actions" and not forge_config["github_actions"]["self_hosted"] - and os.path.basename(forge_dir) not in SERVICE_FEEDSTOCKS ): raise RuntimeError( "Using github_actions as the CI provider inside " @@ -1266,6 +1268,82 @@ def render_appveyor(jinja_env, forge_config, forge_dir, return_metadata=False): def _github_actions_specific_setup( jinja_env, forge_config, forge_dir, platform ): + # Handle GH-hosted and self-hosted runners runs-on config + # Do it before the deepcopy below so these changes can be used by the + # .github/worfkflows/conda-build.yml template + runs_on = { + "osx-64": { + "os": "macos", + "self_hosted_labels": ("macOS", "x64"), + }, + "osx-arm64": { + "os": "macos", + "self_hosted_labels": ("macOS", "arm64"), + }, + "linux-64": { + "os": "ubuntu", + "self_hosted_labels": ("linux", "x64"), + }, + "linux-aarch64": { + "os": "ubuntu", + "self_hosted_labels": ("linux", "ARM64"), + }, + "win-64": { + "os": "windows", + "self_hosted_labels": ("windows", "x64"), + }, + "win-arm64": { + "os": "windows", + "self_hosted_labels": ("windows", "ARM64"), + }, + } + for data in forge_config["configs"]: + if not data["build_platform"].startswith(platform): + continue + # This Github Actions specific configs are prefixed with "gha_" + # because we are not deepcopying the data dict intentionally + # so it can be used in the general "render_github_actions" function + # This avoid potential collisions with other CI providers :crossed_fingers: + data["gha_os"] = runs_on[data["build_platform"]]["os"] + data["gha_with_gpu"] = False + + self_hosted_default = list( + runs_on[data["build_platform"]]["self_hosted_labels"] + ) + self_hosted_default += ["self-hosted"] + hosted_default = [data["gha_os"] + "-latest"] + + labels_default = ( + ["hosted"] + if forge_config["github_actions"]["self_hosted"] + else ["self-hosted"] + ) + labels = conda_build.utils.ensure_list( + data["config"].get("github_actions_labels", [labels_default])[0] + ) + + if len(labels) == 1 and labels[0] == "hosted": + labels = hosted_default + elif len(labels) == 1 and labels[0] in "self-hosted": + labels = self_hosted_default + else: + # Prepend the required ones + labels += self_hosted_default + + if forge_config["github_actions"]["self_hosted"]: + data["gha_runs_on"] = [] + # labels provided in conda-forge.yml + for label in labels: + if label.startswith("cirun-"): + label += ( + "--${{ github.run_id }}-" + data["short_config_name"] + ) + if "gpu" in label.lower(): + data["gha_with_gpu"] = True + data["gha_runs_on"].append(label) + else: + data["gha_runs_on"] = hosted_default + build_setup = _get_build_setup_line(forge_dir, platform, forge_config) if platform == "linux": @@ -1288,11 +1366,13 @@ def _github_actions_specific_setup( ".scripts/run_win_build.bat", ], } - if forge_config["github_actions"]["store_build_artifacts"]: - for tmpls in platform_templates.values(): - tmpls.append(".scripts/create_conda_build_artifacts.sh") + template_files = platform_templates.get(platform, []) + # Templates for all platforms + if forge_config["github_actions"]["store_build_artifacts"]: + template_files.append(".scripts/create_conda_build_artifacts.sh") + _render_template_exe_files( forge_config=forge_config, jinja_env=jinja_env, @@ -1307,7 +1387,7 @@ def render_github_actions( target_path = os.path.join( forge_dir, ".github", "workflows", "conda-build.yml" ) - template_filename = "github-actions.tmpl" + template_filename = "github-actions.yml.tmpl" fast_finish_text = "" ( @@ -1317,7 +1397,7 @@ def render_github_actions( upload_packages, ) = _get_platforms_of_provider("github_actions", forge_config) - logger.debug("github platforms retreived") + logger.debug("github platforms retrieved") remove_file_or_dir(target_path) return _render_ci_provider( @@ -1841,6 +1921,9 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): }, "github_actions": { "self_hosted": False, + "triggers": [], + "timeout_minutes": 360, + "cancel_in_progress": True, # Set maximum parallel jobs "max_parallel": None, # Toggle creating artifacts for conda build_artifacts dir @@ -2001,6 +2084,12 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): if config["test"] is None: config["test"] = "all" + if not config["github_actions"]["triggers"]: + self_hosted = config["github_actions"]["self_hosted"] + config["github_actions"]["triggers"] = ( + ["push"] if self_hosted else ["push", "pull_request"] + ) + # An older conda-smithy used to have some files which should no longer exist, # remove those now. old_files = [ diff --git a/conda_smithy/templates/github-actions.tmpl b/conda_smithy/templates/github-actions.yml.tmpl similarity index 83% rename from conda_smithy/templates/github-actions.tmpl rename to conda_smithy/templates/github-actions.yml.tmpl index 986792192..bc4d25224 100644 --- a/conda_smithy/templates/github-actions.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -11,20 +11,19 @@ {%- endfor %} name: Build conda package -{%- if github_actions.self_hosted %} -on: [push] -{%- else %} -on: [push, pull_request] +on: {{ github_actions.triggers }} + +{%- if github_actions.cancel_in_progress %} +concurrency: + group: {% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}{% endraw %} + cancel-in-progress: true {%- endif %} jobs: build: name: {% raw %}${{ matrix.CONFIG }}{% endraw %} - {%- if github_actions.self_hosted %} - runs-on: {% raw %}${{ matrix.labels }}{% endraw %} - {%- else %} - runs-on: {% raw %}${{ matrix.os }}{% endraw %}-latest - {%- endif %} + runs-on: {% raw %}${{ matrix.runs_on }}{% endraw %} + timeout-minutes: {{ github_actions.timeout_minutes }} strategy: fail-fast: false {%- if github_actions.max_parallel %} @@ -38,36 +37,13 @@ jobs: SHORT_CONFIG: {{ data.short_config_name }} {%- endif %} UPLOAD_PACKAGES: {{ data.upload }} - {%- if data.build_platform.startswith("osx-64") %} - os: macos - {%- if github_actions.self_hosted %} - labels: ['macOS', 'self-hosted', 'x64'] - {%- endif %} - {%- elif data.build_platform.startswith("osx-arm64") %} - os: macos - {%- if github_actions.self_hosted %} - labels: ['macOS', 'self-hosted', 'ARM64'] - {%- endif %} - {%- elif data.build_platform.startswith("linux-64") %} - DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - os: ubuntu - {%- if github_actions.self_hosted %} - labels: ['linux', 'self-hosted', 'x64'] - {%- endif %} - {%- elif data.build_platform.startswith("linux-aarch64") %} - DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - os: ubuntu - {%- if github_actions.self_hosted %} - labels: ['linux', 'self-hosted', 'ARM64'] - {%- endif %} - {%- elif data.build_platform.startswith("linux") %} + os: {{ data.gha_os }} + runs_on: {{ data.gha_runs_on }} + {%- if data.build_platform.startswith("linux") %} DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - os: ubuntu - {%- elif data.build_platform.startswith("win") %} - os: windows - {%- if github_actions.self_hosted %} - labels: ['windows', 'self-hosted', 'x64'] - {%- endif %} + {%- if data.gha_with_gpu %} + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + {%- endif %} {%- endif %} {%- endfor %} steps: @@ -116,7 +92,9 @@ jobs: UPLOAD_ON_BRANCH: {{ upload_on_branch }} {%- endif %} {%- if docker.run_args is defined %} - CONDA_FORGE_DOCKER_RUN_ARGS: "{{ docker.run_args }}" + CONDA_FORGE_DOCKER_RUN_ARGS: "{{ docker.run_args }} {% raw %}${{ matrix.CONDA_FORGE_DOCKER_RUN_ARGS }}{% endraw %}" +{%- else %} + CONDA_FORGE_DOCKER_RUN_ARGS: "{% raw %}${{ matrix.CONDA_FORGE_DOCKER_RUN_ARGS }}{% endraw %}" {%- endif %} {%- for secret in secrets %} {{ secret }}: {% raw %}${{{% endraw %} secrets.{{ secret }} {% raw %}}}{% endraw %} diff --git a/conda_smithy/utils.py b/conda_smithy/utils.py index f50c14025..b0be1a656 100644 --- a/conda_smithy/utils.py +++ b/conda_smithy/utils.py @@ -5,7 +5,6 @@ import datetime import time import os -import sys from pathlib import Path from collections import defaultdict from contextlib import contextmanager diff --git a/environment.yml b/environment.yml index c809f6d07..1084ca6d4 100644 --- a/environment.yml +++ b/environment.yml @@ -14,7 +14,7 @@ dependencies: - mock - pytest - pytest-cov - # Runtime dependencies, duplicated in conda_smithy.recipe/meta.yaml + # Runtime dependencies - conda >=4.2 - conda-build >=3.21.8 - conda-package-handling >=1.9.0 @@ -31,3 +31,4 @@ dependencies: - scrypt - license-expression - libarchive + - cirun diff --git a/news/1703-cirun-self-hosted.rst b/news/1703-cirun-self-hosted.rst new file mode 100644 index 000000000..fdf075a0a --- /dev/null +++ b/news/1703-cirun-self-hosted.rst @@ -0,0 +1,43 @@ +**Added:** + +* For self-hosted github actions runs, a user can add custom labels + by adding `github_actions_labels` yaml key in `recipe/conda_build_config.yaml`. + The value `hosted` can be used for Microsoft hosted free runners + and the value `self-hosted` can be used for the default self-hosted labels. + +* `github_actions: timeout_minutes` option added to change the timeout in minutes. + The default value is `360`. + +* `github_actions: triggers` is a list of triggers which defaults to + `push, pull_request` when not self-hosted and `push` when self-hosted. + +* Added a `--cirun` argument to `conda-smithy ci-register` command to register + `cirun` as a CI service. This makes `cirun` conda package a dependency of + conda-smithy. + +* Added support for `cirun` by generating a unique label when the self-hosted + label starts with `cirun`. + +* When a label is added that has the string with `gpu` or `GPU` for a self-hosted + runner, the docker build will pass the GPUs to the docker instance. + +**Changed:** + +* `github_actions: cancel_in_progress` option added to cancel in progress runs. + The default value was changed to `true`. + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +*