diff --git a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst index 858e9353d..abf95e0e6 100644 --- a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst +++ b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst @@ -1,20 +1,27 @@ Loading External Actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Robusta can load playbook actions from external git repositories. This extends Robusta with additional actions for -use in :ref:`customPlaybooks`. +Robusta can load playbook actions from external git repositories and externally hosted +Python packages provided as .tgz or .tar.gz files. This extends Robusta with additional +actions for use in :ref:`customPlaybooks`. .. warning:: - Robusta does not watch for changes in git repositories. Playbooks are reloaded when: + Robusta does not watch for changes in git repositories/externally hosted Python packages. + Playbooks are reloaded when: * Robusta starts * Robusta's configuration changes * ``robusta playbooks reload`` is run -External actions are loaded using the ``playbookRepos`` Helm value, with either HTTPs or SSH. +External actions are loaded using the ``playbookRepos`` Helm value, with either HTTPs or SSH +in the case of git repositories, and appropriate URLs in the case of externally hosted +Python packages. The way Robusta distinguishes between the case of git repository and an +external package is to check if the URL ends with `.tgz` or `.tar.gz` +- if that is the case, the source is treated as an external package; otherwise the +URL is treated as a git repository address. -If you are going to be using an external repository via HTTPS, you just need to configure +If you are going to be using an external git repository via HTTPS, you just need to configure correct read access credentials (see below). When connecting via SSH, however, there is an additional requirement to verify the remote host's identity on the client side, as SSH generally does not provide any method of doing that automatically (in contrast with HTTPS, @@ -109,6 +116,23 @@ The ``key`` parameter must contain a ``base64`` encoded deployment key with ``re ewfrcfsfvC1rZXktdjEAAAAABG5vb..... -----END OPENSSH PRIVATE KEY----- +Loading Actions from an external Python Package +--------------------------------------------------- + +For external Python packages, just specify an URL starting with http(s), and ending with +either .tar.gz or .tgz. + +.. code-block:: yaml + + playbookRepos: + web_playbooks: + url: "https://my-domain.com/bla/web-playbooks.tgz" + http_headers: # optional, may be used for auth + Authorization: Bearer XXXYYY + # pip_install: true # optional: load this playbook's dependencies (default True) + # build_isolation: false + +The `http_headers` option is only available for this method of loading actions. Handling Secrets ******************* @@ -131,6 +155,14 @@ Then reference it using an environment variable: url: "git@github.com:robusta-dev/robusta-chaos.git" key: "{{env.GITHUB_SSH_KEY}}" +Build Isolation +***************** + +``build_isolation`` is optional (defaults to `true`). If specified as `false`, the `pip` +install command for the package being installed will be run with `--no-build-isolation` (see +the `pip docs `_ +for details). + Baking Actions into a Custom Image -------------------------------------- diff --git a/src/robusta/core/model/runner_config.py b/src/robusta/core/model/runner_config.py index 3b7ffbb26..60a4aa091 100644 --- a/src/robusta/core/model/runner_config.py +++ b/src/robusta/core/model/runner_config.py @@ -1,7 +1,7 @@ import base64 from typing import Dict, List, Optional, Union -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, root_validator, validator from robusta.core.playbooks.playbook_utils import get_env_replacement, replace_env_vars_values from robusta.core.sinks.datadog.datadog_sink_params import DataDogSinkConfigWrapper @@ -36,6 +36,14 @@ class PlaybookRepo(BaseModel): key: Optional[SecretStr] = SecretStr("") branch: Optional[str] = None # when url is a git repo pip_install: bool = True # Set to False, if the playbooks package is already in site-packages. + http_headers: Optional[Dict[str, str]] = None + build_isolation: bool = False + + @root_validator + def validate_pip_build_isolation(cls, values: Dict) -> Dict: + if values.get("build_isolation") and not values.get("pip_install"): + raise ValueError("build_isolation should not be specified for non-pip builds") + return values class RunnerConfig(BaseModel): diff --git a/src/robusta/runner/config_loader.py b/src/robusta/runner/config_loader.py index 732c2186e..3f98a5311 100644 --- a/src/robusta/runner/config_loader.py +++ b/src/robusta/runner/config_loader.py @@ -6,14 +6,17 @@ import signal import subprocess import sys +import tarfile +import tempfile import threading from inspect import getmembers from typing import Dict, Optional +import dpath.util +import requests +import toml import yaml -import toml -import dpath.util from robusta.core.model.env_vars import ( CUSTOM_PLAYBOOKS_ROOT, DEFAULT_PLAYBOOKS_PIP_INSTALL, @@ -116,25 +119,33 @@ def __load_playbooks_repos( for playbook_package, playbooks_repo in playbooks_repos.items(): try: if playbooks_repo.pip_install: # skip playbooks that are already in site-packages - if playbooks_repo.url.startswith(GIT_SSH_PREFIX) or playbooks_repo.url.startswith(GIT_HTTPS_PREFIX): - repo = GitRepo(playbooks_repo.url, playbooks_repo.key.get_secret_value(), playbooks_repo.branch) - local_path = repo.repo_local_path - elif playbooks_repo.url.startswith(LOCAL_PATH_URL_PREFIX): - local_path = playbooks_repo.url.replace(LOCAL_PATH_URL_PREFIX, "") + # Check that the config specifies an external Python package to be downloaded + # and installed in Robusta. + url = playbooks_repo.url + if url.startswith(("https://", "http://")) and url.endswith((".tar.gz", ".tgz")): + if url.startswith("http://"): + logging.warning(f"Downloading a playbook package from non-https source f{url}") + + playbook_package = self.install_package_remote_tgz( + url=url, headers=playbooks_repo.http_headers, build_isolation=playbooks_repo.build_isolation + ) + elif url.startswith((GIT_SSH_PREFIX, GIT_HTTPS_PREFIX)): + repo = GitRepo(url, playbooks_repo.key.get_secret_value(), playbooks_repo.branch) + pkg_path = repo.repo_local_path + self.install_package( + pkg_path=repo.repo_local_path, build_isolation=playbooks_repo.build_isolation + ) + playbook_package = self.__get_package_name(local_path=pkg_path) + elif url.startswith(LOCAL_PATH_URL_PREFIX): + pkg_path = url.replace(LOCAL_PATH_URL_PREFIX, "") + self.install_package(pkg_path=pkg_path, build_isolation=playbooks_repo.build_isolation) + playbook_package = self.__get_package_name(local_path=pkg_path) else: raise Exception( - f"Illegal playbook repo url {playbooks_repo.url}. " + f"Illegal playbook repo url {url}. " f"Must start with '{GIT_SSH_PREFIX}', '{GIT_HTTPS_PREFIX}' or '{LOCAL_PATH_URL_PREFIX}'" ) - if not os.path.exists(local_path): # in case the repo url was defined before it was actually loaded - logging.error(f"Playbooks local path {local_path} does not exist. Skipping") - continue - - # Adding to pip the playbooks repo from local_path - subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-build-isolation", local_path]) - playbook_package = self.__get_package_name(local_path=local_path) - playbook_packages.append(playbook_package) except Exception: logging.error(f"Failed to add playbooks repo {playbook_package}", exc_info=True) @@ -142,6 +153,20 @@ def __load_playbooks_repos( for package_name in playbook_packages: self.__import_playbooks_package(actions_registry, package_name) + @classmethod + def install_package(cls, pkg_path: str, build_isolation: bool) -> str: + logging.debug(f"Installing package {pkg_path}") + if not os.path.exists(pkg_path): + # In case the repo url was defined before it was actually loaded. Note we don't + # perform this check when the pkg is sourced from an externally hosted tgz etc. + logging.error(f"Playbook local path {pkg_path} does not exist. Skipping") + raise Exception(f"Playbook local path {pkg_path} does not exist. Skipping") + + # Adding to pip the playbooks repo from local_path + extra_pip_args = ["--no-build-isolation"] if build_isolation else [] + + subprocess.check_call([sys.executable, "-m", "pip", "install"] + extra_pip_args + [pkg_path]) + @classmethod def __import_playbooks_package(cls, actions_registry: ActionsRegistry, package_name: str): logging.info(f"Importing actions package {package_name}") @@ -292,3 +317,24 @@ def __load_runner_config(cls, config_file_path) -> Optional[RunnerConfig]: with open(config_file_path) as file: yaml_content = yaml.safe_load(file) return RunnerConfig(**yaml_content) + + @classmethod + def install_package_remote_tgz(cls, url: str, headers, build_isolation: bool) -> Optional[str]: + with tempfile.NamedTemporaryFile(suffix=".tgz") as f: + r = requests.get(url, stream=True, headers=headers) + r.raise_for_status() + for chunk in r.iter_content(chunk_size=65536): + f.write(chunk) + + f.flush() + + with tarfile.open(f.name, "r:gz") as tar, tempfile.TemporaryDirectory() as temp_dir: + tar.extractall(path=temp_dir) + extracted_items = os.listdir(temp_dir) + + pkg_path = temp_dir + if len(extracted_items) == 1: + pkg_path = os.path.join(temp_dir, extracted_items[0]) + + cls.install_package(pkg_path=pkg_path, build_isolation=build_isolation) + return cls.__get_package_name(local_path=pkg_path)