From f11d3b4313e5a6a13781a4406d17345eb785bac0 Mon Sep 17 00:00:00 2001 From: Robert Szefler Date: Thu, 27 Jun 2024 14:16:32 +0200 Subject: [PATCH 1/9] Handle playbooks from external tar/tgz files etc --- .../external-playbook-repositories.rst | 17 ++-- src/robusta/core/model/runner_config.py | 10 ++- src/robusta/runner/config_loader.py | 88 ++++++++++++++----- 3 files changed, 89 insertions(+), 26 deletions(-) diff --git a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst index 858e9353d..5fadef673 100644 --- a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst +++ b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst @@ -1,20 +1,27 @@ Loading External Actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Robusta can load playbook actions from external git repositories. This extends Robusta with additional actions for -use in :ref:`customPlaybooks`. +Robusta can load playbook actions from external git repositories and externally hosted +Python packages provided as .tar/.tar.gz files. This extends Robusta with additional +actions for use in :ref:`customPlaybooks`. .. warning:: - Robusta does not watch for changes in git repositories. Playbooks are reloaded when: + Robusta does not watch for changes in git repositories/externally hosted Python packages. + Playbooks are reloaded when: * Robusta starts * Robusta's configuration changes * ``robusta playbooks reload`` is run -External actions are loaded using the ``playbookRepos`` Helm value, with either HTTPs or SSH. +External actions are loaded using the ``playbookRepos`` Helm value, with either HTTPs or SSH +in the case of git repositories, and appropriate URLs in the case of externally hosted +Python packages. The way Robusta distinguishes between the case of git repository and an +external package is to check if the URL ends with `.tar`, `.tar.gz`, or `.tar.gz` - if that +is the case, the source is an external package; otherwise the URL is treated as a git +address. -If you are going to be using an external repository via HTTPS, you just need to configure +If you are going to be using an external git repository via HTTPS, you just need to configure correct read access credentials (see below). When connecting via SSH, however, there is an additional requirement to verify the remote host's identity on the client side, as SSH generally does not provide any method of doing that automatically (in contrast with HTTPS, diff --git a/src/robusta/core/model/runner_config.py b/src/robusta/core/model/runner_config.py index 3b7ffbb26..8bc390091 100644 --- a/src/robusta/core/model/runner_config.py +++ b/src/robusta/core/model/runner_config.py @@ -1,7 +1,7 @@ import base64 from typing import Dict, List, Optional, Union -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, root_validator, validator from robusta.core.playbooks.playbook_utils import get_env_replacement, replace_env_vars_values from robusta.core.sinks.datadog.datadog_sink_params import DataDogSinkConfigWrapper @@ -36,6 +36,14 @@ class PlaybookRepo(BaseModel): key: Optional[SecretStr] = SecretStr("") branch: Optional[str] = None # when url is a git repo pip_install: bool = True # Set to False, if the playbooks package is already in site-packages. + http_headers: Optional[Dict[str, str]] = None + build_isolation: Optional[bool] = False + + @root_validator + def validate_pip_build_isolation(cls, values: Dict) -> Dict: + if values.get("build_isolation") and not values.get("pip_install"): + raise ValueError("build_isolation should not be specified for non-pip builds") + return values class RunnerConfig(BaseModel): diff --git a/src/robusta/runner/config_loader.py b/src/robusta/runner/config_loader.py index 732c2186e..222d6e082 100644 --- a/src/robusta/runner/config_loader.py +++ b/src/robusta/runner/config_loader.py @@ -3,17 +3,21 @@ import logging import os import pkgutil +import shutil import signal import subprocess import sys +import tempfile import threading from inspect import getmembers from typing import Dict, Optional +from urllib.parse import urlparse +import dpath.util +import requests +import toml import yaml -import toml -import dpath.util from robusta.core.model.env_vars import ( CUSTOM_PLAYBOOKS_ROOT, DEFAULT_PLAYBOOKS_PIP_INSTALL, @@ -116,25 +120,58 @@ def __load_playbooks_repos( for playbook_package, playbooks_repo in playbooks_repos.items(): try: if playbooks_repo.pip_install: # skip playbooks that are already in site-packages - if playbooks_repo.url.startswith(GIT_SSH_PREFIX) or playbooks_repo.url.startswith(GIT_HTTPS_PREFIX): - repo = GitRepo(playbooks_repo.url, playbooks_repo.key.get_secret_value(), playbooks_repo.branch) - local_path = repo.repo_local_path - elif playbooks_repo.url.startswith(LOCAL_PATH_URL_PREFIX): - local_path = playbooks_repo.url.replace(LOCAL_PATH_URL_PREFIX, "") - else: - raise Exception( - f"Illegal playbook repo url {playbooks_repo.url}. " - f"Must start with '{GIT_SSH_PREFIX}', '{GIT_HTTPS_PREFIX}' or '{LOCAL_PATH_URL_PREFIX}'" - ) - - if not os.path.exists(local_path): # in case the repo url was defined before it was actually loaded - logging.error(f"Playbooks local path {local_path} does not exist. Skipping") - continue - - # Adding to pip the playbooks repo from local_path - subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-build-isolation", local_path]) - playbook_package = self.__get_package_name(local_path=local_path) + remove_pkg_path = None + try: + check_pkg_path = True + if ( + ( + playbooks_repo.url.startswith("https://") + or playbooks_repo.url.startswith("http://") + ) and ( + playbooks_repo.url.endswith(".tgz") + or playbooks_repo.url.endswith(".tar.gz") + or playbooks_repo.url.endswith(".tar") + ) + ): + if playbooks_repo.url.startswith("http://"): + logging.warning( + f"Downloading a playbook package from non-https source f{playbooks_repo.url}" + ) + temp_dir = tempfile.mkdtemp() + # after installing the downloaded package, remove it along with its directory + remove_pkg_path = temp_dir + pkg_name = urlparse(playbooks_repo.url).path.rsplit('/', 1)[-1] + filename = os.path.join(temp_dir, pkg_name) + self.download_file(playbooks_repo.url, filename, playbooks_repo.http_headers) + pkg_path = f"file://{filename}" + check_pkg_path = False + elif playbooks_repo.url.startswith(GIT_SSH_PREFIX) or playbooks_repo.url.startswith(GIT_HTTPS_PREFIX): + repo = GitRepo(playbooks_repo.url, playbooks_repo.key.get_secret_value(), playbooks_repo.branch) + pkg_path = repo.repo_local_path + elif playbooks_repo.url.startswith(LOCAL_PATH_URL_PREFIX): + pkg_path = playbooks_repo.url.replace(LOCAL_PATH_URL_PREFIX, "") + else: + raise Exception( + f"Illegal playbook repo url {playbooks_repo.url}. " + f"Must start with '{GIT_SSH_PREFIX}', '{GIT_HTTPS_PREFIX}' or '{LOCAL_PATH_URL_PREFIX}'" + ) + if check_pkg_path and not os.path.exists(pkg_path): + # In case the repo url was defined before it was actually loaded. Note we don't + # perform this check when the pkg is sourced from an externally hosted tgz etc. + logging.error(f"Playbook local path {pkg_path} does not exist. Skipping") + continue + + # Adding to pip the playbooks repo from local_path + if playbooks_repo.build_isolation: + extra_pip_args = [] + else: + extra_pip_args = ["--no-build-isolation"] + subprocess.check_call([sys.executable, "-m", "pip", "install"] + extra_pip_args + [pkg_path]) + playbook_package = self.__get_package_name(local_path=pip_path) + finally: + if remove_pkg_path: + shutil.rmtree(remove_pkg_path) playbook_packages.append(playbook_package) except Exception: logging.error(f"Failed to add playbooks repo {playbook_package}", exc_info=True) @@ -142,6 +179,17 @@ def __load_playbooks_repos( for package_name in playbook_packages: self.__import_playbooks_package(actions_registry, package_name) + @classmethod + def download_file(cls, url: str, dest_path: str, headers: Optional[Dict[str, str]]) -> None: + logging.debug(f"Downloading playbook file from {url} to {dest_path}") + # Note we don't need to care about deleting the file if something fails, it's done by + # the caller (__load_playbooks_repos above). + with requests.get(url, stream=True, headers=headers) as r: + r.raise_for_status() + f = open(dest_path, 'wb') + for chunk in r.iter_content(chunk_size=65536): + f.write(chunk) + @classmethod def __import_playbooks_package(cls, actions_registry: ActionsRegistry, package_name: str): logging.info(f"Importing actions package {package_name}") From 326eb092c2c459f9d5bb3b425d35eede344a1a37 Mon Sep 17 00:00:00 2001 From: Robert Szefler Date: Sat, 29 Jun 2024 15:37:30 +0200 Subject: [PATCH 2/9] handle .tar.bz2 & .tbz2; documentation additions --- .../external-playbook-repositories.rst | 14 ++++++++++++++ src/robusta/runner/config_loader.py | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst index 5fadef673..bb779a10f 100644 --- a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst +++ b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst @@ -116,6 +116,20 @@ The ``key`` parameter must contain a ``base64`` encoded deployment key with ``re ewfrcfsfvC1rZXktdjEAAAAABG5vb..... -----END OPENSSH PRIVATE KEY----- +Loading Actions from an external Python Package +--------------------------------------------------- + +For external Python packages, just specify an url starting with http(s), and ending with +either .tar, .tar.gz, .tgz, .tar.bz2, or .tbz2. + +.. code-block:: yaml + + playbookRepos: + web_playbooks: + url: "https://my-domain.com/bla/web-playbooks.tgz" + http_headers: # optional, may be used for auth + Authorization: Bearer XXXYYY + # pip_install: True # optional: load this playbook's dependencies (default True) Handling Secrets ******************* diff --git a/src/robusta/runner/config_loader.py b/src/robusta/runner/config_loader.py index 222d6e082..e08f87bbe 100644 --- a/src/robusta/runner/config_loader.py +++ b/src/robusta/runner/config_loader.py @@ -128,9 +128,11 @@ def __load_playbooks_repos( playbooks_repo.url.startswith("https://") or playbooks_repo.url.startswith("http://") ) and ( - playbooks_repo.url.endswith(".tgz") + playbooks_repo.url.endswith(".tar") or playbooks_repo.url.endswith(".tar.gz") - or playbooks_repo.url.endswith(".tar") + or playbooks_repo.url.endswith(".tgz") + or playbooks_repo.url.endswith(".tar.bz2") + or playbooks_repo.url.endswith(".tbz2") ) ): if playbooks_repo.url.startswith("http://"): From 8a93e2de2b7219552a60a9d12cf2587e032d4a02 Mon Sep 17 00:00:00 2001 From: Robert Szefler Date: Sat, 29 Jun 2024 15:38:27 +0200 Subject: [PATCH 3/9] ignore shutil.rmtree errors --- src/robusta/runner/config_loader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/robusta/runner/config_loader.py b/src/robusta/runner/config_loader.py index e08f87bbe..02f838ccf 100644 --- a/src/robusta/runner/config_loader.py +++ b/src/robusta/runner/config_loader.py @@ -173,7 +173,10 @@ def __load_playbooks_repos( playbook_package = self.__get_package_name(local_path=pip_path) finally: if remove_pkg_path: - shutil.rmtree(remove_pkg_path) + try: + shutil.rmtree(remove_pkg_path) + except: + pass playbook_packages.append(playbook_package) except Exception: logging.error(f"Failed to add playbooks repo {playbook_package}", exc_info=True) From 255337b4d167698c21ee08b6d4eaf3c1beeacaa6 Mon Sep 17 00:00:00 2001 From: Robert Szefler Date: Sat, 29 Jun 2024 16:40:37 +0200 Subject: [PATCH 4/9] minor doc fixes --- .../defining-playbooks/external-playbook-repositories.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst index bb779a10f..d235cb553 100644 --- a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst +++ b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst @@ -17,9 +17,9 @@ actions for use in :ref:`customPlaybooks`. External actions are loaded using the ``playbookRepos`` Helm value, with either HTTPs or SSH in the case of git repositories, and appropriate URLs in the case of externally hosted Python packages. The way Robusta distinguishes between the case of git repository and an -external package is to check if the URL ends with `.tar`, `.tar.gz`, or `.tar.gz` - if that -is the case, the source is an external package; otherwise the URL is treated as a git -address. +external package is to check if the URL ends with `.tar`, `.tar.gz`, `.tar.gz`, `.tar.bz2`, +or `.tbz2` - if that is the case, the source is treated as an external package; otherwise the +URL is treated as a git repository address. If you are going to be using an external git repository via HTTPS, you just need to configure correct read access credentials (see below). When connecting via SSH, however, there is an @@ -119,7 +119,7 @@ The ``key`` parameter must contain a ``base64`` encoded deployment key with ``re Loading Actions from an external Python Package --------------------------------------------------- -For external Python packages, just specify an url starting with http(s), and ending with +For external Python packages, just specify an URL starting with http(s), and ending with either .tar, .tar.gz, .tgz, .tar.bz2, or .tbz2. .. code-block:: yaml From 6d20254e562526bd06ef6efd6ec707d9ffcb355b Mon Sep 17 00:00:00 2001 From: Robert Szefler Date: Tue, 9 Jul 2024 09:54:35 +0200 Subject: [PATCH 5/9] improve docs --- .../external-playbook-repositories.rst | 9 ++++++++- src/robusta/core/model/runner_config.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst index d235cb553..2414d7f4b 100644 --- a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst +++ b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst @@ -129,7 +129,14 @@ either .tar, .tar.gz, .tgz, .tar.bz2, or .tbz2. url: "https://my-domain.com/bla/web-playbooks.tgz" http_headers: # optional, may be used for auth Authorization: Bearer XXXYYY - # pip_install: True # optional: load this playbook's dependencies (default True) + # pip_install: true # optional: load this playbook's dependencies (default True) + # build_isolation: false + +Note that the `http_headers` option is only available for this method of loading actions. Also, +the `build_isolation` is optional (defaults to `true`). If specified as `false`, the `pip` +install command for the package being installed will be run with `--no-build-isolation` (see +the `pip docs `_ +for details). Handling Secrets ******************* diff --git a/src/robusta/core/model/runner_config.py b/src/robusta/core/model/runner_config.py index 8bc390091..60a4aa091 100644 --- a/src/robusta/core/model/runner_config.py +++ b/src/robusta/core/model/runner_config.py @@ -37,7 +37,7 @@ class PlaybookRepo(BaseModel): branch: Optional[str] = None # when url is a git repo pip_install: bool = True # Set to False, if the playbooks package is already in site-packages. http_headers: Optional[Dict[str, str]] = None - build_isolation: Optional[bool] = False + build_isolation: bool = False @root_validator def validate_pip_build_isolation(cls, values: Dict) -> Dict: From a66084980e62d2d2200c75b8ad7dfafcba1e9b31 Mon Sep 17 00:00:00 2001 From: Robert Szefler Date: Tue, 9 Jul 2024 10:07:50 +0200 Subject: [PATCH 6/9] add a comment --- src/robusta/runner/config_loader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/robusta/runner/config_loader.py b/src/robusta/runner/config_loader.py index 02f838ccf..cd36ead9a 100644 --- a/src/robusta/runner/config_loader.py +++ b/src/robusta/runner/config_loader.py @@ -123,6 +123,8 @@ def __load_playbooks_repos( remove_pkg_path = None try: check_pkg_path = True + # Check that the config specifies an external Python package to be downloaded + # and installed in Robusta. if ( ( playbooks_repo.url.startswith("https://") From edc7df3f0e8546e60f6b762d9905dd35e31c35df Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Thu, 18 Jul 2024 18:24:33 +0300 Subject: [PATCH 7/9] rework install remote tgz playbook installation --- .../external-playbook-repositories.rst | 2 +- src/robusta/runner/config_loader.py | 129 ++++++++---------- 2 files changed, 61 insertions(+), 70 deletions(-) diff --git a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst index 2414d7f4b..1ccf2c8fd 100644 --- a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst +++ b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst @@ -120,7 +120,7 @@ Loading Actions from an external Python Package --------------------------------------------------- For external Python packages, just specify an URL starting with http(s), and ending with -either .tar, .tar.gz, .tgz, .tar.bz2, or .tbz2. +either .tar.gz or .tgz. .. code-block:: yaml diff --git a/src/robusta/runner/config_loader.py b/src/robusta/runner/config_loader.py index cd36ead9a..3f98a5311 100644 --- a/src/robusta/runner/config_loader.py +++ b/src/robusta/runner/config_loader.py @@ -3,15 +3,14 @@ import logging import os import pkgutil -import shutil import signal import subprocess import sys +import tarfile import tempfile import threading from inspect import getmembers from typing import Dict, Optional -from urllib.parse import urlparse import dpath.util import requests @@ -120,65 +119,33 @@ def __load_playbooks_repos( for playbook_package, playbooks_repo in playbooks_repos.items(): try: if playbooks_repo.pip_install: # skip playbooks that are already in site-packages - remove_pkg_path = None - try: - check_pkg_path = True - # Check that the config specifies an external Python package to be downloaded - # and installed in Robusta. - if ( - ( - playbooks_repo.url.startswith("https://") - or playbooks_repo.url.startswith("http://") - ) and ( - playbooks_repo.url.endswith(".tar") - or playbooks_repo.url.endswith(".tar.gz") - or playbooks_repo.url.endswith(".tgz") - or playbooks_repo.url.endswith(".tar.bz2") - or playbooks_repo.url.endswith(".tbz2") - ) - ): - if playbooks_repo.url.startswith("http://"): - logging.warning( - f"Downloading a playbook package from non-https source f{playbooks_repo.url}" - ) - temp_dir = tempfile.mkdtemp() - # after installing the downloaded package, remove it along with its directory - remove_pkg_path = temp_dir - pkg_name = urlparse(playbooks_repo.url).path.rsplit('/', 1)[-1] - filename = os.path.join(temp_dir, pkg_name) - self.download_file(playbooks_repo.url, filename, playbooks_repo.http_headers) - pkg_path = f"file://{filename}" - check_pkg_path = False - elif playbooks_repo.url.startswith(GIT_SSH_PREFIX) or playbooks_repo.url.startswith(GIT_HTTPS_PREFIX): - repo = GitRepo(playbooks_repo.url, playbooks_repo.key.get_secret_value(), playbooks_repo.branch) - pkg_path = repo.repo_local_path - elif playbooks_repo.url.startswith(LOCAL_PATH_URL_PREFIX): - pkg_path = playbooks_repo.url.replace(LOCAL_PATH_URL_PREFIX, "") - else: - raise Exception( - f"Illegal playbook repo url {playbooks_repo.url}. " - f"Must start with '{GIT_SSH_PREFIX}', '{GIT_HTTPS_PREFIX}' or '{LOCAL_PATH_URL_PREFIX}'" - ) + # Check that the config specifies an external Python package to be downloaded + # and installed in Robusta. + url = playbooks_repo.url + if url.startswith(("https://", "http://")) and url.endswith((".tar.gz", ".tgz")): + if url.startswith("http://"): + logging.warning(f"Downloading a playbook package from non-https source f{url}") + + playbook_package = self.install_package_remote_tgz( + url=url, headers=playbooks_repo.http_headers, build_isolation=playbooks_repo.build_isolation + ) + elif url.startswith((GIT_SSH_PREFIX, GIT_HTTPS_PREFIX)): + repo = GitRepo(url, playbooks_repo.key.get_secret_value(), playbooks_repo.branch) + pkg_path = repo.repo_local_path + self.install_package( + pkg_path=repo.repo_local_path, build_isolation=playbooks_repo.build_isolation + ) + playbook_package = self.__get_package_name(local_path=pkg_path) + elif url.startswith(LOCAL_PATH_URL_PREFIX): + pkg_path = url.replace(LOCAL_PATH_URL_PREFIX, "") + self.install_package(pkg_path=pkg_path, build_isolation=playbooks_repo.build_isolation) + playbook_package = self.__get_package_name(local_path=pkg_path) + else: + raise Exception( + f"Illegal playbook repo url {url}. " + f"Must start with '{GIT_SSH_PREFIX}', '{GIT_HTTPS_PREFIX}' or '{LOCAL_PATH_URL_PREFIX}'" + ) - if check_pkg_path and not os.path.exists(pkg_path): - # In case the repo url was defined before it was actually loaded. Note we don't - # perform this check when the pkg is sourced from an externally hosted tgz etc. - logging.error(f"Playbook local path {pkg_path} does not exist. Skipping") - continue - - # Adding to pip the playbooks repo from local_path - if playbooks_repo.build_isolation: - extra_pip_args = [] - else: - extra_pip_args = ["--no-build-isolation"] - subprocess.check_call([sys.executable, "-m", "pip", "install"] + extra_pip_args + [pkg_path]) - playbook_package = self.__get_package_name(local_path=pip_path) - finally: - if remove_pkg_path: - try: - shutil.rmtree(remove_pkg_path) - except: - pass playbook_packages.append(playbook_package) except Exception: logging.error(f"Failed to add playbooks repo {playbook_package}", exc_info=True) @@ -187,15 +154,18 @@ def __load_playbooks_repos( self.__import_playbooks_package(actions_registry, package_name) @classmethod - def download_file(cls, url: str, dest_path: str, headers: Optional[Dict[str, str]]) -> None: - logging.debug(f"Downloading playbook file from {url} to {dest_path}") - # Note we don't need to care about deleting the file if something fails, it's done by - # the caller (__load_playbooks_repos above). - with requests.get(url, stream=True, headers=headers) as r: - r.raise_for_status() - f = open(dest_path, 'wb') - for chunk in r.iter_content(chunk_size=65536): - f.write(chunk) + def install_package(cls, pkg_path: str, build_isolation: bool) -> str: + logging.debug(f"Installing package {pkg_path}") + if not os.path.exists(pkg_path): + # In case the repo url was defined before it was actually loaded. Note we don't + # perform this check when the pkg is sourced from an externally hosted tgz etc. + logging.error(f"Playbook local path {pkg_path} does not exist. Skipping") + raise Exception(f"Playbook local path {pkg_path} does not exist. Skipping") + + # Adding to pip the playbooks repo from local_path + extra_pip_args = ["--no-build-isolation"] if build_isolation else [] + + subprocess.check_call([sys.executable, "-m", "pip", "install"] + extra_pip_args + [pkg_path]) @classmethod def __import_playbooks_package(cls, actions_registry: ActionsRegistry, package_name: str): @@ -347,3 +317,24 @@ def __load_runner_config(cls, config_file_path) -> Optional[RunnerConfig]: with open(config_file_path) as file: yaml_content = yaml.safe_load(file) return RunnerConfig(**yaml_content) + + @classmethod + def install_package_remote_tgz(cls, url: str, headers, build_isolation: bool) -> Optional[str]: + with tempfile.NamedTemporaryFile(suffix=".tgz") as f: + r = requests.get(url, stream=True, headers=headers) + r.raise_for_status() + for chunk in r.iter_content(chunk_size=65536): + f.write(chunk) + + f.flush() + + with tarfile.open(f.name, "r:gz") as tar, tempfile.TemporaryDirectory() as temp_dir: + tar.extractall(path=temp_dir) + extracted_items = os.listdir(temp_dir) + + pkg_path = temp_dir + if len(extracted_items) == 1: + pkg_path = os.path.join(temp_dir, extracted_items[0]) + + cls.install_package(pkg_path=pkg_path, build_isolation=build_isolation) + return cls.__get_package_name(local_path=pkg_path) From e50bc2fa697b96b23363532cf574db77a7d45f8b Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Mon, 22 Jul 2024 10:30:21 +0300 Subject: [PATCH 8/9] docs fixes --- .../external-playbook-repositories.rst | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst index 1ccf2c8fd..6e4898fd4 100644 --- a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst +++ b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst @@ -2,7 +2,7 @@ Loading External Actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Robusta can load playbook actions from external git repositories and externally hosted -Python packages provided as .tar/.tar.gz files. This extends Robusta with additional +Python packages provided as .tar or tar.gz files. This extends Robusta with additional actions for use in :ref:`customPlaybooks`. .. warning:: @@ -17,8 +17,8 @@ actions for use in :ref:`customPlaybooks`. External actions are loaded using the ``playbookRepos`` Helm value, with either HTTPs or SSH in the case of git repositories, and appropriate URLs in the case of externally hosted Python packages. The way Robusta distinguishes between the case of git repository and an -external package is to check if the URL ends with `.tar`, `.tar.gz`, `.tar.gz`, `.tar.bz2`, -or `.tbz2` - if that is the case, the source is treated as an external package; otherwise the +external package is to check if the URL ends with `.tar` or `.tar.gz` +- if that is the case, the source is treated as an external package; otherwise the URL is treated as a git repository address. If you are going to be using an external git repository via HTTPS, you just need to configure @@ -132,11 +132,7 @@ either .tar.gz or .tgz. # pip_install: true # optional: load this playbook's dependencies (default True) # build_isolation: false -Note that the `http_headers` option is only available for this method of loading actions. Also, -the `build_isolation` is optional (defaults to `true`). If specified as `false`, the `pip` -install command for the package being installed will be run with `--no-build-isolation` (see -the `pip docs `_ -for details). +The `http_headers` option is only available for this method of loading actions. Handling Secrets ******************* @@ -159,6 +155,14 @@ Then reference it using an environment variable: url: "git@github.com:robusta-dev/robusta-chaos.git" key: "{{env.GITHUB_SSH_KEY}}" +Build Isolation +***************** + +``build_isolation`` is optional (defaults to `true`). If specified as `false`, the `pip` +install command for the package being installed will be run with `--no-build-isolation` (see +the `pip docs `_ +for details). + Baking Actions into a Custom Image -------------------------------------- From 86e76bc13d30675b5cdf9a833a7f0963d7032c09 Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Mon, 22 Jul 2024 11:03:32 +0300 Subject: [PATCH 9/9] fix supported files --- .../defining-playbooks/external-playbook-repositories.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst index 6e4898fd4..abf95e0e6 100644 --- a/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst +++ b/docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst @@ -2,7 +2,7 @@ Loading External Actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Robusta can load playbook actions from external git repositories and externally hosted -Python packages provided as .tar or tar.gz files. This extends Robusta with additional +Python packages provided as .tgz or .tar.gz files. This extends Robusta with additional actions for use in :ref:`customPlaybooks`. .. warning:: @@ -17,7 +17,7 @@ actions for use in :ref:`customPlaybooks`. External actions are loaded using the ``playbookRepos`` Helm value, with either HTTPs or SSH in the case of git repositories, and appropriate URLs in the case of externally hosted Python packages. The way Robusta distinguishes between the case of git repository and an -external package is to check if the URL ends with `.tar` or `.tar.gz` +external package is to check if the URL ends with `.tgz` or `.tar.gz` - if that is the case, the source is treated as an external package; otherwise the URL is treated as a git repository address.