Git playbooks (robusta-dev#124)

* git playbooks

* git playbooks

* self ui registration via cli

Co-authored-by: Robusta Runner <runner@robusta.dev>

Dockerfile

@@ -28,8 +28,6 @@ RUN pip3 install --use-feature=in-tree-build .
 RUN pip3 install git+https://github.com/astanin/python-tabulate.git@b2c26bcb70e497f674b38aa7e29de12c0123708a#egg=tabulate
 
 COPY playbooks/ /etc/robusta/playbooks/defaults
-RUN pip3 install -r /etc/robusta/playbooks/defaults/requirements.txt
-# remove the requirements so that we don't reinstall them at runtime
-RUN rm /etc/robusta/playbooks/defaults/requirements.txt
+
 # -u disables stdout buffering https://stackoverflow.com/questions/107705/disable-output-buffering
 CMD [ "python3", "-u", "-m", "robusta.runner.main"]

README.md

@@ -26,6 +26,8 @@ This is never necessary, but you might find it more convenient to run Robusta lo
 
 If you're on Mac OS and receive errors about `Pillow` or `libjpeg` when running `poetry install` then run `brew install libjpeg` first.
 
+If you encounter `NotADirectoryError: [Errno 20] Not a directory` while trying to debug, you may need to disable the `Attach to subprocess` option on your debugger.
+
 # Running Robusta cli locally as a developer
 This is only necessary if you are developing features for the cli itself.
 

helm/robusta/templates/_helpers.tpl

@@ -1,10 +1,6 @@
 {{ define "robusta.configfile" -}}
-{{- if or .Values.playbook_sets }}
-playbook_sets:
-{{- range $playbook_set := .Values.playbook_sets }}
-- {{ $playbook_set }}
-{{- end }}
-{{- end }}
+playbook_repos:
+{{ toYaml .Values.playbookRepos | indent 2 }}
 {{- if or .Values.slackApiKey .Values.robustaApiKey }}
 sinks_config:
 {{- if .Values.slackApiKey }}

helm/robusta/templates/runner.yaml

@@ -23,8 +23,6 @@ spec:
         securityContext:
           privileged: false
         env:
-          - name: PLAYBOOKS_ROOT
-            value: /etc/robusta/playbooks/
           - name: PLAYBOOKS_CONFIG_FILE_PATH
             value: /etc/robusta/config/active_playbooks.yaml
           - name: ENABLE_MANHOLE
@@ -46,8 +44,10 @@ spec:
         volumeMounts:
           - name: playbooks-config
             mountPath: /etc/robusta/config
-          - name: custom-playbooks
-            mountPath: /etc/robusta/playbooks/custom
+          {{- if .Values.playbooksPersistentVolume }}
+          - name: persistent-playbooks-storage
+            mountPath: /etc/robusta/playbooks/storage
+          {{- end }}
         lifecycle:
           preStop:
             exec:
@@ -78,11 +78,24 @@ spec:
           configMap:
             name: robusta-playbooks-config
             optional: true
-        - name: custom-playbooks
-          configMap:
-            name: robusta-custom-playbooks
-            optional: true
-
+        {{- if .Values.playbooksPersistentVolume }}
+        - name: persistent-playbooks-storage
+          persistentVolumeClaim:
+            claimName: persistent-playbooks-pv-claim
+        {{- end }}
+{{- if .Values.playbooksPersistentVolume }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: persistent-playbooks-pv-claim
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 128Mi
+{{- end }}
 ---
 apiVersion: v1
 kind: Service

helm/robusta/values.yaml

@@ -1,5 +1,7 @@
-# playbooks sets
-playbook_sets: []
+# playbook repositories
+playbookRepos:
+  robusta_playbooks:
+    url: "file:///etc/robusta/playbooks/defaults"
 
 # global parameters
 clusterName: ""
@@ -27,6 +29,9 @@ enableServiceMonitors: false
 # disable messages routed by Robusta cloud
 disableCloudRouting: false
 
+# Enable loading playbooks to a persistent volume
+playbooksPersistentVolume: false
+
 # custom user playbooks
 customPlaybooks: []
 

playbooks/pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "robusta_playbooks"
+version = "0.0.1"
+description = ""
+authors = ["Arik Alon <arikalon1@users.noreply.github.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.7.1"
+CairoSVG = "^2.5.2"
+Flask = "^2.0.2"
+prometheus-api-client = "^0.4.2"
+pygal = "^3.0.0"
+tinycss = "^0.4"
+cssselect = "^1.1.0"
+rsa = "^4.8"
+humanize = "^3.13.1"
+
+[tool.poetry.dev-dependencies]
+robusta-cli = "^0.8.9"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

playbooks/git_change_audit.py → playbooks/robusta_playbooks/git_change_audit.py

@@ -53,21 +53,23 @@ def git_change_audit(event: KubernetesAnyChangeEvent, action_params: GitAuditPar
     git_repo = GitRepoManager.get_git_repo(
         action_params.git_url,
         action_params.git_key.get_secret_value(),
-        action_params.cluster_name,
     )
     name = f"{git_safe_name(event.obj.metadata.name)}.yaml"
     namespace = event.obj.metadata.namespace or "None"
     path = f"{git_safe_name(action_params.cluster_name)}/{git_safe_name(namespace)}"
 
     if event.operation == K8sOperationType.DELETE:
-        git_repo.delete_push(path, name, f"Delete {path}/{name}")
+        git_repo.delete_push(
+            path, name, f"Delete {path}/{name}", action_params.cluster_name
+        )
     elif event.operation == K8sOperationType.CREATE:
         obj_yaml = hikaru.get_yaml(event.obj.spec)
         git_repo.commit_push(
             obj_yaml,
             path,
             name,
             f"Create {event.obj.kind} named {event.obj.metadata.name} on namespace {namespace}",
+            action_params.cluster_name,
         )
     else:  # update
         old_spec = event.old_obj.spec if event.old_obj else None
@@ -79,4 +81,5 @@ def git_change_audit(event: KubernetesAnyChangeEvent, action_params: GitAuditPar
                 path,
                 name,
                 f"Update {event.obj.kind} named {event.obj.metadata.name} on namespace {namespace}",
+                action_params.cluster_name,
             )

playbooks/node_enrichments.py → playbooks/robusta_playbooks/node_enrichments.py

@@ -1,5 +1,5 @@
 from robusta.api import *
-from networking import incluster_ping, PingParams
+from .networking import incluster_ping, PingParams
 
 
 def pod_row(pod: Pod) -> List[str]:

src/robusta/cli/main.py

@@ -125,10 +125,36 @@ def gen_config(
     # asking the question
     if robusta_api_key is None:
         if typer.confirm("Would you like to use Robusta UI?"):
-            robusta_api_key = typer.prompt(
-                "Please insert your Robusta account token",
-                default=None,
-            )
+            if typer.confirm("Do you already have a Robusta account?"):
+                robusta_api_key = typer.prompt(
+                    "Please insert your Robusta account token",
+                    default=None,
+                )
+            else:  # self registration
+                account_name = typer.prompt("Choose your account name")
+                email = typer.prompt(
+                    "Enter a GMail/GSuite address. This will be used to login"
+                )
+                res = requests.post(
+                    "https://robusta.dev/accounts/create",
+                    json={
+                        "account_name": account_name,
+                        "email": email,
+                    },
+                )
+                if res.status_code == 201:
+                    robusta_api_key = res.json().get("token")
+                    typer.echo(
+                        "\nSuccessfully registered. You can login at https://platform.robusta.dev\n",
+                        color="green",
+                    )
+                    typer.echo("A few more questions and we're done...\n")
+                else:
+                    typer.echo(
+                        "Sorry, something didn't work out. Please contact us at support@robusta.dev",
+                        color="red",
+                    )
+                    robusta_api_key = ""
         else:
             robusta_api_key = ""
 
@@ -188,6 +214,10 @@ def gen_config(
             f"Saved configuration to {output_path}",
             fg="green",
         )
+        typer.secho(
+            f"Save this file for future use. It contains your account credentials",
+            fg="red",
+        )
 
 
 @app.command()

src/robusta/cli/playbooks_cmd.py

@@ -20,13 +20,29 @@
     PLAYBOOKS_DIR,
 )
 
+PLAYBOOKS_MOUNT_LOCATION = "/etc/robusta/playbooks/storage"
 
 NAMESPACE_EXPLANATION = (
     "Installation namespace. If none use the namespace currently active with kubectl."
 )
 app = typer.Typer()
 
 
+def get_runner_pod(namespace: str) -> Optional[str]:
+    pods = subprocess.check_output(
+        f'kubectl get pods {namespace_to_kubectl(namespace)} --selector="robustaComponent=runner"',
+        shell=True,
+    )
+    text = pods.decode("utf-8")
+    if not text:
+        return None
+
+    lines = text.split("\n")
+    if len(lines) < 2:
+        return None
+    return lines[1].split(" ")[0]
+
+
 @app.command()
 def push(
     playbooks_directory: str = typer.Argument(
@@ -41,18 +57,30 @@ def push(
     """Load custom playbooks code"""
     log_title("Uploading playbooks code...")
     with fetch_runner_logs(namespace):
+        runner_pod = get_runner_pod(namespace)
+        if not runner_pod:
+            log_title("Runner pod not found.", color="red")
+            return
+
         subprocess.check_call(
-            f"kubectl create configmap {namespace_to_kubectl(namespace)} robusta-custom-playbooks --from-file {playbooks_directory} -o yaml --dry-run | kubectl apply -f -",
+            f"kubectl exec -it {namespace_to_kubectl(namespace)} {runner_pod} -c runner "
+            f"-- bash -c 'mkdir -p {PLAYBOOKS_MOUNT_LOCATION}'",
             shell=True,
         )
         subprocess.check_call(
-            f'kubectl annotate pods {namespace_to_kubectl(namespace)} --selector="robustaComponent=runner" --overwrite "playbooks-last-modified={time.time()}"',
+            f"kubectl cp {namespace_to_kubectl(namespace)} {playbooks_directory} "
+            f"{runner_pod}:{PLAYBOOKS_MOUNT_LOCATION}/ -c runner",
             shell=True,
         )
         time.sleep(
             5
         )  # wait five seconds for the runner to actually reload the playbooks
     log_title("Loaded custom playbooks code!")
+    log_title(
+        f"Make sure your playbook repos configuration contains:\n"
+        f"PLAYBOOKS_PACKAGE_NAME:\n"
+        f'  url: "file://{os.path.join(PLAYBOOKS_MOUNT_LOCATION, os.path.basename(playbooks_directory))}"'
+    )
 
 
 @app.command()
@@ -91,14 +119,6 @@ def get_playbooks_config(namespace: str):
     return yaml.safe_load(configmap_content)
 
 
-def get_custom_playbooks(namespace: str):
-    configmap_content = subprocess.check_output(
-        f"kubectl get configmap {namespace_to_kubectl(namespace)} robusta-custom-playbooks -o yaml",
-        shell=True,
-    )
-    return yaml.safe_load(configmap_content)
-
-
 @app.command()
 def pull(
     playbooks_directory: str = typer.Argument(
@@ -117,17 +137,83 @@ def pull(
     log_title(f"Pulling playbooks into {playbooks_directory} ")
 
     try:
-        playbooks_config = get_custom_playbooks(namespace)
-
-        for file_name in playbooks_config["data"].keys():
-            playbook_file = os.path.join(playbooks_directory, file_name)
-            with open(playbook_file, "w") as f:
-                f.write(playbooks_config["data"][file_name])
+        runner_pod = get_runner_pod(namespace)
+        if not runner_pod:
+            log_title("Runner pod not found.", color="red")
+            return
 
+        subprocess.check_call(
+            f"kubectl cp {namespace_to_kubectl(namespace)} "
+            f"{runner_pod}:{PLAYBOOKS_MOUNT_LOCATION}/ -c runner {playbooks_directory}",
+            shell=True,
+        )
     except Exception as e:
         typer.echo(f"Failed to pull deployed playbooks {e}", traceback.print_exc())
 
 
+@app.command("list-dirs")
+def list_dirs(
+    namespace: str = typer.Option(
+        None,
+        help=NAMESPACE_EXPLANATION,
+    ),
+):
+    """List stored playbooks directories"""
+    log_title(f"Listing playbooks directories ")
+
+    try:
+        runner_pod = get_runner_pod(namespace)
+        if not runner_pod:
+            log_title("Runner pod not found.", color="red")
+            return
+
+        ls_res = subprocess.check_output(
+            f"kubectl exec -it {namespace_to_kubectl(namespace)} {runner_pod} -c runner "
+            f"-- bash -c 'ls {PLAYBOOKS_MOUNT_LOCATION}'",
+            shell=True,
+        )
+
+        log_title(f"Stored playbooks directories: \n { ls_res.decode('utf-8')}")
+
+    except Exception as e:
+        typer.echo(f"Failed to list deployed playbooks {e}", traceback.print_exc())
+
+
+@app.command()
+def delete(
+    playbooks_directory: str = typer.Argument(
+        ...,
+        help="Playbooks directory that should be deleted",
+    ),
+    namespace: str = typer.Option(
+        None,
+        help=NAMESPACE_EXPLANATION,
+    ),
+):
+    """delete playbooks directory from storage"""
+    if not playbooks_directory:
+        log_title("Playbooks directory not specified", "red")
+        return
+
+    log_title(f"Deleting playbooks directory {playbooks_directory} ")
+
+    try:
+        runner_pod = get_runner_pod(namespace)
+        if not runner_pod:
+            log_title("Runner pod not found.", color="red")
+            return
+
+        path_to_delete = os.path.join(PLAYBOOKS_MOUNT_LOCATION, playbooks_directory)
+        subprocess.check_call(
+            f"kubectl exec -it {namespace_to_kubectl(namespace)} {runner_pod} -c runner "
+            f"-- bash -c 'rm -rf {path_to_delete}'",
+            shell=True,
+        )
+
+    except Exception as e:
+        typer.echo(f"Failed to delete deployed playbooks {e}", traceback.print_exc())
+
+
 def print_yaml_if_not_none(key: str, json_dict: dict):
     if json_dict.get(key):
         json = {}