qhub cli tool to save kubernetes logs - qhub support (nebari-dev#818)

* initial commit for qhub support command

* WIP test out different commands

* add zip support to logging

* need sub directory for file writing

* get namespace from qhub-config.yaml file

* add spike notebook for using kubectl

* black formatting

* add missing raise exception

* add kubernetes dependency

* Delete kubectl_spike.ipynb

* Add support for multi-container pods

Co-authored-by: Abraham <utabe11@hotmail.com>
Co-authored-by: t-potts <tylerjacobpotts@gmail.com>
Co-authored-by: Amit Kumar <dtu.amit@gmail.com>
Co-authored-by: iameskild <eskild@doublee.io>

environment-dev.yaml

@@ -8,7 +8,8 @@ dependencies:
   - auth0-python
   - pydantic
   - pynacl
-
+  - bcrypt
+  - python-kubernetes
   # dev dependencies
   - flake8 ==3.8.4
   - black ==20.8b1

qhub/cli/__init__.py

@@ -8,6 +8,7 @@
 from qhub.cli.render import create_render_subcommand
 from qhub.cli.validate import create_validate_subcommand
 from qhub.cli.destroy import create_destroy_subcommand
+from qhub.cli.support import create_support_subcommand
 from qhub.cli.upgrade import create_upgrade_subcommand
 from qhub.cli.keycloak import create_keycloak_subcommand
 from qhub.provider.terraform import TerraformException
@@ -32,6 +33,7 @@ def cli(args):
     create_init_subcommand(subparser)
     create_validate_subcommand(subparser)
     create_destroy_subcommand(subparser)
+    create_support_subcommand(subparser)
     create_upgrade_subcommand(subparser)
     create_keycloak_subcommand(subparser)
 

qhub/cli/support.py

@@ -0,0 +1,83 @@
+from kubernetes import client, config
+from ruamel import yaml
+from pathlib import Path
+from zipfile import ZipFile
+
+QHUB_SUPPORT_LOG_FILE = "./qhub-support-logs.zip"
+
+
+def create_support_subcommand(subparser):
+    subparser = subparser.add_parser("support")
+
+    subparser.add_argument(
+        "-c", "--config", help="qhub configuration yaml file", required=True
+    )
+
+    subparser.add_argument(
+        "-o", "--output", default=QHUB_SUPPORT_LOG_FILE, help="output filename"
+    )
+
+    subparser.set_defaults(func=handle_support)
+
+
+def handle_support(args):
+    config.load_kube_config()
+
+    v1 = client.CoreV1Api()
+
+    namespace = get_config_namespace(config=args.config)
+
+    pods = v1.list_namespaced_pod(namespace=namespace)
+
+    for pod in pods.items:
+        Path(f"./log/{namespace}").mkdir(parents=True, exist_ok=True)
+        path = Path(f"./log/{namespace}/{pod.metadata.name}.txt")
+        with path.open(mode="wt") as file:
+            try:
+                file.write(
+                    "%s\t%s\t%s\n"
+                    % (
+                        pod.status.pod_ip,
+                        namespace,
+                        pod.metadata.name,
+                    )
+                )
+
+                # some pods are running multiple containers
+                containers = [
+                    _.name if len(pod.spec.containers) > 1 else None
+                    for _ in pod.spec.containers
+                ]
+
+                for container in containers:
+                    if container is not None:
+                        file.write(f"Container: {container}\n")
+                    file.write(
+                        v1.read_namespaced_pod_log(
+                            name=pod.metadata.name,
+                            namespace=namespace,
+                            container=container,
+                        )
+                    )
+
+            except client.exceptions.ApiException as e:
+                file.write("%s not available" % pod.metadata.name)
+                raise e
+
+    with ZipFile(QHUB_SUPPORT_LOG_FILE, "w") as zip:
+        for file in list(Path(f"./log/{namespace}").glob("*.txt")):
+            print(file)
+            zip.write(file)
+
+
+def get_config_namespace(config):
+    config_filename = Path(config)
+    if not config_filename.is_file():
+        raise ValueError(
+            f"passed in configuration filename={config_filename} must exist"
+        )
+
+    with config_filename.open() as f:
+        config = yaml.safe_load(f.read())
+
+    return config["namespace"]

setup.py

@@ -39,6 +39,7 @@
         "pydantic",
         "pynacl",
         "bcrypt",
+        "kubernetes",
         "azure-identity==1.6.1",
         "azure-mgmt-containerservice==16.2.0",
         "packaging",