Merge pull request #16 from OmegaVVeapon/fix/init-container

Support for METHOD env var LIST mode

Pipfile

@@ -4,11 +4,11 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-kopf = "==0.28.3"
+kopf = "==1.29.2"
+pykube-ng = "==20.10.0"
 
 [dev-packages]
 kubernetes = "*"
-configargparse = "*"
 
 [requires]
 python_version = "3.8"

README.md

@@ -36,23 +36,42 @@ All tags are automatically built and pushed to [Dockerhub](https://hub.docker.co
 
 | Variable | Required? | Default | Description |
 | --- |:---:|:---:| --- |
-| LABEL | <b>Yes</b> | None | Label that should be used for filtering |
-| FOLDER | <b>Yes</b> | None | Folder where the files should be placed. |
-| FOLDER_ANNOTATION | No | 'k8s-sidecar-target-directory' | The annotation the sidecar will look for in configmaps to override the destination folder for files |
-| LABEL_VALUE | No | None | The value for the label you want to filter your resources on.<br>Don't set a value to filter by any value |
-| NAMESPACE | No | 'ALL' | The namespace from which resources will be watched.<br>If not set or set to `ALL`, it will watch all namespaces. |
-| RESOURCE | No | 'configmap' | The resource type that the operator will filter for. Can be 'configmap', 'secret' or 'both' |
-| DEFAULT_FILE_MODE | No | 644 | The default file system permission for every file. Use three digits (e.g. '500', '440', ...) |
-| VERBOSE | No | False | A value of 'true' will enable the kopf verbose logs |
-| DEBUG | No | False | A value of 'true' will enable the kopf debug logs |
-| LIVENESS | No | True | A value of 'false' will disable the kopf liveness probe |
-| WATCH_CLIENT_TIMEOUT | No | 660 | (seconds) is how long the session with a watching request will exist before closing it from the client side. This includes the connection establishing and event streaming. |
-| WATCH_SERVER_TIMEOUT | No | 600 | (seconds) is how long the session with a watching request will exist before closing it from the server side. This value is passed to the server side in a query string, and the server decides on how to follow it. The watch-stream is then gracefully closed. |
-| EVENT_LOGGING | No | False | A value of 'true' will allow the operator to log directly to k8s events (`kubectl get events`).<br>Note that if you enable this, you'll need to provide the operator with RBAC access to `events` resources, see an example of how to do this in [rbac.yaml](examples/rbac.yaml) |
-| UNIQUE_FILENAMES | No | False | A value of 'true' will produce unique filenames to avoid issues when duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. |
+| LABEL | <b>Yes</b> | `None` | Label that should be used for filtering |
+| FOLDER | <b>Yes</b> | `None` | Folder where the files should be placed. |
+| FOLDER_ANNOTATION | No | `k8s-sidecar-target-directory` | The annotation the sidecar will look for in `ConfigMap`s and/or `Secret`s to override the destination folder for files |
+| LABEL_VALUE | No | `None` | The value for the label you want to filter your resources on.<br>Don't set a value to filter by any value |
+| NAMESPACE | No | `ALL` | The `Namespace`(s) from which resources will be watched. <br>For multiple namespaces, use a comma-separated string like "default,test".<br>If not set or set to `ALL`, it will watch all `Namespace`s. |
+| RESOURCE | No | `configmap` | The resource type that the operator will filter for. Can be `ConfigMap`, `Secret` or `both` |
+| METHOD | No | `WATCH` | Determines how kopf-k8s-sidecar will run. If `WATCH` it will run like like a normal operator **forever**. <br>If `LIST` it will gather the matching configmaps and secrets currently present, write those files to the destination directory and **die** |
+| DEFAULT_FILE_MODE | No | `644` | The default file system permission for every file. Use three digits (e.g. '500', '440', ...) |
+| VERBOSE | No | `False` | A value of `true` will enable the kopf verbose logs |
+| DEBUG | No | `False` | A value of `true` will enable the kopf debug logs |
+| WATCH_CLIENT_TIMEOUT | No | `660` | (seconds) is how long the session with a watching request will exist before closing it from the client side. This includes the connection establishing and event streaming. |
+| WATCH_SERVER_TIMEOUT | No | `600` | (seconds) is how long the session with a watching request will exist before closing it from the server side. This value is passed to the server side in a query string, and the server decides on how to follow it. The watch-stream is then gracefully closed. |
+| EVENT_LOGGING | No | `False` | A value of `true` will allow the operator to log directly to k8s events (`kubectl get events`).<br>Note that if you enable this, you'll need to provide the operator with RBAC access to `Event`s resources, see an example of how to do this in [rbac.yaml](examples/rbac.yaml) |
+| UNIQUE_FILENAMES | No (but recommended!) | `False` | A value of `true` will produce unique filenames to avoid issues when duplicate data keys exist between `ConfigMap`s and/or `Secret`s within the same or multiple `Namespace`s. |
 
 ## Gotchas
 
+### Namespaces
+
+Contrary to the original k8s-sidecar, we will look in `ALL` namespaces by default as explained in the [Configuration Environment Variables](#configuration-environment-variables) section.
+
+If you only want to look for resources in the namespace where the sidecar is installed, feel free to specify it.
+
+### RBAC permissions
+
+If you're using this image with the [Grafana Helm chart](https://github.com/grafana/helm-charts/tree/main/charts/grafana) you will need to provide `patch` permissions for `configmaps` and `secrets` in the `values.yaml`
+
+```
+-  extraClusterRoleRules: []
++  extraClusterRoleRules:
++    - apiGroups: [""]  # "" indicates the core API group
++      resources: ["configmaps", "secrets"]
++      verbs: ["patch"]
+```
+This is because the operator needs to patch the resources to add finalizers, see the [Resource deletion](#resource-deletion) below.
+
 ### Resource deletion
 
 With the usage of k8s operators, we have access to [finalizers](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#finalizers)

app/conditions.py

@@ -0,0 +1,30 @@
+import os
+from misc import get_required_env_var
+
+def label_is_satisfied(meta, **_):
+    """Runs the logic for LABEL and LABEL_VALUE and tells us if we need to watch the resource"""
+    label = get_required_env_var('LABEL')
+    label_value = os.getenv('LABEL_VALUE')
+
+    # if there are no labels in the resource, there's no point in checking further
+    if 'labels' not in meta:
+        return False
+
+    # If LABEL_VALUE wasn't set but we find the LABEL, that's good enough
+    if label_value is None and label in meta['labels'].keys():
+        return True
+
+    # If LABEL_VALUE was set, it needs to be the value of LABEL for one of the key-vars in the dict
+    for key, value in meta['labels'].items():
+        if key == label and value == label_value:
+            return True
+
+    return False
+
+def resource_is_desired(body, **_):
+    """Runs the logic for the RESOURCE environment variable"""
+    resource = os.getenv('RESOURCE', 'configmap')
+
+    kind = body['kind'].lower()
+
+    return resource in (kind, 'both')

app/io_helpers.py

@@ -20,7 +20,7 @@ def create_folder(folder, logger):
                 return
     logger.debug(f"Folder {folder} already exists. Skipping creation.")
 
-async def get_folder(metadata):
+def get_folder(metadata):
     """
     Handles the logic to determine which folder this resource needs to be written to
     and returns it.
@@ -41,7 +41,7 @@ async def get_folder(metadata):
 
     return folder
 
-async def get_filepath(filename, folder, body):
+def get_filepath(filename, folder, kind, body):
     """
     Returns unique path if UNIQUE_FILENAMES are desired.
     Otherwise, simply returns the concatenated filename with the folder.
@@ -52,42 +52,39 @@ async def get_filepath(filename, folder, body):
             namespace = body['metadata']['namespace']
 
         name = body['metadata']['name']
-        kind = body['kind'].lower()
+        kind = kind.lower()
 
         filename = namespace + "." + kind + "_" + name + "." + filename
 
     return os.path.join(folder, filename)
 
 
-async def delete_file(body, logger):
-    resource_kind = body['kind']
-
-    folder = await get_folder(body['metadata'])
+def delete_file(body, kind, logger):
+    folder = get_folder(body['metadata'])
 
     for filename in body['data'].keys():
-        filepath = await get_filepath(filename, folder, body)
-        logger.info(f"[DELETE:{resource_kind}] Deleting file {filepath}.")
+        filepath = get_filepath(filename, folder, kind, body)
+        logger.info(f"[DELETE:{kind}] Deleting file {filepath}.")
         try:
             os.remove(filepath)
         except FileNotFoundError:
-            logger.error(f"[DELETE:{resource_kind}] {filepath} not found.")
+            logger.error(f"[DELETE:{kind}] {filepath} not found.")
         except OSError as e:
             logger.error(e)
 
-async def write_file(event, body, logger):
+def write_file(event, body, kind, logger):
     """
     Write contents to the desired filepath if they have changed.
     """
-    resource_kind = body['kind']
     event = event.upper()
 
-    folder = await get_folder(body['metadata'])
+    folder = get_folder(body['metadata'])
     create_folder(folder, logger)
 
     for filename, content in body['data'].items():
-        filepath = await get_filepath(filename, folder, body)
+        filepath = get_filepath(filename, folder, kind, body)
 
-        if resource_kind == 'Secret':
+        if kind == 'Secret':
             content = get_base64_decoded(content)
 
         if os.path.exists(filepath):
@@ -99,12 +96,12 @@ async def write_file(event, body, logger):
                     sha256_hash_cur.update(byte_block)
 
             if sha256_hash_new.hexdigest() == sha256_hash_cur.hexdigest():
-                logger.info(f"[{event}:{resource_kind}] Contents of {filepath} haven't changed. Not overwriting existing file.")
+                logger.info(f"[{event}:{kind}] Contents of {filepath} haven't changed. Not overwriting existing file.")
                 continue
 
         try:
             with open(filepath, 'w') as f:
-                logger.info(f"[{event}:{resource_kind}] Writing content to file {filepath}")
+                logger.info(f"[{event}:{kind}] Writing content to file {filepath}")
                 f.write(content)
         # TODO: Flesh out IO exception handling here
         except Exception as e:

app/list_mode.py

@@ -0,0 +1,56 @@
+import os
+import pykube
+from io_helpers import write_file
+from conditions import label_is_satisfied
+from misc import log_env_vars, get_scope
+import logging
+
+def _get_configmaps(namespace):
+    try:
+        api = pykube.HTTPClient(pykube.KubeConfig.from_env())
+        configmaps = pykube.ConfigMap.objects(api).filter(namespace=namespace)
+        return configmaps
+    except pykube.exceptions.HTTPError as e:
+        if e.code in [409]:
+            pass
+        else:
+            raise
+
+def _get_secrets(namespace):
+    try:
+        api = pykube.HTTPClient(pykube.KubeConfig.from_env())
+        configmaps = pykube.Secret.objects(api).filter(namespace=namespace)
+        return configmaps
+    except pykube.exceptions.HTTPError as e:
+        if e.code in [409]:
+            pass
+        else:
+            raise
+
+def one_run():
+    """Search through all the ConfigMaps and Secrets in the specified namespaces. If they meet the label requirements,
+    copy the files to the destination. Update and delete operations not needed in this mode"""
+
+    logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
+    logger = logging.getLogger(__name__)
+
+    # Log some useful variables for troubleshooting
+    log_env_vars(logger)
+
+    scope = get_scope()
+
+    resource = os.getenv('RESOURCE', 'configmap')
+
+    if resource in ('configmap', 'both'):
+        for namespace in scope['namespaces']:
+            configmaps = _get_configmaps(namespace)
+            for configmap in configmaps:
+                if label_is_satisfied(configmap.obj['metadata']):
+                    write_file("create", configmap.obj, configmap.kind, logger)
+
+    if resource in ('secret', 'both'):
+        for namespace in scope['namespaces']:
+            secrets = _get_secrets(namespace)
+            for secret in secrets:
+                if label_is_satisfied(secret.obj['metadata']):
+                    write_file("create", secret.obj, secret.kind, logger)