Merge pull request #545 from NethServer/feat-6814

Implement MM relation of modules and user domains

core/agent/README.md

@@ -203,6 +203,7 @@ A running action step receives the **current working directory** value and its o
     * `AGENT_COMFD` (integer) The file descriptor number where **action commands** (see below) can be writte to
     * `AGENT_TASK_ID` (string) The unique identifier of the Task that started the Action
     * `AGENT_TASK_ACTION` (string) The Action name
+    * `AGENT_TASK_USER` (string) The user invoking the action, if available
 
 ## Action commands
 

core/agent/htask.go

@@ -131,6 +131,7 @@ func runAction(rdb *redis.Client, actionCtx context.Context, task *models.Task)
 			"AGENT_COMFD=3", // 3 is the additional FD number where the action step can write its commands for us
 			"AGENT_TASK_ID="+task.ID,
 			"AGENT_TASK_ACTION="+task.Action,
+			"AGENT_TASK_USER="+task.User,
 		)
 		inputData, _ := json.Marshal(task.Data)
 		cmd.Stdin = strings.NewReader(string(inputData))

core/agent/test/actions/read-environment/30printvar

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-echo -n $VAR1
+echo -n "${VAR1}${AGENT_TASK_USER:?}"

core/agent/test/suite/20__environment.robot

@@ -9,4 +9,4 @@ Test Teardown    Stop command monitoring
 Read a variable
     Given The task is submitted    read-environment
     When The command is received    set    /exit_code    0
-    And The task output should be equal to    VAL1
+    And The task output should be equal to    VAL1admin

core/agent/test/suite/taskrun.resource

@@ -66,7 +66,7 @@ The task is submitted
     Set Test Variable    ${LAST_TASK_ID}    id-${action_name}
     Push Item To First Index In List Redis    ${RDB}
     ...   ${AGENT_ID}/tasks
-    ...   {"id":"id-${action_name}","action":"${action_name}","data":${task_data}}
+    ...   {"user":"admin","id":"id-${action_name}","action":"${action_name}","data":${task_data}}
 The task has exit code
     [Arguments]    ${expected_exit_code}
     Redis Key Should Be Exist    ${RDB}    task/${AGENT_ID}/${LAST_TASK_ID}/exit_code

core/imageroot/usr/local/agent/pypkg/agent/__init__.py

@@ -576,3 +576,34 @@ def get_hostname():
     except ValueError:
         hostname = "myserver.example.org"
     return hostname
+
+def bind_user_domains(domain_list, check=True):
+    """Associate the caller module with a new list of user domains. The
+    previous list is discarded."""
+    response = agent.tasks.run(
+        agent_id='cluster',
+        action='bind-user-domains',
+        data={
+            'domains': domain_list,
+        }
+    )
+    if check:
+        assert_exp(response['exit_code'] == 0)
+    else:
+        return response['exit_code'] == 0
+
+def get_bound_domain_list(rdb, module_id=None):
+    """Return an array of domain names, bound to module_id.
+    If the module_id argument is omitted return bound domains
+    for the current module."""
+    if module_id is None:
+        module_id = os.getenv("MODULE_ID")
+
+    if module_id is None:
+        return []
+
+    rval = rdb.hget("cluster/module_domains", module_id)
+    if rval is not None:
+        return rval.split()
+    else:
+        return []

core/imageroot/usr/local/agent/pypkg/agent/ldapproxy.py

@@ -18,6 +18,8 @@
 # along with NethServer.  If not, see COPYING.
 #
 
+import sys
+import agent
 import redis
 import os
 import cluster.userdomains
@@ -94,9 +96,19 @@ def _load_domains(self):
         except TypeError:
             pass
 
+        # Retrieve the list of bound user domains, to check warning
+        # conditions:
+        bound_domain_list = agent.get_bound_domain_list(rdb)
+
         domains = {}
         configured_domains = cluster.userdomains.list_domains(rdb)
         for domain in configured_domains:
+            if "MODULE_ID" in os.environ and domain not in bound_domain_list:
+                # Warn only if the module is running under a module environment
+                print(agent.SD_WARNING + \
+                    f'agent.ldapproxy: domain {domain} should not be used by ' + \
+                    f'{os.getenv("MODULE_ID")}. Invoke agent.bind_user_domains(' + \
+                    f'["{domain}"]) to fix this warning.', file=sys.stderr)
             dhx = configured_domains[domain]
             domains.setdefault(domain, dhx)
 

core/imageroot/usr/local/agent/pypkg/cluster/userdomains.py

@@ -214,3 +214,15 @@ def list_domains(rdb):
     domains = get_external_domains(rdb)
     domains.update(get_internal_domains(rdb))
     return domains
+
+def get_domain_modules(rdb, domain):
+    """Return a list of module identifiers that are bound to the given domain"""
+
+    module_list = []
+    rawrel = rdb.hgetall("cluster/module_domains") or {}
+    for (key, val) in rawrel.items():
+        pdomains = val.split()
+        if domain in pdomains:
+            module_list.append(key)
+
+    return module_list

core/imageroot/var/lib/nethserver/cluster/actions/bind-user-domains/50bind_user_domains

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import sys
+import json
+import agent
+import os
+
+request = json.load(sys.stdin)
+
+domain_list = request["domains"]
+module_id = os.environ["AGENT_TASK_USER"].removeprefix('module/')
+
+rdb = agent.redis_connect(privileged=False)
+try:
+    test = int(rdb.hget(f'module/{module_id}/environment', 'NODE_ID'))
+except Exception as ex:
+    print(f"Error: to validate a module_id instance {ex}", file=sys.stderr)
+    sys.exit(0)
+
+previous_domains = rdb.hget(f'cluster/module_domains', module_id) or ""
+
+rdb = agent.redis_connect(privileged=True)
+rdb.hset(f'cluster/module_domains', module_id, " ".join(domain_list))
+
+union_domains = set(domain_list) | set(previous_domains.split())
+
+agent_id = os.environ['AGENT_ID']
+trx = rdb.pipeline()
+trx.publish(agent_id + '/event/module-domain-changed', json.dumps({
+    "modules": [module_id],
+    "domains": list(union_domains)
+}))
+trx.execute()

core/imageroot/var/lib/nethserver/cluster/actions/bind-user-domains/validate-input.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "bind-user-domains-input",
+  "$id": "http://schema.nethserver.org/cluster/bind-user-domains-input.json",
+  "description": "Input schema of the bind-user-domains action",
+  "examples": [
+    {
+      "domains": [
+        "mydom.test"
+      ]
+    }
+  ],
+  "type": "object",
+  "required": [
+    "domains"
+  ],
+  "properties": {
+    "domains": {
+      "description": "One or more domains to bind with the module calling this action",
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "description": "A user domain name",
+        "type": "string",
+        "minLength": 1
+      }
+    }
+  }
+}

core/imageroot/var/lib/nethserver/cluster/actions/create-cluster/50update

@@ -80,9 +80,6 @@ agent.run_helper('systemctl', 'stop', 'wg-quick@wg0.service')
 cluster.vpn.initialize_wgconf(ip_address)
 agent.run_helper('systemctl', 'start', 'wg-quick@wg0.service').check_returncode()
 
-# (Samba) account provider might want to route some IP addresses through our VPN. Define a role for that:
-cluster.grants.grant(rdb, action_clause="update-routes", to_clause="accountprovider", on_clause='cluster')
-
 #
 # Install core modules
 #

core/imageroot/var/lib/nethserver/cluster/actions/remove-internal-domain/50remove_domain

@@ -67,3 +67,33 @@ if errors > 0:
 
 with agent.redis_connect(privileged=True) as rdb:
     rdb.delete(f"cluster/counters_cache/users/{kdom}", f"cluster/counters_cache/groups/{kdom}")
+
+# Iterate over the hash and find the matching value, remove the domains and  update the value to '' if no more domains
+with agent.redis_connect(privileged=True) as rdb:
+    for key, value in rdb.hscan_iter("cluster/module_domains"):
+        print(f"Checking {key} {value}", file=sys.stderr)
+        values = value.split()
+        if kdom in values:
+            values.remove(kdom)
+            new_value = ' '.join(values)
+            print(f"Updating {key} to {new_value} in cluster/module_domains", file=sys.stderr)
+            rdb.hset("cluster/module_domains", key, new_value)
+
+# iterate over the hash and trigger one time the event
+with agent.redis_connect(privileged=True) as rdb:
+    keys_set = set()
+    values_set = set()
+    for key, value in rdb.hscan_iter("cluster/module_domains"):
+        keys_set.add(key)
+        # Iterate over the value and push each element into values_set
+        for element in value.split():
+            values_set.add(element)
+    print(f"Checking cluster/module_domains keys:{list(keys_set)} values:{list(values_set)}", file=sys.stderr)
+
+agent_id = os.environ['AGENT_ID']
+trx = rdb.pipeline()
+trx.publish(agent_id + '/event/module-domain-changed', json.dumps({
+    "modules": list(keys_set),
+    "domains": list(values_set)
+}))
+trx.execute()

core/imageroot/var/lib/nethserver/cluster/actions/remove-module/50update

@@ -132,6 +132,16 @@ trx.publish(agent_id + '/event/module-removed', json.dumps({
     'node': node_id,
 }))
 
+# we get the list of domains from the cluster/module_domains hash
+values = rdb.hget("cluster/module_domains", module_id) or ""
+# Delete module_id key in module_domains hash
+trx.hdel("cluster/module_domains", module_id)
+# we trigger the event module-domain-changed with the list of domains
+trx.publish(agent_id + '/event/module-domain-changed', json.dumps({
+    "modules": [module_id],
+    "domains": values.split()
+}))
+
 trx.execute()
 
 json.dump({}, fp=sys.stdout)

core/imageroot/var/lib/nethserver/node/install-finalize.sh

@@ -131,6 +131,10 @@ cluster.grants.grant(rdb, action_clause="add-public-service",  to_clause="tunadm
 cluster.grants.grant(rdb, action_clause="remove-public-service",  to_clause="tunadm", on_clause='node/1')
 cluster.grants.grant(rdb, action_clause="add-custom-zone",  to_clause="tunadm", on_clause='node/1')
 cluster.grants.grant(rdb, action_clause="remove-custom-zone",  to_clause="tunadm", on_clause='node/1')
+
+cluster.grants.grant(rdb, action_clause="update-routes", to_clause="accountprovider", on_clause='cluster')
+cluster.grants.grant(rdb, action_clause="bind-user-domains",  to_clause="accountconsumer", on_clause='cluster')
+cluster.grants.grant(rdb, action_clause="list-modules",  to_clause="accountprovider", on_clause='cluster')
 EOF
 
 for arg in "${@}"; do

core/imageroot/var/lib/nethserver/node/update-core.d/50update_grants

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+
+import agent
+import cluster.grants
+
+rdb = agent.redis_connect(privileged=True)
+
+cluster.grants.grant(rdb, action_clause="bind-user-domains",  to_clause="accountconsumer", on_clause='cluster')
+cluster.grants.grant(rdb, action_clause="list-modules",  to_clause="accountprovider", on_clause='cluster')

docs/core/database.md

@@ -168,6 +168,7 @@ subsections for more information.
 |cluster/node_sequence                  |INTEGER    |generate node IDs, default: `0` |
 |cluster/module_sequence/{image}        |INTEGER    |module sequence to generate instances of image ID, default: `0` |
 |cluster/module_node                    |HASH       |The module-node association, used for roles assignment|
+|cluster/module_domains                 |HASH       |Store relation of modules with user domains. Hash key is MODULE_ID, hash value is a list of domains separated by spaces, e.g. `mydom1.tld mydom2.tld mydom3.tld`|
 |cluster/authorizations/{agent_id}      |SET        |Authorization labels persistence, to enforce labels on future modules|
 |cluster/roles/{role}                   |SET        |glob patterns matching the actions that {role} can run. {role} is one of "owner", "reader"...|
 |cluster/environment                    |HASH       |Cluster environment variables|

docs/core/events.md

@@ -26,6 +26,10 @@ Well known events:
 - `ldap-provider-changed`: an external LDAP account provider was removed
   or added to a user-domain. The JSON parameter format is
   `{"domain":STRING,"key":STRING}`.
+- `module-domain-changed`: the relation between modules and user domains
+  has been changed. The JSON parameter format is `{"domains":[DOMAIN1,
+  DOMAIN2 ...], "modules":[MODULE_ID1, MODULE_ID2...]}` and reflects the
+  domains and modules affected by the latest change.
 
 ## Cluster events
 

docs/core/user_domains.md

@@ -106,3 +106,77 @@ print(users_filter)
 groups_filter = lp.get_ldap_groups_search_filter_clause("mydomain")
 print(groups_filter)
 ```
+
+## Bind modules and account domains
+
+If a module wants to use an account domain it must be granted API
+permissions. Add the `accountconsumer` role to the
+`org.nethserver.authorizations` label of the module image. For instance
+set
+
+    org.nethserver.authorizations=cluster:accountconsumer
+
+Then the module can execute a bind procedure, so the core is aware of
+existing relations between modules and account domains. When such
+relations are formally established the core can
+
+- limit/grant access to LDAP resources
+- show the relations in the web user interfaces
+
+For example, a module that uses one domain at a time can unbind the old
+domain and bind the new one with a script like this:
+
+```python
+import agent
+import json
+import os
+import sys
+
+request = json.load(sys.stdin)
+
+# Store domain name for services configuration:
+agent.set_env("LDAP_USER_DOMAIN", request["ldap_domain"])
+
+# Bind the new domain, overriding previous values (unbind)
+agent.bind_user_domains([request["ldap_domain"]])
+```
+
+At any time, retrieve the list of domains currently bound:
+
+```python
+import agent
+rdb = agent.redis_connect(use_replica=True)
+domlist = agent.get_bound_domain_list(rdb)
+```
+
+When the module or the domain is removed from the cluster, the relation
+cleanup occurs automatically.
+
+If the module wants to be notified of any change to the relation between
+modules and user domains it can subscribe the `module-domain-changed`
+event. For instance, this is the payload of such event:
+
+```json
+{
+    "modules": ["mymodule1"],
+    "domains": ["mydomain.test"]
+}
+```
+
+The event paylod contains a list of module and domains that were affected
+by the relation change. Modules and domains can be either added or
+removed: they are listed to ease the implementation of event handlers in
+both account provider and account client modules.
+
+For instance, the following Python excerpt checks if the module domain was
+changed:
+
+```python
+event = json.load(sys.stdin)
+if not os.environ["LDAP_USER_DOMAIN"] in event["domains"]:
+    sys.exit(0) # nothing to do if our domain is among affected domains
+
+# Handle the event by some means, for example
+# - rewrite some config file
+# - reload some service running in a container
+```

docs/modules/agent.md

@@ -66,6 +66,7 @@ Other available variables:
 - `AGENT_COMFD`: the file descriptor to talk to the agent via [action commands](#action-commands)
 - `AGENT_TASK_ID`: the unique ID of current task, i.e. `c0b8b976-9444-42d5-a40b-142a6a483a84`
 - `AGENT_TASK_ACTION`: the name of executed action, i.e.: `create-module`
+- `AGENT_TASK_USER`: the name of the user that created the action, if available. For example, `admin`
 - `AGENT_INSTALL_DIR`
 - `AGENT_STATE_DIR`
 - `AGENT_ID`