config-sync: T6185: T6146: combined backport of config-sync extensions and priority data

interface-definitions/service_config-sync.xml.in

@@ -495,6 +495,12 @@
                       <valueless/>
                     </properties>
                   </leafNode>
+                  <leafNode name="time-zone">
+                    <properties>
+                      <help>Local time zone</help>
+                      <valueless/>
+                    </properties>
+                  </leafNode>
                 </children>
               </node>
               <leafNode name="vpn">

python/vyos/configsession.py

@@ -176,6 +176,25 @@ def load_section(self, path: list, d: dict):
         except (ValueError, ConfigSessionError) as e:
             raise ConfigSessionError(e)
 
+    def set_section_tree(self, d: dict):
+        try:
+            if d:
+                for p in dict_to_paths(d):
+                    self.set(p)
+        except (ValueError, ConfigSessionError) as e:
+            raise ConfigSessionError(e)
+
+    def load_section_tree(self, mask: dict, d: dict):
+        try:
+            if mask:
+                for p in dict_to_paths(mask):
+                    self.delete(p)
+            if d:
+                for p in dict_to_paths(d):
+                    self.set(p)
+        except (ValueError, ConfigSessionError) as e:
+            raise ConfigSessionError(e)
+
     def comment(self, path, value=None):
         if not value:
             value = [""]

python/vyos/configtree.py

@@ -401,6 +401,30 @@ def union(left, right, libpath=LIBPATH):
 
     return tree
 
+def mask_inclusive(left, right, libpath=LIBPATH):
+    if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)):
+        raise TypeError("Arguments must be instances of ConfigTree")
+
+    try:
+        __lib = cdll.LoadLibrary(libpath)
+        __mask_tree = __lib.mask_tree
+        __mask_tree.argtypes = [c_void_p, c_void_p]
+        __mask_tree.restype = c_void_p
+        __get_error = __lib.get_error
+        __get_error.argtypes = []
+        __get_error.restype = c_char_p
+
+        res = __mask_tree(left._get_config(), right._get_config())
+    except Exception as e:
+        raise ConfigTreeError(e)
+    if not res:
+        msg = __get_error().decode()
+        raise ConfigTreeError(msg)
+
+    tree = ConfigTree(address=res)
+
+    return tree
+
 def reference_tree_to_json(from_dir, to_file, libpath=LIBPATH):
     try:
         __lib = cdll.LoadLibrary(libpath)

python/vyos/priority.py

@@ -0,0 +1,75 @@
+# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+from pathlib import Path
+from typing import List
+
+from vyos.xml_ref import load_reference
+from vyos.base import Warning as Warn
+
+def priority_data(d: dict) -> list:
+    def func(d, path, res, hier):
+        for k,v in d.items():
+            if not 'node_data' in v:
+                continue
+            subpath = path + [k]
+            hier_prio = hier
+            data = v.get('node_data')
+            o = data.get('owner')
+            p = data.get('priority')
+            # a few interface-definitions have priority preceding owner
+            # attribute, instead of within properties; pass in descent
+            if p is not None and o is None:
+                hier_prio = p
+            if o is not None and p is None:
+                p = hier_prio
+            if o is not None and p is not None:
+                o = Path(o.split()[0]).name
+                p = int(p)
+                res.append((subpath, o, p))
+            if isinstance(v, dict):
+                func(v, subpath, res, hier_prio)
+        return res
+    ret = func(d, [], [], 0)
+    ret = sorted(ret, key=lambda x: x[0])
+    ret = sorted(ret, key=lambda x: x[2])
+    return ret
+
+def get_priority_data() -> list:
+    xml = load_reference()
+    return priority_data(xml.ref)
+
+def priority_sort(sections: List[list[str]] = None,
+                  owners: List[str] = None,
+                  reverse=False) -> List:
+    if sections is not None:
+        index = 0
+        collection: List = sections
+    elif owners is not None:
+        index = 1
+        collection = owners
+    else:
+        raise ValueError('one of sections or owners is required')
+
+    l = get_priority_data()
+    m = [item for item in l if item[index] in collection]
+    n = sorted(m, key=lambda x: x[2], reverse=reverse)
+    o = [item[index] for item in n]
+    # sections are unhashable; use comprehension
+    missed = [j for j in collection if j not in o]
+    if missed:
+        Warn(f'No priority available for elements {missed}')
+
+    return o

python/vyos/xml_ref/generate_cache.py

@@ -33,7 +33,8 @@
 pkg_cache = abspath(join(_here, 'pkg_cache'))
 ref_cache = abspath(join(_here, 'cache.py'))
 
-node_data_fields = ("node_type", "multi", "valueless", "default_value")
+node_data_fields = ("node_type", "multi", "valueless", "default_value",
+                    "owner", "priority")
 
 def trim_node_data(cache: dict):
     for k in list(cache):

src/helpers/priority.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2024 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import sys
+from argparse import ArgumentParser
+from tabulate import tabulate
+
+from vyos.priority import get_priority_data
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('--legacy-format', action='store_true',
+                        help="format output for comparison with legacy 'priority.pl'")
+    args = parser.parse_args()
+
+    prio_list = get_priority_data()
+    if args.legacy_format:
+        for p in prio_list:
+            print(f'{p[2]} {"/".join(p[0])}')
+        sys.exit(0)
+
+    l = []
+    for p in prio_list:
+        l.append((p[2], p[1], p[0]))
+    headers = ['priority', 'owner', 'path']
+    out = tabulate(l, headers, numalign='right')
+    print(out)

src/helpers/vyos_config_sync.py

@@ -21,9 +21,11 @@
 import requests
 import urllib3
 import logging
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Tuple, Dict, Any
 
 from vyos.config import Config
+from vyos.configtree import ConfigTree
+from vyos.configtree import mask_inclusive
 from vyos.template import bracketize_ipv6
 
 
@@ -61,39 +63,45 @@ def post_request(url: str,
 
 
 
-def retrieve_config(section: Optional[List[str]] = None) -> Optional[Dict[str, Any]]:
+def retrieve_config(sections: List[list[str]]) -> Tuple[Dict[str, Any], Dict[str, Any]]:
     """Retrieves the configuration from the local server.
 
     Args:
-        section: List[str]: The section of the configuration to retrieve.
-        Default is None.
+        sections: List[list[str]]: The list of sections of the configuration
+        to retrieve, given as list of paths.
 
     Returns:
-        Optional[Dict[str, Any]]: The retrieved configuration as a
-        dictionary, or None if an error occurred.
+        Tuple[Dict[str, Any],Dict[str,Any]]: The tuple (mask, config) where:
+            - mask: The tree of paths of sections, as a dictionary.
+            - config: The subtree of masked config data, as a dictionary.
     """
-    if section is None:
-        section = []
 
-    conf = Config()
-    config = conf.get_config_dict(section, get_first_key=True)
-    if config:
-        return config
-    return None
+    mask = ConfigTree('')
+    for section in sections:
+        mask.set(section)
+    mask_dict = json.loads(mask.to_json())
+
+    config = Config()
+    config_tree = config.get_config_tree()
+    masked = mask_inclusive(config_tree, mask)
+    config_dict = json.loads(masked.to_json())
 
+    return mask_dict, config_dict
 
 def set_remote_config(
         address: str,
         key: str,
-        commands: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+        op: str,
+        mask: Dict[str, Any],
+        config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
     """Loads the VyOS configuration in JSON format to a remote host.
 
     Args:
         address (str): The address of the remote host.
         key (str): The key to use for loading the configuration.
-        commands (list): List of set/load commands for request, given as:
-                         [{'op': str, 'path': list[str], 'section': dict},
-                         ...]
+        op (str): The operation to perform (set or load).
+        mask (dict): The dict of paths in sections.
+        config (dict): The dict of masked config data.
 
     Returns:
         Optional[Dict[str, Any]]: The response from the remote host as a
@@ -107,7 +115,9 @@ def set_remote_config(
 
     url = f'https://{address}/configure-section'
     data = json.dumps({
-        'commands': commands,
+        'op': op,
+        'mask': mask,
+        'config': config,
         'key': key
     })
 
@@ -140,23 +150,15 @@ def config_sync(secondary_address: str,
     )
 
     # Sync sections ("nat", "firewall", etc)
-    commands = []
-    for section in sections:
-        config_json = retrieve_config(section=section)
-        # Check if config path deesn't exist, for example "set nat"
-        # we set empty value for config_json data
-        # As we cannot send to the remote host section "nat None" config
-        if not config_json:
-            config_json = {}
-        logger.debug(
-            f"Retrieved config for section '{section}': {config_json}")
-
-        d = {'op': mode, 'path': section, 'section': config_json}
-        commands.append(d)
+    mask_dict, config_dict = retrieve_config(sections)
+    logger.debug(
+        f"Retrieved config for sections '{sections}': {config_dict}")
 
     set_config = set_remote_config(address=secondary_address,
                                    key=secondary_key,
-                                   commands=commands)
+                                   op=mode,
+                                   mask=mask_dict,
+                                   config=config_dict)
 
     logger.debug(f"Set config for sections '{sections}': {set_config}")