Add NTP placed and overridden states to NTP module (#151)

plugins/module_utils/network/sonic/argspec/ntp/ntp.py

@@ -84,7 +84,7 @@ def __init__(self, **kwargs):
             'type': 'dict'
         },
         'state': {
-            'choices': ['merged', 'deleted'],
+            'choices': ['merged', "replaced", "overridden", 'deleted'],
             'default': 'merged',
             'type': 'str'
         }

plugins/module_utils/network/sonic/config/ntp/ntp.py

@@ -29,6 +29,8 @@
 )
 from ansible_collections.dellemc.enterprise_sonic.plugins.module_utils.network.sonic.utils.utils import (
     get_diff,
+    get_replaced_config,
+    send_requests,
     update_states,
     normalize_interface_name,
     normalize_interface_name_list
@@ -39,10 +41,8 @@
 DELETE = 'DELETE'
 
 TEST_KEYS = [
-    {
-        "vrf": "", "enable_ntp_auth": "", "source_interfaces": "", "trusted_keys": "",
-        "servers": {"address": ""}, "ntp_keys": {"key_id": ""}
-    }
+    {"servers": {"address": ""}},
+    {"ntp_keys": {"key_id": ""}}
 ]
 
 
@@ -145,6 +145,10 @@ def set_state(self, want, have):
             commands, requests = self._state_deleted(want, have)
         elif state == 'merged':
             commands, requests = self._state_merged(want, have)
+        elif state == 'overridden':
+            commands, requests = self._state_overridden(want, have)
+        elif state == 'replaced':
+            commands, requests = self._state_replaced(want, have)
 
         return commands, requests
 
@@ -209,6 +213,78 @@ def _state_deleted(self, want, have):
 
         return commands, requests
 
+    def _state_replaced(self, want, have):
+        """ The command generator when state is replaced
+
+        :param want: the desired configuration as a dictionary
+        :param have: the current configuration as a dictionary
+        :param diff: the difference between want and have
+        :rtype: A list
+        :returns: the commands necessary to migrate the current configuration
+                  to the desired configuration
+        """
+        replaced_config = get_replaced_config(want, have, TEST_KEYS)
+
+        if replaced_config:
+            self.sort_lists_in_config(replaced_config)
+            self.sort_lists_in_config(have)
+            delete_all = (replaced_config == have)
+            requests = self.get_delete_requests(replaced_config, delete_all)
+            send_requests(self._module, requests)
+
+            commands = want
+        else:
+            diff = get_diff(want, have, TEST_KEYS)
+            commands = diff
+
+        requests = []
+
+        if commands:
+            self.preprocess_merge_commands(commands, want)
+            requests = self.get_merge_requests(commands, have)
+
+            if len(requests) > 0:
+                commands = update_states(commands, "replaced")
+            else:
+                commands = []
+        else:
+            commands = []
+
+        return commands, requests
+
+    def _state_overridden(self, want, have):
+        """ The command generator when state is overridden
+
+        :param want: the desired configuration as a dictionary
+        :param have: the current configuration as a dictionary
+        :param diff: the difference between want and have
+        :rtype: A list
+        :returns: the commands necessary to migrate the current configuration
+                  to the desired configuration
+        """
+        self.sort_lists_in_config(want)
+        self.sort_lists_in_config(have)
+
+        if have and have != want:
+            delete_all = True
+            requests = self.get_delete_requests(have, delete_all)
+            send_requests(self._module, requests)
+            have = []
+
+        commands = []
+        requests = []
+
+        if not have and want:
+            commands = want
+            requests = self.get_merge_requests(commands, have)
+
+            if len(requests) > 0:
+                commands = update_states(commands, "overridden")
+            else:
+                commands = []
+
+        return commands, requests
+
     def validate_want(self, want, state):
 
         if state == 'deleted':
@@ -254,6 +330,15 @@ def preprocess_want(self, want, state):
                     if 'prefer' in server and server['prefer'] is None:
                         server.pop('prefer')
 
+        if state == 'replaced' or state == 'overridden':
+            enable_auth_want = want.get('enable_ntp_auth', None)
+            if enable_auth_want is None:
+                want['enable_ntp_auth'] = False
+            if 'servers' in want and want['servers'] is not None:
+                for server in want['servers']:
+                    if 'prefer' in server and server['prefer'] is None:
+                        server['prefer'] = False
+
     def search_servers(self, svr_address, servers):
 
         found_server = dict()
@@ -592,3 +677,20 @@ def get_delete_keys_requests(self, configs):
             requests.append(request)
 
         return requests
+
+    def get_server_address(self, ntp_server):
+        return ntp_server.get('address')
+
+    def get_ntp_key_id(self, ntp_key):
+        return ntp_key.get('key_id')
+
+    def sort_lists_in_config(self, config):
+
+        if 'source_interfaces' in config and config['source_interfaces'] is not None:
+            config['source_interfaces'].sort()
+        if 'servers' in config and config['servers'] is not None:
+            config['servers'].sort(key=self.get_server_address)
+        if 'trusted_keys' in config and config['trusted_keys'] is not None:
+            config['trusted_keys'].sort()
+        if 'ntp_keys' in config and config['ntp_keys'] is not None:
+            config['ntp_keys'].sort(key=self.get_ntp_key_id)

plugins/module_utils/network/sonic/facts/ntp/ntp.py

@@ -113,16 +113,16 @@ def get_ntp_configuration(self):
 
         ntp_config = dict()
 
-        if 'network-instance' in ntp_global_config:
+        if 'network-instance' in ntp_global_config and ntp_global_config['network-instance']:
             ntp_config['vrf'] = ntp_global_config['network-instance']
 
         if 'enable-ntp-auth' in ntp_global_config:
             ntp_config['enable_ntp_auth'] = ntp_global_config['enable-ntp-auth']
 
-        if 'source-interface' in ntp_global_config:
+        if 'source-interface' in ntp_global_config and ntp_global_config['source-interface']:
             ntp_config['source_interfaces'] = ntp_global_config['source-interface']
 
-        if 'trusted-key' in ntp_global_config:
+        if 'trusted-key' in ntp_global_config and ntp_global_config['trusted-key']:
             ntp_config['trusted_keys'] = ntp_global_config['trusted-key']
 
         servers = []
@@ -136,7 +136,8 @@ def get_ntp_configuration(self):
                 server['maxpoll'] = ntp_server['config'].get('maxpoll', None)
                 server['prefer'] = ntp_server['config'].get('prefer', None)
                 servers.append(server)
-        ntp_config['servers'] = servers
+        if servers:
+            ntp_config['servers'] = servers
 
         keys = []
         for ntp_key in ntp_keys:
@@ -149,6 +150,7 @@ def get_ntp_configuration(self):
                 key['key_type'] = key_type
                 key['key_value'] = ntp_key['config'].get('key-value', None)
                 keys.append(key)
-        ntp_config['ntp_keys'] = keys
+        if keys:
+            ntp_config['ntp_keys'] = keys
 
         return ntp_config

plugins/module_utils/network/sonic/utils/utils.py

@@ -509,3 +509,170 @@ def command_list_str_to_dict(module, warnings, cmd_list_in, exec_cmd=False):
             cmd_list_out.append(cmd_out)
 
     return cmd_list_out
+
+
+def send_requests(module, requests):
+
+    reply = dict()
+    response = []
+    if not module.check_mode and requests:
+        try:
+            response = edit_config(module, to_request(module, requests))
+        except ConnectionError as exc:
+            module.fail_json(msg=str(exc), code=exc.code)
+
+        reply = response[0][1]
+
+    return reply
+
+
+def get_replaced_config(new_conf, exist_conf, test_keys=None):
+
+    replace_conf = []
+
+    if isinstance(new_conf, list) and isinstance(exist_conf, list):
+
+        replace_conf_dict = get_replaced_config_dict({"config": new_conf},
+                                                     {"config": exist_conf},
+                                                     test_keys)
+        replaced_conf = replace_conf_dict.get("config", [])
+    else:
+        replaced_conf = get_replaced_config_dict(new_conf, exist_conf, test_keys)
+
+    return replaced_conf
+
+
+def get_replaced_config_dict(new_conf, exist_conf, test_keys=None, key_set=None):
+
+    replaced_conf = dict()
+
+    if test_keys is None:
+        test_keys = []
+    if key_set is None:
+        key_set = []
+
+    if not new_conf:
+        return replaced_conf
+
+    new_key_set = set(new_conf.keys())
+    exist_key_set = set(exist_conf.keys())
+
+    trival_new_key_set = set()
+    dict_list_new_key_set = set()
+    for key in new_key_set:
+        if new_conf[key] not in [None, [], {}]:
+            if isinstance(new_conf[key], (list, dict)):
+                dict_list_new_key_set.add(key)
+            else:
+                trival_new_key_set.add(key)
+
+    trival_exist_key_set = set()
+    dict_list_exist_key_set = set()
+    for key in exist_key_set:
+        if exist_conf[key] not in [None, [], {}]:
+            if isinstance(exist_conf[key], (list, dict)):
+                dict_list_exist_key_set.add(key)
+            else:
+                trival_exist_key_set.add(key)
+
+    common_trival_key_set = trival_new_key_set.intersection(trival_exist_key_set)
+
+    key_matched_cnt = 0
+    common_trival_key_matched = True
+    for key in common_trival_key_set:
+        if new_conf[key] == exist_conf[key]:
+            if key in key_set:
+                key_matched_cnt += 1
+        else:
+            if key not in key_set:
+                common_trival_key_matched = False
+
+    key_matched = (key_matched_cnt == len(key_set))
+    if key_matched:
+        extra_trival_new_key_set = trival_new_key_set - common_trival_key_set
+        extra_trival_exist_key_set = trival_exist_key_set - common_trival_key_set
+        if extra_trival_new_key_set or extra_trival_exist_key_set or \
+           not common_trival_key_matched:
+            # Replace whole dict.
+            replaced_conf = exist_conf
+            return replaced_conf
+    else:
+        replaced_conf = []
+        return replaced_conf
+
+    replace_whole_dict = False
+    replace_some_list = False
+    replace_some_dict = False
+    common_dict_list_key_set = dict_list_new_key_set.intersection(dict_list_exist_key_set)
+    for key in common_dict_list_key_set:
+
+        new_value = new_conf[key]
+        exist_value = exist_conf[key]
+
+        if (isinstance(new_value, list) and isinstance(exist_value, list)):
+            n_list = new_value
+            e_list = exist_value
+            t_keys = next((t_key_item[key] for t_key_item in test_keys if key in t_key_item), None)
+            t_key_set = set()
+            if t_keys:
+                t_key_set = set(t_keys.keys())
+
+            replaced_list = list()
+            not_dict_item = False
+            dict_no_key_item = False
+            for n_item in n_list:
+                for e_item in e_list:
+                    if (isinstance(n_item, dict) and isinstance(e_item, dict)):
+                        if t_keys:
+                            remaining_keys = [t_key_item for t_key_item in test_keys if key not in t_key_item]
+                            replaced_dict = get_replaced_config_dict(n_item, e_item,
+                                                                     remaining_keys, t_key_set)
+                        else:
+                            dict_no_key_item = True
+                            break
+
+                        if replaced_dict:
+                            replaced_list.append(replaced_dict)
+                            break
+                    else:
+                        not_dict_item = True
+                        break
+
+                if not_dict_item or dict_no_key_item:
+                    break
+
+            if dict_no_key_item:
+                replaced_list = e_list
+
+            if not_dict_item:
+                n_set = set(n_list)
+                e_set = set(e_list)
+                diff_set = n_set.symmetric_difference(e_set)
+                if diff_set:
+                    replaced_conf[key] = e_list
+                    replace_some_list = True
+
+            elif replaced_list:
+                replaced_conf[key] = replaced_list
+                replace_some_list = True
+
+        elif (isinstance(new_value, dict) and isinstance(exist_value, dict)):
+            replaced_dict = get_replaced_config_dict(new_conf[key], exist_conf[key], test_keys)
+            if replaced_dict:
+                replaced_conf[key] = replaced_dict
+                replace_some_dict = True
+
+        elif (isinstance(new_value, (list, dict)) or isinstance(exist_value, (list, dict))):
+            # Replace whole dict.
+            replaced_conf = exist_conf
+            replace_whole_dict = True
+            break
+
+        else:
+            continue
+
+    if ((replace_some_dict or replace_some_list) and (not replace_whole_dict)):
+        for key in key_set:
+            replaced_conf[key] = exist_conf[key]
+
+    return replaced_conf