[testbed-cli-add-topo] Code change for deploying testbed with peers on multiple servers

ansible/library/announce_routes.py

@@ -96,6 +96,12 @@
 M1_ASN_START = 65200
 
 
+def get_ports_by_offset(offset, v4_base=IPV4_BASE_PORT, v6_base=IPV6_BASE_PORT):
+    port = v4_base + offset
+    port6 = v6_base + offset
+    return port, port6
+
+
 def wait_for_http(host_ip, http_port, timeout=10):
     """
     Waits for HTTP server to open.
@@ -130,12 +136,20 @@ def get_topo_type(topo_name):
     return topo_type
 
 
-def read_topo(topo_name, path):
+def read_topo(topo_name, path, server_index=-1):
     topo_file_path = os.path.join(
         path, TOPO_FILE_FOLDER, TOPO_FILENAME_TEMPLATE.format(topo_name))
     try:
         with open(topo_file_path) as f:
-            return yaml.safe_load(f)
+            _ret = yaml.safe_load(f)
+            if server_index != -1:
+                for _name, _attr in _ret['topology']['VMs'].items():
+                    if _attr['vm_offset'].startswith(str(server_index) + ','):
+                        _offset = int(_attr['vm_offset'].split(',')[1])
+                        _ret['topology']['VMs'][_name]['vm_offset'] = _offset
+                    else:
+                        _ret['topology']['VMs'].pop(_name)
+            return _ret
     except IOError:
         return {}
 
@@ -468,9 +482,7 @@ def fib_t0(topo, ptf_ip, no_default_route=False, action="announce"):
 
     vms = topo['topology']['VMs']
     for vm in vms.values():
-        vm_offset = vm['vm_offset']
-        port = IPV4_BASE_PORT + vm_offset
-        port6 = IPV6_BASE_PORT + vm_offset
+        port, port6 = get_ports_by_offset(vm['vm_offset'])
 
         routes_v4 = generate_routes("v4", podset_number, tor_number, tor_subnet_number,
                                     spine_asn, leaf_asn_start, tor_asn_start,
@@ -503,9 +515,9 @@ def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce"):
     vms_config = topo['configuration']
 
     for k, v in vms_config.items():
-        vm_offset = vms[k]['vm_offset']
-        port = IPV4_BASE_PORT + vm_offset
-        port6 = IPV6_BASE_PORT + vm_offset
+        if k not in vms:
+            continue
+        port, port6 = get_ports_by_offset(vms[k]['vm_offset'])
 
         router_type = None
         if 'spine' in v['properties']:
@@ -646,9 +658,9 @@ def fib_m0(topo, ptf_ip, action="announce"):
     m1_routes_v6 = None
     mx_index = -1
     for k, v in vms_config.items():
-        vm_offset = vms[k]['vm_offset']
-        port = IPV4_BASE_PORT + vm_offset
-        port6 = IPV6_BASE_PORT + vm_offset
+        if k not in vms:
+            continue
+        port, port6 = get_ports_by_offset(vms[k]['vm_offset'])
 
         router_type = None
         # Upstream
@@ -817,9 +829,9 @@ def fib_mx(topo, ptf_ip, action="announce"):
     m0_routes_v4 = None
     m0_routes_v6 = None
     for k, v in vms_config.items():
-        vm_offset = vms[k]['vm_offset']
-        port = IPV4_BASE_PORT + vm_offset
-        port6 = IPV6_BASE_PORT + vm_offset
+        if k not in vms:
+            continue
+        port, port6 = get_ports_by_offset(vms[k]['vm_offset'])
 
         # Routes announced by different M0s are the same, can reuse generated routes
         if m0_routes_v4 is not None:
@@ -934,9 +946,7 @@ def generate_t2_routes(dut_vm_dict, topo, ptf_ip, action="announce"):
                 set_num = 0
             else:
                 set_num = 1
-            vm_offset = vms[a_vm]['vm_offset']
-            port = IPV4_BASE_PORT + vm_offset
-            port6 = IPV6_BASE_PORT + vm_offset
+            port, port6 = get_ports_by_offset(vms[a_vm]['vm_offset'])
 
             router_type = None
             if 'leaf' in vms_config[a_vm]['properties']:
@@ -995,9 +1005,7 @@ def fib_t0_mclag(topo, ptf_ip, action="announce"):
             set_num = 0
         else:
             set_num = 1
-        vm_offset = vms[vm]['vm_offset']
-        port = IPV4_BASE_PORT + vm_offset
-        port6 = IPV6_BASE_PORT + vm_offset
+        port, port6 = get_ports_by_offset(vms[vm]['vm_offset'])
 
         routes_v4 = generate_routes("v4", podset_number, tor_number, tor_subnet_number,
                                     spine_asn, leaf_asn_start, tor_asn_start,
@@ -1025,9 +1033,7 @@ def fib_dpu(topo, ptf_ip, action="announce"):
     all_vms = sorted(vms.keys())
 
     for vm in all_vms:
-        vm_offset = vms[vm]['vm_offset']
-        port = IPV4_BASE_PORT + vm_offset
-        port6 = IPV6_BASE_PORT + vm_offset
+        port, port6 = get_ports_by_offset(vms[vm]['vm_offset'])
 
         change_routes(action, ptf_ip, port, routes_v4)
         change_routes(action, ptf_ip, port6, routes_v6)
@@ -1041,6 +1047,7 @@ def main():
             action=dict(required=False, type='str',
                         default='announce', choices=["announce", "withdraw"]),
             path=dict(required=False, type='str', default=''),
+            server_index=dict(required=False, type='int', default=-1),
             log_path=dict(required=False, type='str', default='')
         ),
         supports_check_mode=False)
@@ -1052,8 +1059,9 @@ def main():
     ptf_ip = module.params['ptf_ip']
     action = module.params['action']
     path = module.params['path']
+    server_index = int(module.params['server_index'])
 
-    topo = read_topo(topo_name, path)
+    topo = read_topo(topo_name, path, server_index)
     if not topo:
         module.fail_json(msg='Unable to load topology "{}"'.format(topo_name))
 

ansible/plugins/filter/filters.py

@@ -8,6 +8,7 @@
 class FilterModule(object):
     def filters(self):
         return {
+            'filter_vms_by_server_index': filter_vms_by_server_index,
             'extract_by_prefix': extract_by_prefix,
             'filter_by_prefix': filter_by_prefix,
             'filter_vm_targets': filter_vm_targets,
@@ -19,6 +20,19 @@ def filters(self):
         }
 
 
+def filter_vms_by_server_index(vms, server_index=-1, remove_index=False):
+    _ret_vms = {}
+    for _name, _value in vms.items():
+        if server_index != -1 and not _value['vm_offset'].startswith(str(server_index) + ','):
+            continue
+
+        _ret_vms[_name] = _value
+        if server_index != -1 and remove_index:
+            _ret_vms[_name]['vm_offset'] = int(_ret_vms[_name]['vm_offset'].split(',')[1])
+
+    return _ret_vms
+
+
 def extract_by_prefix(values, prefix):
     """
     This function takes a list as 'values' parameter and extract a first value from the list which contains prefix.
@@ -88,7 +102,7 @@ def first_n_elements(values, num):
     return values[0:int(num)]
 
 
-def filter_vm_targets(values, topology, vm_base):
+def filter_vm_targets(values, topology, vm_base, server_index=-1):
     """
     This function takes a list of host VMs as parameter 'values' and then extract a list of host VMs
     which starts with 'vm_base' and contains all VMs which mentioned in 'vm_offset' keys inside of 'topology' structure
@@ -117,9 +131,14 @@ def filter_vm_targets(values, topology, vm_base):
     result = []
     base = values.index(vm_base)
     for hostname, attr in topology.items():
-        if base + attr['vm_offset'] >= len(values):
+        offset = attr['vm_offset']
+        if server_index != -1 and not offset.startswith(str(server_index) + ','):
+            continue
+        elif server_index != -1:
+            offset = int(offset.split(',')[1])
+        if base + offset >= len(values):  # if the vm is not defined in iventory files veos
             continue
-        result.append(values[base + attr['vm_offset']])
+        result.append(values[base + offset])
 
     return result
 

ansible/roles/eos/tasks/main.yml

@@ -17,7 +17,7 @@
   set_fact: VM_host={{ groups[current_server] | difference(VM_list) }}
 
 - name: Generate hostname for target VM
-  set_fact: hostname={{ VM_list | extract_hostname(topology['VMs'], VM_base, inventory_hostname) }}
+  set_fact: hostname={{ VM_list | extract_hostname(topology['VMs'] | filter_vms_by_server_index(server_index | default(-1), True), VM_base, inventory_hostname) }}
   when: topology['VMs'] is defined
 
 - fail:

ansible/roles/vm_set/library/vm_topology.py

@@ -213,7 +213,7 @@ def adaptive_temporary_interface(vm_set_name, interface_name, reserved_space=0):
 
 class VMTopology(object):
 
-    def __init__(self, vm_names, vm_properties, fp_mtu, max_fp_num, topo):
+    def __init__(self, vm_names, vm_properties, fp_mtu, max_fp_num, topo, server_index=-1):
         self.vm_names = vm_names
         self.vm_properties = vm_properties
         self.fp_mtu = fp_mtu
@@ -222,6 +222,7 @@ def __init__(self, vm_names, vm_properties, fp_mtu, max_fp_num, topo):
         self._host_interfaces = None
         self._disabled_host_interfaces = None
         self._host_interfaces_active_active = None
+        self.server_index = server_index
         return
 
     def init(self, vm_set_name, vm_base, duts_fp_ports, duts_name, ptf_exists=True, check_bridge=True):
@@ -242,7 +243,13 @@ def init(self, vm_set_name, vm_base, duts_fp_ports, duts_name, ptf_exists=True,
                 raise Exception('VM_base "%s" should be presented in current vm_names: %s' % (
                     vm_base, str(self.vm_names)))
             for k, v in self.topo['VMs'].items():
-                if self.vm_base_index + v['vm_offset'] < len(self.vm_names):
+                offset = v['vm_offset']
+                if self.server_index != -1 and not offset.startswith(str(self.server_index) + ','):
+                    continue
+                elif self.server_index != -1:
+                    offset = int(offset.split(',')[1])
+                    v['vm_offset'] = offset
+                if self.vm_base_index + offset < len(self.vm_names):
                     self.VMs[k] = v
             if check_bridge:
                 for hostname, attrs in self.VMs.items():
@@ -351,6 +358,14 @@ def _parse_host_interfaces(self, host_interfaces):
                     _host_interfaces.append(
                         tuple(map(int, re.split(r'\.|@', intfs[0].strip()))))
             return _host_interfaces
+        elif self.server_index != -1:
+            _host_interfaces = []
+            for _host_interface in host_interfaces:
+                _server_idx, _intf = _host_interface.split(',')
+                if int(_server_idx) != self.server_index:
+                    continue
+                _host_interfaces.append(int(_intf))
+            return _host_interfaces
         else:
             return host_interfaces
 
@@ -1766,7 +1781,7 @@ def parse_vm_vlan_port(vlan):
         return (dut_index, vlan_index, ptf_index)
 
 
-def check_topo(topo, is_multi_duts=False):
+def check_topo(topo, is_multi_duts=False, is_multi_servers=False):
 
     def _assert(condition, exctype, msg):
         if not condition:
@@ -1793,6 +1808,14 @@ def _assert(condition, exctype, msg):
                     _assert(p not in all_intfs, ValueError,
                             "topo['host_interfaces'] double use of host interface: %s" % p)
                     all_intfs.add(p)
+            elif is_multi_servers:
+                condition = (isinstance(host_intf, str) and re.match(r"^\d+,\d+$", host_intf))
+                _assert(condition, ValueError,
+                        "topo['host_interfaces'] should be a "
+                        "list of strings of format '<server_index>,<dut_inft>'")
+                _assert(host_intf not in all_intfs, ValueError,
+                        "topo['host_interfaces'] double use of host interface: %s" % host_intf)
+                all_intfs.add(host_intf)
             else:
                 condition = isinstance(host_intf, int) and host_intf >= 0
                 _assert(condition, ValueError,
@@ -1815,12 +1838,17 @@ def _assert(condition, exctype, msg):
                     ValueError,
                     "topo['VMs']['%s'] should contain "
                     "'vlans' with a list of vlans" % hostname)
-
-            _assert(('vm_offset' in attrs and
-                     isinstance(attrs['vm_offset'], int)),
-                    ValueError,
-                    "topo['VMs']['%s'] should contain "
-                    "'vm_offset' with a number" % hostname)
+            if is_multi_servers:
+                offset = attrs['vm_offset']
+                condition = (isinstance(offset, str) and re.match(r"^\d+,\d+$", offset))
+                _assert(condition, ValueError,
+                        "topo['VMs']['vm_offset'] for multi-servers should has format '<server_index>,<dut_inft>'")
+            else:
+                _assert(('vm_offset' in attrs and
+                        isinstance(attrs['vm_offset'], int)),
+                        ValueError,
+                        "topo['VMs']['%s'] should contain "
+                        "'vm_offset' with a number" % hostname)
 
             for vlan in attrs['vlans']:
                 if is_multi_duts:
@@ -1886,6 +1914,7 @@ def main():
                      'renumber', 'unbind', 'destroy', "connect-vms", "disconnect-vms"]),
             vm_set_name=dict(required=False, type='str'),
             topo=dict(required=False, type='dict'),
+            server_index=dict(required=False, type='int', default=-1),
             vm_names=dict(required=True, type='list'),
             vm_base=dict(required=False, type='str'),
             vm_type=dict(required=False, type='str'),
@@ -1914,6 +1943,7 @@ def main():
     fp_mtu = module.params['fp_mtu']
     max_fp_num = module.params['max_fp_num']
     vm_properties = module.params['vm_properties']
+    server_index = int(module.params['server_index'])
 
     config_module_logging(construct_log_filename(cmd, vm_set_name))
 
@@ -1923,7 +1953,7 @@ def main():
     try:
 
         topo = module.params['topo']
-        net = VMTopology(vm_names, vm_properties, fp_mtu, max_fp_num, topo)
+        net = VMTopology(vm_names, vm_properties, fp_mtu, max_fp_num, topo, server_index)
 
         if cmd == 'create':
             net.create_bridges()
@@ -1951,7 +1981,7 @@ def main():
                 raise Exception("vm_set_name can't be longer than %d characters: %s (%d)" % (
                     VM_SET_NAME_MAX_LEN, vm_set_name, len(vm_set_name)))
 
-            hostif_exists, vms_exists = check_topo(topo, is_multi_duts)
+            hostif_exists, vms_exists = check_topo(topo, is_multi_duts, module.params['server_index'] != -1)
             devices_interconnect_exists = check_devices_interconnect(
                 topo, is_multi_duts)
 

ansible/roles/vm_set/tasks/add_topo.yml

@@ -227,6 +227,7 @@
       fp_mtu: "{{ fp_mtu_size }}"
       max_fp_num: "{{ max_fp_num }}"
       netns_mgmt_ip_addr: "{{ netns_mgmt_ip if netns_mgmt_ip is defined else omit }}"
+      server_index: "{{ server_index | default(-1) }}"
     become: yes
 
   - name: Change MAC address for PTF interfaces