Updated test_po_cleanup.py for multi-asic platforms. (#3069)

What/Why I did:

Enhanced test_po_cleanup.py for multi-asic platforms.

Also instead of stopping teamd stopping swss. This is done because when we stop teamd swss sends continuously lot of Error message like below because of which logAnalyzer is not able to capture end marker message in /tmp/syslog which is resulting VS test failure. This does not happen consistently and will need further debug. With this change /tmp/syslog size reduce from Mb to Kb

tests/common/devices/multi_asic.py

@@ -169,6 +169,12 @@ def get_vtysh_cmd_for_namespace(self, cmd, namespace):
             return cmd
         ns_cmd = cmd.replace('vtysh', 'vtysh -n {}'.format(asic_id))
         return ns_cmd
+
+    def get_linux_ip_cmd_for_namespace(self, cmd, namespace):
+        if not namespace:
+            return cmd
+        ns_cmd = cmd.replace('ip', 'ip -n {}'.format(namespace))
+        return ns_cmd
 
     def get_route(self, prefix, namespace=DEFAULT_NAMESPACE):
         asic_id = self.get_asic_id_from_namespace(namespace)

tests/pc/test_po_cleanup.py

@@ -27,36 +27,38 @@ def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
         ]
         loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
         expectRegex = [
-            ".*teamd#teammgrd: :- cleanTeamProcesses.*",
-            ".*teamd#teamsyncd: :- cleanTeamSync.*"
+            ".*teammgrd: :- cleanTeamProcesses.*",
+            ".*teamsyncd: :- cleanTeamSync.*"
         ]
         loganalyzer[rand_one_dut_hostname].expect_regex.extend(expectRegex)
 
-
-def check_kernel_po_interface_cleaned(duthost):
-    res = duthost.shell("ip link show | grep -c PortChannel",  module_ignore_errors=True)["stdout_lines"][0].decode("utf-8")
+def check_kernel_po_interface_cleaned(duthost, asic_index):
+    namespace = duthost.get_namespace_from_asic_id(asic_index)
+    res = duthost.shell(duthost.get_linux_ip_cmd_for_namespace("ip link show | grep -c PortChannel", namespace),module_ignore_errors=True)["stdout_lines"][0].decode("utf-8")
     return res == '0'
 
+@pytest.fixture(scope="module", autouse=True)
+def check_topo_and_restore(duthosts, rand_one_dut_hostname, tbinfo):
+
+    duthost = duthosts[rand_one_dut_hostname]
+    mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
+
+    if len(mg_facts['minigraph_portchannels'].keys()) == 0 and not duthost.is_multi_asic:
+        pytest.skip("Skip test due to there is no portchannel exists in current topology.")
+    yield 
+    # Do config reload to restore everything back
+    logging.info("Reloading config..")
+    config_reload(duthost)
 
-def test_po_cleanup(duthosts, rand_one_dut_hostname, tbinfo):
+def test_po_cleanup(duthosts, rand_one_dut_hostname, enum_asic_index):
     """
     test port channel are cleaned up correctly and teammgrd and teamsyncd process
     handle  SIGTERM gracefully
     """
     duthost = duthosts[rand_one_dut_hostname]
-    mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
-
-    if len(mg_facts['minigraph_portchannels'].keys()) == 0:
-        pytest.skip("Skip test due to there is no portchannel exists in current topology.")
-
-    try:
-        logging.info("Disable Teamd Feature")
-        duthost.shell("sudo systemctl stop teamd")
-        # Check if Linux Kernel Portchannel Interface teamdev are clean up
-        if not wait_until(10, 1, check_kernel_po_interface_cleaned, duthost):
-            fail_msg = "PortChannel interface still exists in kernel"
-            pytest.fail(fail_msg)
-    finally:
-        # Do config reload to restor everything back
-        logging.info("Reloading config..")
-        config_reload(duthost)
+    logging.info("Disable swss/teamd Feature")
+    duthost.asic_instance(enum_asic_index).stop_service("swss")
+    # Check if Linux Kernel Portchannel Interface teamdev are clean up
+    if not wait_until(10, 1, check_kernel_po_interface_cleaned, duthost, enum_asic_index):
+        fail_msg = "PortChannel interface still exists in kernel"
+        pytest.fail(fail_msg)