Merge remote-tracking branch 'msazure/arlakshm/internal-202205/chassi…

…s_testing_latest' into arlakshm/internal-202205/chassis_testing_latest

Signed-off-by: Arvindsrinivasan Lakshminarasimhan <arlakshm@microsoft.com>

.azure-pipelines/get_dut_version.py

@@ -0,0 +1,171 @@
+import argparse
+import logging
+import os
+import sys
+import json
+import yaml
+
+_self_dir = os.path.dirname(os.path.abspath(__file__))
+base_path = os.path.realpath(os.path.join(_self_dir, ".."))
+if base_path not in sys.path:
+    sys.path.append(base_path)
+ansible_path = os.path.realpath(os.path.join(_self_dir, "../ansible"))
+if ansible_path not in sys.path:
+    sys.path.append(ansible_path)
+
+from devutil.devices import init_localhost, init_testbed_sonichosts  # noqa E402
+
+logger = logging.getLogger(__name__)
+
+RC_INIT_FAILED = 1
+RC_GET_DUT_VERSION_FAILED = 2
+
+ASIC_NAME_PATH = '../ansible/group_vars/sonic/variables'
+
+
+def read_asic_name(hwsku):
+    asic_name_file = os.path.join(os.path.dirname(__file__), ASIC_NAME_PATH)
+    try:
+        with open(asic_name_file) as f:
+            asic_name = yaml.safe_load(f)
+
+        asic_name_dict = {}
+        for key, value in asic_name.items():
+            if "hwskus" in key:
+                asic_name_dict[key] = value
+
+        for name, hw in asic_name_dict.items():
+            if hwsku in hw:
+                return name.split('_')[1]
+
+        return "unknown"
+
+    except IOError:
+        return None
+
+
+def get_duts_version(sonichosts, output=None):
+    try:
+        ret = {}
+        duts_version = sonichosts.command("show version")
+        for dut, version in duts_version.items():
+            ret[dut] = {}
+            dut_version = version["stdout_lines"]
+
+            for line in dut_version:
+                if ":" in line:
+                    line_splitted = line.split(":", 1)
+                    key = line_splitted[0].strip()
+                    value = line_splitted[1].strip()
+                    if key == "Docker images":
+                        ret[dut]["Docker images"] = []
+                        continue
+                    elif key == "ASIC":
+                        ret[dut]["ASIC TYPE"] = value
+                        continue
+                    elif key == "HwSKU":
+                        ret[dut]["ASIC"] = read_asic_name(value)
+                    ret[dut][key] = value
+                elif "docker" in line:
+                    line_splitted = line.split()
+                    ret[dut]["Docker images"].append({"REPOSITORY": line_splitted[0],
+                                                      "TAG": line_splitted[1],
+                                                      "IMAGE ID": line_splitted[2],
+                                                      "SIZE": line_splitted[3]})
+
+        if output:
+            with open(output, "w") as f:
+                f.write(json.dumps(ret))
+                f.close()
+        else:
+            print(ret)
+    except Exception as e:
+        logger.error("Failed to get DUT version: {}".format(e))
+        sys.exit(RC_GET_DUT_VERSION_FAILED)
+
+
+def validate_args(args):
+    _log_level_map = {
+        "debug": logging.DEBUG,
+        "info": logging.INFO,
+        "warning": logging.WARNING,
+        "error": logging.ERROR,
+        "critical": logging.CRITICAL
+    }
+    logging.basicConfig(
+        stream=sys.stdout,
+        level=_log_level_map[args.log_level],
+        format="%(asctime)s %(filename)s#%(lineno)d %(levelname)s - %(message)s"
+    )
+
+
+def main(args):
+    logger.info("Validating arguments")
+    validate_args(args)
+
+    logger.info("Initializing hosts")
+    localhost = init_localhost(args.inventory, options={"verbosity": args.verbosity})
+    sonichosts = init_testbed_sonichosts(
+        args.inventory, args.testbed_name, testbed_file=args.tbfile, options={"verbosity": args.verbosity}
+    )
+
+    if not localhost or not sonichosts:
+        sys.exit(RC_INIT_FAILED)
+
+    get_duts_version(sonichosts, args.output)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Tool for getting sonic device version.")
+
+    parser.add_argument(
+        "-i", "--inventory",
+        dest="inventory",
+        nargs="+",
+        help="Ansible inventory file")
+
+    parser.add_argument(
+        "-t", "--testbed-name",
+        type=str,
+        required=True,
+        dest="testbed_name",
+        help="Testbed name."
+    )
+
+    parser.add_argument(
+        "--tbfile",
+        type=str,
+        dest="tbfile",
+        default="testbed.yaml",
+        help="Testbed definition file."
+    )
+
+    parser.add_argument(
+        "-v", "--verbosity",
+        type=int,
+        dest="verbosity",
+        default=2,
+        help="Log verbosity (0-3)."
+    )
+
+    parser.add_argument(
+        "--log-level",
+        type=str,
+        dest="log_level",
+        choices=["debug", "info", "warning", "error", "critical"],
+        default="debug",
+        help="Loglevel"
+    )
+
+    parser.add_argument(
+        "-o", "--output",
+        type=str,
+        dest="output",
+        required=False,
+        help="Output duts version to the specified file."
+    )
+
+    args = parser.parse_args()
+    main(args)

.azure-pipelines/pr_test_scripts.yaml

@@ -2,6 +2,7 @@ t0:
   - arp/test_arp_extended.py
   - arp/test_neighbor_mac.py
   - arp/test_neighbor_mac_noptf.py
+  - bgp/test_bgp_dual_asn.py
   - bgp/test_bgp_fact.py
   - bgp/test_bgp_gr_helper.py::test_bgp_gr_helper_routes_perserved
   - bgp/test_bgp_speaker.py
@@ -28,6 +29,7 @@ t0:
   - generic_config_updater/test_syslog.py
   - generic_config_updater/test_vlan_interface.py
   - generic_config_updater/test_mmu_dynamic_threshold_config_update.py
+  - generic_config_updater/test_ecn_config_update.py
   - iface_namingmode/test_iface_namingmode.py
   - lldp/test_lldp.py
   - monit/test_monit_status.py
@@ -122,7 +124,6 @@ multi-asic-t1-lag:
   - iface_namingmode/test_iface_namingmode.py
   - scp/test_scp_copy.py
   - test_interfaces.py
-  - override_config_table/test_override_config_table.py
   - process_monitoring/test_critical_process_monitoring.py
   - container_checker/test_container_checker.py
   - http/test_http_copy.py

.azure-pipelines/run-test-scheduler-template.yml

@@ -92,15 +92,18 @@ parameters:
 
 - name: DUMP_KVM_IF_FAIL
   type: string
-  default: ""
+  default: "True"
+  values:
+  - "True"
+  - "False"
 
 - name: REQUESTER
   type: string
   default: ""
 
-- name: MAX_EXECUTE_SECONDS
-  type: string
-  default: ""
+- name: MAX_RUN_TEST_MINUTES
+  type: number
+  default: 480
 
 - name: SPECIFIED_PARAMS
   type: string
@@ -135,7 +138,7 @@ steps:
       --retry-times ${{ parameters.RETRY_TIMES }} \
       --dump-kvm-if-fail ${{ parameters.DUMP_KVM_IF_FAIL }} \
       --requester "${{ parameters.REQUESTER }}" \
-      --max-execute-seconds ${{ parameters.MAX_EXECUTE_SECONDS }}
+      --max-execute-seconds $((${{ parameters.MAX_RUN_TEST_MINUTES }} * 60))
 
       TEST_PLAN_ID=`cat new_test_plan_id.txt`
 
@@ -183,18 +186,20 @@ steps:
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Run test
-
-  - script: |
-      set -e
-      echo "KVM dump"
-      echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
-      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID "
-      # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
-    condition: succeededOrFailed()
-    env:
-      TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
-    displayName: KVM dump
+    timeoutInMinutes: ${{ parameters.MAX_RUN_TEST_MINUTES }}
+
+  - ${{ if eq(parameters.DUMP_KVM_IF_FAIL, 'True') }}:
+    - script: |
+        set -e
+        echo "KVM dump"
+        echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
+        echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID "
+        # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
+        python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
+      condition: succeededOrFailed()
+      env:
+        TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
+      displayName: KVM dump
 
   - script: |
       set -e

.azure-pipelines/upgrade_image.py

@@ -211,9 +211,8 @@ def main(args):
 
     parser.add_argument(
         "-i", "--inventory",
-        type=str,
+        nargs="+",
         dest="inventory",
-        required=True,
         help="Ansible inventory file")
 
     parser.add_argument(

.hooks/pre_commit_hooks/check_conditional_mark_sort.py

@@ -0,0 +1,28 @@
+import re
+import sys
+
+
+def main():
+    stage_files = sys.argv[1:]
+    retval = 0
+    for stage_file in stage_files:
+        if "tests_mark_conditions" in stage_file:
+            conditions = []
+            with open(stage_file, 'r') as f:
+                file_contents = f.readlines()
+                if not file_contents:
+                    continue
+                for line in file_contents:
+                    if re.match('^[a-zA-Z]', line):
+                        conditions.append(line.strip().rstrip(":"))
+                sorted_conditions = conditions[:]
+                sorted_conditions.sort()
+                if conditions != sorted_conditions:
+                    print("The entries in tests/common/plugins/conditional_mark/tests_mark_conditions*.yaml "
+                          "are not sorted in alphabetic order.")
+                    retval = 1
+    return retval
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.pre-commit-config.yaml

@@ -15,3 +15,8 @@ repos:
     hooks:
     -   id: flake8
         args: ["--max-line-length=120"]
+
+-   repo: https://github.com/sonic-net/sonic-mgmt
+    rev: 1.0.0+pre_commit
+    hooks:
+    -   id: check-conditional-mark-sort

.pre-commit-hooks.yaml

@@ -0,0 +1,6 @@
+-   id: check-conditional-mark-sort
+    name: check conditional mark sort
+    description: check conditional mark yaml file sort from A to Z.
+    entry: check-conditional-mark-sort
+    language: python
+    types: [yaml]

ansible/devutil/sonic_helpers.py

@@ -89,8 +89,8 @@ def patch_rsyslog(sonichosts):
             path=conf_file,
             state="present",
             insertafter="# Define a custom template",
-            line='$template RemoteSONiCFileFormat,"<%PRI%>1 %TIMESTAMP:::date-rfc3339% %HOSTNAME% %APP-NAME% '
-                 '%PROCID% %MSGID% [origin swVersion="{}"] %msg%\n"'.format(sonic_build_version),
+            line=r'$template RemoteSONiCFileFormat,"<%PRI%>1 %TIMESTAMP:::date-rfc3339% %HOSTNAME% %APP-NAME% '
+                 r'%PROCID% %MSGID% [origin swVersion=\"{}\"] %msg%\n"'.format(sonic_build_version),
             module_attrs={"become": True}
         )