Merge remote-tracking branch 'upstream/202012' into vxlan_counters_2012

acl_loader/main.py

@@ -4,6 +4,7 @@
 import ipaddress
 import json
 import syslog
+import operator
 
 import openconfig_acl
 import tabulate
@@ -754,7 +755,7 @@ def incremental_update(self):
                 namespace_configdb.mod_entry(self.ACL_RULE, key, None)
 
         for key in existing_controlplane_rules:
-            if cmp(self.rules_info[key], self.rules_db_info[key]) != 0:
+            if not operator.eq(self.rules_info[key], self.rules_db_info[key]):
                 self.configdb.set_entry(self.ACL_RULE, key, self.rules_info[key])
                 # Program for per-asic namespace corresponding to front asic also if present.
                 # For control plane ACL it's not needed but to keep all db in sync program everywhere

config/vlan.py

@@ -209,21 +209,13 @@ def add_vlan_dhcp_relay_destination(db, vid, dhcp_relay_destination_ip):
     if len(vlan) == 0:
         ctx.fail("{} doesn't exist".format(vlan_name))
 
-    # Verify all ip addresses are valid and not exist in DB
-    dhcp_servers = vlan.get('dhcp_servers', [])
-    dhcpv6_servers = vlan.get('dhcpv6_servers', [])
-
-    if dhcp_relay_destination_ip in dhcp_servers + dhcpv6_servers:
+    dhcp_relay_dests = vlan.get('dhcp_servers', [])
+    if dhcp_relay_destination_ip in dhcp_relay_dests:
         click.echo("{} is already a DHCP relay destination for {}".format(dhcp_relay_destination_ip, vlan_name))
         return
 
-    if clicommon.ipaddress_type(dhcp_relay_destination_ip) == 6:
-        dhcpv6_servers.append(dhcp_relay_destination_ip)
-        vlan['dhcpv6_servers'] = dhcpv6_servers
-    else:
-        dhcp_servers.append(dhcp_relay_destination_ip)
-        vlan['dhcp_servers'] = dhcp_servers
-
+    dhcp_relay_dests.append(dhcp_relay_destination_ip)
+    vlan['dhcp_servers'] = dhcp_relay_dests
     db.cfgdb.set_entry('VLAN', vlan_name, vlan)
     click.echo("Added DHCP relay destination address {} to {}".format(dhcp_relay_destination_ip, vlan_name))
     try:
@@ -251,26 +243,15 @@ def del_vlan_dhcp_relay_destination(db, vid, dhcp_relay_destination_ip):
     if len(vlan) == 0:
         ctx.fail("{} doesn't exist".format(vlan_name))
 
-    # Remove dhcp servers if they exist in the DB
-    dhcp_servers = vlan.get('dhcp_servers', [])
-    dhcpv6_servers = vlan.get('dhcpv6_servers', [])
-
-    if not dhcp_relay_destination_ip in dhcp_servers + dhcpv6_servers:
+    dhcp_relay_dests = vlan.get('dhcp_servers', [])
+    if not dhcp_relay_destination_ip in dhcp_relay_dests:
         ctx.fail("{} is not a DHCP relay destination for {}".format(dhcp_relay_destination_ip, vlan_name))
 
-    if clicommon.ipaddress_type(dhcp_relay_destination_ip) == 6:
-        dhcpv6_servers.remove(dhcp_relay_destination_ip)
-        if len(dhcpv6_servers) == 0:
-            del vlan['dhcpv6_servers']
-        else:
-            vlan['dhcpv6_servers'] = dhcpv6_servers
+    dhcp_relay_dests.remove(dhcp_relay_destination_ip)
+    if len(dhcp_relay_dests) == 0:
+        del vlan['dhcp_servers']
     else:
-        dhcp_servers.remove(dhcp_relay_destination_ip)
-        if len(dhcp_servers) == 0:
-            del vlan['dhcp_servers']
-        else:
-            vlan['dhcp_servers'] = dhcp_servers
-
+        vlan['dhcp_servers'] = dhcp_relay_dests
     db.cfgdb.set_entry('VLAN', vlan_name, vlan)
     click.echo("Removed DHCP relay destination address {} from {}".format(dhcp_relay_destination_ip, vlan_name))
     try:

scripts/route_check.py

@@ -413,6 +413,45 @@ def filter_out_vnet_routes(routes):
     return updated_routes
 
 
+def filter_out_standalone_tunnel_routes(routes):
+    config_db = swsscommon.ConfigDBConnector()
+    config_db.connect()
+    device_metadata = config_db.get_table('DEVICE_METADATA')
+    subtype = device_metadata['localhost'].get('subtype', '')
+
+    if subtype.lower() != 'dualtor':
+        return routes
+
+    app_db = swsscommon.DBConnector('APPL_DB', 0)
+    neigh_table = swsscommon.Table(app_db, 'NEIGH_TABLE')
+    neigh_keys = neigh_table.getKeys()
+    standalone_tunnel_route_ips = []
+    updated_routes = []
+
+    for neigh in neigh_keys:
+        _, mac = neigh_table.hget(neigh, 'neigh')
+        if mac == '00:00:00:00:00:00':
+            # remove preceding 'VlanXXXX' to get just the neighbor IP
+            neigh_ip = ':'.join(neigh.split(':')[1:])
+            standalone_tunnel_route_ips.append(neigh_ip)
+
+    if not standalone_tunnel_route_ips:
+        return routes
+
+    for route in routes:
+        ip, subnet = route.split('/')
+        ip_version = ipaddress.ip_address(ip).version
+
+        # we want to keep the route if it is not a standalone tunnel route.
+        # if the route subnet contains more than one address, it is not a
+        # standalone tunnel route
+        if (ip not in standalone_tunnel_route_ips) or \
+           ((ip_version == 6 and subnet != '128') or (ip_version == 4 and subnet != '32')):
+            updated_routes.append(route)
+
+    return updated_routes
+
+
 def check_routes():
     """
     The heart of this script which runs the checks.
@@ -449,6 +488,7 @@ def check_routes():
     _, rt_asic_miss = diff_sorted_lists(intf_appl, rt_asic_miss)
     rt_asic_miss = filter_out_default_routes(rt_asic_miss)
     rt_asic_miss = filter_out_vnet_routes(rt_asic_miss)
+    rt_asic_miss = filter_out_standalone_tunnel_routes(rt_asic_miss)
 
     # Check APPL-DB INTF_TABLE with ASIC table route entries
     intf_appl_miss, _ = diff_sorted_lists(intf_appl, rt_asic)

scripts/route_check_test.sh

@@ -2,22 +2,36 @@
 
 # add a route, interface & route-entry to simulate error
 #
-sonic-db-cli APPL_DB hmset "ROUTE_TABLE:20c0:d9b8:99:80::/64" "nexthop" "fc00::72,fc00::76,fc00::7a,fc00::7e" "ifname" "PortChannel01,PortChannel02,PortChannel03,PortChannel04"
+sonic-db-cli APPL_DB hmset "ROUTE_TABLE:20c0:d9b8:99:80::/64" "nexthop" "fc00::72,fc00::76,fc00::7a,fc00::7e" "ifname" "PortChannel01,PortChannel02,PortChannel03,PortChannel04" > /dev/null
+sonic-db-cli ASIC_DB hmset "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:{\"dest\":\"192.193.120.255/25\",\"switch_id\":\"oid:0x21000000000000\",\"vr\":\"oid:0x3000000000022\"}" "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID" "oid:0x5000000000614" > /dev/null
+sonic-db-cli APPL_DB hmset  "INTF_TABLE:PortChannel01:10.0.0.99/31" "scope" "global" "family" "IPv4" > /dev/null
 
+echo "------"
+echo "expect errors!"
+echo "Running Route Check..."
+./route_check.py
+echo "return value: $?"
 
-sonic-db-cli ASIC_DB hmset "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:{\"dest\":\"192.193.120.255/25\",\"switch_id\":\"oid:0x21000000000000\",\"vr\":\"oid:0x3000000000022\"}" "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID" "oid:0x5000000000614"
+sonic-db-cli APPL_DB del "ROUTE_TABLE:20c0:d9b8:99:80::/64" > /dev/null
+sonic-db-cli ASIC_DB del "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:{\"dest\":\"192.193.120.255/25\",\"switch_id\":\"oid:0x21000000000000\",\"vr\":\"oid:0x3000000000022\"}" > /dev/null
+sonic-db-cli APPL_DB del "INTF_TABLE:PortChannel01:10.0.0.99/31" > /dev/null
 
-sonic-db-cli APPL_DB hmset  "INTF_TABLE:PortChannel01:10.0.0.99/31" "scope" "global" "family" "IPv4"
+# add standalone tunnel route to simulate unreachable neighbor scenario on dual ToR
+# in this scenario, we expect the route mismatch to be ignored
+sonic-db-cli APPL_DB hmset "NEIGH_TABLE:Vlan1000:fc02:1000::99" "neigh" "00:00:00:00:00:00" "family" "IPv6" > /dev/null
+sonic-db-cli ASIC_DB hmset 'ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:{"dest":"fc02:1000::99/128","switch_id":"oid:0x21000000000000","vr":"oid:0x300000000007c"}' "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID" "oid:0x400000000167d" "SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION" "SAI_PACKET_ACTION_FORWARD" > /dev/null
 
-echo "expect errors!\n------\nRunning Route Check...\n"
+echo "------"
+echo "expect success on dualtor, expect error on all other devices!"
+echo "Running Route Check..."
 ./route_check.py
 echo "return value: $?"
 
-sonic-db-cli APPL_DB del "ROUTE_TABLE:20c0:d9b8:99:80::/64"
-sonic-db-cli ASIC_DB del "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:{\"dest\":\"192.193.120.255/25\",\"switch_id\":\"oid:0x21000000000000\",\"vr\":\"oid:0x3000000000022\"}"
-sonic-db-cli APPL_DB del "INTF_TABLE:PortChannel01:10.0.0.99/31"
-
+sonic-db-cli APPL_DB del "NEIGH_TABLE:Vlan1000:fc02:1000::99" > /dev/null
+sonic-db-cli ASIC_DB del 'ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:{"dest":"fc02:1000::99/128","switch_id":"oid:0x21000000000000","vr":"oid:0x300000000007c"}' > /dev/null
 
-echo "expect success!\n------\nRunning Route Check...\n"
+echo "------"
+echo "expect success!"
+echo "Running Route Check..."
 ./route_check.py
 echo "return value: $?"

show/vlan.py

@@ -32,10 +32,11 @@ def brief(db, verbose):
 
     # Parsing DHCP Helpers info
     for key in natsorted(list(vlan_dhcp_helper_data.keys())):
-        dhcp_helpers = vlan_dhcp_helper_data.get(key, {}).get("dhcp_servers", [])
-        dhcpv6_helpers = vlan_dhcp_helper_data.get(key, {}).get("dhcpv6_servers", [])
-        all_helpers = dhcp_helpers + dhcpv6_helpers
-        vlan_dhcp_helper_dict[key.strip('Vlan')] = all_helpers
+        try:
+            if vlan_dhcp_helper_data[key]['dhcp_servers']:
+                vlan_dhcp_helper_dict[key.strip('Vlan')] = vlan_dhcp_helper_data[key]['dhcp_servers']
+        except KeyError:
+            vlan_dhcp_helper_dict[key.strip('Vlan')] = " "
 
     # Parsing VLAN Gateway info
     for key in vlan_ip_data:

tests/acl_input/incremental_1.json

@@ -0,0 +1,32 @@
+{
+    "acl": {
+        "acl-sets": {
+            "acl-set": {
+                "ntp-acl": {
+                    "acl-entries": {
+                        "acl-entry": {
+                            "1": {
+                                "ip": {
+                                    "config": {
+                                        "source-ip-address": "20.0.0.12/32"
+                                    }
+                                }, 
+                                "config": {
+                                    "sequence-id": 1
+                                }, 
+                                "actions": {
+                                    "config": {
+                                        "forwarding-action": "ACCEPT"
+                                    }
+                                }
+                            }
+                        }
+                    }, 
+                    "config": {
+                        "name": "ntp-acl"
+                    }
+                }
+            }
+        }
+    }
+}

tests/acl_input/incremental_2.json

@@ -0,0 +1,32 @@
+{
+    "acl": {
+        "acl-sets": {
+            "acl-set": {
+                "ntp-acl": {
+                    "acl-entries": {
+                        "acl-entry": {
+                            "1": {
+                                "ip": {
+                                    "config": {
+                                        "source-ip-address": "20.0.0.12/32"
+                                    }
+                                }, 
+                                "config": {
+                                    "sequence-id": 1
+                                }, 
+                                "actions": {
+                                    "config": {
+                                        "forwarding-action": "DROP"
+                                    }
+                                }
+                            }
+                        }
+                    }, 
+                    "config": {
+                        "name": "ntp-acl"
+                    }
+                }
+            }
+        }
+    }
+}

tests/acl_loader_test.py

@@ -1,6 +1,7 @@
 import sys
 import os
 import pytest
+from unittest import mock
 
 test_path = os.path.dirname(os.path.abspath(__file__))
 modules_path = os.path.dirname(test_path)
@@ -200,3 +201,19 @@ def test_icmp_fields_with_non_tcp_protocol(self, acl_loader):
         acl_loader.rules_info = {}
         acl_loader.load_rules_from_file(os.path.join(test_path, 'acl_input/tcp_bad_protocol_number.json'))
         assert not acl_loader.rules_info.get("RULE_1")
+
+    def test_incremental_update(self, acl_loader):
+        acl_loader.rules_info = {}
+        acl_loader.tables_db_info['NTP_ACL'] = {
+            "stage": "INGRESS",
+            "type": "CTRLPLANE"
+        }
+        acl_loader.load_rules_from_file(os.path.join(test_path, 'acl_input/incremental_1.json'))
+        acl_loader.rules_db_info = acl_loader.rules_info
+        assert acl_loader.rules_info[(('NTP_ACL', 'RULE_1'))]["PACKET_ACTION"] == "ACCEPT"
+        acl_loader.per_npu_configdb = None
+        acl_loader.configdb.mod_entry = mock.MagicMock(return_value=True)
+        acl_loader.configdb.set_entry = mock.MagicMock(return_value=True)
+        acl_loader.load_rules_from_file(os.path.join(test_path, 'acl_input/incremental_2.json'))
+        acl_loader.incremental_update()
+        assert acl_loader.rules_info[(('NTP_ACL', 'RULE_1'))]["PACKET_ACTION"] == "DROP"

tests/mock_tables/config_db.json

@@ -498,7 +498,6 @@
     },
     "VLAN|Vlan1000": {
         "dhcp_servers@": "192.0.0.1,192.0.0.2,192.0.0.3,192.0.0.4",
-        "dhcpv6_servers@": "fc02:2000::1,fc02:2000::2",
         "vlanid": "1000"
     },
     "VLAN|Vlan2000": {
@@ -757,7 +756,8 @@
         "mac": "1d:34:db:16:a6:00",
         "platform": "x86_64-mlnx_msn3800-r0",
         "peer_switch": "sonic-switch",
-        "type": "ToRRouter"
+        "type": "ToRRouter",
+        "subtype": "DualToR"
     },
     "SNMP_COMMUNITY|msft": {
         "TYPE": "RO"

tests/pgdropstat_test.py

@@ -93,7 +93,7 @@ def test_show_pg_drop_config_reload(self):
         self.test_show_pg_drop_clear()
 
         # simulate 'config reload' to provoke counters recalculation (remove  backup from /tmp folder)
-        result = runner.invoke(config.config.commands["reload"], [ "--no_service_restart",  "-y"])
+        result = runner.invoke(config.config.commands["reload"], [ "--no_service_restart", "--disable_arp_cache",  "-y"])
 
         print(result.exit_code)
         print(result.output)

tests/route_check_test.py

@@ -23,6 +23,7 @@
 OP_SET = "SET"
 OP_DEL = "DEL"
 
+NEIGH_TABLE = 'NEIGH_TABLE'
 ROUTE_TABLE = 'ROUTE_TABLE'
 VNET_ROUTE_TABLE = 'VNET_ROUTE_TABLE'
 INTF_TABLE = 'INTF_TABLE'
@@ -217,6 +218,22 @@
         }
     },
     "6": {
+        DESCR: "dualtor standalone tunnel route case",
+        ARGS: "route_check",
+        PRE: {
+            APPL_DB: {
+                NEIGH_TABLE: {
+                    "Vlan1000:fc02:1000::99": { "neigh": "00:00:00:00:00:00", "family": "IPv6"}
+                }
+            },
+            ASIC_DB: {
+                RT_ENTRY_TABLE: {
+                    RT_ENTRY_KEY_PREFIX + "fc02:1000::99/128" + RT_ENTRY_KEY_SUFFIX: {},
+                }
+            }
+        }
+    },
+    "7": {
         DESCR: "Good one with VNET routes",
         ARGS: "route_check",
         PRE: {
@@ -337,6 +354,11 @@ def get(self, key):
         return (True, ret)
 
 
+    def hget(self, key, field):
+        ret = copy.deepcopy(self.data.get(key, {}).get(field, {}))
+        return True, ret
+
+
 db_conns = {"APPL_DB": APPL_DB, "ASIC_DB": ASIC_DB}
 def conn_side_effect(arg, _):
     return db_conns[arg]