Merge branch 'develop' into issue_2012_add_huge_tree_junos

docs/conf.py

@@ -1,4 +1,5 @@
 """conf.py."""
+
 # -*- coding: utf-8 -*-
 #
 # napalm documentation build configuration file, created by

docs/requirements.txt

@@ -1,8 +1,8 @@
-urllib3==1.26.15 # https://github.com/readthedocs/readthedocs.org/issues/10290
+urllib3==2.2.1 # https://github.com/readthedocs/readthedocs.org/issues/10290
 sphinx==1.8.6
 sphinx-rtd-theme==1.2.0
 sphinxcontrib-napoleon==0.7
-invoke==2.0.0
+invoke==2.2.0
 jinja2==2.11.3
 MarkupSafe==2.0.1
 pytest==7.2.2

docs/validate/index.rst

@@ -95,7 +95,9 @@ A few notes:
     * We can also use comparison on the conditions of numerical validate. For example, if you want 
       to validate there that the ``cpu``and ``memory`` into ``get_environment`` are ``15%`` or less.
       We can use writing comparison operators such as ``<15.0`` or ``>10.0`` in this case, or range
-      with the operator syntax of ``<->`` such as ``10.0<->20.0`` or ``10<->20``.
+      with the operator syntax of ``<->`` such as ``10.0<->20.0`` or ``10<->20``. In a similar vain
+      a percentage tolerance can be validated upon, for example ``10%20`` allows a 10% tolerance
+      either side of 20 (a range of 18 to 22).
     * Some methods require extra arguments, for example ``ping``. You can pass arguments to those
       methods using the magic keyword ``_kwargs``. In addition, an optional keyword ``_name`` can
       be specified to override the name in the report. Useful for having a more descriptive report

napalm/base/helpers.py

@@ -1,4 +1,5 @@
 """Helper functions for the NAPALM base."""
+
 import ipaddress
 import itertools
 import logging

napalm/base/test/conftest.py

@@ -1,4 +1,5 @@
 """Test fixtures."""
+
 import ast
 import json
 import os

napalm/base/test/double.py

@@ -1,4 +1,5 @@
 """Base class for Test doubles."""
+
 import json
 import re
 import os

napalm/base/test/getters.py

@@ -1,4 +1,5 @@
 """Testing framework."""
+
 import functools
 from itertools import zip_longest
 import inspect

napalm/base/utils/jinja_filters.py

@@ -1,4 +1,5 @@
 """Some common jinja filters."""
+
 from typing import Dict, Any
 
 

napalm/base/utils/string_parsers.py

@@ -1,4 +1,5 @@
 """ Common methods to normalize a string """
+
 import re
 import struct
 from typing import Union, List, Iterable, Dict, Optional, Tuple

napalm/base/validate.py

@@ -3,9 +3,11 @@
 
 See: https://napalm.readthedocs.io/en/latest/validate.html
 """
+
 import yaml
 import copy
 import re
+from math import isclose
 from typing import Dict, List, Union, TypeVar, Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -16,6 +18,7 @@
 
 # We put it here to compile it only once
 numeric_compare_regex = re.compile(r"^(<|>|<=|>=|==|!=)(\d+(\.\d+){0,1})$")
+numeric_tolerance_regex = re.compile(r"^(\d+)%(\d+)$")
 
 
 def _get_validation_file(validation_file: str) -> Dict[str, Dict]:
@@ -155,6 +158,9 @@ def compare(
         elif "<->" in src and len(src.split("<->")) == 2:
             cmp_result = _compare_range(src, dst)
             return cmp_result
+        elif re.search(r"^\d+%\d+$", src):
+            cmp_result = _compare_tolerance(src, dst)
+            return cmp_result
         else:
             m = re.search(src, str(dst))
             if m:
@@ -214,6 +220,22 @@ def _compare_range(src_num: str, dst_num: str) -> bool:
         return False
 
 
+def _compare_tolerance(src_num: str, dst_num: str) -> bool:
+    """Compare against a tolerance percentage either side. You can use 't%%d'."""
+    dst_num = float(dst_num)
+
+    match = numeric_tolerance_regex.match(src_num)
+    if not match:
+        error = "Failed tolerance comparison. Collected: {}. Expected: {}".format(
+            dst_num, src_num
+        )
+        raise ValueError(error)
+
+    src_num = float(match.group(2))
+    max_diff = src_num * int(match.group(1)) / 100
+    return isclose(src_num, dst_num, abs_tol=max_diff)
+
+
 def empty_tree(input_list: List) -> bool:
     """Recursively iterate through values in nested lists."""
     for item in input_list:

napalm/eos/eos.py

@@ -1395,7 +1395,7 @@ def get_interfaces_ip(self):
                         interface_details.get("linkLocal", {})
                         .get("subnet", "::/0")
                         .split("/")[-1]
-                    )
+                    ),
                     # when no link-local set, address will be None and maslken 0
                 }
             )

napalm/eos/utils/versions.py

@@ -1,4 +1,5 @@
 """Some functions to work with EOS version numbers"""
+
 import re
 
 

napalm/ios/ios.py

@@ -1,4 +1,5 @@
 """NAPALM Cisco IOS Handler."""
+
 # Copyright 2015 Spotify AB. All rights reserved.
 #
 # The contents of this file are licensed under the Apache License, Version 2.0
@@ -1957,12 +1958,17 @@ def get_bgp_neighbors(self):
             # get neighbor_entry out of neighbor data
             neighbor_entry = None
             for neighbor in neighbor_data:
-                if (
-                    neighbor["afi"].lower() == afi
-                    and napalm.base.helpers.ip(neighbor["remote_addr"]) == remote_addr
-                ):
-                    neighbor_entry = neighbor
-                    break
+                current_neighbor = napalm.base.helpers.ip(neighbor["remote_addr"])
+                if neighbor["afi"].lower() == afi and current_neighbor == remote_addr:
+                    # Neighbor IPs in VRFs can overlap, so make sure
+                    # we haven't covered this VRF + IP already
+                    vrf = neighbor["vrf"] or "global"
+                    if (
+                        vrf == "global"
+                        or current_neighbor not in bgp_neighbor_data[vrf]["peers"]
+                    ):
+                        neighbor_entry = neighbor
+                        break
             # check for proper session data for the afi
             if neighbor_entry is None:
                 continue
@@ -2095,46 +2101,54 @@ def get_bgp_neighbors_detail(self, neighbor_address=""):
                 "up": neigh["up"] != "never",
                 "local_as": napalm.base.helpers.as_number(neigh["local_as"]),
                 "remote_as": napalm.base.helpers.as_number(neigh["remote_as"]),
-                "router_id": napalm.base.helpers.ip(bgp_neigh["router_id"])
-                if bgp_neigh["router_id"]
-                else "",
-                "local_address": napalm.base.helpers.ip(bgp_neigh["local_address"])
-                if bgp_neigh["local_address"]
-                else "",
+                "router_id": (
+                    napalm.base.helpers.ip(bgp_neigh["router_id"])
+                    if bgp_neigh["router_id"]
+                    else ""
+                ),
+                "local_address": (
+                    napalm.base.helpers.ip(bgp_neigh["local_address"])
+                    if bgp_neigh["local_address"]
+                    else ""
+                ),
                 "local_address_configured": False,
-                "local_port": napalm.base.helpers.as_number(bgp_neigh["local_port"])
-                if bgp_neigh["local_port"]
-                else 0,
+                "local_port": (
+                    napalm.base.helpers.as_number(bgp_neigh["local_port"])
+                    if bgp_neigh["local_port"]
+                    else 0
+                ),
                 "routing_table": bgp_neigh["vrf"] if bgp_neigh["vrf"] else "global",
                 "remote_address": napalm.base.helpers.ip(bgp_neigh["neighbor"]),
-                "remote_port": napalm.base.helpers.as_number(bgp_neigh["remote_port"])
-                if bgp_neigh["remote_port"]
-                else 0,
+                "remote_port": (
+                    napalm.base.helpers.as_number(bgp_neigh["remote_port"])
+                    if bgp_neigh["remote_port"]
+                    else 0
+                ),
                 "multihop": False,
                 "multipath": False,
                 "remove_private_as": False,
                 "import_policy": "",
                 "export_policy": "",
-                "input_messages": napalm.base.helpers.as_number(
-                    bgp_neigh["msg_total_in"]
-                )
-                if bgp_neigh["msg_total_in"]
-                else 0,
-                "output_messages": napalm.base.helpers.as_number(
-                    bgp_neigh["msg_total_out"]
-                )
-                if bgp_neigh["msg_total_out"]
-                else 0,
-                "input_updates": napalm.base.helpers.as_number(
-                    bgp_neigh["msg_update_in"]
-                )
-                if bgp_neigh["msg_update_in"]
-                else 0,
-                "output_updates": napalm.base.helpers.as_number(
-                    bgp_neigh["msg_update_out"]
-                )
-                if bgp_neigh["msg_update_out"]
-                else 0,
+                "input_messages": (
+                    napalm.base.helpers.as_number(bgp_neigh["msg_total_in"])
+                    if bgp_neigh["msg_total_in"]
+                    else 0
+                ),
+                "output_messages": (
+                    napalm.base.helpers.as_number(bgp_neigh["msg_total_out"])
+                    if bgp_neigh["msg_total_out"]
+                    else 0
+                ),
+                "input_updates": (
+                    napalm.base.helpers.as_number(bgp_neigh["msg_update_in"])
+                    if bgp_neigh["msg_update_in"]
+                    else 0
+                ),
+                "output_updates": (
+                    napalm.base.helpers.as_number(bgp_neigh["msg_update_out"])
+                    if bgp_neigh["msg_update_out"]
+                    else 0
+                ),
                 "messages_queued_out": napalm.base.helpers.as_number(neigh["out_q"]),
                 "connection_state": bgp_neigh["bgp_state"],
                 "previous_connection_state": "",
@@ -2145,13 +2159,17 @@ def get_bgp_neighbors_detail(self, neighbor_address=""):
                     else False
                 ),
                 "local_as_prepend": False,
-                "holdtime": napalm.base.helpers.as_number(bgp_neigh["holdtime"])
-                if bgp_neigh["holdtime"]
-                else 0,
+                "holdtime": (
+                    napalm.base.helpers.as_number(bgp_neigh["holdtime"])
+                    if bgp_neigh["holdtime"]
+                    else 0
+                ),
                 "configured_holdtime": 0,
-                "keepalive": napalm.base.helpers.as_number(bgp_neigh["keepalive"])
-                if bgp_neigh["keepalive"]
-                else 0,
+                "keepalive": (
+                    napalm.base.helpers.as_number(bgp_neigh["keepalive"])
+                    if bgp_neigh["keepalive"]
+                    else 0
+                ),
                 "configured_keepalive": 0,
                 "active_prefix_count": 0,
                 "received_prefix_count": 0,
@@ -3204,10 +3222,10 @@ def get_route_to(self, destination="", protocol="", longer=False):
                             # was not specified
                             if protocol == "" or protocol == route_entry["protocol"]:
                                 if route_proto == "bgp":
-                                    route_entry[
-                                        "protocol_attributes"
-                                    ] = self._get_bgp_route_attr(
-                                        destination, _vrf, nh, ip_version
+                                    route_entry["protocol_attributes"] = (
+                                        self._get_bgp_route_attr(
+                                            destination, _vrf, nh, ip_version
+                                        )
                                     )
                                 nh_line_found = (
                                     False  # for next RT entry processing ...
@@ -3300,12 +3318,16 @@ def get_users(self):
             output = self._send_command(command)
         for match in re.finditer(username_regex, output, re.M):
             users[match.groupdict()["username"]] = {
-                "level": int(match.groupdict()["priv_level"])
-                if match.groupdict()["priv_level"]
-                else 1,
-                "password": match.groupdict()["pwd_hash"]
-                if match.groupdict()["pwd_hash"]
-                else "",
+                "level": (
+                    int(match.groupdict()["priv_level"])
+                    if match.groupdict()["priv_level"]
+                    else 1
+                ),
+                "password": (
+                    match.groupdict()["pwd_hash"]
+                    if match.groupdict()["pwd_hash"]
+                    else ""
+                ),
                 "sshkeys": [],
             }
         for match in re.finditer(pub_keychain_regex, output, re.M):
@@ -3722,7 +3744,11 @@ def get_vlans(self):
             return self._get_vlan_all_ports(output)
 
     def _get_vlan_all_ports(self, output):
-        find_regexp = re.compile(r"^(\d+)\s+(\S+)\s+\S+(\s+[A-Z][a-z].*)?$")
+        find_regexp = re.compile(
+            r"^(\d+)\s+"  # vlan id
+            r"(.*?(?=active|act\/[isl]{1}shut|act\/unsup))"  # vlan name
+            r"\w+(?:\/\w+)?\S+(\s+[A-Z][a-z].*)?$"  # ports
+        )
         continuation_regexp = re.compile(r"^\s+([A-Z][a-z].*)$")
         output = output.splitlines()
         vlans = {}
@@ -3736,7 +3762,7 @@ def _get_vlan_all_ports(self, output):
             if vlan_m:
                 was_vlan_or_cont = True
                 vlan_id = vlan_m.group(1)
-                vlan_name = vlan_m.group(2)
+                vlan_name = vlan_m.group(2).strip()
                 interfaces = vlan_m.group(3) or ""
                 vlans[vlan_id] = {"name": vlan_name, "interfaces": []}
 
@@ -3763,7 +3789,7 @@ def _get_vlan_all_ports(self, output):
     def _get_vlan_from_id(self):
         command = "show vlan brief"
         output = self._send_command(command)
-        vlan_regexp = r"^(\d+)\s+(\S+)\s+\S+.*$"
+        vlan_regexp = r"^(\d+)\W+(.*?(?=active|act\/[isl]{1}shut|act\/unsup))"
         find_vlan = re.findall(vlan_regexp, output, re.MULTILINE)
         vlans = {}
         for vlan_id, vlan_name in find_vlan: