From 2e5cd9feae4947c479ecae7331c746376eabf6d7 Mon Sep 17 00:00:00 2001 From: Andrea Agosti Date: Mon, 24 Jun 2019 16:16:00 +0100 Subject: [PATCH 1/4] Use ss filter to match TCP connections on Linux Parsing the output of all TCP connections might be really slow for high number of connections. This patch address the problem using the filtering provided by `ss` to reduce the number of line returned by the command. Fixes #53580 --- salt/utils/network.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/salt/utils/network.py b/salt/utils/network.py index d6fc6a98c6ee..908eb62c9fb6 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -1470,8 +1470,9 @@ def _netlink_tool_remote_on(port, which_end): ''' remotes = set() valid = False + tcp_end = 'dst' if which_end == 'remote_port' else 'src' try: - data = subprocess.check_output(['ss', '-ant']) # pylint: disable=minimum-python-version + data = subprocess.check_output(['ss', '-ant', tcp_end, ':{0}'.format(port)]) # pylint: disable=minimum-python-version except subprocess.CalledProcessError: log.error('Failed ss') raise @@ -1486,13 +1487,8 @@ def _netlink_tool_remote_on(port, which_end): elif 'ESTAB' not in line: continue chunks = line.split() - local_host, local_port = chunks[3].rsplit(':', 1) remote_host, remote_port = chunks[4].rsplit(':', 1) - if which_end == 'remote_port' and int(remote_port) != port: - continue - if which_end == 'local_port' and int(local_port) != port: - continue remotes.add(remote_host) if valid is False: From 02ac01c8bf1eaedb9953de1f9193251ad9788441 Mon Sep 17 00:00:00 2001 From: Andrea Agosti Date: Mon, 24 Jun 2019 16:25:19 +0100 Subject: [PATCH 2/4] Fix test_netlink_tool_remote_on Fix the wrong parameter passed to the function --- tests/unit/utils/test_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/utils/test_network.py b/tests/unit/utils/test_network.py index af5cbbab2b59..6a940300b841 100644 --- a/tests/unit/utils/test_network.py +++ b/tests/unit/utils/test_network.py @@ -678,5 +678,5 @@ def test_generate_minion_id_with_long_hostname(self): def test_netlink_tool_remote_on(self): with patch('subprocess.check_output', return_value=NETLINK_SS): - remotes = network._netlink_tool_remote_on('4505', 'remote') + remotes = network._netlink_tool_remote_on('4505', 'remote_port') self.assertEqual(remotes, set(['127.0.0.1', '::ffff:1.2.3.4'])) From 0aa5f6e2140b4977cd62841433129a9edd0424ff Mon Sep 17 00:00:00 2001 From: Andrea Agosti Date: Wed, 25 Sep 2019 15:25:09 +0100 Subject: [PATCH 3/4] Fix CI Lint test failure Fix all the variables and other problems found during the CI Lint pass --- salt/utils/network.py | 83 ++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/salt/utils/network.py b/salt/utils/network.py index 908eb62c9fb6..1836cb809d67 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -47,8 +47,8 @@ try: import ctypes import ctypes.util - libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library("c")) - res_init = libc.__res_init + LIBC = ctypes.cdll.LoadLibrary(ctypes.util.find_library("c")) + RES_INIT = LIBC.__res_init except (ImportError, OSError, AttributeError, TypeError): pass @@ -249,29 +249,29 @@ def is_reachable_host(entity_name): return ret -def is_ip(ip): +def is_ip(ip_addr): ''' Returns a bool telling if the passed IP is a valid IPv4 or IPv6 address. ''' - return is_ipv4(ip) or is_ipv6(ip) + return is_ipv4(ip_addr) or is_ipv6(ip_addr) -def is_ipv4(ip): +def is_ipv4(ip_addr): ''' Returns a bool telling if the value passed to it was a valid IPv4 address ''' try: - return ipaddress.ip_address(ip).version == 4 + return ipaddress.ip_address(ip_addr).version == 4 except ValueError: return False -def is_ipv6(ip): +def is_ipv6(ip_addr): ''' Returns a bool telling if the value passed to it was a valid IPv6 address ''' try: - return ipaddress.ip_address(ip).version == 6 + return ipaddress.ip_address(ip_addr).version == 6 except ValueError: return False @@ -304,11 +304,11 @@ def is_ipv6_subnet(cidr): @jinja_filter('is_ip') -def is_ip_filter(ip, options=None): +def is_ip_filter(ip_addr, options=None): ''' Returns a bool telling if the passed IP is a valid IPv4 or IPv6 address. ''' - return is_ipv4_filter(ip, options=options) or is_ipv6_filter(ip, options=options) + return is_ipv4_filter(ip_addr, options=options) or is_ipv6_filter(ip_addr, options=options) def _ip_options_global(ip_obj, version): @@ -381,7 +381,7 @@ def _ip_options(ip_obj, version, options=None): return six.text_type(ip_obj) -def _is_ipv(ip, version, options=None): +def _is_ipv(ip_addr, version, options=None): if not version: version = 4 @@ -390,11 +390,11 @@ def _is_ipv(ip, version, options=None): return None try: - ip_obj = ipaddress.ip_address(ip) + ip_obj = ipaddress.ip_address(ip_addr) except ValueError: # maybe it is an IP network try: - ip_obj = ipaddress.ip_interface(ip) + ip_obj = ipaddress.ip_interface(ip_addr) except ValueError: # nope, still not :( return None @@ -407,7 +407,7 @@ def _is_ipv(ip, version, options=None): @jinja_filter('is_ipv4') -def is_ipv4_filter(ip, options=None): +def is_ipv4_filter(ip_addr, options=None): ''' Returns a bool telling if the value passed to it was a valid IPv4 address. @@ -420,12 +420,12 @@ def is_ipv4_filter(ip, options=None): options CSV of options regarding the nature of the IP address. E.g.: loopback, multicast, private etc. ''' - _is_ipv4 = _is_ipv(ip, 4, options=options) + _is_ipv4 = _is_ipv(ip_addr, 4, options=options) return isinstance(_is_ipv4, six.string_types) @jinja_filter('is_ipv6') -def is_ipv6_filter(ip, options=None): +def is_ipv6_filter(ip_addr, options=None): ''' Returns a bool telling if the value passed to it was a valid IPv6 address. @@ -438,7 +438,7 @@ def is_ipv6_filter(ip, options=None): options CSV of options regarding the nature of the IP address. E.g.: loopback, multicast, private etc. ''' - _is_ipv6 = _is_ipv(ip, 6, options=options) + _is_ipv6 = _is_ipv(ip_addr, 6, options=options) return isinstance(_is_ipv6, six.string_types) @@ -569,11 +569,11 @@ def network_size(value, options=None, version=None): ] -def natural_ipv4_netmask(ip, fmt='prefixlen'): +def natural_ipv4_netmask(ip_addr, fmt='prefixlen'): ''' Returns the "natural" mask of an IPv4 address ''' - bits = _ipv4_to_bits(ip) + bits = _ipv4_to_bits(ip_addr) if bits.startswith('11'): mask = '24' @@ -588,14 +588,14 @@ def natural_ipv4_netmask(ip, fmt='prefixlen'): return '/' + mask -def rpad_ipv4_network(ip): +def rpad_ipv4_network(ip_addr): ''' Returns an IP network address padded with zeros. Ex: '192.168.3' -> '192.168.3.0' '10.209' -> '10.209.0.0' ''' - return '.'.join(itertools.islice(itertools.chain(ip.split('.'), '0000'), 0, + return '.'.join(itertools.islice(itertools.chain(ip_addr.split('.'), '0000'), 0, 4)) @@ -959,6 +959,7 @@ def _interfaces_ipconfig(out): ''' ifaces = dict() iface = None + addr = None adapter_iface_regex = re.compile(r'adapter (\S.+):$') for line in out.splitlines(): @@ -1303,16 +1304,16 @@ def hex2ip(hex_ip, invert=False): returned. If 'invert=True' assume that ip from /proc/net/ ''' if len(hex_ip) == 32: # ipv6 - ip = [] + ip_addr = [] for i in range(0, 32, 8): ip_part = hex_ip[i:i + 8] ip_part = [ip_part[x:x + 2] for x in range(0, 8, 2)] if invert: - ip.append("{0[3]}{0[2]}:{0[1]}{0[0]}".format(ip_part)) + ip_addr.append("{0[3]}{0[2]}:{0[1]}{0[0]}".format(ip_part)) else: - ip.append("{0[0]}{0[1]}:{0[2]}{0[3]}".format(ip_part)) + ip_addr.append("{0[0]}{0[1]}:{0[2]}{0[3]}".format(ip_part)) try: - address = ipaddress.IPv6Address(":".join(ip)) + address = ipaddress.IPv6Address(":".join(ip_addr)) if address.ipv4_mapped: return str(address.ipv4_mapped) else: @@ -1370,10 +1371,10 @@ def active_tcp(): if line.strip().startswith('sl'): continue iret = _parse_tcp_line(line) - sl = next(iter(iret)) - if iret[sl]['state'] == 1: # 1 is ESTABLISHED - del iret[sl]['state'] - ret[len(ret)] = iret[sl] + slot = next(iter(iret)) + if iret[slot]['state'] == 1: # 1 is ESTABLISHED + del iret[slot]['state'] + ret[len(ret)] = iret[slot] return ret @@ -1414,9 +1415,9 @@ def _remotes_on(port, which_end): if line.strip().startswith('sl'): continue iret = _parse_tcp_line(line) - sl = next(iter(iret)) - if iret[sl][which_end] == port and iret[sl]['state'] == 1: # 1 is ESTABLISHED - ret.add(iret[sl]['remote_addr']) + slot = next(iter(iret)) + if iret[slot][which_end] == port and iret[slot]['state'] == 1: # 1 is ESTABLISHED + ret.add(iret[slot]['remote_addr']) if not proc_available: # Fallback to use OS specific tools if salt.utils.platform.is_sunos(): @@ -1443,15 +1444,15 @@ def _parse_tcp_line(line): ''' ret = {} comps = line.strip().split() - sl = comps[0].rstrip(':') - ret[sl] = {} + slot = comps[0].rstrip(':') + ret[slot] = {} l_addr, l_port = comps[1].split(':') r_addr, r_port = comps[2].split(':') - ret[sl]['local_addr'] = hex2ip(l_addr, True) - ret[sl]['local_port'] = int(l_port, 16) - ret[sl]['remote_addr'] = hex2ip(r_addr, True) - ret[sl]['remote_port'] = int(r_port, 16) - ret[sl]['state'] = int(comps[3], 16) + ret[slot]['local_addr'] = hex2ip(l_addr, True) + ret[slot]['local_port'] = int(l_port, 16) + ret[slot]['remote_addr'] = hex2ip(r_addr, True) + ret[slot]['remote_port'] = int(r_port, 16) + ret[slot]['state'] = int(comps[3], 16) return ret @@ -1886,9 +1887,9 @@ def refresh_dns(): issue #21397: force glibc to re-read resolv.conf ''' try: - res_init() + RES_INIT() except NameError: - # Exception raised loading the library, thus res_init is not defined + # Exception raised loading the library, thus RES_INIT is not defined pass From 0ed511bdcf3681b2b35cd0c676a85c8f2eb63f4e Mon Sep 17 00:00:00 2001 From: Andrea Agosti Date: Wed, 25 Sep 2019 16:45:47 +0100 Subject: [PATCH 4/4] Remove a level of indentation Make codeclimate happy about not going too deep into indentation --- salt/utils/network.py | 46 ++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/salt/utils/network.py b/salt/utils/network.py index 1836cb809d67..df299773e8ff 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -1364,17 +1364,18 @@ def active_tcp(): ''' ret = {} for statf in ['/proc/net/tcp', '/proc/net/tcp6']: - if os.path.isfile(statf): - with salt.utils.files.fopen(statf, 'rb') as fp_: - for line in fp_: - line = salt.utils.stringutils.to_unicode(line) - if line.strip().startswith('sl'): - continue - iret = _parse_tcp_line(line) - slot = next(iter(iret)) - if iret[slot]['state'] == 1: # 1 is ESTABLISHED - del iret[slot]['state'] - ret[len(ret)] = iret[slot] + if not os.path.isfile(statf): + continue + with salt.utils.files.fopen(statf, 'rb') as fp_: + for line in fp_: + line = salt.utils.stringutils.to_unicode(line) + if line.strip().startswith('sl'): + continue + iret = _parse_tcp_line(line) + slot = next(iter(iret)) + if iret[slot]['state'] == 1: # 1 is ESTABLISHED + del iret[slot]['state'] + ret[len(ret)] = iret[slot] return ret @@ -1407,17 +1408,18 @@ def _remotes_on(port, which_end): ret = set() proc_available = False for statf in ['/proc/net/tcp', '/proc/net/tcp6']: - if os.path.isfile(statf): - proc_available = True - with salt.utils.files.fopen(statf, 'r') as fp_: - for line in fp_: - line = salt.utils.stringutils.to_unicode(line) - if line.strip().startswith('sl'): - continue - iret = _parse_tcp_line(line) - slot = next(iter(iret)) - if iret[slot][which_end] == port and iret[slot]['state'] == 1: # 1 is ESTABLISHED - ret.add(iret[slot]['remote_addr']) + if not os.path.isfile(statf): + continue + proc_available = True + with salt.utils.files.fopen(statf, 'r') as fp_: + for line in fp_: + line = salt.utils.stringutils.to_unicode(line) + if line.strip().startswith('sl'): + continue + iret = _parse_tcp_line(line) + slot = next(iter(iret)) + if iret[slot][which_end] == port and iret[slot]['state'] == 1: # 1 is ESTABLISHED + ret.add(iret[slot]['remote_addr']) if not proc_available: # Fallback to use OS specific tools if salt.utils.platform.is_sunos():