Replace ipaddr() with ips_in_ranges()

This change implements a filter_plugin that is used in the
ceph-facts, ceph-validate roles and infrastucture-playbooks.
The new filter plugin will return a list of all IP address
that reside in any one of the given IP ranges. The new filter
replaces the use of the ipaddr filter.

ceph.conf already support a comma separated list of CIDRs
for the public_network and cluster_network options.

Changes: [1] and [2] introduced a regression in ceph-ansible
where public_network can no longer be a comma separated list
of cidrs.

With this change a comma separated list of subnet CIDRs can
also be used for monitor_address_block and radosgw_address_block.

[1] commit: d67230b
[2] commit: 20e4852

Related-To: https://bugs.launchpad.net/tripleo/+bug/1840030
Related-To: https://bugzilla.redhat.com/show_bug.cgi?id=1740283

Closes: ceph#4333
Please backport to stable-4.0

Signed-off-by: Harald Jensås <hjensas@redhat.com>

.travis.yml

@@ -9,5 +9,5 @@ install:
   - pip install ansible-lint pytest
 script:
   - if [[ -n $(grep --exclude-dir=.git -P "\xa0" -r .) ]]; then echo 'NBSP characters found'; exit 1; fi
-  - pytest -vvvv library/
+  - pytest -vvvv library/ plugins/
   - for i in $(ls -1 roles/); do ansible-lint -x 204 -v roles/$i/ ; done

ansible.cfg

@@ -6,6 +6,7 @@ ansible_managed = Please do not change this file directly since it is managed by
 library = ./library
 action_plugins = plugins/actions
 callback_plugins = plugins/callback
+filter_plugins = plugins/filter
 roles_path = ./roles
 # Be sure the user running Ansible has permissions on the logfile
 log_path = $HOME/ansible/ansible.log

infrastructure-playbooks/purge-cluster.yml

@@ -67,7 +67,7 @@
           run_once: true
 
         - name: get all nfs-ganesha mount points
-          command: grep "{{ hostvars[item]['ansible_all_ipv4_addresses'] | ipaddr(public_network) | first }}" /proc/mounts
+          command: grep "{{ hostvars[item]['ansible_all_ipv4_addresses'] | ips_in_ranges(public_network.split(',')) | first }}" /proc/mounts
           register: nfs_ganesha_mount_points
           failed_when: false
           with_items: "{{ groups[nfs_group_name] }}"

infrastructure-playbooks/purge-docker-cluster.yml

@@ -51,7 +51,7 @@
           run_once: true
 
         - name: get all nfs-ganesha mount points
-          command: grep "{{ hostvars[item]['ansible_all_ipv4_addresses'] | ipaddr(public_network) | first }}" /proc/mounts
+          command: grep "{{ hostvars[item]['ansible_all_ipv4_addresses'] | ips_in_ranges(public_network.split(',')) | first }}" /proc/mounts
           register: nfs_ganesha_mount_points
           failed_when: false
           with_items: "{{ groups[nfs_group_name] }}"

plugins/filter/ipaddrs_in_ranges.py

@@ -0,0 +1,30 @@
+from ansible import errors
+
+try:
+    import netaddr
+except ImportError:
+    # in this case, we'll make the filter return an error message (see bottom)
+    netaddr = None
+
+
+class FilterModule(object):
+   ''' IP addresses within IP ranges '''
+
+   def ips_in_ranges(self, ip_addresses, ip_ranges):
+       ips_in_ranges = list()
+       for ip_addr in ip_addresses:
+           for ip_range in ip_ranges:
+               if netaddr.IPAddress(ip_addr) in netaddr.IPNetwork(ip_range):
+                   ips_in_ranges.append(ip_addr)
+       return ips_in_ranges
+
+   def filters(self):
+       if netaddr:
+           return {
+               'ips_in_ranges': self.ips_in_ranges
+           }
+       else:
+           # Need to install python's netaddr for these filters to work
+           raise errors.AnsibleFilterError(
+               "The ips_in_ranges filter requires python's netaddr be "
+               "installed on the ansible controller.")

plugins/filter/test_ipaddrs_in_ranges.py

@@ -0,0 +1,43 @@
+from . import ipaddrs_in_ranges
+
+filter_plugin = ipaddrs_in_ranges.FilterModule()
+
+class TestIpaddrsInRanges(object):
+
+    def test_one_ip_one_range(self):
+        ips = ['10.10.10.1']
+        ranges = ['10.10.10.1/24']
+        result = filter_plugin.ips_in_ranges(ips, ranges)
+        assert ips[0] in result
+        assert len(result) == 1
+
+    def test_two_ip_one_range(self):
+        ips = ['192.168.1.1', '10.10.10.1']
+        ranges = ['10.10.10.1/24']
+        result = filter_plugin.ips_in_ranges(ips, ranges)
+        assert ips[0] not in result
+        assert ips[1] in result
+        assert len(result) == 1
+
+    def test_one_ip_two_ranges(self):
+        ips = ['10.10.10.1']
+        ranges = ['192.168.1.0/24', '10.10.10.1/24']
+        result = filter_plugin.ips_in_ranges(ips, ranges)
+        assert ips[0] in result
+        assert len(result) == 1
+
+    def test_multiple_ips_multiple_ranges(self):
+        ips = ['10.10.10.1', '192.168.1.1', '172.16.10.1']
+        ranges = ['192.168.1.0/24', '10.10.10.1/24', '172.16.17.0/24']
+        result = filter_plugin.ips_in_ranges(ips, ranges)
+        assert ips[0] in result
+        assert ips[1] in result
+        assert ips[2] not in result
+        assert len(result) == 2
+
+    def test_no_ips_in_ranges(self):
+        ips = ['10.10.20.1', '192.168.2.1', '172.16.10.1']
+        ranges = ['192.168.1.0/24', '10.10.10.1/24', '172.16.17.0/24']
+        result = filter_plugin.ips_in_ranges(ips, ranges)
+        assert result == []
+

roles/ceph-facts/tasks/facts.yml

@@ -313,7 +313,7 @@
 
 - name: set grafana_server_addr fact - ipv4
   set_fact:
-    grafana_server_addr: "{{ hostvars[inventory_hostname]['ansible_all_ipv4_addresses'] | ipaddr(public_network) | first }}"
+    grafana_server_addr: "{{ hostvars[inventory_hostname]['ansible_all_ipv4_addresses'] | ips_in_ranges(public_network.split(',')) | first }}"
   when:
     - groups.get(grafana_server_group_name, []) | length > 0
     - ip_version == 'ipv4'
@@ -322,7 +322,7 @@
 
 - name: set grafana_server_addr fact - ipv6
   set_fact:
-    grafana_server_addr: "{{ hostvars[inventory_hostname]['ansible_all_ipv6_addresses'] | ipaddr(public_network) | last | ipwrap }}"
+    grafana_server_addr: "{{ hostvars[inventory_hostname]['ansible_all_ipv6_addresses'] | ips_in_ranges(public_network.split(',')) | last | ipwrap }}"
   when:
     - groups.get(grafana_server_group_name, []) | length > 0
     - ip_version == 'ipv6'

roles/ceph-facts/tasks/set_monitor_address.yml

@@ -1,7 +1,7 @@
 ---
 - name: set_fact _monitor_address to monitor_address_block ipv4
   set_fact:
-    _monitor_addresses: "{{ _monitor_addresses | default([]) + [{ 'name': item, 'addr': hostvars[item]['ansible_all_ipv4_addresses'] | ipaddr(hostvars[item]['monitor_address_block']) | first }] }}"
+    _monitor_addresses: "{{ _monitor_addresses | default([]) + [{ 'name': item, 'addr': hostvars[item]['ansible_all_ipv4_addresses'] | ips_in_ranges(hostvars[item]['monitor_address_block'].split(',')) | first }] }}"
   with_items: "{{ groups.get(mon_group_name, []) }}"
   when:
     - "item not in _monitor_addresses | default([]) | selectattr('name', 'defined') |  map(attribute='name') | list"
@@ -11,7 +11,7 @@
 
 - name: set_fact _monitor_address to monitor_address_block ipv6
   set_fact:
-    _monitor_addresses: "{{ _monitor_addresses | default([]) + [{ 'name': item, 'addr': hostvars[item]['ansible_all_ipv6_addresses'] | ipaddr(hostvars[item]['monitor_address_block']) | last | ipwrap }] }}"
+    _monitor_addresses: "{{ _monitor_addresses | default([]) + [{ 'name': item, 'addr': hostvars[item]['ansible_all_ipv6_addresses'] | ips_in_ranges(hostvars[item]['monitor_address_block'].split(',')) | last | ipwrap }] }}"
   with_items: "{{ groups.get(mon_group_name, []) }}"
   when:
     - "item not in _monitor_addresses | default([]) | selectattr('name', 'defined') |  map(attribute='name') | list"

roles/ceph-facts/tasks/set_radosgw_address.yml

@@ -1,15 +1,15 @@
 ---
 - name: set_fact _radosgw_address to radosgw_address_block ipv4
   set_fact:
-    _radosgw_address: "{{ hostvars[inventory_hostname]['ansible_all_ipv4_addresses'] | ipaddr(radosgw_address_block) | first }}"
+    _radosgw_address: "{{ hostvars[inventory_hostname]['ansible_all_ipv4_addresses'] | ips_in_ranges(hostvars[item]['radosgw_address_block'].split(',')) | first }}"
   when:
     - radosgw_address_block is defined
     - radosgw_address_block != 'subnet'
     - ip_version == 'ipv4'
 
 - name: set_fact _radosgw_address to radosgw_address_block ipv6
   set_fact:
-    _radosgw_address: "{{ hostvars[inventory_hostname]['ansible_all_ipv6_addresses'] | ipaddr(radosgw_address_block) | last | ipwrap }}"
+    _radosgw_address: "{{ hostvars[inventory_hostname]['ansible_all_ipv6_addresses'] | ips_in_ranges(hostvars[item]['radosgw_address_block'].split(',')) | last | ipwrap }}"
   when:
     - radosgw_address_block is defined
     - radosgw_address_block != 'subnet'

roles/ceph-validate/tasks/check_ipaddr_mon.yml

@@ -2,4 +2,4 @@
 - name: "fail if {{ inventory_hostname }} does not have any {{ ip_version }} address on {{ monitor_address_block }}"
   fail:
     msg: "{{ inventory_hostname }} does not have any {{ ip_version }} address on {{ monitor_address_block }}"
-  when: hostvars[inventory_hostname]['ansible_all_' + ip_version + '_addresses'] | ipaddr(hostvars[inventory_hostname]['monitor_address_block']) | length == 0
+  when: hostvars[inventory_hostname]['ansible_all_' + ip_version + '_addresses'] | ips_in_ranges(hostvars[inventory_hostname]['monitor_address_block'].split(',')) | length == 0

tox-podman.ini

@@ -15,8 +15,6 @@ sitepackages=True
 setenv=
   ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config -o ControlMaster=auto -o ControlPersist=600s -o PreferredAuthentications=publickey
   ANSIBLE_CONFIG = {toxinidir}/ansible.cfg
-  ANSIBLE_ACTION_PLUGINS = {toxinidir}/plugins/actions
-  ANSIBLE_CALLBACK_PLUGINS = {toxinidir}/plugins/callback
   ANSIBLE_CALLBACK_WHITELIST = profile_tasks
   ANSIBLE_KEEP_REMOTE_FILES = 1
   ANSIBLE_CACHE_PLUGIN = memory

tox-update.ini

@@ -13,8 +13,6 @@ passenv=*
 setenv=
   ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config -o ControlMaster=auto -o ControlPersist=600s -o PreferredAuthentications=publickey
   ANSIBLE_CONFIG = {toxinidir}/ansible.cfg
-  ANSIBLE_ACTION_PLUGINS = {toxinidir}/plugins/actions
-  ANSIBLE_CALLBACK_PLUGINS = {toxinidir}/plugins/callback
   ANSIBLE_CALLBACK_WHITELIST = profile_tasks
   ANSIBLE_CACHE_PLUGIN = memory
   ANSIBLE_GATHERING = implicit

tox.ini

@@ -22,6 +22,7 @@ setenv=
   ANSIBLE_ACTION_PLUGINS = {toxinidir}/plugins/actions
   ANSIBLE_CALLBACK_PLUGINS = {toxinidir}/plugins/callback
   ANSIBLE_CALLBACK_WHITELIST = profile_tasks
+  ANSIBLE_FILTER_PLUGINS = {toxinidir}/plugins/filter
   CEPH_STABLE_RELEASE = luminous
   # only available for ansible >= 2.5
   ANSIBLE_STDOUT_CALLBACK = yaml
@@ -370,8 +371,6 @@ sitepackages=True
 setenv=
   ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config -o ControlMaster=auto -o ControlPersist=600s -o PreferredAuthentications=publickey
   ANSIBLE_CONFIG = {toxinidir}/ansible.cfg
-  ANSIBLE_ACTION_PLUGINS = {toxinidir}/plugins/actions
-  ANSIBLE_CALLBACK_PLUGINS = {toxinidir}/plugins/callback
   ANSIBLE_CALLBACK_WHITELIST = profile_tasks
   ANSIBLE_KEEP_REMOTE_FILES = 1
   ANSIBLE_CACHE_PLUGIN = memory