Fix strict filtering for ip-addresses

.github/workflows/ci.yml

@@ -64,6 +64,9 @@ jobs:
           - python-version: "3.8"
             nautobot-version: "1.3"
             ansible-release: "2.12"
+          - python-version: "3.8"
+            nautobot-version: "1.4"
+            ansible-release: "2.12"
     env:
       INVOKE_NAUTOBOT_ANSIBLE_PYTHON_VER: "${{ matrix.python-version }}"
       INVOKE_NAUTOBOT_ANSIBLE_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
@@ -80,7 +83,7 @@ jobs:
         run: "poetry remove ansible-base"
       - name: "Add ansible-core"
         if: "${{ matrix.ansible-release == '2.11' }}"
-        run: "poetry add ansible-core@^2.11"
+        run: "poetry add ansible-core@~2.11"
       - name: "Install poetry"
         if: "${{ matrix.ansible-release == '2.9' }}"
         run: "pip install poetry"
@@ -89,7 +92,7 @@ jobs:
         run: "poetry remove ansible-base"
       - name: "Add Ansible 2.9"
         if: "${{ matrix.ansible-release == '2.9' }}"
-        run: "poetry add ansible@^2.9"
+        run: "poetry add ansible@~2.9"
       - name: "Install poetry"
         if: "${{ matrix.ansible-release == '2.12' }}"
         run: "pip install poetry"
@@ -98,7 +101,7 @@ jobs:
         run: "poetry remove ansible-base"
       - name: "Add Ansible 2.12"
         if: "${{ matrix.ansible-release == '2.12' }}"
-        run: "poetry add ansible-core@^2.12 --python ^${{ matrix.python-version }}"
+        run: "poetry add ansible-core@~2.12 --python ^${{ matrix.python-version }}"
       - name: "Start containers"
         run: "invoke start"
       - name: "Tests"

Dockerfile

@@ -79,7 +79,7 @@ RUN ansible-galaxy collection install community.general
 RUN ansible-galaxy collection build --output-path ./dist/ .
 
 # Install built library
-RUN ansible-galaxy collection install ./dist/networktocode*.tar.gz
+RUN ansible-galaxy collection install ./dist/networktocode*.tar.gz -p ${ANSIBLE_COLLECTIONS_PATH}
 
 # Switch to the collection path for tests
 WORKDIR ${ANSIBLE_COLLECTIONS_PATH}/ansible_collections/networktocode/nautobot
@@ -100,6 +100,8 @@ FROM unittests AS integration
 
 ARG ANSIBLE_INTEGRATION_ARGS
 ENV ANSIBLE_INTEGRATION_ARGS=${ANSIBLE_INTEGRATION_ARGS}
+ARG NAUTOBOT_VER
+ENV NAUTOBOT_VER=${NAUTOBOT_VER}
 
 # Integration test entrypoint
 ENTRYPOINT ${ANSIBLE_COLLECTIONS_PATH}/ansible_collections/networktocode/nautobot/tests/integration/entrypoint.sh

development/docker-compose.yml

@@ -27,6 +27,6 @@ services:
     image: "redis:6-alpine"
     command:
       - "sh"
-      - "-c"  # this is to evaluate the $REDIS_PASSWORD from the env
-      - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD"  ## $$ because of docker-compose
-    env_file: "./dev.env"
+      - "-c" # this is to evaluate the $REDIS_PASSWORD from the env
+      - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD" ## $$ because of docker-compose
+    env_file: "./dev.env"

docker-compose.yml

@@ -15,6 +15,7 @@ x-service:
     ANSIBLE_INTEGRATION_ARGS: ${ANSIBLE_INTEGRATION_ARGS:-}
     ANSIBLE_SANITY_ARGS: ${ANSIBLE_SANITY_ARGS:-}
     ANSIBLE_UNIT_ARGS: ${ANSIBLE_UNIT_ARGS:-}
+    NAUTOBOT_VER: ${INVOKE_NAUTOBOT_ANSIBLE_NAUTOBOT_VER:-}
 
 services:
   unit:

plugins/module_utils/utils.py

@@ -284,14 +284,14 @@
     "device_role": set(["slug"]),
     "device_type": set(["slug"]),
     "front_port": set(["name", "device", "rear_port"]),
-    "front_port_template": set(["name", "device_type", "rear_port"]),
+    "front_port_template": set(["name", "device_type", "rear_port_template"]),
     "installed_device": set(["name"]),
     "interface": set(["name", "device", "virtual_machine"]),
     "interface_template": set(["name", "device_type"]),
     "inventory_item": set(["name", "device"]),
-    "ip_address": set(["address", "vrf", "device", "interface", "assigned_object"]),
-    "ip_addresses": set(["address", "vrf", "device", "interface", "assigned_object"]),
-    "ipaddresses": set(["address", "vrf", "device", "interface", "assigned_object"]),
+    "ip_address": set(["address", "vrf", "device", "interface", "vminterface"]),
+    "ip_addresses": set(["address", "vrf", "device", "interface", "vminterface"]),
+    "ipaddresses": set(["address", "vrf", "device", "interface", "vminterface"]),
     "lag": set(["name"]),
     "manufacturer": set(["slug"]),
     "master": set(["name"]),
@@ -337,7 +337,7 @@
     "vrf": set(["name", "tenant"]),
 }
 
-QUERY_PARAMS_IDS = set(["circuit", "cluster", "device", "group", "interface", "rir", "vrf", "site", "tenant", "type", "virtual_machine"])
+QUERY_PARAMS_IDS = set(["circuit", "cluster", "device", "group", "interface", "rir", "vrf", "site", "tenant", "type", "virtual_machine", "vminterface"])
 
 REQUIRED_ID_FIND = {
     "cables": set(["status", "type", "length_unit"]),
@@ -713,7 +713,7 @@ def _build_query_params(self, parent, module_data, user_query_params=None, child
             else:
                 query_dict.update({"device": module_data["device"]})
 
-        elif parent == "ip_address" and "assigned_object" in matches and module_data.get("assigned_object_type"):
+        elif parent == "ip_address" and module_data.get("assigned_object_type"):
             if module_data["assigned_object_type"] == "virtualization.vminterface":
                 query_dict.update({"vminterface_id": module_data.get("assigned_object_id")})
             elif module_data["assigned_object_type"] == "dcim.interface":

tests/integration/entrypoint.sh

@@ -32,7 +32,11 @@ function main {
 
     echo "# Running..."
     # shellcheck disable=SC2086
-    ansible-test integration $ANSIBLE_INTEGRATION_ARGS --coverage --python "$PYTHON_VERSION" inventory "$@"
+    if [[ "${NAUTOBOT_VER:-}" == "1.4" ]]; then
+      ansible-test integration $ANSIBLE_INTEGRATION_ARGS --coverage --python "$PYTHON_VERSION" inventory-1.4 "$@"
+    else
+      ansible-test integration $ANSIBLE_INTEGRATION_ARGS --coverage --python "$PYTHON_VERSION" inventory "$@"
+    fi
     ansible-test integration $ANSIBLE_INTEGRATION_ARGS --coverage --python "$PYTHON_VERSION" regression-latest "$@"
     ansible-test integration $ANSIBLE_INTEGRATION_ARGS --coverage --python "$PYTHON_VERSION" latest "$@"
     ansible-test coverage report

tests/integration/nautobot-populate.py

@@ -294,11 +294,11 @@ def make_nautobot_calls(endpoint, payload):
 
 created_ip_addresses = make_nautobot_calls(nb.ipam.ip_addresses, ip_addresses)
 # Grab first two IPs
-ip1 = nb.ipam.ip_addresses.get(address="172.16.180.1/24", assigned_object_id=test100_gi1.id)
-ip2 = nb.ipam.ip_addresses.get(address="2001::1:1/64", assigned_object_id=test100_gi2.id)
+ip1 = nb.ipam.ip_addresses.get(address="172.16.180.1/24", interface_id=test100_gi1.id)
+ip2 = nb.ipam.ip_addresses.get(address="2001::1:1/64", interface_id=test100_gi2.id)
 
 # Assign Primary IP
-nexus_eth1_ip = nb.ipam.ip_addresses.get(address="172.16.180.11/24", assigned_object_id=nexus_eth1.id)
+nexus_eth1_ip = nb.ipam.ip_addresses.get(address="172.16.180.11/24", interface_id=nexus_eth1.id)
 nexus.update({"primary_ip4": nexus_eth1_ip})
 
 # Create RIRs

tests/integration/targets/inventory-1.4/.gitignore

@@ -0,0 +1 @@
+runme_config

tests/integration/targets/inventory-1.4/aliases

@@ -0,0 +1 @@
+# https://docs.ansible.com/ansible/devel/dev_guide/testing/sanity/integration-aliases.html

tests/integration/targets/inventory-1.4/compare_inventory_json.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+
+# Inspired by community.aws collection script_inventory_ec2 test
+# https://github.com/ansible-collections/community.aws/blob/master/tests/integration/targets/script_inventory_ec2/inventory_diff.py
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import sys
+import json
+import argparse
+from jsondiff import diff
+from operator import itemgetter
+
+# Nautobot includes "created" and "last_updated" times on objects. These end up in the interfaces objects that are included verbatim from the Nautobot API.
+# "url" may be different if local tests use a different host/port
+# Remove these from files saved in git as test data
+KEYS_REMOVE = frozenset(["created", "id", "last_updated", "rack_id", "url", "notes_url"])
+
+# Ignore these when performing diffs as they will be different for each test run
+KEYS_IGNORE = frozenset()
+
+
+def all_keys_to_ignore():
+    keys = KEYS_REMOVE.union(KEYS_IGNORE)
+
+    return keys
+
+
+# Assume the object will not be recursive, as it originally came from JSON
+def remove_keys(obj, keys):
+
+    if isinstance(obj, dict):
+        keys_to_remove = keys.intersection(obj.keys())
+        for key in keys_to_remove:
+            del obj[key]
+
+        for (key, value) in obj.items():
+            remove_keys(value, keys)
+
+    elif isinstance(obj, list):
+        # Iterate over temporary copy, as we may remove items
+        for item in obj[:]:
+            if isinstance(item, str) and item in keys:
+                # List contains a string that we want to remove
+                # eg. a group name in list of groups
+                obj.remove(item)
+            remove_keys(item, keys)
+
+
+def sort_hostvar_arrays(obj):
+    meta = obj.get("_meta")
+    if not meta:
+        return
+
+    hostvars = meta.get("hostvars")
+    if not hostvars:
+        return
+
+    for hostname, host in hostvars.items():
+        interfaces = host.get("interfaces")
+        if interfaces:
+            host["interfaces"] = sorted(interfaces, key=itemgetter("name"))
+
+        services = host.get("services")
+        if services:
+            host["services"] = sorted(services, key=itemgetter("name"))
+
+
+def read_json(filename):
+    with open(filename, "r") as f:
+        return json.loads(f.read())
+
+
+def write_json(filename, data):
+    with open(filename, "w") as f:
+        json.dump(data, f, indent=4)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Diff Ansible inventory JSON output")
+    parser.add_argument(
+        "filename_a",
+        metavar="ORIGINAL.json",
+        type=str,
+        help="Original json to test against",
+    )
+    parser.add_argument(
+        "filename_b",
+        metavar="NEW.json",
+        type=str,
+        help="Newly generated json to compare against original",
+    )
+    parser.add_argument(
+        "--write",
+        action="store_true",
+        help=(
+            "When comparing files, various keys are removed. "
+            "This option will not compare the files, and instead writes ORIGINAL.json to NEW.json after removing these keys. "
+            "This is used to clean the test json files before saving to the git repo. "
+            "For example, this removes dates. "
+        ),
+    )
+    parser.add_argument(
+        "--nautobot-version",
+        metavar="VERSION",
+        type=str,
+        help=(
+            "Apply comparison specific to Nautobot version. "
+            "For example, rack_groups arrays will only contain a single item in v2.7, so are ignored in the comparison."
+        ),
+    )
+
+    args = parser.parse_args()
+
+    data_a = read_json(args.filename_a)
+
+    if args.write:
+        # When writing test data, only remove "remove_keys" that will change on every git commit.
+        # This makes diffs more easily readable to ensure changes to test data look correct.
+        remove_keys(data_a, KEYS_REMOVE)
+        sort_hostvar_arrays(data_a)
+        write_json(args.filename_b, data_a)
+
+    else:
+        data_b = read_json(args.filename_b)
+
+        # Ignore keys that we don't want to diff, in addition to the ones removed that change on every commit
+        keys = all_keys_to_ignore()
+        remove_keys(data_a, keys)
+        remove_keys(data_b, keys)
+
+        sort_hostvar_arrays(data_a)
+        sort_hostvar_arrays(data_b)
+
+        # Perform the diff
+        # syntax='symmetric' will produce output that prints both the before and after as "$insert" and "$delete"
+        # marshal=True removes any special types, allowing to be dumped as json
+        result = diff(data_a, data_b, marshal=True, syntax="symmetric")
+
+        if result:
+            # Dictionary is not empty - print differences
+            print(json.dumps(result, sort_keys=True, indent=4))
+            sys.exit(1)
+        else:
+            # Success, no differences
+            sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()