diff --git a/README.md b/README.md index aa4e7d1..a4b529c 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ The Ansible ``ansible.utils`` collection includes a variety of plugins that aid This collection has been tested against following Ansible versions: **>=2.9.10**. -For collections that support Ansible 2.9, please ensure you update your `network_os` to use the -fully qualified collection name (for example, `cisco.ios.ios`). +For collections that support Ansible 2.9, please ensure you update your `network_os` to use the +fully qualified collection name (for example, `cisco.ios.ios`). Plugins and modules within a collection may be tested with only specific Ansible versions. A collection may contain metadata that identifies these versions. PEP440 is the schema used to describe the versions of Ansible. diff --git a/changelogs/fragments/consolidate_filter_plugin.yml b/changelogs/fragments/consolidate_filter_plugin.yml index e57e811..66e016c 100644 --- a/changelogs/fragments/consolidate_filter_plugin.yml +++ b/changelogs/fragments/consolidate_filter_plugin.yml @@ -1,5 +1,3 @@ --- minor_changes: - "'consolidate' filter plugin added." -trivial: - - Fix sanity issues and update black version. diff --git a/docs/ansible.utils.consolidate_filter.rst b/docs/ansible.utils.consolidate_filter.rst index d1a588a..1b4eb22 100644 --- a/docs/ansible.utils.consolidate_filter.rst +++ b/docs/ansible.utils.consolidate_filter.rst @@ -19,7 +19,7 @@ Synopsis -------- - This plugin presents collective structured data including all supplied facts grouping on common attributes mentioned. - All other boolean parameter defaults to False unless parameters is explicitly mentioned. -- Using the parameters below- ``data_source|ansible.utils.consolidate(fail_missing_match_key=False``)) +- Using the parameters below- ``data_sources|ansible.utils.consolidate(fail_missing_match_key=False``)) @@ -39,7 +39,7 @@ Parameters
- data_source + data_sources
list @@ -98,7 +98,7 @@ Parameters
- prefix + name
string @@ -110,7 +110,7 @@ Parameters -
Specify the prefix with which the result set be created.
+
Specify the name with which the result set be created.
@@ -126,13 +126,13 @@ Parameters -
Fail if duplicate values for any key is found.
+
Fail if the match key's value exists more than once in a given data set.
@@ -147,7 +147,7 @@ Parameters @@ -168,13 +168,13 @@ Parameters -
Fail if a keys to match in not same accross all data sets.
+
Fail if the match key's value is not found in every data source.
@@ -280,25 +280,25 @@ Examples tasks: - name: Build the facts collection set_fact: - data_source: + data_sources: - data: "{{ interfaces }}" match_key: name - prefix: interfaces + name: interfaces - data: "{{ l2_interfaces }}" match_key: name - prefix: l2_interfaces + name: l2_interfaces - data: "{{ l3_interfaces }}" match_key: name - prefix: l3_interfaces + name: l3_interfaces - name: Combine all the facts based on match_keys set_fact: - combined: "{{ data_source|ansible.utils.consolidate(fail_missing_match_value=False) }}" + combined: "{{ data_sources|ansible.utils.consolidate(fail_missing_match_value=False) }}" ##Output # ok: [localhost] => { # "ansible_facts": { - # "data_source": [ + # "data_sources": [ # { # "data": [ # { @@ -344,7 +344,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "interfaces" + # "name": "interfaces" # }, # { # "data": [ @@ -402,7 +402,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "l2_interfaces" + # "name": "l2_interfaces" # }, # { # "data": [ @@ -428,7 +428,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "l3_interfaces" + # "name": "l3_interfaces" # } # ] # }, @@ -663,25 +663,25 @@ Examples tasks: - name: Build the facts collection set_fact: - data_source: + data_sources: - data: "{{ interfaces }}" match_key: name - prefix: interfaces + name: interfaces - data: "{{ l2_interfaces }}" match_key: name - prefix: l2_interfaces + name: l2_interfaces - data: "{{ l3_interfaces }}" match_key: name - prefix: l3_interfaces + name: l3_interfaces - name: Combine all the facts based on match_keys set_fact: - combined: "{{ data_source|ansible.utils.consolidate(fail_missing_match_value=True) }}" + combined: "{{ data_sources|ansible.utils.consolidate(fail_missing_match_value=True) }}" ##Output # ok: [localhost] => { # "ansible_facts": { - # "data_source": [ + # "data_sources": [ # { # "data": [ # { @@ -727,7 +727,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "interfaces" + # "name": "interfaces" # }, # { # "data": [ @@ -785,7 +785,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "l2_interfaces" + # "name": "l2_interfaces" # }, # { # "data": [ @@ -811,7 +811,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "l3_interfaces" + # "name": "l3_interfaces" # } # ] # }, @@ -916,25 +916,25 @@ Examples tasks: - name: Build the facts collection set_fact: - data_source: + data_sources: - data: "{{ interfaces }}" match_key: name - prefix: interfaces + name: interfaces - data: "{{ l2_interfaces }}" match_key: name - prefix: l2_interfaces + name: l2_interfaces - data: "{{ l3_interfaces }}" match_key: name - prefix: l3_interfaces + name: l3_interfaces - name: Combine all the facts based on match_keys set_fact: - combined: "{{ data_source|ansible.utils.consolidate(fail_missing_match_key=True) }}" + combined: "{{ data_sources|ansible.utils.consolidate(fail_missing_match_key=True) }}" ##Output # ok: [localhost] => { # "ansible_facts": { - # "data_source": [ + # "data_sources": [ # { # "data": [ # { @@ -980,7 +980,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "interfaces" + # "name": "interfaces" # }, # { # "data": [ @@ -1038,7 +1038,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "l2_interfaces" + # "name": "l2_interfaces" # }, # { # "data": [ @@ -1064,7 +1064,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "l3_interfaces" + # "name": "l3_interfaces" # } # ] # }, @@ -1173,25 +1173,25 @@ Examples tasks: - name: Build the facts collection set_fact: - data_source: + data_sources: - data: "{{ interfaces }}" match_key: name - prefix: interfaces + name: interfaces - data: "{{ l2_interfaces }}" match_key: name - prefix: l2_interfaces + name: l2_interfaces - data: "{{ l3_interfaces }}" match_key: name - prefix: l3_interfaces + name: l3_interfaces - name: Combine all the facts based on match_keys set_fact: - combined: "{{ data_source|ansible.utils.consolidate(fail_duplicate=True) }}" + combined: "{{ data_sources|ansible.utils.consolidate(fail_duplicate=True) }}" ##Output # ok: [localhost] => { # "ansible_facts": { - # "data_source": [ + # "data_sources": [ # { # "data": [ # { @@ -1237,7 +1237,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "interfaces" + # "name": "interfaces" # }, # { # "data": [ @@ -1298,7 +1298,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "l2_interfaces" + # "name": "l2_interfaces" # }, # { # "data": [ @@ -1324,7 +1324,7 @@ Examples # } # ], # "match_key": "name", - # "prefix": "l3_interfaces" + # "name": "l3_interfaces" # } # ] # }, diff --git a/plugins/filter/consolidate.py b/plugins/filter/consolidate.py index 1219b18..cb6f8e8 100644 --- a/plugins/filter/consolidate.py +++ b/plugins/filter/consolidate.py @@ -20,9 +20,9 @@ DOCUMENTATION = """ description: - This plugin presents collective structured data including all supplied facts grouping on common attributes mentioned. - All other boolean parameter defaults to False unless parameters is explicitly mentioned. - - Using the parameters below- C(data_source|ansible.utils.consolidate(fail_missing_match_key=False))) + - Using the parameters below- C(data_sources|ansible.utils.consolidate(fail_missing_match_key=False))) options: - data_source: + data_sources: description: - This option represents a list of dictionaries to perform the operation on. - For example C(facts_source|ansible.utils.consolidate(fail_missing_match_key=False))), in this case C(facts_source) represents this option. @@ -38,19 +38,22 @@ DOCUMENTATION = """ description: Specify key to match on. type: str required: True - prefix: - description: Specify the prefix with which the result set be created. + name: + description: Specify the name with which the result set be created. type: str required: True fail_missing_match_key: description: Fail if match_key is not found in a specific data set. type: bool + default: True fail_missing_match_value: - description: Fail if a keys to match in not same accross all data sets. + description: Fail if the match key's value is not found in every data source. type: bool + default: True fail_duplicate: - description: Fail if duplicate values for any key is found. + description: Fail if the match key's value exists more than once in a given data set. type: bool + default: True """ EXAMPLES = r""" @@ -147,25 +150,25 @@ vars_files: tasks: - name: Build the facts collection set_fact: - data_source: + data_sources: - data: "{{ interfaces }}" match_key: name - prefix: interfaces + name: interfaces - data: "{{ l2_interfaces }}" match_key: name - prefix: l2_interfaces + name: l2_interfaces - data: "{{ l3_interfaces }}" match_key: name - prefix: l3_interfaces + name: l3_interfaces - name: Combine all the facts based on match_keys set_fact: - combined: "{{ data_source|ansible.utils.consolidate(fail_missing_match_value=False) }}" + combined: "{{ data_sources|ansible.utils.consolidate(fail_missing_match_value=False) }}" ##Output # ok: [localhost] => { # "ansible_facts": { -# "data_source": [ +# "data_sources": [ # { # "data": [ # { @@ -211,7 +214,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "interfaces" +# "name": "interfaces" # }, # { # "data": [ @@ -269,7 +272,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "l2_interfaces" +# "name": "l2_interfaces" # }, # { # "data": [ @@ -295,7 +298,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "l3_interfaces" +# "name": "l3_interfaces" # } # ] # }, @@ -530,25 +533,25 @@ vars_files: tasks: - name: Build the facts collection set_fact: - data_source: + data_sources: - data: "{{ interfaces }}" match_key: name - prefix: interfaces + name: interfaces - data: "{{ l2_interfaces }}" match_key: name - prefix: l2_interfaces + name: l2_interfaces - data: "{{ l3_interfaces }}" match_key: name - prefix: l3_interfaces + name: l3_interfaces - name: Combine all the facts based on match_keys set_fact: - combined: "{{ data_source|ansible.utils.consolidate(fail_missing_match_value=True) }}" + combined: "{{ data_sources|ansible.utils.consolidate(fail_missing_match_value=True) }}" ##Output # ok: [localhost] => { # "ansible_facts": { -# "data_source": [ +# "data_sources": [ # { # "data": [ # { @@ -594,7 +597,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "interfaces" +# "name": "interfaces" # }, # { # "data": [ @@ -652,7 +655,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "l2_interfaces" +# "name": "l2_interfaces" # }, # { # "data": [ @@ -678,7 +681,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "l3_interfaces" +# "name": "l3_interfaces" # } # ] # }, @@ -783,25 +786,25 @@ vars_files: tasks: - name: Build the facts collection set_fact: - data_source: + data_sources: - data: "{{ interfaces }}" match_key: name - prefix: interfaces + name: interfaces - data: "{{ l2_interfaces }}" match_key: name - prefix: l2_interfaces + name: l2_interfaces - data: "{{ l3_interfaces }}" match_key: name - prefix: l3_interfaces + name: l3_interfaces - name: Combine all the facts based on match_keys set_fact: - combined: "{{ data_source|ansible.utils.consolidate(fail_missing_match_key=True) }}" + combined: "{{ data_sources|ansible.utils.consolidate(fail_missing_match_key=True) }}" ##Output # ok: [localhost] => { # "ansible_facts": { -# "data_source": [ +# "data_sources": [ # { # "data": [ # { @@ -847,7 +850,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "interfaces" +# "name": "interfaces" # }, # { # "data": [ @@ -905,7 +908,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "l2_interfaces" +# "name": "l2_interfaces" # }, # { # "data": [ @@ -931,7 +934,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "l3_interfaces" +# "name": "l3_interfaces" # } # ] # }, @@ -1040,25 +1043,25 @@ vars_files: tasks: - name: Build the facts collection set_fact: - data_source: + data_sources: - data: "{{ interfaces }}" match_key: name - prefix: interfaces + name: interfaces - data: "{{ l2_interfaces }}" match_key: name - prefix: l2_interfaces + name: l2_interfaces - data: "{{ l3_interfaces }}" match_key: name - prefix: l3_interfaces + name: l3_interfaces - name: Combine all the facts based on match_keys set_fact: - combined: "{{ data_source|ansible.utils.consolidate(fail_duplicate=True) }}" + combined: "{{ data_sources|ansible.utils.consolidate(fail_duplicate=True) }}" ##Output # ok: [localhost] => { # "ansible_facts": { -# "data_source": [ +# "data_sources": [ # { # "data": [ # { @@ -1104,7 +1107,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "interfaces" +# "name": "interfaces" # }, # { # "data": [ @@ -1165,7 +1168,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "l2_interfaces" +# "name": "l2_interfaces" # }, # { # "data": [ @@ -1191,7 +1194,7 @@ tasks: # } # ], # "match_key": "name", -# "prefix": "l3_interfaces" +# "name": "l3_interfaces" # } # ] # }, @@ -1224,7 +1227,7 @@ def _consolidate(*args, **kwargs): """Consolidate facts together on common attributes""" keys = [ - "data_source", + "data_sources", "fail_missing_match_key", "fail_missing_match_value", "fail_duplicate", diff --git a/plugins/plugin_utils/consolidate.py b/plugins/plugin_utils/consolidate.py index 30ea170..e77de83 100644 --- a/plugins/plugin_utils/consolidate.py +++ b/plugins/plugin_utils/consolidate.py @@ -33,7 +33,7 @@ def _raise_error(filter, msg): def fail_on_filter(validator_func): - """decorator to fail on supplied filters + """Decorator to fail on supplied filters Args: validator_func (func): Function that generates failure messages @@ -43,7 +43,11 @@ def fail_on_filter(validator_func): """ def update_err(*args, **kwargs): + """Filters return value or raises error as per supplied parameters + Returns: + any: Return value to the function call + """ res, err = validator_func(*args, **kwargs) if err.get("match_key_err"): _raise_error( @@ -64,8 +68,8 @@ def fail_on_filter(validator_func): def check_missing_match_key_duplicate( data_sources, fail_missing_match_key, fail_duplicate ): - """Checks if the match_key specified is present in all the supplied data, - also checks for duplicate data accross all the data sources + """Check if the match_key specified is present in all the supplied data, + also check for duplicate data accross all the data sources Args: data_sources (list): list of dicts as data sources @@ -75,11 +79,11 @@ def check_missing_match_key_duplicate( list: list of unique keys based on specified match_keys """ results, errors_match_key, errors_duplicate = [], [], [] - for ds_idx, data_source in enumerate(data_sources): + for ds_idx, data_source in enumerate(data_sources, start=1): match_key = data_source["match_key"] ds_values = [] - for dd_idx, data_dict in enumerate(data_source["data"]): + for dd_idx, data_dict in enumerate(data_source["data"], start=1): try: ds_values.append(data_dict[match_key]) except KeyError: @@ -117,7 +121,7 @@ def check_missing_match_values(matched_keys, fail_missing_match_value): errors_match_values = [] all_values = set(itertools.chain.from_iterable(matched_keys)) if fail_missing_match_value: - for ds_idx, ds_values in enumerate(matched_keys): + for ds_idx, ds_values in enumerate(matched_keys, start=1): missing_match = all_values - ds_values if missing_match: m_matches = ", ".join(missing_match) @@ -143,7 +147,7 @@ def consolidate_facts(data_sources, all_values): consolidated_facts = {} for data_source in data_sources: match_key = data_source["match_key"] - source = data_source["prefix"] + source = data_source["name"] data_dict = { d[match_key]: d for d in data_source["data"] if match_key in d } @@ -155,7 +159,7 @@ def consolidate_facts(data_sources, all_values): def consolidate( - data_source, + data_sources, fail_missing_match_key=False, fail_missing_match_value=False, fail_duplicate=False, @@ -173,8 +177,8 @@ def consolidate( """ key_sets = check_missing_match_key_duplicate( - data_source, fail_missing_match_key, fail_duplicate + data_sources, fail_missing_match_key, fail_duplicate ) key_vals = check_missing_match_values(key_sets, fail_missing_match_value) - consolidated_facts = consolidate_facts(data_source, key_vals) + consolidated_facts = consolidate_facts(data_sources, key_vals) return consolidated_facts diff --git a/tests/integration/targets/utils_consolidate/tasks/simple.yaml b/tests/integration/targets/utils_consolidate/tasks/simple.yaml index 1bb0b2f..19081e7 100644 --- a/tests/integration/targets/utils_consolidate/tasks/simple.yaml +++ b/tests/integration/targets/utils_consolidate/tasks/simple.yaml @@ -1,14 +1,14 @@ --- - name: Build the data structure ansible.builtin.set_fact: - data_source: + data_sources: - data: [ { "name": "GigabitEthernet0/1" }, { "name": "GigabitEthernet0/2" }, ] match_key: name - prefix: acl_interfaces + name: acl_interfaces - data: [ { @@ -23,11 +23,11 @@ }, ] match_key: name - prefix: interfaces + name: interfaces - name: Combine all the dictionaries based on match_keys ansible.builtin.set_fact: - combined: "{{ data_source|ansible.utils.consolidate(fail_missing_match_value=False) }}" + combined: "{{ data_sources|ansible.utils.consolidate(fail_missing_match_value=False) }}" - name: Assert result dicts assert: diff --git a/tests/unit/plugins/filter/test_consolidate.py b/tests/unit/plugins/filter/test_consolidate.py index b3213af..876d0ea 100644 --- a/tests/unit/plugins/filter/test_consolidate.py +++ b/tests/unit/plugins/filter/test_consolidate.py @@ -19,7 +19,7 @@ class TestConsolidate(unittest.TestCase): pass def test_consolidate_plugin(self): - data_source = [ + data_sources = [ { "data": [ { @@ -60,7 +60,7 @@ class TestConsolidate(unittest.TestCase): }, ], "match_key": "name", - "prefix": "interfaces", + "name": "interfaces", }, { "data": [ @@ -116,7 +116,7 @@ class TestConsolidate(unittest.TestCase): }, ], "match_key": "name", - "prefix": "l2_interfaces", + "name": "l2_interfaces", }, { "data": [ @@ -130,7 +130,7 @@ class TestConsolidate(unittest.TestCase): {"name": "Loopback999"}, ], "match_key": "name", - "prefix": "l3_interfaces", + "name": "l3_interfaces", }, ] @@ -244,13 +244,13 @@ class TestConsolidate(unittest.TestCase): "l3_interfaces": {"name": "Loopback999"}, }, } - args = ["", data_source] + args = ["", data_sources] result = _consolidate(*args) self.assertEqual(result, output) def test_fail_missing_match_key(self): - data_source = [ + data_sources = [ { "data": [ { @@ -291,7 +291,7 @@ class TestConsolidate(unittest.TestCase): }, ], "match_key": "name", - "prefix": "interfaces", + "name": "interfaces", }, { "data": [ @@ -347,7 +347,7 @@ class TestConsolidate(unittest.TestCase): }, ], "match_key": "name", - "prefix": "l2_interfaces", + "name": "l2_interfaces", }, { "data": [ @@ -361,21 +361,21 @@ class TestConsolidate(unittest.TestCase): {"name": "Loopback999"}, ], "match_key": "name", - "prefix": "l3_interfaces", + "name": "l3_interfaces", }, ] fail_missing_match_key = True - args = ["", data_source, fail_missing_match_key] + args = ["", data_sources, fail_missing_match_key] with self.assertRaises(AnsibleFilterError) as error: _consolidate(*args) self.assertIn( - "Error when using plugin 'consolidate': 'fail_missing_match_key' reported Missing match key 'name' in data source 2 in list entry 0", + "Error when using plugin 'consolidate': 'fail_missing_match_key' reported Missing match key 'name' in data source 3 in list entry 1", str(error.exception), ) def test_fail_missing_match_value(self): - data_source = [ + data_sources = [ { "data": [ { @@ -416,7 +416,7 @@ class TestConsolidate(unittest.TestCase): }, ], "match_key": "name", - "prefix": "interfaces", + "name": "interfaces", }, { "data": [ @@ -472,7 +472,7 @@ class TestConsolidate(unittest.TestCase): }, ], "match_key": "name", - "prefix": "l2_interfaces", + "name": "l2_interfaces", }, { "data": [ @@ -490,7 +490,7 @@ class TestConsolidate(unittest.TestCase): {"name": "Loopback999"}, ], "match_key": "name", - "prefix": "l3_interfaces", + "name": "l3_interfaces", }, ] @@ -499,7 +499,7 @@ class TestConsolidate(unittest.TestCase): fail_duplicate = True args = [ "", - data_source, + data_sources, fail_missing_match_key, fail_missing_match_value, fail_duplicate, @@ -507,6 +507,6 @@ class TestConsolidate(unittest.TestCase): with self.assertRaises(AnsibleFilterError) as error: _consolidate(*args) self.assertIn( - "Error when using plugin 'consolidate': 'fail_duplicate' reported Duplicate values in data source 2", + "Error when using plugin 'consolidate': 'fail_duplicate' reported Duplicate values in data source 3", str(error.exception), )