Move sub plugins under plugins/sub_plugins directory and common code in plugin_utils (#33)

Move sub plugins under plugins/sub_plugins directory and common code in plugin_utils

Reviewed-by: https://github.com/apps/ansible-zuul

changelogs/fragments/sub_plugins_dir_change.yaml

@@ -0,0 +1,7 @@
+---
+breaking_changes:
+  - Move sub plugins cli_parsers, fact_diff and validate to `plugins/sub_plugins` folder
+  - If added custom sub plugins in your collection move from old location `plugins/<sub-plugin-name>` to the
+    new location `plugins/sub_plugins/<sub-plugin-name>` and update the imports as required
+  - The `cli_parsers` sub plugins folder name is changed to `cli_parse` to have consistent naming convention,
+    that is all the cli_parse subplugins will now be in `plugins/sub_plugins/cli_parse` folder

plugins/action/cli_parse.py

@@ -128,7 +128,7 @@ class ActionModule(ActionBase):
             )
             self._display.warning(msg)
 
-        parserlib = "ansible_collections.{corg}.{cname}.plugins.cli_parsers.{plugin}_parser".format(
+        parserlib = "ansible_collections.{corg}.{cname}.plugins.sub_plugins.cli_parser.{plugin}_parser".format(
             **cref
         )
         try:

plugins/action/fact_diff.py

@@ -69,7 +69,7 @@ class ActionModule(ActionBase):
             return None
         cref = dict(zip(["corg", "cname", "plugin"], plugin.split(".")))
         cref.update(directory=directory)
-        parserlib = "ansible_collections.{corg}.{cname}.plugins.{directory}.{plugin}".format(
+        parserlib = "ansible_collections.{corg}.{cname}.plugins.sub_plugins.{directory}.{plugin}".format(
             **cref
         )
         try:

plugins/action/validate.py

@@ -17,8 +17,8 @@ from ansible.plugins.action import ActionBase
 from ansible_collections.ansible.utils.plugins.modules.validate import (
     DOCUMENTATION,
 )
-from ansible_collections.ansible.utils.plugins.module_utils.validate.base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.validate import (
-    load_validator,
+    _load_validator,
 )
 from ansible_collections.ansible.utils.plugins.module_utils.common.argspec_validate import (
     check_argspec,
@@ -72,7 +72,7 @@ class ActionModule(ActionBase):
             task_vars.get("inventory_hostname") if task_vars else None
         )
 
-        self._validator_engine, validator_result = load_validator(
+        self._validator_engine, validator_result = _load_validator(
             engine=updated_params["engine"],
             data=updated_params["data"],
             criteria=updated_params["criteria"],

plugins/filter/get_path.py

@@ -25,7 +25,7 @@ DOCUMENTATION = """
         description:
         - The variable from which the value should be extracted.
         - This option represents the value that is passed to the filter plugin in pipe format.
-        - For example C(config_data|ansible.utils.get_path()), in this case B(config_data) represents this option.
+        - For example C(config_data|ansible.utils.get_path()), in this case C(config_data) represents this option.
         type: raw
         required: True
       path:

plugins/filter/index_of.py

@@ -26,7 +26,7 @@ DOCUMENTATION = """
         description:
         - A list of items to enumerate and test against.
         - This option represents the value that is passed to the filter plugin in pipe format.
-        - For example C(config_data|ansible.utils.index_of('x')), in this case B(config_data) represents this option.
+        - For example C(config_data|ansible.utils.index_of('x')), in this case C(config_data) represents this option.
         type: list
         required: True
       test:

plugins/filter/to_paths.py

@@ -28,7 +28,7 @@ DOCUMENTATION = """
         description:
         - The value of I(var) will be will be used.
         - This option represents the value that is passed to the filter plugin in pipe format.
-        - For example C(config_data|ansible.utils.to_paths()), in this case B(config_data) represents this option.
+        - For example C(config_data|ansible.utils.to_paths()), in this case C(config_data) represents this option.
         type: raw
         required: True
       prepend:

plugins/filter/validate.py

@@ -15,7 +15,7 @@ DOCUMENTATION = """
         description:
         - Data that will be validated against I(criteria).
         - This option represents the value that is passed to the filter plugin in pipe format.
-          For example B(config_data|ansible.utils.validate()), in this case B(config_data)
+          For example C(config_data|ansible.utils.validate()), in this case C(config_data)
           represents this option.
         - For the type of I(data) that represents this value refer to the documentation of individual validator plugins.
         required: True
@@ -24,8 +24,8 @@ DOCUMENTATION = """
         description:
         - The criteria used for validation of value that represents I(data) options.
         - This option represents the first argument passed in the filter plugin.
-          For example B(config_data|ansible.utils.validate(config_criteria)), in
+          For example C(config_data|ansible.utils.validate(config_criteria)), in
-          this case the value of B(config_criteria) represents this option.
+          this case the value of C(config_criteria) represents this option.
         - For the type of I(criteria) that represents this value refer to the  documentation of individual validator plugins.
         required: True
       engine:
@@ -44,10 +44,10 @@ DOCUMENTATION = """
       documentation that is represented in the value of I(engine) option.
     - For additional plugin configuration options refer to the individual validate plugin
       documentation that is represented by the value of I(engine) option.
-    - The plugin configuration option can be either passed as B(key=value) pairs within filter plugin
+    - The plugin configuration option can be either passed as C(key=value) pairs within filter plugin
       or environment variables.
     - The precedence of the I(validate) plugin configurable option is the variable passed within filter plugin
-      as B(key=value) pairs followed by the environment variables.
+      as C(key=value) pairs followed by the environment variables.
 """
 
 EXAMPLES = r"""
@@ -70,8 +70,8 @@ RETURN = """
 
 from ansible.errors import AnsibleError, AnsibleFilterError
 from ansible.module_utils._text import to_text
-from ansible_collections.ansible.utils.plugins.module_utils.validate.base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.validate import (
-    load_validator,
+    _load_validator,
 )
 from ansible_collections.ansible.utils.plugins.module_utils.common.utils import (
     to_list,
@@ -109,7 +109,7 @@ def validate(*args, **kwargs):
             )
         )
 
-    validator_engine, validator_result = load_validator(
+    validator_engine, validator_result = _load_validator(
         engine=updated_params["engine"],
         data=updated_params["data"],
         criteria=updated_params["criteria"],

plugins/lookup/get_path.py

@@ -35,7 +35,7 @@ DOCUMENTATION = """
       wantlist:
         description: >
             If set to C(True), the return value will always be a list.
-            This can also be accomplished using B(query) or B(q) instead of B(lookup).
+            This can also be accomplished using C(query) or C(q) instead of C(lookup).
             U(https://docs.ansible.com/ansible/latest/plugins/lookup.html).
         type: bool
 

plugins/lookup/index_of.py

@@ -53,7 +53,7 @@ DOCUMENTATION = """
         description: >
             When only a single entry in the I(data) is matched, the index of that entry is returned as an integer.
             If set to C(True), the return value will always be a list, even if only a single entry is matched.
-            This can also be accomplished using B(query) or B(q) instead of B(lookup).
+            This can also be accomplished using C(query) or C(q) instead of C(lookup).
             U(https://docs.ansible.com/ansible/latest/plugins/lookup.html)
         type: bool
 

plugins/lookup/to_paths.py

@@ -35,7 +35,7 @@ DOCUMENTATION = """
       wantlist:
         description: >
             If set to I(True), the return value will always be a list.
-            This can also be accomplished using B(query) or B(q) instead of B(lookup).
+            This can also be accomplished using C(query) or B(q) instead of C(lookup).
             U(https://docs.ansible.com/ansible/latest/plugins/lookup.html)
         type: bool
 

plugins/lookup/validate.py

@@ -21,7 +21,7 @@ DOCUMENTATION = """
         - Data that will be validated against I(criteria).
         - This option represents the value that is passed to the lookup plugin as the first argument.
           For example C(lookup(config_data, config_criteria, engine='ansible.utils.jsonschema')),
-          in this case B(config_data) represents this option.
+          in this case C(config_data) represents this option.
         - For the type of I(data) that represents this value refer to the documentation of individual validate plugins.
         required: True
       criteria:
@@ -30,7 +30,7 @@ DOCUMENTATION = """
         - The criteria used for validation of value that represents I(data) options.
         - This option represents the second argument passed in the lookup plugin
           For example C(lookup(config_data, config_criteria, engine='ansible.utils.jsonschema')),
-          in this case the value of B(config_criteria) represents this option.
+          in this case the value of C(config_criteria) represents this option.
         - For the type of I(criteria) that represents this value refer to the documentation of individual
           validate plugins.
         required: True
@@ -50,10 +50,10 @@ DOCUMENTATION = """
       documentation that is represented in the value of I(engine) option.
     - For additional plugin configuration options refer to the individual validate plugin
       documentation that is represented by the value of I(engine) option.
-    - The plugin configuration option can be either passed as B(key=value) pairs within lookup plugin
+    - The plugin configuration option can be either passed as C(key=value) pairs within lookup plugin
       or task or environment variables.
     - The precedence the validate plugin configurable option is the variable passed within lookup plugin
-      as B(key=value) pairs followed by task variables followed by environment variables.
+      as C(key=value) pairs followed by task variables followed by environment variables.
 """
 
 EXAMPLES = r"""
@@ -83,8 +83,8 @@ RETURN = """
 from ansible.errors import AnsibleError, AnsibleLookupError
 from ansible.module_utils._text import to_text
 from ansible.plugins.lookup import LookupBase
-from ansible_collections.ansible.utils.plugins.module_utils.validate.base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.validate import (
-    load_validator,
+    _load_validator,
 )
 from ansible_collections.ansible.utils.plugins.module_utils.common.utils import (
     to_list,
@@ -123,7 +123,7 @@ class LookupModule(LookupBase):
                 )
             )
 
-        validator_engine, validator_result = load_validator(
+        validator_engine, validator_result = _load_validator(
             engine=updated_params["engine"],
             data=updated_params["data"],
             criteria=updated_params["criteria"],

plugins/module_utils/validate/base.py

@@ -1,64 +0,0 @@
-"""
-The base class for validator
-"""
-from __future__ import absolute_import, division, print_function
-
-__metaclass__ = type
-
-try:
-    from importlib import import_module
-except ImportError:
-    pass
-
-from ansible.module_utils._text import to_native
-
-
-def load_validator(
-    engine, data, criteria, plugin_vars=None, cls_name="Validate", kwargs=None
-):
-    """
-    Load the validate plugin from engine name
-    :param engine: Name of the validate engine in format <org-name>.<collection-name>.<validate-plugin>
-    :param vars: Variables for validate plugins. The variable information for each validate plugins can
-                 be referred in individual plugin documentation.
-    :param cls_name: Base class name for validate plugin. Defaults to ``Validate``.
-    :param kwargs: The base name of the class for validate plugin
-    :return:
-    """
-    result = {}
-    if plugin_vars is None:
-        plugin_vars = {}
-
-    if kwargs is None:
-        kwargs = {}
-
-    if len(engine.split(".")) != 3:
-        result["failed"] = True
-        result[
-            "msg"
-        ] = "Parser name should be provided as a full name including collection"
-        return None, result
-
-    cref = dict(zip(["corg", "cname", "plugin"], engine.split(".")))
-    validatorlib = "ansible_collections.{corg}.{cname}.plugins.validate.{plugin}".format(
-        **cref
-    )
-
-    try:
-        validatorcls = getattr(import_module(validatorlib), cls_name)
-        validator = validatorcls(
-            data=data,
-            criteria=criteria,
-            engine=engine,
-            plugin_vars=plugin_vars,
-            kwargs=kwargs,
-        )
-        return validator, result
-    except Exception as exc:
-        result["failed"] = True
-        result[
-            "msg"
-        ] = "For engine '{engine}' error loading the corresponding validate plugin: {err}".format(
-            engine=engine, err=to_native(exc)
-        )
-        return None, result

plugins/plugin_utils/base/validate.py

@@ -11,7 +11,7 @@ from importlib import import_module
 
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import iteritems
-from ansible.module_utils._text import to_text
+from ansible.module_utils._text import to_text, to_native
 
 from ansible_collections.ansible.utils.plugins.module_utils.common.argspec_validate import (
     check_argspec,
@@ -48,7 +48,7 @@ class ValidateBase(object):
         self._sub_plugin_options = {}
 
         cref = dict(zip(["corg", "cname", "plugin"], engine.split(".")))
-        validatorlib = "ansible_collections.{corg}.{cname}.plugins.validate.{plugin}".format(
+        validatorlib = "ansible_collections.{corg}.{cname}.plugins.sub_plugins.validate.{plugin}".format(
             **cref
         )
 
@@ -138,3 +138,54 @@ class ValidateBase(object):
 
     def _get_sub_plugin_options(self, name):
         return self._sub_plugin_options.get(name)
+
+
+def _load_validator(
+    engine, data, criteria, plugin_vars=None, cls_name="Validate", kwargs=None
+):
+    """
+    Load the validate plugin from engine name
+    :param engine: Name of the validate engine in format <org-name>.<collection-name>.<validate-plugin>
+    :param vars: Variables for validate plugins. The variable information for each validate plugins can
+                 be referred in individual plugin documentation.
+    :param cls_name: Base class name for validate plugin. Defaults to ``Validate``.
+    :param kwargs: The base name of the class for validate plugin
+    :return:
+    """
+    result = {}
+    if plugin_vars is None:
+        plugin_vars = {}
+
+    if kwargs is None:
+        kwargs = {}
+
+    if len(engine.split(".")) != 3:
+        result["failed"] = True
+        result[
+            "msg"
+        ] = "Parser name should be provided as a full name including collection"
+        return None, result
+
+    cref = dict(zip(["corg", "cname", "plugin"], engine.split(".")))
+    validatorlib = "ansible_collections.{corg}.{cname}.plugins.sub_plugins.validate.{plugin}".format(
+        **cref
+    )
+
+    try:
+        validatorcls = getattr(import_module(validatorlib), cls_name)
+        validator = validatorcls(
+            data=data,
+            criteria=criteria,
+            engine=engine,
+            plugin_vars=plugin_vars,
+            kwargs=kwargs,
+        )
+        return validator, result
+    except Exception as exc:
+        result["failed"] = True
+        result[
+            "msg"
+        ] = "For engine '{engine}' error loading the corresponding validate plugin: {err}".format(
+            engine=engine, err=to_native(exc)
+        )
+        return None, result

plugins/sub_plugins/cli_parser/json_parser.py

@@ -10,7 +10,7 @@ __metaclass__ = type
 DOCUMENTATION = """
     author: Bradley Thornton (@cidrblock)
     name: json
-    short_description: Define configurable options for B(json) sub-plugin of B(cli_parse) module
+    short_description: Define configurable options for B(json) sub-plugin of M(ansible.utils.cli_parse) module
     description:
     - This plugin documentation provides the configurable options that can be passed
       to the M(ansible.utils.cli_parse) plugins when I(ansible.utils.json) is used as a value for
@@ -38,7 +38,7 @@ import json
 
 from ansible.module_utils._text import to_native
 from ansible.module_utils.six import string_types
-from ansible_collections.ansible.utils.plugins.cli_parsers._base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.cli_parser import (
     CliParserBase,
 )
 

plugins/sub_plugins/cli_parser/textfsm_parser.py

@@ -11,7 +11,7 @@ __metaclass__ = type
 DOCUMENTATION = """
     author: Bradley Thornton (@cidrblock)
     name: textfsm
-    short_description: Define configurable options for C(textfsm) sub-plugin of B(cli_parse) module
+    short_description: Define configurable options for C(textfsm) sub-plugin of M(ansible.utils.cli_parse) module
     description:
     - This plugin documentation provides the configurable options that can be passed
       to the M(ansible.utils.cli_parse) plugins when I(ansible.utils.textfsm) is used as a value for
@@ -40,7 +40,7 @@ import os
 
 from ansible.module_utils._text import to_native
 from ansible.module_utils.basic import missing_required_lib
-from ansible_collections.ansible.utils.plugins.cli_parsers._base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.cli_parser import (
     CliParserBase,
 )
 

plugins/sub_plugins/cli_parser/ttp_parser.py

@@ -11,7 +11,7 @@ __metaclass__ = type
 DOCUMENTATION = """
     author: Bradley Thornton (@cidrblock)
     name: ttp
-    short_description: Define configurable options for C(ttp) sub-plugin of B(cli_parse) module
+    short_description: Define configurable options for C(ttp) sub-plugin of M(ansible.utils.cli_parse) module
     description:
     - This plugin documentation provides the configurable options that can be passed
       to the M(ansible.utils.cli_parse) plugins when I(ansible.utils.ttp) is used as a value for
@@ -40,7 +40,7 @@ import os
 
 from ansible.module_utils._text import to_native
 from ansible.module_utils.basic import missing_required_lib
-from ansible_collections.ansible.utils.plugins.cli_parsers._base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.cli_parser import (
     CliParserBase,
 )
 

plugins/sub_plugins/cli_parser/xml_parser.py

@@ -11,7 +11,7 @@ __metaclass__ = type
 DOCUMENTATION = """
     author: Bradley Thornton (@cidrblock)
     name: xml
-    short_description: Define configurable options for C(xml) sub-plugin of B(cli_parse) module
+    short_description: Define configurable options for C(xml) sub-plugin of M(ansible.utils.cli_parse) module
     description:
     - This plugin documentation provides the configurable options that can be passed
       to the M(ansible.utils.cli_parse) plugins when I(ansible.utils.xml) is used as a value for
@@ -38,7 +38,7 @@ EXAMPLES = r"""
 
 from ansible.module_utils._text import to_native
 from ansible.module_utils.basic import missing_required_lib
-from ansible_collections.ansible.utils.plugins.cli_parsers._base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.cli_parser import (
     CliParserBase,
 )
 

plugins/sub_plugins/fact_diff/native.py

@@ -10,7 +10,7 @@ __metaclass__ = type
 DOCUMENTATION = """
     author: Bradley Thornton (@cidrblock)
     name: native
-    short_description: Define configurable options for C(native) sub-plugin of B(fact_diff) module
+    short_description: Define configurable options for C(native) sub-plugin of M(ansible.utils.fact_diff) module
     description:
     - This plugin documentation provides the configurable options that can be passed
       to the I(ansible.utils.fact_diff) plugins when I(ansible.utils.native) is used as a value for
@@ -29,7 +29,7 @@ EXAMPLES = r"""
 
 import re
 from ansible.plugins.callback import CallbackBase
-from ansible_collections.ansible.utils.plugins.module_utils.base_classes.fact_diff import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.fact_diff import (
     FactDiffBase,
 )
 

plugins/sub_plugins/validate/jsonschema.py

@@ -12,7 +12,7 @@ DOCUMENTATION = """
     name: jsonschema
     short_description: Define configurable options for jsonschema validate plugin
     description:
-    - This plugin documentation provides the configurable options that can be passed
+    - This sub plugin documentation provides the configurable options that can be passed
       to the validate plugins when C(ansible.utils.jsonschema) is used as a value for
       engine option.
     version_added: 1.0.0
@@ -46,7 +46,7 @@ from ansible.module_utils.basic import missing_required_lib
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import string_types
 
-from ansible_collections.ansible.utils.plugins.validate._base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.validate import (
     ValidateBase,
 )
 

plugins/test/validate.py

@@ -39,7 +39,7 @@ DOCUMENTATION = """
         - This option can be passed in test plugin as a key, value pair
           For example C(config_data is ansible.utils.validate(engine='ansible.utils.jsonschema', criteria=criteria)), in
           this case the value of I(engine) key represents the engine to be use for data validation.
-          If the value is not provided the default value that is B(ansible.uitls.jsonschema) will be used.
+          If the value is not provided the default value that is C(ansible.utils.jsonschema) will be used.
         - The value should be in fully qualified collection name format that is
           B(<org-name>.<collection-name>.<validate-plugin-name>).
         default: ansible.utils.jsonschema
@@ -48,10 +48,10 @@ DOCUMENTATION = """
       documentation that is represented in the value of I(engine) option.
     - For additional plugin configuration options refer the individual validate plugin
       documentation that is represented by the value of I(engine) option.
-    - The plugin configuration option can be either passed as B(key=value) pairs within test plugin
+    - The plugin configuration option can be either passed as C(key=value) pairs within test plugin
       or set as environment variables.
     - The precedence the validate plugin configurable option is the variable passed within test plugin
-      as B(key=value) pairs followed by task variables followed by environment variables.
+      as C(key=value) pairs followed by task variables followed by environment variables.
 """
 
 EXAMPLES = r"""
@@ -74,8 +74,8 @@ RETURN = """
 
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_text
-from ansible_collections.ansible.utils.plugins.module_utils.validate.base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.validate import (
-    load_validator,
+    _load_validator,
 )
 from ansible_collections.ansible.utils.plugins.module_utils.common.utils import (
     to_list,
@@ -114,7 +114,7 @@ def validate(*args, **kwargs):
             )
         )
 
-    validator_engine, validator_result = load_validator(
+    validator_engine, validator_result = _load_validator(
         engine=updated_params["engine"],
         data=updated_params["data"],
         criteria=updated_params["criteria"],

tests/unit/plugins/action/test_cli_parse.py

@@ -30,7 +30,7 @@ from ansible_collections.ansible.utils.plugins.module_utils.common.argspec_valid
 from ansible_collections.ansible.utils.plugins.action.cli_parse import (
     ARGSPEC_CONDITIONALS,
 )
-from ansible_collections.ansible.utils.plugins.cli_parsers._base import (
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.cli_parser import (
     CliParserBase,
 )
 from ansible.module_utils.connection import (

tests/unit/plugins/sub_plugins/cli_parsers/test_json_parser.py

@@ -8,7 +8,7 @@ __metaclass__ = type
 import json
 
 from ansible_collections.ansible.utils.tests.unit.compat import unittest
-from ansible_collections.ansible.utils.plugins.cli_parsers.json_parser import (
+from ansible_collections.ansible.utils.plugins.sub_plugins.cli_parser.json_parser import (
     CliParser,
 )
 

tests/unit/plugins/sub_plugins/cli_parsers/test_textfsm_parser.py

@@ -10,7 +10,7 @@ import os
 import pytest
 
 from ansible_collections.ansible.utils.tests.unit.compat import unittest
-from ansible_collections.ansible.utils.plugins.cli_parsers.textfsm_parser import (
+from ansible_collections.ansible.utils.plugins.sub_plugins.cli_parser.textfsm_parser import (
     CliParser,
 )
 

tests/unit/plugins/sub_plugins/cli_parsers/test_ttp_parser.py

@@ -10,7 +10,7 @@ import os
 import pytest
 
 from ansible_collections.ansible.utils.tests.unit.compat import unittest
-from ansible_collections.ansible.utils.plugins.cli_parsers.ttp_parser import (
+from ansible_collections.ansible.utils.plugins.sub_plugins.cli_parser.ttp_parser import (
     CliParser,
 )
 

tests/unit/plugins/sub_plugins/cli_parsers/test_xml_parser.py

@@ -10,7 +10,7 @@ from collections import OrderedDict
 import pytest
 
 from ansible_collections.ansible.utils.tests.unit.compat import unittest
-from ansible_collections.ansible.utils.plugins.cli_parsers.xml_parser import (
+from ansible_collections.ansible.utils.plugins.sub_plugins.cli_parser.xml_parser import (
     CliParser,
 )