New validate sub-plugin "config" (#112)

New validate sub-plugin "config"

SUMMARY


Implement ansible-collections/ansible.network#15 as a validate sub-plugin.
ISSUE TYPE


Feature Pull Request

COMPONENT NAME

validate

Reviewed-by: Ganesh Nalawade <None>
Reviewed-by: Nilashish Chakraborty <nilashishchakraborty8@gmail.com>
Reviewed-by: Nathaniel Case <this.is@nathanielca.se>
Reviewed-by: None <None>

changelogs/fragments/112-validate-config-plugin.yaml

@@ -0,0 +1,4 @@
+---
+minor_changes:
+  - New validate sub-plugin "config" to validate device configuration against user-defined rules
+    (https://github.com/ansible-collections/ansible.network/issues/15).

docs/ansible.utils.validate_module.rst

@@ -114,6 +114,12 @@ Examples
       vars:
         ansible_jsonschema_draft: draft7
 
+    - name: validate configuration with config plugin (see config plugin for criteria examples)
+      ansible.utils.validate:
+        data: "{{ lookup('ansible.builtin.file', './backup/eos.config' }}"
+        criteria: "{{ lookup('ansible.builtin.file', './validate/criteria/config/eos_config_rules.yaml' }}"
+        engine: ansible.utils.config
+
 
 
 Return Values

plugins/action/validate.py

@@ -95,6 +95,7 @@ class ActionModule(ActionBase):
                 )
             )
 
+        self._result["msg"] = ""
         if result.get("errors"):
             self._result["errors"] = result["errors"]
             self._result.update({"failed": True})
@@ -104,6 +105,13 @@ class ActionModule(ActionBase):
                 )
             else:
                 self._result["msg"] = "Validation errors were found."
-        else:
+
-            self._result["msg"] = "all checks passed"
+        if result.get("warnings", []):
+            self._result["warnings"] = result["warnings"]
+            if not self._result["msg"]:
+                self._result["msg"] = "Non-fatal validation errors were found."
+
+        if not self._result["msg"]:
+            self._result["msg"] = "All checks passed."
+
         return self._result

plugins/modules/validate.py

@@ -61,6 +61,12 @@ EXAMPLES = r"""
     engine: ansible.utils.jsonschema
   vars:
     ansible_jsonschema_draft: draft7
+
+- name: validate configuration with config plugin (see config plugin for criteria examples)
+  ansible.utils.validate:
+    data: "{{ lookup('ansible.builtin.file', './backup/eos.config' }}"
+    criteria: "{{ lookup('ansible.builtin.file', './validate/criteria/config/eos_config_rules.yaml' }}"
+    engine: ansible.utils.config
 """
 
 RETURN = r"""

plugins/sub_plugins/validate/config.py

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+# Copyright 2021 Red Hat
+# GNU General Public License v3.0+
+# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+DOCUMENTATION = """
+    author: Nathaniel Case (@Qalthos)
+    name: config
+    short_description: Define configurable options for configuration validate plugin
+    description:
+    - This sub plugin documentation provides the configurable options that can be passed
+      to the validate plugins when C(ansible.utils.config) is used as a value for
+      engine option.
+    version_added: 2.1.0
+    notes:
+    - The value of I(data) option should be a candidate device configuration.
+    - The value of I(criteria) should be a B(list) of rules the candidate configuration
+      will be checked against, or a yaml document containing those rules.
+"""
+
+
+EXAMPLES = r"""
+- name: Interface description should not be more than 8 chars
+  example: "Matches description this-is-a-long-description"
+  rule: 'description\s(.{9,})'
+  action: warn
+
+- name: Ethernet interface names should be in format Ethernet[Slot/chassis number].[sub-intf number (optional)]
+  example: "Matches interface Eth1/1, interface Eth 1/1, interface Ethernet 1/1, interface Ethernet 1/1.100"
+  rule: 'interface\s[eE](?!\w{7}\d/\d(.\d+)?)'
+  action: fail
+
+- name: Loopback interface names should be in format loopback[Virtual Interface Number]
+  example: "Matches interface Lo10, interface Loopback 10"
+  rule: 'interface\s[lL](?!\w{7}\d)'
+  action: fail
+
+- name: Port Channel names should be in format port-channel[Port Channel number].[sub-intf number (optional)]
+  example: "Matches interface port-channel 10, interface po10, interface port-channel 10.1"
+  rule: 'interface\s[pP](?!\w{3}-\w{7}\d(.\d+)?)'
+  action: fail
+"""
+
+import re
+
+from ansible.module_utils._text import to_text
+from ansible.errors import AnsibleError
+from ansible.module_utils.six import string_types
+
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.validate import (
+    ValidateBase,
+)
+
+from ansible_collections.ansible.utils.plugins.module_utils.common.utils import (
+    to_list,
+)
+
+try:
+    import yaml
+
+    # use C version if possible for speedup
+    try:
+        from yaml import CSafeLoader as SafeLoader
+    except ImportError:
+        from yaml import SafeLoader
+    HAS_YAML = True
+except ImportError:
+    HAS_YAML = False
+
+
+def format_message(match, line_number, criteria):
+    """Format warning or error message based on given line and criteria."""
+    return 'At line {line_number}: {message}\nFound "{line}"'.format(
+        line_number=line_number + 1,
+        message=criteria["name"],
+        line=match.string,
+    )
+
+
+class Validate(ValidateBase):
+    def _check_args(self):
+        """Ensure specific args are set
+
+        :return: None: In case all arguments passed are valid
+        """
+
+        try:
+            if isinstance(self._criteria, string_types):
+                self._criteria = yaml.load(
+                    str(self._criteria), Loader=SafeLoader
+                )
+        except yaml.parser.ParserError as exc:
+            msg = (
+                "'criteria' option value is invalid, value should be valid YAML."
+                " Failed to read with error '{err}'".format(
+                    err=to_text(exc, errors="surrogate_then_replace")
+                )
+            )
+            raise AnsibleError(msg)
+
+        issues = []
+        for item in to_list(self._criteria):
+            if "name" not in item:
+                issues.append(
+                    'Criteria {item} missing "name" key'.format(item=item)
+                )
+            if "action" not in item:
+                issues.append(
+                    'Criteria {item} missing "action" key'.format(item=item)
+                )
+            elif item["action"] not in ("warn", "fail"):
+                issues.append(
+                    'Action in criteria {item} is not one of "warn" or "fail"'.format(
+                        item=item
+                    )
+                )
+            if "rule" not in item:
+                issues.append(
+                    'Criteria {item} missing "rule" key'.format(item=item)
+                )
+            else:
+                try:
+                    item["rule"] = re.compile(item["rule"])
+                except re.error as exc:
+                    issues.append(
+                        'Failed to compile regex "{rule}": {exc}'.format(
+                            rule=item["rule"], exc=exc
+                        )
+                    )
+
+        if issues:
+            msg = "\n".join(issues)
+            raise AnsibleError(msg)
+
+    def validate(self):
+        """Std entry point for a validate execution
+
+        :return: Errors or parsed text as structured data
+        :rtype: dict
+
+        :example:
+
+        The parse function of a parser should return a dict:
+        {"errors": [a list of errors]}
+        or
+        {"parsed": obj}
+        """
+        self._check_args()
+
+        try:
+            self._validate_config()
+        except Exception as exc:
+            return {"errors": to_text(exc, errors="surrogate_then_replace")}
+
+        return self._result
+
+    def _validate_config(self):
+        warnings = []
+        errors = []
+        error_messages = []
+
+        for criteria in self._criteria:
+            for line_number, line in enumerate(self._data.split("\n")):
+                match = criteria["rule"].search(line)
+                if match:
+                    if criteria["action"] == "warn":
+                        warnings.append(
+                            format_message(match, line_number, criteria)
+                        )
+                    if criteria["action"] == "fail":
+                        errors.append(
+                            {"message": criteria["name"], "found": line}
+                        )
+                        error_messages.append(
+                            format_message(match, line_number, criteria)
+                        )
+
+        if errors:
+            if "errors" not in self._result:
+                self._result["errors"] = []
+            self._result["errors"].extend(errors)
+        if error_messages:
+            if "msg" not in self._result:
+                self._result["msg"] = "\n".join(error_messages)
+            else:
+                self._result["msg"] += "\n".join(error_messages)
+        if warnings:
+            if "warnings" not in self._result:
+                self._result["warnings"] = []
+            self._result["warnings"].extend(warnings)

tests/integration/targets/utils_validate/files/criteria/rules.yaml

@@ -0,0 +1,20 @@
+---
+- name: Interface description should not be more than 8 chars
+  example: "Matches description this-is-a-long-description"
+  rule: 'description\s(.{9,})'
+  action: warn
+
+- name: Ethernet interface names should be in format Ethernet[Slot/chassis number].[sub-intf number (optional)]
+  example: "Matches interface Eth1/1, interface eth 1/1, interface Ethernet 1/1, interface ethernet 1/1.100"
+  rule: 'interface\s[eE](?!\w{7}\d/\d(.\d+)?)'
+  action: fail
+
+- name: Loopback interface names should be in format loopback[Virtual Interface Number]
+  example: "Matches interface Lo10, interface loopback 10"
+  rule: 'interface\s[lL](?!\w{7}\d)'
+  action: fail
+
+- name: Port Channel names should be in format port-channel[Port Channel number].[sub-intf number (optional)]
+  example: "Matches interface Port-channel 10, interface po10, interface Port-channel 10.1"
+  rule: 'interface\s[pP](?!\w{3}-\w{7}\d(.\d+)?)'
+  action: fail

tests/integration/targets/utils_validate/files/data/fail.cfg

@@ -0,0 +1,15 @@
+interface Eth1/1
+  description test-description-too-long
+  no switchport
+
+interface ethernet1/2
+  description intf-2
+
+interface port-channel1
+  description po-1
+
+interface po2.1
+  description po2
+
+interface Loopback 10
+  description lo10

tests/integration/targets/utils_validate/files/data/pass.cfg

@@ -0,0 +1,15 @@
+interface Ethernet1/1
+  description test
+  no switchport
+
+interface ethernet1/2
+  description intf-2
+
+interface port-channel1
+  description po-1
+
+interface port-channel2.1
+  description po2
+
+interface loopback10
+  description lo10

tests/integration/targets/utils_validate/files/data/warn.cfg

@@ -0,0 +1,15 @@
+interface Ethernet1/1
+  description test-description-too-long
+  no switchport
+
+interface ethernet1/2
+  description intf-2
+
+interface port-channel1
+  description po-1
+
+interface port-channel2.1
+  description po2
+
+interface loopback10
+  description lo10

tests/integration/targets/utils_validate/tasks/main.yaml

@@ -2,11 +2,8 @@
 - name: Recursively find all test files
   find:
     file_type: file
-    paths: "{{ role_path }}/tasks"
+    paths: "{{ role_path }}/tests"
-    recurse: false
+    recurse: true
-    use_regex: true
-    patterns:
-      - '^(?!_|main).+$'
   delegate_to: localhost
   register: found
 

tests/integration/targets/utils_validate/tests/config/filter.yaml

@@ -0,0 +1,57 @@
+---
+- name: Set up data and criteria
+  ansible.builtin.set_fact:
+   fail_config: "{{ lookup('ansible.builtin.file', 'data/fail.cfg') }}"
+   warn_config: "{{ lookup('ansible.builtin.file', 'data/warn.cfg') }}"
+   pass_config: "{{ lookup('ansible.builtin.file', 'data/pass.cfg') }}"
+   rules: "{{ lookup('ansible.builtin.file', 'criteria/rules.yaml') }}"
+   bad_rules:
+     - name: Invalid action
+       action: flunge
+       rule: Flunge it!
+     - name: No action
+       rule: Rule
+     - name: No rule
+       action: fail
+     - rule: No name
+       action: fail
+
+- name: validate configuration using config (with errors)
+  ansible.builtin.set_fact:
+    data_criteria_checks: "{{ fail_config|ansible.utils.validate(rules, engine='ansible.utils.config') }}"
+
+- assert:
+    that:
+      - "data_criteria_checks[0].found == 'interface Eth1/1'"
+      - "data_criteria_checks[1].found == 'interface Loopback 10'"
+      - "data_criteria_checks[2].found == 'interface po2.1'"
+
+- name: validate configuration using config (with warnings)
+  ansible.builtin.set_fact:
+    data_criteria_checks: "{{ warn_config|ansible.utils.validate(rules, engine='ansible.utils.config') }}"
+
+- assert:
+    that:
+      - "data_criteria_checks == []"
+
+- name: validate configuration using config (all pass)
+  ansible.builtin.set_fact:
+    data_criteria_checks: "{{ pass_config|ansible.utils.validate(rules, engine='ansible.utils.config') }}"
+
+- assert:
+    that:
+      - "data_criteria_checks == []"
+
+- name: invalid rules
+  ansible.builtin.set_fact:
+    data_criteria_checks: "{{ pass_config|ansible.utils.validate(bad_rules, engine='ansible.utils.config') }}"
+  ignore_errors: true
+  register: result
+
+- assert:
+    that:
+      - "result['failed'] == true"
+      - '"is not one of \"warn\" or \"fail\"" in result["msg"]'
+      - '"missing \"action\" key" in result["msg"]'
+      - '"missing \"rule\" key" in result["msg"]'
+      - '"missing \"name\" key" in result["msg"]'

tests/integration/targets/utils_validate/tests/config/lookup.yaml

@@ -0,0 +1,57 @@
+---
+- name: Set up data and criteria
+  ansible.builtin.set_fact:
+   fail_config: "{{ lookup('ansible.builtin.file', 'data/fail.cfg') }}"
+   warn_config: "{{ lookup('ansible.builtin.file', 'data/warn.cfg') }}"
+   pass_config: "{{ lookup('ansible.builtin.file', 'data/pass.cfg') }}"
+   rules: "{{ lookup('ansible.builtin.file', 'criteria/rules.yaml') }}"
+   bad_rules:
+     - name: Invalid action
+       action: flunge
+       rule: Flunge it!
+     - name: No action
+       rule: Rule
+     - name: No rule
+       action: fail
+     - rule: No name
+       action: fail
+
+- name: validate configuration using config (with errors)
+  ansible.builtin.set_fact:
+    data_criteria_checks: "{{ lookup('ansible.utils.validate', fail_config, rules, engine='ansible.utils.config') }}"
+
+- assert:
+    that:
+      - "data_criteria_checks[0].found == 'interface Eth1/1'"
+      - "data_criteria_checks[1].found == 'interface Loopback 10'"
+      - "data_criteria_checks[2].found == 'interface po2.1'"
+
+- name: validate configuration using config (with warnings)
+  ansible.builtin.set_fact:
+    data_criteria_checks: "{{ lookup('ansible.utils.validate', warn_config, rules, engine='ansible.utils.config') }}"
+
+- assert:
+    that:
+      - "data_criteria_checks == []"
+
+- name: validate configuration using config (all pass)
+  ansible.builtin.set_fact:
+    data_criteria_checks: "{{ lookup('ansible.utils.validate', pass_config, rules, engine='ansible.utils.config') }}"
+
+- assert:
+    that:
+      - "data_criteria_checks == []"
+
+- name: invalid rules
+  ansible.builtin.set_fact:
+    data_criteria_checks: "{{ lookup('ansible.utils.validate', pass_config, bad_rules, engine='ansible.utils.config') }}"
+  ignore_errors: true
+  register: result
+
+- assert:
+    that:
+      - "result['failed'] == true"
+      - '"is not one of \"warn\" or \"fail\"" in result["msg"]'
+      - '"missing \"action\" key" in result["msg"]'
+      - '"missing \"rule\" key" in result["msg"]'
+      - '"missing \"name\" key" in result["msg"]'

tests/integration/targets/utils_validate/tests/config/module.yaml

@@ -0,0 +1,84 @@
+---
+- name: Set up data and criteria
+  ansible.builtin.set_fact:
+   fail_config: "{{ lookup('ansible.builtin.file', 'data/fail.cfg') }}"
+   warn_config: "{{ lookup('ansible.builtin.file', 'data/warn.cfg') }}"
+   pass_config: "{{ lookup('ansible.builtin.file', 'data/pass.cfg') }}"
+   rules: "{{ lookup('ansible.builtin.file', 'criteria/rules.yaml') }}"
+   bad_rules:
+     - name: Invalid action
+       action: flunge
+       rule: Flunge it!
+     - name: No action
+       rule: Rule
+     - name: No rule
+       action: fail
+     - rule: No name
+       action: fail
+
+- name: validate configuration using config (with errors)
+  ansible.utils.validate:
+    data: "{{ fail_config }}"
+    criteria: "{{ rules }}"
+    engine: ansible.utils.config
+  ignore_errors: true
+  register: result
+
+- assert:
+    that:
+      - "'errors' in result"
+      - "result['errors'][0].found == 'interface Eth1/1'"
+      - "result['errors'][1].found == 'interface Loopback 10'"
+      - "result['errors'][2].found == 'interface po2.1'"
+      - "result['failed'] == true"
+      - "'Validation errors were found' in result.msg"
+      - "'Ethernet interface names should be in format' in result.msg"
+      - "'Loopback interface names should be in format' in result.msg"
+      - "'Port Channel names should be in format' in result.msg"
+      - "'warnings' in result"
+      - "'At line 2: Interface description should not be more than 8 chars' in result['warnings'][0]"
+
+- name: validate configuration using config (with warnings)
+  ansible.utils.validate:
+    data: "{{ warn_config }}"
+    criteria: "{{ rules }}"
+    engine: ansible.utils.config
+  register: result
+
+- assert:
+    that:
+      - "'errors' not in result"
+      - "'warnings' in result"
+      - "'At line 2: Interface description should not be more than 8 chars' in result['warnings'][0]"
+      - "result['failed'] == false"
+      - "'Non-fatal validation errors were found.' in result.msg"
+
+- name: validate configuration using config (all passed)
+  ansible.utils.validate:
+    data: "{{ pass_config }}"
+    criteria: "{{ rules }}"
+    engine: ansible.utils.config
+  register: result
+
+- assert:
+    that:
+      - "'errors' not in result"
+      - "'warnings' not in result"
+      - "result['failed'] == false"
+      - "'All checks passed' in result.msg"
+
+- name: invalid rules
+  ansible.utils.validate:
+    data: "{{ pass_config }}"
+    criteria: "{{ bad_rules }}"
+    engine: ansible.utils.config
+  ignore_errors: true
+  register: result
+
+- assert:
+    that:
+      - "result['failed'] == true"
+      - '"is not one of \"warn\" or \"fail\"" in result["msg"]'
+      - '"missing \"action\" key" in result["msg"]'
+      - '"missing \"rule\" key" in result["msg"]'
+      - '"missing \"name\" key" in result["msg"]'

tests/integration/targets/utils_validate/tests/config/test.yaml

@@ -0,0 +1,55 @@
+---
+- name: Set up data and criteria
+  ansible.builtin.set_fact:
+   fail_config: "{{ lookup('ansible.builtin.file', 'data/fail.cfg') }}"
+   warn_config: "{{ lookup('ansible.builtin.file', 'data/warn.cfg') }}"
+   pass_config: "{{ lookup('ansible.builtin.file', 'data/pass.cfg') }}"
+   rules: "{{ lookup('ansible.builtin.file', 'criteria/rules.yaml') }}"
+   bad_rules:
+     - name: Invalid action
+       action: flunge
+       rule: Flunge it!
+     - name: No action
+       rule: Rule
+     - name: No rule
+       action: fail
+     - rule: No name
+       action: fail
+
+- name: validate configuration using config (with errors)
+  ansible.builtin.set_fact:
+    is_data_valid: "{{ fail_config is ansible.utils.validate(criteria=rules, engine='ansible.utils.config') }}"
+
+- assert:
+    that:
+      - "is_data_valid == false"
+
+- name: validate configuration using config (with warnings)
+  ansible.builtin.set_fact:
+    is_data_valid: "{{ warn_config is ansible.utils.validate(criteria=rules, engine='ansible.utils.config') }}"
+
+- assert:
+    that:
+      - "is_data_valid == true"
+
+- name: validate configuration using config (all pass)
+  ansible.builtin.set_fact:
+    is_data_valid: "{{ pass_config is ansible.utils.validate(criteria=rules, engine='ansible.utils.config') }}"
+
+- assert:
+    that:
+      - "is_data_valid == true"
+
+- name: invalid rules
+  ansible.builtin.set_fact:
+    is_data_valid: "{{ pass_config is ansible.utils.validate(criteria=bad_rules, engine='ansible.utils.config') }}"
+  ignore_errors: true
+  register: result
+
+- assert:
+    that:
+      - "result['failed'] == true"
+      - '"is not one of \"warn\" or \"fail\"" in result["msg"]'
+      - '"missing \"action\" key" in result["msg"]'
+      - '"missing \"rule\" key" in result["msg"]'
+      - '"missing \"name\" key" in result["msg"]'

tests/integration/targets/utils_validate/tests/jsonschema/module.yaml

@@ -36,7 +36,7 @@
 - assert:
     that:
       - "'errors' not in result"
-      - "'all checks passed' in result.msg"
+      - "'All checks passed' in result.msg"
 
 - name: test invalid plugin configuration option
   ansible.utils.validate:
@@ -131,7 +131,7 @@
 - assert:
     that:
       - "'errors' not in result"
-      - "'all checks passed' in result.msg"
+      - "'All checks passed' in result.msg"
 
 - name: validate list data using jsonschema
   ansible.utils.validate:
@@ -142,4 +142,4 @@
 - assert:
     that:
       - "'errors' not in result"
-      - "'all checks passed' in result.msg"
+      - "'All checks passed' in result.msg"

tests/unit/plugins/action/test_validate.py

@@ -215,7 +215,7 @@ class TestValidate(unittest.TestCase):
         result = self._plugin.run(
             task_vars={"ansible_validate_jsonschema_draft": "draft3"}
         )
-        self.assertIn("all checks passed", result["msg"])
+        self.assertIn("All checks passed", result["msg"])
 
     def test_validate_plugin_config_options_with_draft4(self):
         """Check passing invalid validate plugin options"""
@@ -229,7 +229,7 @@ class TestValidate(unittest.TestCase):
         result = self._plugin.run(
             task_vars={"ansible_validate_jsonschema_draft": "draft4"}
         )
-        self.assertIn("all checks passed", result["msg"])
+        self.assertIn("All checks passed", result["msg"])
 
     def test_validate_plugin_config_options_with_draft6(self):
         """Check passing invalid validate plugin options"""
@@ -243,7 +243,7 @@ class TestValidate(unittest.TestCase):
         result = self._plugin.run(
             task_vars={"ansible_validate_jsonschema_draft": "draft6"}
         )
-        self.assertIn("all checks passed", result["msg"])
+        self.assertIn("All checks passed", result["msg"])
 
     def test_invalid_data(self):
         """Check passing invalid data as per criteria"""
@@ -281,7 +281,7 @@ class TestValidate(unittest.TestCase):
         }
 
         result = self._plugin.run(task_vars=None)
-        self.assertIn("all checks passed", result["msg"])
+        self.assertIn("All checks passed", result["msg"])
 
     def test_support_for_format(self):
         """Check passing valid data as per criteria"""
@@ -293,7 +293,7 @@ class TestValidate(unittest.TestCase):
         }
 
         result = self._plugin.run(task_vars=None)
-        self.assertIn("all checks passed", result["msg"])
+        self.assertIn("All checks passed", result["msg"])
 
     def test_support_for_format_with_invalid_data(self):
         """Check passing valid data as per criteria"""

tests/unit/plugins/sub_plugins/validate/test_config.py

@@ -0,0 +1,111 @@
+# (c) 2022 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+from ansible.module_utils._text import to_text
+from ansible.errors import AnsibleError
+from ansible_collections.ansible.utils.plugins.plugin_utils.base.validate import (
+    _load_validator,
+)
+import pytest
+
+
+@pytest.fixture(name="test_rule")
+def criterion():
+    return {"name": "Rule name", "rule": "Rule regex", "action": "warn"}
+
+
+@pytest.fixture(name="validator")
+def config_validator():
+    engine, result = _load_validator(
+        engine="ansible.utils.config", data="", criteria=[]
+    )
+    return engine
+
+
+@pytest.mark.parametrize("key", ["name", "rule", "action"])
+def test_check_args_missing_key(validator, test_rule, key):
+    del test_rule[key]
+    original = to_text(test_rule)
+    validator._criteria.append(test_rule)
+
+    try:
+        validator._check_args()
+        error = ""
+    except AnsibleError as exc:
+        error = to_text(exc)
+
+    assert error == 'Criteria {rule} missing "{key}" key'.format(
+        rule=original, key=key
+    )
+
+
+def test_invalid_yaml(validator):
+    test_rule = "[This is not valid YAML"
+    validator._criteria = test_rule
+
+    try:
+        validator._check_args()
+        error = ""
+    except AnsibleError as exc:
+        error = to_text(exc)
+
+    expected_error = (
+        "'criteria' option value is invalid, value should be valid YAML."
+    )
+    # Don't test for exact error string, varies with Python version
+    assert error.startswith(expected_error)
+
+
+def test_invalid_action(validator, test_rule):
+    test_rule["action"] = "flunge"
+    original = to_text(test_rule)
+    validator._criteria.append(test_rule)
+
+    try:
+        validator._check_args()
+        error = ""
+    except AnsibleError as exc:
+        error = to_text(exc)
+
+    expected_error = 'Action in criteria {item} is not one of "warn" or "fail"'.format(
+        item=original
+    )
+    assert error == expected_error
+
+
+def test_invalid_regex(validator, test_rule):
+    test_rule["rule"] = "reg(ex"
+    validator._criteria.append(test_rule)
+
+    try:
+        validator._check_args()
+        error = ""
+    except AnsibleError as exc:
+        error = to_text(exc)
+
+    expected_error = 'Failed to compile regex "reg(ex":'
+    # Don't test for exact error string, varies with Python version
+    assert error.startswith(expected_error)
+
+
+def test_valid_warning(validator, test_rule):
+    validator._criteria.append(test_rule)
+    validator._data = "This line matches Rule regex."
+
+    validator.validate()
+    assert "errors" not in validator._result
+    assert len(validator._result["warnings"]) == 1
+
+
+def test_valid_error(validator, test_rule):
+    test_rule["action"] = "fail"
+    validator._criteria.append(test_rule)
+    validator._data = "This line matches Rule regex."
+
+    validator.validate()
+    assert len(validator._result["errors"]) == 1
+    assert "warnings" not in validator._result