Fix docs syntax highlighting errors (#50836)

* Add support for [WARNING]: ...

* Fix unreachable/failed output lexing.

* Detecting retry/--limit lines.

* Removing strange (invisible) characters which cause lexing problems.

* Using better-fitting lexers.

* Improve YAML lexing: don't accept quotes in keys.

* Add Django lexer (unchanged) from Pygments.

* Add support for != and % operators.

docs/docsite/_extensions/pygments_lexer.py

@@ -37,9 +37,11 @@
 from __future__ import absolute_import, print_function
 
 from pygments.lexer import LexerContext, ExtendedRegexLexer, DelegatingLexer, RegexLexer, bygroups, include
-from pygments.lexers import DjangoLexer, DiffLexer
+from pygments.lexers import DiffLexer
 from pygments import token
 
+import re
+
 
 class AnsibleYamlLexerContext(LexerContext):
     """Indentation context for the YAML lexer."""
@@ -255,7 +257,7 @@ class AnsibleYamlLexer(ExtendedRegexLexer):
             # whitespaces separating tokens
             (r'[ ]+', token.Text),
             # key with colon
-            (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)',
+            (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
              bygroups(token.Name.Tag, set_indent(token.Punctuation, implicit=True))),
             # tags, anchors and aliases,
             include('descriptors'),
@@ -334,7 +336,7 @@ class AnsibleYamlLexer(ExtendedRegexLexer):
         # a flow mapping indicated by '{' and '}'
         'flow-mapping': [
             # key with colon
-            (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)',
+            (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
              bygroups(token.Name.Tag, token.Punctuation)),
             # include flow collection rules
             include('flow-collection'),
@@ -458,6 +460,89 @@ class AnsibleYamlLexer(ExtendedRegexLexer):
         return super(AnsibleYamlLexer, self).get_tokens_unprocessed(text, context)
 
 
+class AnsibleDjangoLexer(RegexLexer):
+    """
+    Generic `django <http://www.djangoproject.com/documentation/templates/>`_
+    and `jinja <http://wsgiarea.pocoo.org/jinja/>`_ template lexer.
+
+    It just highlights django/jinja code between the preprocessor directives,
+    other data is left untouched by the lexer.
+    """
+
+    name = 'Django/Jinja'
+    aliases = ['django', 'jinja']
+    mimetypes = ['application/x-django-templating', 'application/x-jinja']
+
+    flags = re.M | re.S
+
+    tokens = {
+        'root': [
+            (r'[^{]+', token.Other),
+            (r'\{\{', token.Comment.Preproc, 'var'),
+            # jinja/django comments
+            (r'\{[*#].*?[*#]\}', token.Comment),
+            # django comments
+            (r'(\{%)(-?\s*)(comment)(\s*-?)(%\})(.*?)'
+             r'(\{%)(-?\s*)(endcomment)(\s*-?)(%\})',
+             bygroups(token.Comment.Preproc, token.Text, token.Keyword, token.Text, token.Comment.Preproc,
+                      token.Comment, token.Comment.Preproc, token.Text, token.Keyword, token.Text,
+                      token.Comment.Preproc)),
+            # raw jinja blocks
+            (r'(\{%)(-?\s*)(raw)(\s*-?)(%\})(.*?)'
+             r'(\{%)(-?\s*)(endraw)(\s*-?)(%\})',
+             bygroups(token.Comment.Preproc, token.Text, token.Keyword, token.Text, token.Comment.Preproc,
+                      token.Text, token.Comment.Preproc, token.Text, token.Keyword, token.Text,
+                      token.Comment.Preproc)),
+            # filter blocks
+            (r'(\{%)(-?\s*)(filter)(\s+)([a-zA-Z_]\w*)',
+             bygroups(token.Comment.Preproc, token.Text, token.Keyword, token.Text, token.Name.Function),
+             'block'),
+            (r'(\{%)(-?\s*)([a-zA-Z_]\w*)',
+             bygroups(token.Comment.Preproc, token.Text, token.Keyword), 'block'),
+            (r'\{', token.Other)
+        ],
+        'varnames': [
+            (r'(\|)(\s*)([a-zA-Z_]\w*)',
+             bygroups(token.Operator, token.Text, token.Name.Function)),
+            (r'(is)(\s+)(not)?(\s+)?([a-zA-Z_]\w*)',
+             bygroups(token.Keyword, token.Text, token.Keyword, token.Text, token.Name.Function)),
+            (r'(_|true|false|none|True|False|None)\b', token.Keyword.Pseudo),
+            (r'(in|as|reversed|recursive|not|and|or|is|if|else|import|'
+             r'with(?:(?:out)?\s*context)?|scoped|ignore\s+missing)\b',
+             token.Keyword),
+            (r'(loop|block|super|forloop)\b', token.Name.Builtin),
+            (r'[a-zA-Z_][\w-]*', token.Name.Variable),
+            (r'\.\w+', token.Name.Variable),
+            (r':?"(\\\\|\\"|[^"])*"', token.String.Double),
+            (r":?'(\\\\|\\'|[^'])*'", token.String.Single),
+            (r'([{}()\[\]+\-*/%,:~]|[><=]=?|!=)', token.Operator),
+            (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|"
+             r"0[xX][0-9a-fA-F]+[Ll]?", token.Number),
+        ],
+        'var': [
+            (r'\s+', token.Text),
+            (r'(-?)(\}\})', bygroups(token.Text, token.Comment.Preproc), '#pop'),
+            include('varnames')
+        ],
+        'block': [
+            (r'\s+', token.Text),
+            (r'(-?)(%\})', bygroups(token.Text, token.Comment.Preproc), '#pop'),
+            include('varnames'),
+            (r'.', token.Punctuation)
+        ]
+    }
+
+    def analyse_text(text):
+        rv = 0.0
+        if re.search(r'\{%\s*(block|extends)', text) is not None:
+            rv += 0.4
+        if re.search(r'\{%\s*if\s*.*?%\}', text) is not None:
+            rv += 0.1
+        if re.search(r'\{\{.*?\}\}', text) is not None:
+            rv += 0.1
+        return rv
+
+
 class AnsibleYamlJinjaLexer(DelegatingLexer):
     """
     Subclass of the `DjangoLexer` that highlights unlexed data with the
@@ -474,7 +559,7 @@ class AnsibleYamlJinjaLexer(DelegatingLexer):
     mimetypes = ['text/x-yaml+jinja']
 
     def __init__(self, **options):
-        super(AnsibleYamlJinjaLexer, self).__init__(AnsibleYamlLexer, DjangoLexer, **options)
+        super(AnsibleYamlJinjaLexer, self).__init__(AnsibleYamlLexer, AnsibleDjangoLexer, **options)
 
 
 class AnsibleOutputPrimaryLexer(RegexLexer):
@@ -555,8 +640,8 @@ class AnsibleOutputPrimaryLexer(RegexLexer):
         ],
 
         'host-error': [
-            (r'(?:( )(UNREACHABLE|FAILED)(!))?',
+            (r'(?:(:)( )(UNREACHABLE|FAILED)(!))?',
-                bygroups(token.Text, token.Keyword, token.Punctuation),
+                bygroups(token.Punctuation, token.Text, token.Keyword, token.Punctuation),
                 'host-postfix'),
             (r'', token.Text, 'host-postfix'),
         ],
@@ -579,9 +664,12 @@ class AnsibleOutputPrimaryLexer(RegexLexer):
             (r'(fatal|ok|changed|skipping)(:)( )',
                 bygroups(token.Keyword, token.Punctuation, token.Text),
                 'host-name'),
+            (r'(\[)(WARNING)(\]:)([^\n]+)',
+                bygroups(token.Punctuation, token.Keyword, token.Punctuation, token.Text)),
             (r'([^ ]+)( +)(:)',
                 bygroups(token.Name, token.Text, token.Punctuation),
                 'host-result'),
+            (r'(\tto retry, use: )(.*)(\n)', bygroups(token.Text, token.Literal.String, token.Text)),
             (r'.*\n', token.Other),
         ],
     }
@@ -609,7 +697,12 @@ def setup(app):
     """ Initializer for Sphinx extension API.
         See http://www.sphinx-doc.org/en/stable/extdev/index.html#dev-extensions.
     """
-    for lexer in [AnsibleYamlLexer(startinline=True), AnsibleYamlJinjaLexer(startinline=True), AnsibleOutputLexer(startinline=True)]:
+    for lexer in [
+        AnsibleDjangoLexer(startinline=True),
+        AnsibleYamlLexer(startinline=True),
+        AnsibleYamlJinjaLexer(startinline=True),
+        AnsibleOutputLexer(startinline=True)
+    ]:
         app.add_lexer(lexer.name, lexer)
         for alias in lexer.aliases:
             app.add_lexer(alias, lexer)

docs/docsite/rst/dev_guide/developing_modules_general.rst

@@ -56,7 +56,7 @@ To create a new module:
 3. Paste the content below into your new module file. It includes the :ref:`required Ansible format and documentation <developing_modules_documenting>` and some example code.
 4. Modify and extend the code to do what you want your new module to do. See the :ref:`programming tips <developing_modules_best_practices>` and :ref:`Python 3 compatibility <developing_python_3>` pages for pointers on writing clean, concise module code.
 
-.. code:: python
+.. code-block:: python
 
     #!/usr/bin/python
 

docs/docsite/rst/dev_guide/style_guide/index.rst

@@ -63,34 +63,46 @@ Sphinx will 'learn on the fly' when creating a hierarchy of headers.
 To make our documents easy to read and to edit, we follow a standard set of header notations.
 We use:
 
-* ``###`` with overline, for parts::
+* ``###`` with overline, for parts:
+
+.. code-block:: rst
 
       ###############
       Developer guide
       ###############
 
-* ``***`` with overline, for chapters::
+* ``***`` with overline, for chapters:
+
+.. code-block:: rst
 
       *******************
       Ansible style guide
       *******************
 
-* ``===`` for sections::
+* ``===`` for sections:
+
+.. code-block:: rst
 
       Mechanical guidelines
       =====================
 
-* ``---`` for subsections::
+* ``---`` for subsections:
+
+.. code-block:: rst
 
       Internal navigation
       -------------------
 
-* ``^^^`` for sub-subsections::
+* ``^^^`` for sub-subsections:
+
+.. code-block:: rst
 
       Adding anchors
       ^^^^^^^^^^^^^^
 
-* ``"""`` for paragraphs::
+* ``"""`` for paragraphs:
+
+.. code-block:: rst
 
       Paragraph that needs a title
       """"""""""""""""""""""""""""
@@ -120,7 +132,9 @@ Adding anchors
 Adding internal links
 ^^^^^^^^^^^^^^^^^^^^^
 
-* All internal links must use ``:ref:`` syntax. These links both point to the anchor defined above::
+* All internal links must use ``:ref:`` syntax. These links both point to the anchor defined above:
+
+.. code-block:: rst
 
    :ref:`unique_page`
    :ref:`this page <unique_page>`
@@ -137,7 +151,9 @@ If you include a local TOC:
 * use the ``:local:`` directive so the page's main header is not included
 * do not include a title
 
-The syntax is::
+The syntax is:
+
+.. code-block:: rst
 
    .. contents::
       :local:

docs/docsite/rst/plugins/inventory.rst

@@ -55,7 +55,9 @@ Or for the openstack plugin:
     # clouds.yml
     plugin: openstack
 
-The ``auto`` inventory plugin is enabled by default and works by using the ``plugin`` field to indicate the plugin that should attempt to parse it. You can configure the whitelist/precedence of inventory plugins used to parse source using the `ansible.cfg` ['inventory'] ``enable_plugins`` list. After enabling the plugin and providing any required options you can view the populated inventory with ``ansible-inventory -i demo.aws_ec2.yml --graph``::
+The ``auto`` inventory plugin is enabled by default and works by using the ``plugin`` field to indicate the plugin that should attempt to parse it. You can configure the whitelist/precedence of inventory plugins used to parse source using the `ansible.cfg` ['inventory'] ``enable_plugins`` list. After enabling the plugin and providing any required options you can view the populated inventory with ``ansible-inventory -i demo.aws_ec2.yml --graph``:
+
+.. code-block:: text
 
     @all:
       |--@aws_ec2:
@@ -88,7 +90,9 @@ You can create dynamic groups using host variables with the constructed ``keyed_
       # set the ansible_host variable to connect with the private IP address without changing the hostname
       ansible_host: private_ip_address
 
-Now the output of ``ansible-inventory -i demo.aws_ec2.yml --graph``::
+Now the output of ``ansible-inventory -i demo.aws_ec2.yml --graph``:
+
+.. code-block:: text
 
     @all:
       |--@aws_ec2:

docs/docsite/rst/plugins/lookup.rst

@@ -41,24 +41,20 @@ Using lookup plugins
 
 Lookup plugins can be used anywhere you can use templating in Ansible: in a play, in variables file, or in a Jinja2 template for the :ref:`template <template_module>` module.
 
-.. code-block:: yaml
+.. code-block:: YAML+Jinja
 
   vars:
     file_contents: "{{lookup('file', 'path/to/file.txt')}}"
 
 Lookups are an integral part of loops. Wherever you see ``with_``, the part after the underscore is the name of a lookup.
-This is also the reason most lookups output lists and take lists as input; for example, ``with_items`` uses the :ref:`items <items_lookup>` lookup:
+This is also the reason most lookups output lists and take lists as input; for example, ``with_items`` uses the :ref:`items <items_lookup>` lookup::
-
-.. code-block:: yaml
 
   tasks:
     - name: count to 3
       debug: msg={{item}}
       with_items: [1, 2, 3]
 
-You can combine lookups with :ref:`playbooks_filters`, :ref:`playbooks_tests` and even each other to do some complex data generation and manipulation. For example:
+You can combine lookups with :ref:`playbooks_filters`, :ref:`playbooks_tests` and even each other to do some complex data generation and manipulation. For example::
-
-.. code-block:: yaml
 
   tasks:
     - name: valid but useless and over complicated chained lookups and filters
@@ -77,6 +73,8 @@ To ignore errors::
     - name: file doesnt exist, but i dont care .. file plugin itself warns anyways ...
       debug: msg="{{ lookup('file', '/idontexist', errors='ignore') }}"
 
+.. code-block:: ansible-output
+
     [WARNING]: Unable to find '/idontexist' in expected paths (use -vvvvv to see paths)
 
     ok: [localhost] => {
@@ -89,6 +87,8 @@ To get a warning instead of a failure::
     - name: file doesnt exist, let me know, but continue
       debug: msg="{{ lookup('file', '/idontexist', errors='warn') }}"
 
+.. code-block:: ansible-output
+
     [WARNING]: Unable to find '/idontexist' in expected paths (use -vvvvv to see paths)
 
     [WARNING]: An unhandled exception occurred while running the lookup plugin 'file'. Error was a <class 'ansible.errors.AnsibleError'>, original message: could not locate file in lookup: /idontexist
@@ -103,6 +103,8 @@ Fatal error (the default)::
     - name: file doesnt exist, FAIL (this is the default)
       debug: msg="{{ lookup('file', '/idontexist', errors='strict') }}"
 
+.. code-block:: ansible-output
+
     [WARNING]: Unable to find '/idontexist' in expected paths (use -vvvvv to see paths)
 
     fatal: [localhost]: FAILED! => {"msg": "An unhandled exception occurred while running the lookup plugin 'file'. Error was a <class 'ansible.errors.AnsibleError'>, original message: could not locate file in lookup: /idontexist"}
@@ -120,7 +122,9 @@ The default behavior of ``lookup`` is to return a string of comma separated valu
 
 This was done primarily to provide an easier and more consistent interface for interacting with the new ``loop`` keyword, while maintaining backwards compatibility with other uses of ``lookup``.
 
-The following examples are equivalent::
+The following examples are equivalent:
+
+.. code-block:: jinja
 
     lookup('dict', dict_variable, wantlist=True)
 
@@ -128,7 +132,9 @@ The following examples are equivalent::
 
 As demonstrated above the behavior of ``wantlist=True`` is implicit when using ``query``.
 
-Additionally, ``q`` was introduced as a shortform of ``query``::
+Additionally, ``q`` was introduced as a shortform of ``query``:
+
+.. code-block:: jinja
 
     q('dict', dict_variable)
 

docs/docsite/rst/reference_appendices/YAMLSyntax.rst

@@ -170,7 +170,9 @@ you can use escapes::
 
 The list of allowed escapes can be found in the YAML Specification under "Escape Sequences" (YAML 1.1) or "Escape Characters" (YAML 1.2).
 
-The following is invalid YAML::
+The following is invalid YAML:
+
+.. code-block:: text
 
     foo: "an escaped \' single quote"
 

docs/docsite/rst/scenario_guides/guide_scaleway.rst

@@ -193,10 +193,14 @@ Create a file named ``scaleway_inventory.yml`` with the following content:
 This inventory means that we want all hosts that got the tag ``web_server`` on the zones ``ams1`` and ``par1``.
 Once you have configured this file, you can get the information using the following command:
 
-::
+.. code-block:: bash
 
     $ ansible-inventory --list -i scaleway_inventory.yml
 
+The output will be:
+
+.. code-block:: yaml
+
     {
         "_meta": {
             "hostvars": {
@@ -251,7 +255,7 @@ As the Scaleway API is S3 compatible, Ansible supports it natively through the m
 
 You can find many examples in ``./test/legacy/roles/scaleway_s3``
 
-.. code-block:: yaml
+.. code-block:: yaml+jinja
 
     - hosts: myserver
       vars:

docs/docsite/rst/user_guide/playbooks_async.rst

@@ -100,7 +100,7 @@ of tasks running concurrently, you can do it this way::
           - 5
         durations: "{{ item }}"
       include_tasks: execute_batch.yml
-      loop: "{{ sleep_durations | batch(2) | list }}"
+      loop: "{{ sleep_durations | batch(2) | list }}"
 
     #####################
     # execute_batch.yml

docs/docsite/rst/user_guide/playbooks_filters.rst

@@ -904,7 +904,9 @@ style. For example the following::
 
     {{ "Plain style (default)" | comment }}
 
-will produce this output::
+will produce this output:
+
+.. code-block:: text
 
     #
     # Plain style (default)
@@ -923,7 +925,9 @@ above, you can customize it with::
 
   {{ "My Special Case" | comment(decoration="! ") }}
 
-producing::
+producing:
+
+.. code-block:: text
 
   !
   ! My Special Case
@@ -935,7 +939,7 @@ It is also possible to fully customize the comment style::
 
 That will create the following output:
 
-.. code-block:: sh
+.. code-block:: text
 
     #######
     #
@@ -1040,7 +1044,7 @@ To search a string with a regex, use the "regex_search" filter::
     {{ 'ansible' | regex_search('(foobar)') }}
 
     # case insensitive search in multiline mode
-    {{ 'foo\nBAR' | regex_search("^bar", multiline=True, ignorecase=True) }}
+    {{ 'foo\nBAR' | regex_search("^bar", multiline=True, ignorecase=True) }}
 
 
 To search for all occurrences of regex matches, use the "regex_findall" filter::