Improve handling of IDNA/Unicode domains (#436)

* Prepare IDNA/Unicode conversion code. Use to normalize input.

* Use IDNA library first (IDNA2008) and Python's IDNA2003 implementation as a fallback.

* Make sure idna is installed.

* Add changelog fragment.

* 'punycode' → 'idna'.

* Add name_encoding options and tests.

* Avoid invalid character for IDNA2008.

* Linting.

* Forgot to upate value.

* Work around cryptography bug. Fix port handling for URIs.

* Forgot other place sensitive to cryptography bug.

* Forgot one. (Will likely still fail.)

* Decode IDNA in _compress_entry() to avoid comparison screw-ups.

* Work around Python 3.5 problem in Ansible 2.9's default test container.

* Update changelog fragment.

* Fix error, add tests.

* Python 2 compatibility.

* Update requirements.

changelogs/fragments/436-idns.yml

@@ -0,0 +1,12 @@
+minor_changes:
+  - "Support automatic conversion for Internalionalized Domain Names (IDNs).
+     When passing general names, for example Subject Altenative Names to ``community.crypto.openssl_csr``, these will automatically be converted to IDNA.
+     Conversion will be done per label to IDNA2008 if possible, and IDNA2003 if IDNA2008 conversion fails for that label.
+     Note that IDNA conversion requires `the Python idna library <https://pypi.org/project/idna/>`_ to be installed.
+     Please note that depending on which versions of the cryptography library are used, it could try to process the converted IDNA
+     another time with the Python ``idna`` library and reject IDNA2003 encoded values. Using a new enough ``cryptography`` version avoids this
+     (https://github.com/ansible-collections/community.crypto/issues/426, https://github.com/ansible-collections/community.crypto/pull/436)."
+  - "openssl_csr_info - add ``name_encoding`` option to control the encoding (IDNA, Unicode) used to return domain names in general names (https://github.com/ansible-collections/community.crypto/pull/436)."
+  - "x509_certificate_info - add ``name_encoding`` option to control the encoding (IDNA, Unicode) used to return domain names in general names (https://github.com/ansible-collections/community.crypto/pull/436)."
+  - "x509_crl - add ``name_encoding`` option to control the encoding (IDNA, Unicode) used to return domain names in general names (https://github.com/ansible-collections/community.crypto/pull/436)."
+  - "x509_crl_info - add ``name_encoding`` option to control the encoding (IDNA, Unicode) used to return domain names in general names (https://github.com/ansible-collections/community.crypto/pull/436)."

plugins/doc_fragments/name_encoding.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+# Copyright: (c) 2022, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+class ModuleDocFragment(object):
+    DOCUMENTATION = r'''
+options:
+    name_encoding:
+        description:
+            - How to encode names (DNS names, URIs, email addresses) in return values.
+            - C(ignore) will use the encoding returned by the backend.
+            - C(idna) will convert all labels of domain names to IDNA encoding.
+              IDNA2008 will be preferred, and IDNA2003 will be used if IDNA2008 encoding fails.
+            - C(unicode) will convert all labels of domain names to Unicode.
+              IDNA2008 will be preferred, and IDNA2003 will be used if IDNA2008 decoding fails.
+            - B(Note) that C(idna) and C(unicode) require the L(idna Python library,https://pypi.org/project/idna/) to be installed.
+        type: str
+        default: ignore
+        choices:
+            - ignore
+            - idna
+            - unicode
+requirements:
+    - If I(name_encoding) is set to another value than C(ignore), the L(idna Python library,https://pypi.org/project/idna/) needs to be installed.
+'''

plugins/module_utils/crypto/cryptography_crl.py

@@ -95,12 +95,12 @@ def cryptography_decode_revoked_certificate(cert):
     return result
 
 
-def cryptography_dump_revoked(entry):
+def cryptography_dump_revoked(entry, idn_rewrite='ignore'):
     return {
         'serial_number': entry['serial_number'],
         'revocation_date': entry['revocation_date'].strftime(TIMESTAMP_FORMAT),
         'issuer':
-            [cryptography_decode_name(issuer) for issuer in entry['issuer']]
+            [cryptography_decode_name(issuer, idn_rewrite=idn_rewrite) for issuer in entry['issuer']]
             if entry['issuer'] is not None else None,
         'issuer_critical': entry['issuer_critical'],
         'reason': REVOCATION_REASON_MAP_INVERSE.get(entry['reason']) if entry['reason'] is not None else None,

plugins/module_utils/crypto/cryptography_support.py

@@ -23,8 +23,11 @@ import base64
 import binascii
 import re
 import sys
+import traceback
 
 from ansible.module_utils.common.text.converters import to_text, to_bytes
+from ansible.module_utils.six.moves.urllib.parse import urlparse, urlunparse, ParseResult
+
 from ._asn1 import serialize_asn1_string_as_der
 
 from ansible_collections.community.crypto.plugins.module_utils.version import LooseVersion
@@ -80,6 +83,16 @@ except ImportError:
     # Error handled in the calling module.
     _load_pkcs12 = None
 
+try:
+    import idna
+
+    HAS_IDNA = True
+except ImportError:
+    HAS_IDNA = False
+    IDNA_IMP_ERROR = traceback.format_exc()
+
+from ansible.module_utils.basic import missing_required_lib
+
 from .basic import (
     CRYPTOGRAPHY_HAS_DSA_SIGN,
     CRYPTOGRAPHY_HAS_EC_SIGN,
@@ -359,6 +372,80 @@ def cryptography_parse_relative_distinguished_name(rdn):
     return cryptography.x509.RelativeDistinguishedName(names)
 
 
+def _is_ascii(value):
+    '''Check whether the Unicode string `value` contains only ASCII characters.'''
+    try:
+        value.encode("ascii")
+        return True
+    except UnicodeEncodeError:
+        return False
+
+
+def _adjust_idn(value, idn_rewrite):
+    if idn_rewrite == 'ignore' or not value:
+        return value
+    if idn_rewrite == 'idna' and _is_ascii(value):
+        return value
+    if idn_rewrite not in ('idna', 'unicode'):
+        raise ValueError('Invalid value for idn_rewrite: "{0}"'.format(idn_rewrite))
+    if not HAS_IDNA:
+        raise OpenSSLObjectError(
+            missing_required_lib('idna', reason='to transform {what} DNS name "{name}" to {dest}'.format(
+                name=value,
+                what='IDNA' if idn_rewrite == 'unicode' else 'Unicode',
+                dest='Unicode' if idn_rewrite == 'unicode' else 'IDNA',
+            )))
+    # Since IDNA does not like '*' or empty labels (except one empty label at the end),
+    # we split and let IDNA only handle labels that are neither empty or '*'.
+    parts = value.split(u'.')
+    for index, part in enumerate(parts):
+        if part in (u'', u'*'):
+            continue
+        try:
+            if idn_rewrite == 'idna':
+                parts[index] = idna.encode(part).decode('ascii')
+            elif idn_rewrite == 'unicode' and part.startswith(u'xn--'):
+                parts[index] = idna.decode(part)
+        except idna.IDNAError as exc2008:
+            try:
+                if idn_rewrite == 'idna':
+                    parts[index] = part.encode('idna').decode('ascii')
+                elif idn_rewrite == 'unicode' and part.startswith(u'xn--'):
+                    parts[index] = part.encode('ascii').decode('idna')
+            except Exception as exc2003:
+                raise OpenSSLObjectError(
+                    u'Error while transforming part "{part}" of {what} DNS name "{name}" to {dest}.'
+                    u' IDNA2008 transformation resulted in "{exc2008}", IDNA2003 transformation resulted in "{exc2003}".'.format(
+                        part=part,
+                        name=value,
+                        what='IDNA' if idn_rewrite == 'unicode' else 'Unicode',
+                        dest='Unicode' if idn_rewrite == 'unicode' else 'IDNA',
+                        exc2003=exc2003,
+                        exc2008=exc2008,
+                    ))
+    return u'.'.join(parts)
+
+
+def _adjust_idn_email(value, idn_rewrite):
+    idx = value.find(u'@')
+    if idx < 0:
+        return value
+    return u'{0}@{1}'.format(value[:idx], _adjust_idn(value[idx + 1:], idn_rewrite))
+
+
+def _adjust_idn_url(value, idn_rewrite):
+    url = urlparse(value)
+    host = _adjust_idn(url.hostname, idn_rewrite)
+    if url.username is not None and url.password is not None:
+        host = u'{0}:{1}@{2}'.format(url.username, url.password, host)
+    elif url.username is not None:
+        host = u'{0}@{1}'.format(url.username, host)
+    if url.port is not None:
+        host = u'{0}:{1}'.format(host, url.port)
+    return urlunparse(
+        ParseResult(scheme=url.scheme, netloc=host, path=url.path, params=url.params, query=url.query, fragment=url.fragment))
+
+
 def cryptography_get_name(name, what='Subject Alternative Name'):
     '''
     Given a name string, returns a cryptography x509.GeneralName object.
@@ -366,16 +453,16 @@ def cryptography_get_name(name, what='Subject Alternative Name'):
     '''
     try:
         if name.startswith('DNS:'):
-            return x509.DNSName(to_text(name[4:]))
+            return x509.DNSName(_adjust_idn(to_text(name[4:]), 'idna'))
         if name.startswith('IP:'):
             address = to_text(name[3:])
             if '/' in address:
                 return x509.IPAddress(ipaddress.ip_network(address))
             return x509.IPAddress(ipaddress.ip_address(address))
         if name.startswith('email:'):
-            return x509.RFC822Name(to_text(name[6:]))
+            return x509.RFC822Name(_adjust_idn_email(to_text(name[6:]), 'idna'))
         if name.startswith('URI:'):
-            return x509.UniformResourceIdentifier(to_text(name[4:]))
+            return x509.UniformResourceIdentifier(_adjust_idn_url(to_text(name[4:]), 'idna'))
         if name.startswith('RID:'):
             m = re.match(r'^([0-9]+(?:\.[0-9]+)*)$', to_text(name[4:]))
             if not m:
@@ -422,21 +509,23 @@ def _dn_escape_value(value):
     return value
 
 
-def cryptography_decode_name(name):
+def cryptography_decode_name(name, idn_rewrite='ignore'):
     '''
     Given a cryptography x509.GeneralName object, returns a string.
     Raises an OpenSSLObjectError if the name is not supported.
     '''
+    if idn_rewrite not in ('ignore', 'idna', 'unicode'):
+        raise AssertionError('idn_rewrite must be one of "ignore", "idna", or "unicode"')
     if isinstance(name, x509.DNSName):
-        return u'DNS:{0}'.format(name.value)
+        return u'DNS:{0}'.format(_adjust_idn(name.value, idn_rewrite))
     if isinstance(name, x509.IPAddress):
         if isinstance(name.value, (ipaddress.IPv4Network, ipaddress.IPv6Network)):
             return u'IP:{0}/{1}'.format(name.value.network_address.compressed, name.value.prefixlen)
         return u'IP:{0}'.format(name.value.compressed)
     if isinstance(name, x509.RFC822Name):
-        return u'email:{0}'.format(name.value)
+        return u'email:{0}'.format(_adjust_idn_email(name.value, idn_rewrite))
     if isinstance(name, x509.UniformResourceIdentifier):
-        return u'URI:{0}'.format(name.value)
+        return u'URI:{0}'.format(_adjust_idn_url(name.value, idn_rewrite))
     if isinstance(name, x509.DirectoryName):
         # According to https://datatracker.ietf.org/doc/html/rfc4514.html#section-2.1 the
         # list needs to be reversed, and joined by commas

plugins/module_utils/crypto/module_backends/certificate_info.py

@@ -207,6 +207,7 @@ class CertificateInfoRetrievalCryptography(CertificateInfoRetrieval):
     """Validate the supplied cert, using the cryptography backend"""
     def __init__(self, module, content):
         super(CertificateInfoRetrievalCryptography, self).__init__(module, 'cryptography', content)
+        self.name_encoding = module.params.get('name_encoding', 'ignore')
 
     def _get_der_bytes(self):
         return self.cert.public_bytes(serialization.Encoding.DER)
@@ -309,7 +310,7 @@ class CertificateInfoRetrievalCryptography(CertificateInfoRetrieval):
     def _get_subject_alt_name(self):
         try:
             san_ext = self.cert.extensions.get_extension_for_class(x509.SubjectAlternativeName)
-            result = [cryptography_decode_name(san) for san in san_ext.value]
+            result = [cryptography_decode_name(san, idn_rewrite=self.name_encoding) for san in san_ext.value]
             return result, san_ext.critical
         except cryptography.x509.ExtensionNotFound:
             return None, False
@@ -341,7 +342,7 @@ class CertificateInfoRetrievalCryptography(CertificateInfoRetrieval):
             ext = self.cert.extensions.get_extension_for_class(x509.AuthorityKeyIdentifier)
             issuer = None
             if ext.value.authority_cert_issuer is not None:
-                issuer = [cryptography_decode_name(san) for san in ext.value.authority_cert_issuer]
+                issuer = [cryptography_decode_name(san, idn_rewrite=self.name_encoding) for san in ext.value.authority_cert_issuer]
             return ext.value.key_identifier, issuer, ext.value.authority_cert_serial_number
         except cryptography.x509.ExtensionNotFound:
             return None, None, None

plugins/module_utils/crypto/module_backends/crl_info.py

@@ -51,6 +51,7 @@ class CRLInfoRetrieval(object):
         self.module = module
         self.content = content
         self.list_revoked_certificates = list_revoked_certificates
+        self.name_encoding = module.params.get('name_encoding', 'ignore')
 
     def get_info(self):
         self.crl_pem = identify_pem_format(self.content)
@@ -86,7 +87,7 @@ class CRLInfoRetrieval(object):
             result['revoked_certificates'] = []
             for cert in self.crl:
                 entry = cryptography_decode_revoked_certificate(cert)
-                result['revoked_certificates'].append(cryptography_dump_revoked(entry))
+                result['revoked_certificates'].append(cryptography_dump_revoked(entry, idn_rewrite=self.name_encoding))
 
         return result
 

plugins/module_utils/crypto/module_backends/csr_info.py

@@ -174,6 +174,7 @@ class CSRInfoRetrievalCryptography(CSRInfoRetrieval):
     """Validate the supplied CSR, using the cryptography backend"""
     def __init__(self, module, content, validate_signature):
         super(CSRInfoRetrievalCryptography, self).__init__(module, 'cryptography', content, validate_signature)
+        self.name_encoding = module.params.get('name_encoding', 'ignore')
 
     def _get_subject_ordered(self):
         result = []
@@ -256,7 +257,7 @@ class CSRInfoRetrievalCryptography(CSRInfoRetrieval):
     def _get_subject_alt_name(self):
         try:
             san_ext = self.csr.extensions.get_extension_for_class(x509.SubjectAlternativeName)
-            result = [cryptography_decode_name(san) for san in san_ext.value]
+            result = [cryptography_decode_name(san, idn_rewrite=self.name_encoding) for san in san_ext.value]
             return result, san_ext.critical
         except cryptography.x509.ExtensionNotFound:
             return None, False
@@ -264,8 +265,8 @@ class CSRInfoRetrievalCryptography(CSRInfoRetrieval):
     def _get_name_constraints(self):
         try:
             nc_ext = self.csr.extensions.get_extension_for_class(x509.NameConstraints)
-            permitted = [cryptography_decode_name(san) for san in nc_ext.value.permitted_subtrees or []]
-            excluded = [cryptography_decode_name(san) for san in nc_ext.value.excluded_subtrees or []]
+            permitted = [cryptography_decode_name(san, idn_rewrite=self.name_encoding) for san in nc_ext.value.permitted_subtrees or []]
+            excluded = [cryptography_decode_name(san, idn_rewrite=self.name_encoding) for san in nc_ext.value.excluded_subtrees or []]
             return permitted, excluded, nc_ext.critical
         except cryptography.x509.ExtensionNotFound:
             return None, None, False
@@ -291,7 +292,7 @@ class CSRInfoRetrievalCryptography(CSRInfoRetrieval):
             ext = self.csr.extensions.get_extension_for_class(x509.AuthorityKeyIdentifier)
             issuer = None
             if ext.value.authority_cert_issuer is not None:
-                issuer = [cryptography_decode_name(san) for san in ext.value.authority_cert_issuer]
+                issuer = [cryptography_decode_name(san, idn_rewrite=self.name_encoding) for san in ext.value.authority_cert_issuer]
             return ext.value.key_identifier, issuer, ext.value.authority_cert_serial_number
         except cryptography.x509.ExtensionNotFound:
             return None, None, None

plugins/modules/openssl_csr_info.py

@@ -44,6 +44,9 @@ options:
         default: auto
         choices: [ auto, cryptography ]
 
+extends_documentation_fragment:
+    - community.crypto.name_encoding
+
 seealso:
 - module: community.crypto.openssl_csr
 - module: community.crypto.openssl_csr_pipe
@@ -124,7 +127,9 @@ key_usage_critical:
     returned: success
     type: bool
 subject_alt_name:
-    description: Entries in the C(subject_alt_name) extension, or C(none) if extension is not present.
+    description:
+        - Entries in the C(subject_alt_name) extension, or C(none) if extension is not present.
+        - See I(name_encoding) for how IDNs are handled.
     returned: success
     type: list
     elements: str
@@ -152,6 +157,7 @@ name_constraints_excluded:
     description:
         - List of excluded subtrees the CA cannot sign certificates for.
         - Is C(none) if extension is not present.
+        - See I(name_encoding) for how IDNs are handled.
     returned: success
     type: list
     elements: str
@@ -281,6 +287,7 @@ authority_cert_issuer:
     description:
         - The CSR's authority cert issuer as a list of general names.
         - Is C(none) if the C(AuthorityKeyIdentifier) extension is not present.
+        - See I(name_encoding) for how IDNs are handled.
     returned: success
     type: list
     elements: str
@@ -312,6 +319,7 @@ def main():
         argument_spec=dict(
             path=dict(type='path'),
             content=dict(type='str'),
+            name_encoding=dict(type='str', default='ignore', choices=['ignore', 'idna', 'unicode']),
             select_crypto_backend=dict(type='str', default='auto', choices=['auto', 'cryptography']),
         ),
         required_one_of=(

plugins/modules/x509_certificate_info.py

@@ -62,6 +62,9 @@ options:
         default: auto
         choices: [ auto, cryptography ]
 
+extends_documentation_fragment:
+    - community.crypto.name_encoding
+
 notes:
     - All timestamp values are provided in ASN.1 TIME format, in other words, following the C(YYYYMMDDHHMMSSZ) pattern.
       They are all in UTC.
@@ -168,7 +171,9 @@ key_usage_critical:
     returned: success
     type: bool
 subject_alt_name:
-    description: Entries in the C(subject_alt_name) extension, or C(none) if extension is not present.
+    description:
+        - Entries in the C(subject_alt_name) extension, or C(none) if extension is not present.
+        - See I(name_encoding) for how IDNs are handled.
     returned: success
     type: list
     elements: str
@@ -355,6 +360,7 @@ authority_cert_issuer:
     description:
         - The certificate's authority cert issuer as a list of general names.
         - Is C(none) if the C(AuthorityKeyIdentifier) extension is not present.
+        - See I(name_encoding) for how IDNs are handled.
     returned: success
     type: list
     elements: str
@@ -397,6 +403,7 @@ def main():
             path=dict(type='path'),
             content=dict(type='str'),
             valid_at=dict(type='dict'),
+            name_encoding=dict(type='str', default='ignore', choices=['ignore', 'idna', 'unicode']),
             select_crypto_backend=dict(type='str', default='auto', choices=['auto', 'cryptography']),
         ),
         required_one_of=(

plugins/modules/x509_crl.py

@@ -242,6 +242,7 @@ options:
 
 extends_documentation_fragment:
     - files
+    - community.crypto.name_encoding
 
 notes:
     - All ASN.1 TIME values should be specified following the YYYYMMDDHHMMSSZ pattern.
@@ -297,6 +298,7 @@ issuer:
     description:
         - The CRL's issuer.
         - Note that for repeated values, only the last one will be returned.
+        - See I(name_encoding) for how IDNs are handled.
     returned: success
     type: dict
     sample: '{"organizationName": "Ansible", "commonName": "ca.example.com"}'
@@ -336,7 +338,9 @@ revoked_certificates:
             type: str
             sample: 20190413202428Z
         issuer:
-            description: The certificate's issuer.
+            description:
+                - The certificate's issuer.
+                - See I(name_encoding) for how IDNs are handled.
             type: list
             elements: str
             sample: '["DNS:ca.example.org"]'
@@ -405,6 +409,7 @@ from ansible_collections.community.crypto.plugins.module_utils.crypto.support im
 )
 
 from ansible_collections.community.crypto.plugins.module_utils.crypto.cryptography_support import (
+    cryptography_decode_name,
     cryptography_get_name,
     cryptography_name_to_oid,
     cryptography_oid_to_name,
@@ -468,6 +473,7 @@ class CRL(OpenSSLObject):
         self.update = module.params['mode'] == 'update'
         self.ignore_timestamps = module.params['ignore_timestamps']
         self.return_content = module.params['return_content']
+        self.name_encoding = module.params['name_encoding']
         self.crl_content = None
 
         self.privatekey_path = module.params['privatekey_path']
@@ -595,11 +601,20 @@ class CRL(OpenSSLObject):
         super(CRL, self).remove(self.module)
 
     def _compress_entry(self, entry):
+        issuer = None
+        if entry['issuer'] is not None:
+            # Normalize to IDNA. If this is used-provided, it was already converted to
+            # IDNA (by cryptography_get_name) and thus the `idna` library is present.
+            # If this is coming from cryptography and isn't already in IDNA (i.e. ascii),
+            # cryptography < 2.1 must be in use, which depends on `idna`. So this should
+            # not require `idna` except if it was already used by code earlier during
+            # this invocation.
+            issuer = tuple(cryptography_decode_name(issuer, idn_rewrite='idna') for issuer in entry['issuer'])
         if self.ignore_timestamps:
             # Throw out revocation_date
             return (
                 entry['serial_number'],
-                tuple(entry['issuer']) if entry['issuer'] is not None else None,
+                issuer,
                 entry['issuer_critical'],
                 entry['reason'],
                 entry['reason_critical'],
@@ -610,7 +625,7 @@ class CRL(OpenSSLObject):
             return (
                 entry['serial_number'],
                 entry['revocation_date'],
-                tuple(entry['issuer']) if entry['issuer'] is not None else None,
+                issuer,
                 entry['issuer_critical'],
                 entry['reason'],
                 entry['reason_critical'],
@@ -765,7 +780,7 @@ class CRL(OpenSSLObject):
                 result['issuer'][k] = v
             result['revoked_certificates'] = []
             for entry in self.revoked_certificates:
-                result['revoked_certificates'].append(cryptography_dump_revoked(entry))
+                result['revoked_certificates'].append(cryptography_dump_revoked(entry, idn_rewrite=self.name_encoding))
         elif self.crl:
             result['last_update'] = self.crl.last_update.strftime(TIMESTAMP_FORMAT)
             result['next_update'] = self.crl.next_update.strftime(TIMESTAMP_FORMAT)
@@ -780,7 +795,7 @@ class CRL(OpenSSLObject):
             result['revoked_certificates'] = []
             for cert in self.crl:
                 entry = cryptography_decode_revoked_certificate(cert)
-                result['revoked_certificates'].append(cryptography_dump_revoked(entry))
+                result['revoked_certificates'].append(cryptography_dump_revoked(entry, idn_rewrite=self.name_encoding))
 
         if self.return_content:
             result['crl'] = self.crl_content
@@ -836,6 +851,7 @@ def main():
                 required_one_of=[['path', 'content', 'serial_number']],
                 mutually_exclusive=[['path', 'content', 'serial_number']],
             ),
+            name_encoding=dict(type='str', default='ignore', choices=['ignore', 'idna', 'unicode']),
         ),
         required_if=[
             ('state', 'present', ['privatekey_path', 'privatekey_content'], True),

plugins/modules/x509_crl_info.py

@@ -40,6 +40,9 @@ options:
         default: true
         version_added: 1.7.0
 
+extends_documentation_fragment:
+    - community.crypto.name_encoding
+
 notes:
     - All timestamp values are provided in ASN.1 TIME format, in other words, following the C(YYYYMMDDHHMMSSZ) pattern.
       They are all in UTC.
@@ -76,6 +79,7 @@ issuer:
     description:
         - The CRL's issuer.
         - Note that for repeated values, only the last one will be returned.
+        - See I(name_encoding) for how IDNs are handled.
     returned: success
     type: dict
     sample: '{"organizationName": "Ansible", "commonName": "ca.example.com"}'
@@ -115,7 +119,9 @@ revoked_certificates:
             type: str
             sample: 20190413202428Z
         issuer:
-            description: The certificate's issuer.
+            description:
+                - The certificate's issuer.
+                - See I(name_encoding) for how IDNs are handled.
             type: list
             elements: str
             sample: '["DNS:ca.example.org"]'
@@ -173,6 +179,7 @@ def main():
             path=dict(type='path'),
             content=dict(type='str'),
             list_revoked_certificates=dict(type='bool', default=True),
+            name_encoding=dict(type='str', default='ignore', choices=['ignore', 'idna', 'unicode']),
         ),
         required_one_of=(
             ['path', 'content'],

tests/integration/targets/openssl_csr_info/tasks/impl.yml

@@ -8,6 +8,20 @@
     select_crypto_backend: '{{ select_crypto_backend }}'
   register: result
 
+- name: "({{ select_crypto_backend }}) Get CSR info (IDNA encoding)"
+  openssl_csr_info:
+    path: '{{ remote_tmp_dir }}/csr_1.csr'
+    name_encoding: idna
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: result_idna
+
+- name: "({{ select_crypto_backend }}) Get CSR info (Unicode encoding)"
+  openssl_csr_info:
+    path: '{{ remote_tmp_dir }}/csr_1.csr'
+    name_encoding: unicode
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: result_unicode
+
 - name: "({{ select_crypto_backend }}) Check whether subject and extensions behaves as expected"
   assert:
     that:
@@ -23,8 +37,11 @@
       - result.extensions_by_oid['2.5.29.15'].critical == true
       - result.extensions_by_oid['2.5.29.15'].value in ['AwMA/4A=', 'AwMH/4A=']
       # Subject Alternative Names
+      - result.subject_alt_name[1] == ("DNS:âņsïbłè.com" if cryptography_version.stdout is version('2.1', '<') else "DNS:xn--sb-oia0a7a53bya.com")
+      - result_unicode.subject_alt_name[1] == "DNS:âņsïbłè.com"
+      - result_idna.subject_alt_name[1] == "DNS:xn--sb-oia0a7a53bya.com"
       - result.extensions_by_oid['2.5.29.17'].critical == false
-      - result.extensions_by_oid['2.5.29.17'].value == 'MGCCD3d3dy5hbnNpYmxlLmNvbYcEAQIDBIcQAAAAAAAAAAAAAAAAAAAAAYEQdGVzdEBleGFtcGxlLm9yZ4YjaHR0cHM6Ly9leGFtcGxlLm9yZy90ZXN0L2luZGV4Lmh0bWw='
+      - result.extensions_by_oid['2.5.29.17'].value == 'MHmCD3d3dy5hbnNpYmxlLmNvbYIXeG4tLXNiLW9pYTBhN2E1M2J5YS5jb22HBAECAwSHEAAAAAAAAAAAAAAAAAAAAAGBEHRlc3RAZXhhbXBsZS5vcmeGI2h0dHBzOi8vZXhhbXBsZS5vcmcvdGVzdC9pbmRleC5odG1s'
       # Basic Constraints
       - result.extensions_by_oid['2.5.29.19'].critical == true
       - result.extensions_by_oid['2.5.29.19'].value == 'MAYBAf8CARc='

tests/integration/targets/openssl_csr_info/tasks/main.yml

@@ -4,6 +4,11 @@
 # and should not be used as examples of how to write Ansible roles #
 ####################################################################
 
+- name: Make sure the Python idna library is installed
+  pip:
+    name: idna
+    state: present
+
 - name: Generate privatekey
   openssl_privatekey:
     path: '{{ remote_tmp_dir }}/privatekey.pem'
@@ -67,6 +72,7 @@
       - biometricInfo
     subject_alt_name:
       - "DNS:www.ansible.com"
+      - "DNS:âņsïbłè.com"
       - "IP:1.2.3.4"
       - "IP:::1"
       - "email:test@example.org"

tests/integration/targets/x509_certificate_info/tasks/impl.yml

@@ -8,6 +8,20 @@
     select_crypto_backend: '{{ select_crypto_backend }}'
   register: result
 
+- name: ({{select_crypto_backend}}) Get certificate info (IDNA encoding)
+  x509_certificate_info:
+    path: '{{ remote_tmp_dir }}/cert_1.pem'
+    name_encoding: idna
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: result_idna
+
+- name: ({{select_crypto_backend}}) Get certificate info (Unicode encoding)
+  x509_certificate_info:
+    path: '{{ remote_tmp_dir }}/cert_1.pem'
+    name_encoding: unicode
+    select_crypto_backend: '{{ select_crypto_backend }}'
+  register: result_unicode
+
 - name: Check whether issuer and subject and extensions behave as expected
   assert:
     that:
@@ -21,7 +35,26 @@
       - result.public_key_data.size == (default_rsa_key_size_certifiates | int)
       - "result.subject_alt_name == [
           'DNS:www.ansible.com',
-          'DNS:' ~ ('öç' if cryptography_version.stdout is version('2.1', '<') else 'xn--74h') ~ '.com',
+          'DNS:' ~ ('öç' if cryptography_version.stdout is version('2.1', '<') else 'xn--7ca3a') ~ '.com',
+          'DNS:' ~ ('www.öç' if cryptography_version.stdout is version('2.1', '<') else 'xn--74h') ~ '.com',
+          'IP:1.2.3.4',
+          'IP:::1',
+          'email:test@example.org',
+          'URI:https://example.org/test/index.html'
+        ]"
+      - "result_idna.subject_alt_name == [
+          'DNS:www.ansible.com',
+          'DNS:xn--7ca3a.com',
+          'DNS:' ~ ('www.xn--7ca3a' if cryptography_version.stdout is version('2.1', '<') else 'xn--74h') ~ '.com',
+          'IP:1.2.3.4',
+          'IP:::1',
+          'email:test@example.org',
+          'URI:https://example.org/test/index.html'
+        ]"
+      - "result_unicode.subject_alt_name == [
+          'DNS:www.ansible.com',
+          'DNS:öç.com',
+          'DNS:' ~ ('www.öç' if cryptography_version.stdout is version('2.1', '<') else '☺') ~ '.com',
           'IP:1.2.3.4',
           'IP:::1',
           'email:test@example.org',
@@ -37,9 +70,9 @@
       - result.extensions_by_oid['2.5.29.17'].critical == false
       - >
         result.extensions_by_oid['2.5.29.17'].value == (
-          'MG+CD3d3dy5hbnNpYmxlLmNvbYINeG4tLTdjYTNhLmNvbYcEAQIDBIcQAAAAAAAAAAAAAAAAAAAAAYEQdGVzdEBleGFtcGxlLm9yZ4YjaHR0cHM6Ly9leGFtcGxlLm9yZy90ZXN0L2luZGV4Lmh0bWw='
+          'MIGCgg93d3cuYW5zaWJsZS5jb22CDXhuLS03Y2EzYS5jb22CEXd3dy54bi0tN2NhM2EuY29thwQBAgMEhxAAAAAAAAAAAAAAAAAAAAABgRB0ZXN0QGV4YW1wbGUub3JnhiNodHRwczovL2V4YW1wbGUub3JnL3Rlc3QvaW5kZXguaHRtbA=='
           if cryptography_version.stdout is version('2.1', '<') else
-          'MG2CD3d3dy5hbnNpYmxlLmNvbYILeG4tLTc0aC5jb22HBAECAwSHEAAAAAAAAAAAAAAAAAAAAAGBEHRlc3RAZXhhbXBsZS5vcmeGI2h0dHBzOi8vZXhhbXBsZS5vcmcvdGVzdC9pbmRleC5odG1s'
+          'MHyCD3d3dy5hbnNpYmxlLmNvbYINeG4tLTdjYTNhLmNvbYILeG4tLTc0aC5jb22HBAECAwSHEAAAAAAAAAAAAAAAAAAAAAGBEHRlc3RAZXhhbXBsZS5vcmeGI2h0dHBzOi8vZXhhbXBsZS5vcmcvdGVzdC9pbmRleC5odG1s'
         )
       # Basic Constraints
       - result.extensions_by_oid['2.5.29.19'].critical == true

tests/integration/targets/x509_certificate_info/tasks/main.yml

@@ -4,6 +4,11 @@
 # and should not be used as examples of how to write Ansible roles #
 ####################################################################
 
+- name: Make sure the Python idna library is installed
+  pip:
+    name: idna
+    state: present
+
 - name: Generate privatekey
   openssl_privatekey:
     path: '{{ remote_tmp_dir }}/privatekey.pem'
@@ -67,7 +72,9 @@
       - biometricInfo
     subject_alt_name:
       - "DNS:www.ansible.com"
-      - "DNS:{{ 'öç' if cryptography_version.stdout is version('2.1', '<') else 'xn--74h' }}.com"
+      - "DNS:öç.com"
+      # cryptography < 2.1 cannot handle certain Unicode characters
+      - "DNS:{{ 'www.öç' if cryptography_version.stdout is version('2.1', '<') else '☺' }}.com"
       - "IP:1.2.3.4"
       - "IP:::1"
       - "email:test@example.org"

tests/integration/targets/x509_crl/tasks/impl.yml

@@ -536,13 +536,91 @@
     revoked_certificates:
       - serial_number: 1234
         revocation_date: 20191001000000Z
+        # * cryptography < 2.1 strips username and password from URIs. To avoid problems, we do
+        #   not pass usernames and passwords for URIs when the cryptography version is < 2.1.
+        # * Python 3.5 before 3.5.8 rc 1 has a bug in urllib.parse.urlparse() that results in an
+        #   error if a Unicode netloc has a username or password included.
+        #   (https://github.com/ansible-collections/community.crypto/pull/436#issuecomment-1101737134)
+        #   This affects the Python 3.5 included in Ansible 2.9's default test container; to avoid
+        #   this, we also do not pass usernames and passwords for Python 3.5.
         issuer:
           - "DNS:ca.example.org"
+          - "DNS:ffóò.ḃâŗ.çøṁ"
+          - "email:foo@ḃâŗ.çøṁ"
+          - "URI:https://{{ '' if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else 'admin:hunter2@' }}ffóò.ḃâŗ.çøṁ/baz?foo=bar"
+          - "URI:https://{{ '' if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else 'goo@' }}www.straße.de"
+          - "URI:https://straße.de:8080"
+          - "URI:http://gefäß.org"
+          - "URI:http://{{ '' if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else 'a:b@' }}ä:1"
         issuer_critical: true
   register: crl_3
 
+- name: Create CRL 3 (IDNA encoding)
+  x509_crl:
+    path: '{{ remote_tmp_dir }}/ca-crl3.crl'
+    privatekey_path: '{{ remote_tmp_dir }}/ca.key'
+    issuer:
+      CN: Ansible
+    last_update: +0d
+    next_update: +0d
+    revoked_certificates:
+      - serial_number: 1234
+        revocation_date: 20191001000000Z
+        issuer:
+          - "DNS:ca.example.org"
+          - "DNS:xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n"
+          - "email:foo@xn--2ca8uh37e.xn--7ca8a981n"
+          - "URI:https://{{ '' if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else 'admin:hunter2@' }}xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n/baz?foo=bar"
+          - "URI:https://{{ '' if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else 'goo@' }}www.xn--strae-oqa.de"
+          - "URI:https://xn--strae-oqa.de:8080"
+          - "URI:http://xn--gef-7kay.org"
+          - "URI:http://{{ '' if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else 'a:b@' }}xn--4ca:1"
+        issuer_critical: true
+    ignore_timestamps: true
+    name_encoding: idna
+  register: crl_3_idna
+
+- name: Create CRL 3 (Unicode encoding)
+  x509_crl:
+    path: '{{ remote_tmp_dir }}/ca-crl3.crl'
+    privatekey_path: '{{ remote_tmp_dir }}/ca.key'
+    issuer:
+      CN: Ansible
+    last_update: +0d
+    next_update: +0d
+    revoked_certificates:
+      - serial_number: 1234
+        revocation_date: 20191001000000Z
+        issuer:
+          - "DNS:ca.example.org"
+          - "DNS:ffóò.ḃâŗ.çøṁ"
+          - "email:foo@ḃâŗ.çøṁ"
+          - "URI:https://{{ '' if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else 'admin:hunter2@' }}ffóò.ḃâŗ.çøṁ/baz?foo=bar"
+          - "URI:https://{{ '' if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else 'goo@' }}www.straße.de"
+          - "URI:https://straße.de:8080"
+          - "URI:http://gefäß.org"
+          - "URI:http://{{ '' if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else 'a:b@' }}ä:1"
+        issuer_critical: true
+    ignore_timestamps: true
+    name_encoding: unicode
+  register: crl_3_unicode
+
 - name: Retrieve CRL 3 infos
   x509_crl_info:
     path: '{{ remote_tmp_dir }}/ca-crl3.crl'
     list_revoked_certificates: true
   register: crl_3_info
+
+- name: Retrieve CRL 3 infos (IDNA encoding)
+  x509_crl_info:
+    path: '{{ remote_tmp_dir }}/ca-crl3.crl'
+    name_encoding: idna
+    list_revoked_certificates: true
+  register: crl_3_info_idna
+
+- name: Retrieve CRL 3 infos (Unicode encoding)
+  x509_crl_info:
+    path: '{{ remote_tmp_dir }}/ca-crl3.crl'
+    name_encoding: unicode
+    list_revoked_certificates: true
+  register: crl_3_info_unicode

tests/integration/targets/x509_crl/tasks/main.yml

@@ -4,6 +4,11 @@
 # and should not be used as examples of how to write Ansible roles #
 ####################################################################
 
+- name: Make sure the Python idna library is installed
+  pip:
+    name: idna
+    state: present
+
 - set_fact:
     certificates:
       - name: ca

tests/integration/targets/x509_crl/tests/validate.yml

@@ -107,6 +107,73 @@
   assert:
     that:
       - crl_3.revoked_certificates == crl_3_info.revoked_certificates
-      - crl_3.revoked_certificates[0].issuer == [
+      - crl_3.revoked_certificates[0].issuer == ([
           "DNS:ca.example.org",
-        ]
+          "DNS:ffóò.ḃâŗ.çøṁ",
+          "email:foo@ḃâŗ.çøṁ",
+          "URI:https://ffóò.ḃâŗ.çøṁ/baz?foo=bar",
+          "URI:https://www.straße.de",
+          "URI:https://straße.de:8080",
+          "URI:http://gefäß.org",
+          "URI:http://ä:1",
+        ] if cryptography_version.stdout is version('2.1', '<') else [
+          "DNS:ca.example.org",
+          "DNS:xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n",
+          "email:foo@xn--2ca8uh37e.xn--7ca8a981n",
+          "URI:https://xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n/baz?foo=bar",
+          "URI:https://www.xn--strae-oqa.de",
+          "URI:https://xn--strae-oqa.de:8080",
+          "URI:http://xn--gef-7kay.org",
+          "URI:http://xn--4ca:1",
+        ] if ansible_facts.python.version.minor == 5 else [
+          "DNS:ca.example.org",
+          "DNS:xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n",
+          "email:foo@xn--2ca8uh37e.xn--7ca8a981n",
+          "URI:https://admin:hunter2@xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n/baz?foo=bar",
+          "URI:https://goo@www.xn--strae-oqa.de",
+          "URI:https://xn--strae-oqa.de:8080",
+          "URI:http://xn--gef-7kay.org",
+          "URI:http://a:b@xn--4ca:1",
+        ])
+      - crl_3_idna is not changed
+      - crl_3_idna.revoked_certificates == crl_3_info_idna.revoked_certificates
+      - crl_3_idna.revoked_certificates[0].issuer == ([
+          "DNS:ca.example.org",
+          "DNS:xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n",
+          "email:foo@xn--2ca8uh37e.xn--7ca8a981n",
+          "URI:https://xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n/baz?foo=bar",
+          "URI:https://www.xn--strae-oqa.de",
+          "URI:https://xn--strae-oqa.de:8080",
+          "URI:http://xn--gef-7kay.org",
+          "URI:http://xn--4ca:1",
+        ] if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else [
+          "DNS:ca.example.org",
+          "DNS:xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n",
+          "email:foo@xn--2ca8uh37e.xn--7ca8a981n",
+          "URI:https://admin:hunter2@xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n/baz?foo=bar",
+          "URI:https://goo@www.xn--strae-oqa.de",
+          "URI:https://xn--strae-oqa.de:8080",
+          "URI:http://xn--gef-7kay.org",
+          "URI:http://a:b@xn--4ca:1",
+        ])
+      - crl_3_unicode is not changed
+      - crl_3_unicode.revoked_certificates == crl_3_info_unicode.revoked_certificates
+      - crl_3_unicode.revoked_certificates[0].issuer == ([
+          "DNS:ca.example.org",
+          "DNS:ffóò.ḃâŗ.çøṁ",
+          "email:foo@ḃâŗ.çøṁ",
+          "URI:https://ffóò.ḃâŗ.çøṁ/baz?foo=bar",
+          "URI:https://www.straße.de",
+          "URI:https://straße.de:8080",
+          "URI:http://gefäß.org",
+          "URI:http://ä:1",
+        ] if cryptography_version.stdout is version('2.1', '<') or ansible_facts.python.version.minor == 5 else [
+          "DNS:ca.example.org",
+          "DNS:ffóò.ḃâŗ.çøṁ",
+          "email:foo@ḃâŗ.çøṁ",
+          "URI:https://admin:hunter2@ffóò.ḃâŗ.çøṁ/baz?foo=bar",
+          "URI:https://goo@www.straße.de",
+          "URI:https://straße.de:8080",
+          "URI:http://gefäß.org",
+          "URI:http://a:b@ä:1",
+        ])

tests/unit/plugins/module_utils/crypto/test_cryptography_support.py

@@ -20,6 +20,7 @@ from ansible_collections.community.crypto.plugins.module_utils.crypto.basic impo
 
 from ansible_collections.community.crypto.plugins.module_utils.crypto.cryptography_support import (
     cryptography_get_name,
+    _adjust_idn,
     _parse_dn_component,
     _parse_dn,
 )
@@ -27,6 +28,67 @@ from ansible_collections.community.crypto.plugins.module_utils.crypto.cryptograp
 from ansible_collections.community.crypto.plugins.module_utils.version import LooseVersion
 
 
+@pytest.mark.parametrize('unicode, idna, cycled_unicode', [
+    (u'..', u'..', None),
+    (u'foo.com', u'foo.com', None),
+    (u'.foo.com.', u'.foo.com.', None),
+    (u'*.foo.com', u'*.foo.com', None),
+    (u'straße', u'xn--strae-oqa', None),
+    (u'ffóò.ḃâŗ.çøṁ', u'xn--ff-3jad.xn--2ca8uh37e.xn--7ca8a981n', u'ffóò.ḃâŗ.çøṁ'),
+    (u'*.☺.', u'*.xn--74h.', None),
+])
+def test_adjust_idn(unicode, idna, cycled_unicode):
+    if cycled_unicode is None:
+        cycled_unicode = unicode
+
+    result = _adjust_idn(unicode, 'ignore')
+    print(result, unicode)
+    assert result == unicode
+
+    result = _adjust_idn(idna, 'ignore')
+    print(result, idna)
+    assert result == idna
+
+    result = _adjust_idn(unicode, 'unicode')
+    print(result, unicode)
+    assert result == unicode
+
+    result = _adjust_idn(idna, 'unicode')
+    print(result, cycled_unicode)
+    assert result == cycled_unicode
+
+    result = _adjust_idn(unicode, 'idna')
+    print(result, idna)
+    assert result == idna
+
+    result = _adjust_idn(idna, 'idna')
+    print(result, idna)
+    assert result == idna
+
+
+@pytest.mark.parametrize('value, idn_rewrite, message', [
+    (u'bar', 'foo', re.escape(u'Invalid value for idn_rewrite: "foo"')),
+])
+def test_adjust_idn_fail_valueerror(value, idn_rewrite, message):
+    with pytest.raises(ValueError, match=message):
+        result = _adjust_idn(value, idn_rewrite)
+
+
+@pytest.mark.parametrize('value, idn_rewrite, message', [
+    (
+        u'xn--a',
+        'unicode',
+        u'''^Error while transforming part u?"xn\\-\\-a" of IDNA DNS name u?"xn\\-\\-a" to Unicode\\.'''
+        u''' IDNA2008 transformation resulted in "Codepoint U\\+0080 at position 1 of u?'\\\\x80' not allowed",'''
+        u''' IDNA2003 transformation resulted in "(decoding with 'idna' codec failed'''
+        u''' \\(UnicodeError: )?Invalid character u?'\\\\x80'\\)?"\\.$'''
+    ),
+])
+def test_adjust_idn_fail_user_error(value, idn_rewrite, message):
+    with pytest.raises(OpenSSLObjectError, match=message):
+        result = _adjust_idn(value, idn_rewrite)
+
+
 def test_cryptography_get_name_invalid_prefix():
     with pytest.raises(OpenSSLObjectError, match="^Cannot parse Subject Alternative Name"):
         cryptography_get_name('fake:value')

tests/unit/requirements.txt

@@ -1,5 +1,6 @@
 bcrypt
 cryptography
+idna
 ipaddress ; python_version < '3.0'
 
 unittest2 ; python_version < '2.7'