Improve CI (#281)

* Install PyOpenSSL and cryptography from PyPi if target Python != system Python.

* Work around some CentOS6, 7, Ubuntu 16.04 problems. Improve jinja2 compatibility handling.

* Skip tasks that require properties that aren't always there.

* Only install OpenSSL when not present.

* Improve output.

* Improve get_certificate integration test graceful failing.

* Fix tests.

* Fix assert.

* OpenSSL peculiarities.

* Fix condition.

tests/integration/targets/acme_account/tasks/impl.yml

@@ -2,8 +2,8 @@
   - name: Generate account keys
     openssl_privatekey:
       path: "{{ remote_tmp_dir }}/{{ item.name }}.pem"
-      passphrase: "{{ item.pass | default(omit, true) }}"
-      cipher: "{{ 'auto' if item.pass | default() else omit }}"
+      passphrase: "{{ item.pass | default(omit) | default(omit, true) }}"
+      cipher: "{{ 'auto' if (item.pass | default(false)) else omit }}"
       type: ECC
       curve: secp256r1
       force: true
@@ -12,7 +12,7 @@
   - name: Parse account keys (to ease debugging some test failures)
     openssl_privatekey_info:
       path: "{{ remote_tmp_dir }}/{{ item.name }}.pem"
-      passphrase: "{{ item.pass | default(omit, true) }}"
+      passphrase: "{{ item.pass | default(omit) | default(omit, true) }}"
       return_private_key_data: true
     loop: "{{ account_keys }}"
 

tests/integration/targets/acme_certificate/meta/main.yml

@@ -1,3 +1,4 @@
 dependencies:
   - setup_acme
   - setup_remote_tmp_dir
+  - prepare_jinja2_compat

tests/integration/targets/acme_certificate/tasks/impl.yml

@@ -264,92 +264,98 @@
   set_fact:
     cert_5_recreate_3: "{{ challenge_data is changed }}"
     cert_5d_obtain_results: "{{ certificate_obtain_result }}"
-- name: Obtain cert 6
-  include_tasks: obtain-cert.yml
-  vars:
-    certgen_title: Certificate 6
-    certificate_name: cert-6
-    key_type: rsa
-    rsa_bits: "{{ default_rsa_key_size }}"
-    subject_alt_name: "DNS:example.org"
-    subject_alt_name_critical: no
-    account_key: account-ec256
-    challenge: tls-alpn-01
-    modify_account: yes
-    deactivate_authzs: no
-    force: no
-    remaining_days: 10
-    terms_agreed: yes
-    account_email: "example@example.org"
-    acme_expected_root_number: 0
-    select_chain:
-      # All intermediates have the same subject key identifier, so always
-      # the first chain will be found, and we need a second condition to
-      # make sure that the first condition actually works. (The second
-      # condition has been tested above.)
-      - test_certificates: first
-        subject_key_identifier: "{{ acme_intermediates[0].subject_key_identifier }}"
-      - test_certificates: last
-        issuer: "{{ acme_roots[1].subject }}"
-    use_csr_content: true
-- name: Store obtain results for cert 6
-  set_fact:
-    cert_6_obtain_results: "{{ certificate_obtain_result }}"
-    cert_6_alternate: "{{ 0 if select_crypto_backend == 'cryptography' else 0 }}"
-- name: Obtain cert 7
-  include_tasks: obtain-cert.yml
-  vars:
-    certgen_title: Certificate 7
-    certificate_name: cert-7
-    key_type: rsa
-    rsa_bits: "{{ default_rsa_key_size }}"
-    subject_alt_name:
-    - "IP:127.0.0.1"
-    # - "IP:::1"
-    subject_alt_name_critical: no
-    account_key: account-ec256
-    challenge: http-01
-    modify_account: yes
-    deactivate_authzs: no
-    force: no
-    remaining_days: 10
-    terms_agreed: yes
-    account_email: "example@example.org"
-    acme_expected_root_number: 2
-    select_chain:
-      - test_certificates: last
-        authority_key_identifier: "{{ acme_roots[2].subject_key_identifier }}"
-    use_csr_content: false
-- name: Store obtain results for cert 7
-  set_fact:
-    cert_7_obtain_results: "{{ certificate_obtain_result }}"
-    cert_7_alternate: "{{ 2 if select_crypto_backend == 'cryptography' else 0 }}"
-- name: Obtain cert 8
-  include_tasks: obtain-cert.yml
-  vars:
-    certgen_title: Certificate 8
-    certificate_name: cert-8
-    key_type: rsa
-    rsa_bits: "{{ default_rsa_key_size }}"
-    subject_alt_name:
-    - "IP:127.0.0.1"
-    # IPv4 only since our test validation server doesn't work
-    # with IPv6 (thanks to Python's socketserver).
-    subject_alt_name_critical: no
-    account_key: account-ec256
-    challenge: tls-alpn-01
-    challenge_alpn_tls: acme_challenge_cert_helper
-    modify_account: yes
-    deactivate_authzs: no
-    force: no
-    remaining_days: 10
-    terms_agreed: yes
-    account_email: "example@example.org"
-    use_csr_content: true
-- name: Store obtain results for cert 8
-  set_fact:
-    cert_8_obtain_results: "{{ certificate_obtain_result }}"
-    cert_8_alternate: "{{ 0 if select_crypto_backend == 'cryptography' else 0 }}"
+- block:
+  - name: Obtain cert 6
+    include_tasks: obtain-cert.yml
+    vars:
+      certgen_title: Certificate 6
+      certificate_name: cert-6
+      key_type: rsa
+      rsa_bits: "{{ default_rsa_key_size }}"
+      subject_alt_name: "DNS:example.org"
+      subject_alt_name_critical: no
+      account_key: account-ec256
+      challenge: tls-alpn-01
+      modify_account: yes
+      deactivate_authzs: no
+      force: no
+      remaining_days: 10
+      terms_agreed: yes
+      account_email: "example@example.org"
+      acme_expected_root_number: 0
+      select_chain:
+        # All intermediates have the same subject key identifier, so always
+        # the first chain will be found, and we need a second condition to
+        # make sure that the first condition actually works. (The second
+        # condition has been tested above.)
+        - test_certificates: first
+          subject_key_identifier: "{{ acme_intermediates[0].subject_key_identifier }}"
+        - test_certificates: last
+          issuer: "{{ acme_roots[1].subject }}"
+      use_csr_content: true
+  - name: Store obtain results for cert 6
+    set_fact:
+      cert_6_obtain_results: "{{ certificate_obtain_result }}"
+      cert_6_alternate: "{{ 0 if select_crypto_backend == 'cryptography' else 0 }}"
+  when: acme_intermediates[0].subject_key_identifier is defined
+- block:
+  - name: Obtain cert 7
+    include_tasks: obtain-cert.yml
+    vars:
+      certgen_title: Certificate 7
+      certificate_name: cert-7
+      key_type: rsa
+      rsa_bits: "{{ default_rsa_key_size }}"
+      subject_alt_name:
+      - "IP:127.0.0.1"
+      # - "IP:::1"
+      subject_alt_name_critical: no
+      account_key: account-ec256
+      challenge: http-01
+      modify_account: yes
+      deactivate_authzs: no
+      force: no
+      remaining_days: 10
+      terms_agreed: yes
+      account_email: "example@example.org"
+      acme_expected_root_number: 2
+      select_chain:
+        - test_certificates: last
+          authority_key_identifier: "{{ acme_roots[2].subject_key_identifier }}"
+      use_csr_content: false
+  - name: Store obtain results for cert 7
+    set_fact:
+      cert_7_obtain_results: "{{ certificate_obtain_result }}"
+      cert_7_alternate: "{{ 2 if select_crypto_backend == 'cryptography' else 0 }}"
+  when: acme_roots[2].subject_key_identifier is defined
+- block:
+  - name: Obtain cert 8
+    include_tasks: obtain-cert.yml
+    vars:
+      certgen_title: Certificate 8
+      certificate_name: cert-8
+      key_type: rsa
+      rsa_bits: "{{ default_rsa_key_size }}"
+      subject_alt_name:
+      - "IP:127.0.0.1"
+      # IPv4 only since our test validation server doesn't work
+      # with IPv6 (thanks to Python's socketserver).
+      subject_alt_name_critical: no
+      account_key: account-ec256
+      challenge: tls-alpn-01
+      challenge_alpn_tls: acme_challenge_cert_helper
+      modify_account: yes
+      deactivate_authzs: no
+      force: no
+      remaining_days: 10
+      terms_agreed: yes
+      account_email: "example@example.org"
+      use_csr_content: true
+  - name: Store obtain results for cert 8
+    set_fact:
+      cert_8_obtain_results: "{{ certificate_obtain_result }}"
+      cert_8_alternate: "{{ 0 if select_crypto_backend == 'cryptography' else 0 }}"
+  when: cryptography_version.stdout is version('1.3', '>=')
 ## DISSECT CERTIFICATES #######################################################################
 # Make sure certificates are valid. Root certificate for Pebble equals the chain certificate.
 - name: Verifying cert 1
@@ -376,14 +382,17 @@
   command: '{{ openssl_binary }} verify -CAfile "{{ remote_tmp_dir }}/cert-6-root.pem" -untrusted "{{ remote_tmp_dir }}/cert-6-chain.pem" "{{ remote_tmp_dir }}/cert-6.pem"'
   ignore_errors: yes
   register: cert_6_valid
+  when: acme_intermediates[0].subject_key_identifier is defined
 - name: Verifying cert 7
   command: '{{ openssl_binary }} verify -CAfile "{{ remote_tmp_dir }}/cert-7-root.pem" -untrusted "{{ remote_tmp_dir }}/cert-7-chain.pem" "{{ remote_tmp_dir }}/cert-7.pem"'
   ignore_errors: yes
   register: cert_7_valid
+  when: acme_roots[2].subject_key_identifier is defined
 - name: Verifying cert 8
   command: '{{ openssl_binary }} verify -CAfile "{{ remote_tmp_dir }}/cert-8-root.pem" -untrusted "{{ remote_tmp_dir }}/cert-8-chain.pem" "{{ remote_tmp_dir }}/cert-8.pem"'
   ignore_errors: yes
   register: cert_8_valid
+  when: cryptography_version.stdout is version('1.3', '>=')
 # Dump certificate info
 - name: Dumping cert 1
   command: '{{ openssl_binary }} x509 -in "{{ remote_tmp_dir }}/cert-1.pem" -noout -text'
@@ -403,12 +412,15 @@
 - name: Dumping cert 6
   command: '{{ openssl_binary }} x509 -in "{{ remote_tmp_dir }}/cert-6.pem" -noout -text'
   register: cert_6_text
+  when: acme_intermediates[0].subject_key_identifier is defined
 - name: Dumping cert 7
   command: '{{ openssl_binary }} x509 -in "{{ remote_tmp_dir }}/cert-7.pem" -noout -text'
   register: cert_7_text
+  when: acme_roots[2].subject_key_identifier is defined
 - name: Dumping cert 8
   command: '{{ openssl_binary }} x509 -in "{{ remote_tmp_dir }}/cert-8.pem" -noout -text'
   register: cert_8_text
+  when: cryptography_version.stdout is version('1.3', '>=')
 # Dump certificate info
 - name: Dumping cert 1
   x509_certificate_info:
@@ -434,14 +446,17 @@
   x509_certificate_info:
     path: "{{ remote_tmp_dir }}/cert-6.pem"
   register: cert_6_info
+  when: acme_intermediates[0].subject_key_identifier is defined
 - name: Dumping cert 7
   x509_certificate_info:
     path: "{{ remote_tmp_dir }}/cert-7.pem"
   register: cert_7_info
+  when: acme_roots[2].subject_key_identifier is defined
 - name: Dumping cert 8
   x509_certificate_info:
     path: "{{ remote_tmp_dir }}/cert-8.pem"
   register: cert_8_info
+  when: cryptography_version.stdout is version('1.3', '>=')
 ## GET ACCOUNT ORDERS #########################################################################
 - name: Don't retrieve orders
   acme_account_info:

tests/integration/targets/acme_certificate/tests/validate.yml

@@ -124,14 +124,38 @@
     that:
       - cert_5_recreate_3 == True
 
-- name: Check that certificate 6 is valid
-  assert:
-    that:
-      - cert_6_valid is not failed
-- name: Check that certificate 6 contains correct SANs
-  assert:
-    that:
-      - "'DNS:example.org' in cert_6_text.stdout"
+- block:
+  - name: Check that certificate 6 is valid
+    assert:
+      that:
+        - cert_6_valid is not failed
+  - name: Check that certificate 6 contains correct SANs
+    assert:
+      that:
+        - "'DNS:example.org' in cert_6_text.stdout"
+  when: acme_intermediates[0].subject_key_identifier is defined
+
+- block:
+  - name: Check that certificate 7 is valid
+    assert:
+      that:
+        - cert_7_valid is not failed
+  - name: Check that certificate 7 contains correct SANs
+    assert:
+      that:
+        - "'IP Address:127.0.0.1' in cert_8_text.stdout or 'IP:127.0.0.1' in cert_8_text.stdout"
+  when: acme_roots[2].subject_key_identifier is defined
+
+- block:
+  - name: Check that certificate 8 is valid
+    assert:
+      that:
+        - cert_8_valid is not failed
+  - name: Check that certificate 8 contains correct SANs
+    assert:
+      that:
+        - "'IP Address:127.0.0.1' in cert_8_text.stdout or 'IP:127.0.0.1' in cert_8_text.stdout"
+  when: cryptography_version.stdout is version('1.3', '>=')
 
 - name: Validate that orders were not retrieved
   assert:

tests/integration/targets/acme_challenge_cert_helper/tasks/main.yml

@@ -31,4 +31,4 @@
       terms_agreed: yes
       account_email: "example@example.org"
 
-  when: openssl_version.stdout is version('1.0.0', '>=') or cryptography_version.stdout is version('1.5', '>=')
+  when: cryptography_version.stdout is version('1.5', '>=')

tests/integration/targets/acme_inspect/meta/main.yml

@@ -1,3 +1,4 @@
 dependencies:
   - setup_acme
   - setup_remote_tmp_dir
+  - prepare_jinja2_compat

tests/integration/targets/get_certificate/tasks/main.yml

@@ -4,16 +4,35 @@
 # and should not be used as examples of how to write Ansible roles #
 ####################################################################
 
+- set_fact:
+    skip_tests: false
+
 - block:
 
   - name: Get servers certificate with backend auto-detection
     get_certificate:
       host: "{{ httpbin_host }}"
       port: 443
+    ignore_errors: true
+    register: result
+
+  - set_fact:
+      skip_tests: |
+        {{
+          result is failed and (
+            'error: [Errno 1] _ssl.c:492: error:14094410:SSL routines:SSL3_READ_BYTES:sslv3 alert handshake failure' in result.msg
+            or
+            'error: _ssl.c:314: Invalid SSL protocol variant specified.' in result.msg
+          )
+        }}
+
+  - assert:
+      that:
+        - result is success or skip_tests
 
   when: |
     pyopenssl_version.stdout is version('0.15', '>=') or
-    (cryptography_version.stdout is version('1.6', '>=') and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6))
+    cryptography_version.stdout is version('1.6', '>=')
 
 - block:
 
@@ -21,7 +40,7 @@
     vars:
       select_crypto_backend: pyopenssl
 
-  when: pyopenssl_version.stdout is version('0.15', '>=')
+  when: pyopenssl_version.stdout is version('0.15', '>=') and not skip_tests
 
 - block:
 
@@ -32,6 +51,4 @@
   # The module doesn't work with CentOS 6. Since the pyOpenSSL installed there is too old,
   # we never noticed before. This becomes a problem with the new cryptography backend,
   # since there is a new enough cryptography version...
-  when: |
-    cryptography_version.stdout is version('1.6', '>=') and
-    (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6)
+  when: cryptography_version.stdout is version('1.6', '>=') and not skip_tests

tests/integration/targets/openssl_csr_info/meta/main.yml

@@ -2,3 +2,4 @@ dependencies:
   - setup_openssl
   - setup_pyopenssl
   - setup_remote_tmp_dir
+  - prepare_jinja2_compat

tests/integration/targets/openssl_privatekey_info/meta/main.yml

@@ -2,3 +2,4 @@ dependencies:
   - setup_openssl
   - setup_pyopenssl
   - setup_remote_tmp_dir
+  - prepare_jinja2_compat

tests/integration/targets/openssl_publickey_info/meta/main.yml

@@ -2,3 +2,4 @@ dependencies:
   - setup_openssl
   - setup_pyopenssl
   - setup_remote_tmp_dir
+  - prepare_jinja2_compat

tests/integration/targets/prepare_jinja2_compat/filter_plugins/jinja_compatibility.py

@@ -0,0 +1,135 @@
+# Copyright 2007 Pallets
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# 1.  Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#
+# 2.  Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+# 3.  Neither the name of the copyright holder nor the names of its
+#     contributors may be used to endorse or promote products derived from
+#     this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from jinja2.filters import contextfilter
+from jinja2.runtime import Undefined
+from jinja2.exceptions import TemplateRuntimeError, FilterArgumentError
+
+try:
+    from jinja2.nodes import EvalContext
+    HAS_EVALCONTEXT = True
+except ImportError:
+    HAS_EVALCONTEXT = False
+
+
+def call_test(environment, test_name, value, args, kwargs):
+    try:
+        return environment.call_test(test_name, value, args, kwargs)
+    except AttributeError:
+        # call_test was added together with selectattr...
+        func = environment.tests.get(test_name)
+        if func is None:
+            raise TemplateRuntimeError('no test named %r' % test_name)
+        return func(value, *args, **kwargs)
+
+
+def call_filter(environment, name, value, args=None, kwargs=None,
+                context=None, eval_ctx=None):
+    func = environment.filters.get(name)
+    if func is None:
+        raise TemplateRuntimeError('no filter named %r' % name)
+    args = list(args or ())
+    if getattr(func, 'contextfilter', False):
+        if context is None:
+            raise TemplateRuntimeError('Attempted to invoke context filter without context')
+        args.insert(0, context)
+    elif getattr(func, 'evalcontextfilter', False):
+        if eval_ctx is None:
+            if context is not None:
+                eval_ctx = context.eval_ctx
+            elif HAS_EVALCONTEXT:
+                eval_ctx = EvalContext(environment)
+            else:
+                raise TemplateRuntimeError('Too old Jinja2 does not have EvalContext')
+        args.insert(0, eval_ctx)
+    elif getattr(func, 'environmentfilter', False):
+        args.insert(0, environment)
+    return func(value, *args, **(kwargs or {}))
+
+
+def make_attrgetter(environment, attribute_str, default=None):
+    attributes = [int(attribute) if attribute.isdigit() else attribute for attribute in attribute_str.split(".")]
+
+    def f(item):
+        for attribute in attributes:
+            item = environment.getitem(item, attribute)
+            if default and isinstance(item, Undefined):
+                item = default
+        return item
+
+    return f
+
+
+@contextfilter
+def compatibility_selectattr_filter(context, sequence, attribute_str, test_name, *args, **kwargs):
+    f = make_attrgetter(context.environment, attribute_str)
+    for item in sequence:
+        if call_test(context.environment, test_name, f(item), args, kwargs):
+            yield item
+
+
+def prepare_map(context, args, kwargs):
+    if len(args) == 0 and "attribute" in kwargs:
+        attribute = kwargs.pop("attribute")
+        default = kwargs.pop("default", None)
+        if kwargs:
+            raise FilterArgumentError("Unexpected keyword argument {0!r}".format(next(iter(kwargs))))
+        func = make_attrgetter(context.environment, attribute, default=default)
+    else:
+        try:
+            name = args[0]
+            args = args[1:]
+        except LookupError:
+            raise FilterArgumentError("map requires a filter argument")
+
+        def func(item):
+            return call_filter(context.environment, name, item, args, kwargs, context=context)
+
+    return func
+
+
+@contextfilter
+def compatibility_map_filter(context, seq, *args, **kwargs):
+    func = prepare_map(context, args, kwargs)
+    if seq:
+        for item in seq:
+            yield func(item)
+
+
+class FilterModule:
+    ''' Jinja2 compat filters '''
+
+    def filters(self):
+        return {
+            'selectattr': compatibility_selectattr_filter,
+            'map': compatibility_map_filter,
+        }

tests/integration/targets/prepare_jinja2_compat/tasks/main.yml

@@ -0,0 +1 @@
+---

tests/integration/targets/prepare_jinja2_compat/test_plugins/jinja_compatibility.py

@@ -2,6 +2,10 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 
+def compatibility_equalto_test(a, b):
+    return a == b
+
+
 def compatibility_in_test(a, b):
     return a in b
 
@@ -11,5 +15,6 @@ class TestModule:
 
     def tests(self):
         return {
+            'equalto': compatibility_equalto_test,
             'in': compatibility_in_test,
         }

tests/integration/targets/setup_acme/tasks/obtain-cert.yml

@@ -11,7 +11,7 @@
          'secp384r1' if key_type == 'ec384' else
          'secp521r1' if key_type == 'ec521' else
          'invalid value for key_type!' }}
-    passphrase: "{{ certificate_passphrase | default(omit, true) }}"
+    passphrase: "{{ certificate_passphrase | default(omit) | default(omit, true) }}"
     cipher: "{{ 'auto' if certificate_passphrase | default() else omit }}"
     force: true
 ## CSR ########################################################################################
@@ -19,7 +19,7 @@
   openssl_csr:
     path: "{{ remote_tmp_dir }}/{{ certificate_name }}.csr"
     privatekey_path: "{{ remote_tmp_dir }}/{{ certificate_name }}.key"
-    privatekey_passphrase: "{{ certificate_passphrase | default(omit, true) }}"
+    privatekey_passphrase: "{{ certificate_passphrase | default(omit) | default(omit, true) }}"
     subject_alt_name: "{{ subject_alt_name }}"
     subject_alt_name_critical: "{{ subject_alt_name_critical }}"
     return_content: true
@@ -33,7 +33,7 @@
     validate_certs: no
     account_key: "{{ (remote_tmp_dir ~ '/' ~ account_key ~ '.pem') if account_key_content is not defined else omit }}"
     account_key_content: "{{ account_key_content | default(omit) }}"
-    account_key_passphrase: "{{ account_key_passphrase | default(omit, true) }}"
+    account_key_passphrase: "{{ account_key_passphrase | default(omit) | default(omit, true) }}"
     modify_account: "{{ modify_account }}"
     csr: "{{ omit if use_csr_content | default(false) else remote_tmp_dir ~ '/' ~ certificate_name ~ '.csr' }}"
     csr_content: "{{ csr_result.csr if use_csr_content | default(false) else omit }}"
@@ -68,21 +68,21 @@
     body: "{{ item.value }}"
   with_dict: "{{ challenge_data.challenge_data_dns }}"
   when: "challenge_data is changed and challenge == 'dns-01'"
-- name: ({{ certgen_title }}) Create TLS ALPN challenges (acm_challenge_cert_helper)
+- name: ({{ certgen_title }}) Create TLS ALPN challenges (acme_challenge_cert_helper)
   acme_challenge_cert_helper:
     challenge: tls-alpn-01
     challenge_data: "{{ item.value['tls-alpn-01'] }}"
     private_key_src: "{{ remote_tmp_dir }}/{{ certificate_name }}.key"
-    private_key_passphrase: "{{ certificate_passphrase | default(omit, true) }}"
-  with_dict: "{{ challenge_data.challenge_data if challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls is defined and challenge_alpn_tls == 'acme_challenge_cert_helper') else {} }}"
+    private_key_passphrase: "{{ certificate_passphrase | default(omit) | default(omit, true) }}"
+  with_dict: "{{ challenge_data.challenge_data if challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls | default('der-value-b64') == 'acme_challenge_cert_helper') else {} }}"
   register: tls_alpn_challenges
-  when: "challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls is defined and challenge_alpn_tls == 'acme_challenge_cert_helper')"
+  when: "challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls | default('der-value-b64') == 'acme_challenge_cert_helper')"
 - name: ({{ certgen_title }}) Read private key
   slurp:
     src: '{{ remote_tmp_dir }}/{{ certificate_name }}.key'
   register: slurp
-  when: "challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls is defined and challenge_alpn_tls == 'acme_challenge_cert_helper')"
-- name: ({{ certgen_title }}) Set TLS ALPN challenges (acm_challenge_cert_helper)
+  when: "challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls | default('der-value-b64') == 'acme_challenge_cert_helper')"
+- name: ({{ certgen_title }}) Set TLS ALPN challenges (acme_challenge_cert_helper)
   uri:
     url: "http://{{ acme_host }}:5000/tls-alpn/{{ item.domain }}/{{ item.identifier }}/certificate-and-key"
     method: PUT
@@ -90,8 +90,8 @@
     body: "{{ item.challenge_certificate }}\n{{ slurp.content | b64decode }}"
     headers:
       content-type: "application/pem-certificate-chain"
-  with_items: "{{ tls_alpn_challenges.results if challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls is defined and challenge_alpn_tls == 'acme_challenge_cert_helper') else [] }}"
-  when: "challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls is defined and challenge_alpn_tls == 'acme_challenge_cert_helper')"
+  with_items: "{{ tls_alpn_challenges.results if challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls | default('der-value-b64') == 'acme_challenge_cert_helper') else [] }}"
+  when: "challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls | default('der-value-b64') == 'acme_challenge_cert_helper')"
 - name: ({{ certgen_title }}) Create TLS ALPN challenges (der-value-b64)
   uri:
     url: "http://{{ acme_host }}:5000/tls-alpn/{{ item.value['tls-alpn-01'].resource }}/{{ item.value['tls-alpn-01'].resource_original }}/der-value-b64"
@@ -100,8 +100,8 @@
     body: "{{ item.value['tls-alpn-01'].resource_value }}"
     headers:
       content-type: "application/octet-stream"
-  with_dict: "{{ challenge_data.challenge_data if challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls is not defined or challenge_alpn_tls == 'der-value-b64') else [] }}"
-  when: "challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls is not defined or challenge_alpn_tls == 'der-value-b64')"
+  with_dict: "{{ challenge_data.challenge_data if challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls | default('der-value-b64') == 'der-value-b64') else {} }}"
+  when: "challenge_data is changed and challenge == 'tls-alpn-01' and (challenge_alpn_tls | default('der-value-b64') == 'der-value-b64')"
 ## ACME STEP 2 ################################################################################
 - name: ({{ certgen_title }}) Obtain cert, step 2
   acme_certificate:
@@ -111,7 +111,7 @@
     validate_certs: no
     account_key: "{{ (remote_tmp_dir ~ '/' ~ account_key ~ '.pem') if account_key_content is not defined else omit }}"
     account_key_content: "{{ account_key_content | default(omit) }}"
-    account_key_passphrase: "{{ account_key_passphrase | default(omit, true) }}"
+    account_key_passphrase: "{{ account_key_passphrase | default(omit) | default(omit, true) }}"
     account_uri: "{{ challenge_data.account_uri }}"
     modify_account: "{{ modify_account }}"
     csr: "{{ omit if use_csr_content | default(false) else remote_tmp_dir ~ '/' ~ certificate_name ~ '.csr' }}"

tests/integration/targets/setup_openssl/meta/main.yml

@@ -1,3 +1,4 @@
 dependencies:
+  - setup_python_info
   - setup_remote_constraints
   - setup_pkg_mgr

tests/integration/targets/setup_openssl/tasks/main.yml

@@ -8,7 +8,9 @@
   command: "{{ ansible_python.executable }} -c 'import os; print(dict(os.environ))'"
   register: sys_environment
 
-- debug: var=sys_environment
+- name: Show system environment
+  debug:
+    var: sys_environment.stdout_lines
 
 - name: Default value for OpenSSL binary path
   set_fact:
@@ -18,14 +20,19 @@
   include_vars: '{{ ansible_os_family }}.yml'
   when: not ansible_os_family == "Darwin"
 
+- name: Check whether OpenSSL is there
+  command: "{{ openssl_binary }} version"
+  register: openssl_version_full
+  ignore_errors: true
+
 - name: Install OpenSSL
   become: true
   package:
     name: '{{ openssl_package_name }}'
-  when: not ansible_os_family == 'Darwin'
+  when: not ansible_os_family == 'Darwin' and openssl_version_full is failed
 
 - name: Register openssl version (full)
-  shell: "{{ openssl_binary }} version"
+  command: "{{ openssl_binary }} version"
   register: openssl_version_full
 
 - name: Show openssl version (full)
@@ -60,7 +67,7 @@
         openssl_binary: "{{ brew_openssl_prefix.stdout }}/bin/openssl"
 
     - name: MACOS | Register openssl version (full)
-      shell: "{{ openssl_binary }} version"
+      command: "{{ openssl_binary }} version"
       register: openssl_version_full_again
       # We must use a different variable to prevent the 'when' condition of the surrounding block to fail
 
@@ -69,29 +76,37 @@
         var: openssl_version_full_again.stdout_lines
 
 - name: Register openssl version
-  shell: "{{ openssl_binary }}  version | cut -d' ' -f2"
+  shell: "{{ openssl_binary }} version | cut -d' ' -f2"
   register: openssl_version
 
 - when: ansible_facts.distribution ~ ansible_facts.distribution_major_version not in  ['CentOS6', 'RedHat6']
   block:
-  - name: Install cryptography (Python 3)
-    become: true
-    package:
-      name: '{{ cryptography_package_name_python3 }}'
-    when: not ansible_os_family == 'Darwin' and ansible_python_version is version('3.0', '>=')
 
-  - name: Install cryptography (Python 2)
-    become: true
-    package:
-      name: '{{ cryptography_package_name }}'
-    when: not ansible_os_family == 'Darwin' and ansible_python_version is version('3.0', '<')
+  - name: Install from system packages
+    when: ansible_os_family != "Darwin" and target_system_python
+    block:
 
-  - name: Install cryptography (Darwin)
-    become: true
-    pip:
-      name: cryptography>=3.3
-      extra_args: "-c {{ remote_constraints }}"
-    when: ansible_os_family == 'Darwin'
+    - name: Install cryptography (Python 3 from system packages)
+      become: true
+      package:
+        name: '{{ cryptography_package_name_python3 }}'
+      when: ansible_python_version is version('3.0', '>=')
+
+    - name: Install cryptography (Python 2 from system packages)
+      become: true
+      package:
+        name: '{{ cryptography_package_name }}'
+      when: ansible_python_version is version('3.0', '<')
+
+  - name: Install from PyPi
+    when: ansible_os_family == "Darwin" or not target_system_python
+    block:
+
+    - name: Install cryptography (PyPi)
+      become: true
+      pip:
+        name: 'cryptography{% if ansible_os_family == "Darwin" %}>=3.3{% endif %}'
+        extra_args: "-c {{ remote_constraints }}"
 
 - name: Register cryptography version
   command: "{{ ansible_python.executable }} -c 'import cryptography; print(cryptography.__version__)'"

tests/integration/targets/setup_pyopenssl/meta/main.yml

@@ -1,3 +1,4 @@
 dependencies:
+  - setup_python_info
   - setup_remote_constraints
   - setup_pkg_mgr

tests/integration/targets/setup_pyopenssl/tasks/main.yml

@@ -4,28 +4,34 @@
 # and should not be used as examples of how to write Ansible roles #
 ####################################################################
 
-- name: Include OS-specific variables
-  include_vars: '{{ ansible_os_family }}.yml'
-  when: not ansible_os_family == "Darwin"
+- name: Install from system packages
+  when: ansible_os_family != "Darwin" and target_system_python
+  block:
 
-- name: Install pyOpenSSL (Python 3)
-  become: true
-  package:
-    name: '{{ pyopenssl_package_name_python3 }}'
-  when: not ansible_os_family == 'Darwin' and ansible_python_version is version('3.0', '>=')
+  - name: Include OS-specific variables
+    include_vars: '{{ ansible_os_family }}.yml'
 
-- name: Install pyOpenSSL (Python 2)
-  become: true
-  package:
-    name: '{{ pyopenssl_package_name }}'
-  when: not ansible_os_family == 'Darwin' and ansible_python_version is version('3.0', '<')
+  - name: Install pyOpenSSL (Python 3 from system packages)
+    become: true
+    package:
+      name: '{{ pyopenssl_package_name_python3 }}'
+    when: ansible_python_version is version('3.0', '>=')
 
-- name: Install pyOpenSSL (Darwin)
-  become: true
-  pip:
-    name: pyOpenSSL
-    extra_args: "-c {{ remote_constraints }}"
-  when: ansible_os_family == 'Darwin'
+  - name: Install pyOpenSSL (Python 2 from system packages)
+    become: true
+    package:
+      name: '{{ pyopenssl_package_name }}'
+    when: ansible_python_version is version('3.0', '<')
+
+- name: Install from PyPi
+  when: ansible_os_family == "Darwin" or not target_system_python
+  block:
+
+  - name: Install pyOpenSSL (PyPi)
+    become: true
+    pip:
+      name: pyOpenSSL
+      extra_args: "-c {{ remote_constraints }}"
 
 - name: Register pyOpenSSL version
   command: "{{ ansible_python.executable }} -c 'import OpenSSL; print(OpenSSL.__version__)'"

tests/integration/targets/setup_python_info/filter_plugins/version_filter.py

@@ -0,0 +1,33 @@
+# (c) 2021, Felix Fontein <felix@fontein.de>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+def get_major_minor_version(version):
+    parts = version.split('.')[:2]
+    return '.'.join(parts)
+
+
+class FilterModule(object):
+    """ IP address and network manipulation filters """
+
+    def filters(self):
+        return {
+            'internal__get_major_minor_version': get_major_minor_version,
+        }

tests/integration/targets/setup_python_info/tasks/main.yml

@@ -0,0 +1,55 @@
+---
+- name: Gather facts on controller
+  setup:
+    gather_subset: '!all'
+  delegate_to: localhost
+  delegate_facts: true
+  run_once: true
+- name: Show variables
+  debug:
+    msg: |-
+      Target:
+        Python: {{ ansible_facts.python.version.major ~ '.' ~ ansible_facts.python.version.minor }}
+        OS family: {{ ansible_facts.os_family }}
+        Distribution: {{ ansible_facts.distribution }}
+        Distribution version: {{ ansible_facts.distribution_version | internal__get_major_minor_version }}
+        Distribution major version: {{ ansible_facts.distribution_major_version }}
+
+      Controller:
+        Python: {{ hostvars['localhost'].ansible_facts.python.version.major ~ '.' ~ hostvars['localhost'].ansible_facts.python.version.minor }}
+        OS family: {{ hostvars['localhost'].ansible_facts.os_family }}
+        Distribution: {{ hostvars['localhost'].ansible_facts.distribution }}
+        Distribution version: {{ hostvars['localhost'].ansible_facts.distribution_version | internal__get_major_minor_version }}
+        Distribution major version: {{ hostvars['localhost'].ansible_facts.distribution_major_version }}
+- name: Record information
+  set_fact:
+    target_system_python: >-
+      {{
+        (ansible_facts.python.version.major ~ '.' ~ ansible_facts.python.version.minor)
+        in
+        (
+          system_python_version_data[ansible_facts.distribution] |
+          default(system_python_version_data[ansible_facts.os_family])
+        )[ansible_facts.distribution_version | internal__get_major_minor_version]
+        | default(
+          (
+            system_python_version_data[ansible_facts.distribution] |
+            default(system_python_version_data[ansible_facts.os_family])
+          )[ansible_facts.distribution_major_version]
+        )
+      }}
+    controller_system_python: >-
+      {{
+        (hostvars['localhost'].ansible_facts.python.version.major ~ '.' ~ hostvars['localhost'].ansible_facts.python.version.minor)
+        in
+        (
+          system_python_version_data[hostvars['localhost'].ansible_facts.distribution] |
+          default(system_python_version_data[hostvars['localhost'].ansible_facts.os_family])
+        )[ansible_facts.distribution_version | internal__get_major_minor_version]
+        | default(
+          (
+            system_python_version_data[hostvars['localhost'].ansible_facts.distribution] |
+            default(system_python_version_data[hostvars['localhost'].ansible_facts.os_family])
+          )[hostvars['localhost'].ansible_facts.distribution_major_version]
+        )
+      }}

tests/integration/targets/setup_python_info/vars/main.yml

@@ -0,0 +1,50 @@
+---
+system_python_version_data:
+  CentOS:
+    '6':
+      - '2.6'
+    '7':
+      - '2.7'
+    '8':
+      - '3.6'
+  Fedora:
+    '30':
+      - '3.7'
+    '31':
+      - '3.7'
+    '32':
+      - '3.8'
+    '33':
+      - '3.9'
+    '34':
+      - '3.9'
+  Ubuntu:
+    '16':
+      - '2.7'
+    '18':
+      - '3.6'
+    '20':
+      - '3.8'
+  Darwin:
+    '10.11':
+      - '2.7'
+    '10.15':
+      - '3.8'
+    '11.1':
+      - '3.9'
+  FreeBSD:
+    '12.1':
+      - '3.6'
+    '12.2':
+      - '3.7'
+    '13.0':
+      - '3.7'
+  RedHat:
+    '7':
+      - '2.7'
+    '8':
+      - '3.6'
+  Suse:
+    '15':
+      - '2.7'
+      - '3.6'

tests/integration/targets/x509_certificate-acme/meta/main.yml

@@ -1,3 +1,4 @@
 dependencies:
   - setup_acme
   - setup_remote_tmp_dir
+  - prepare_jinja2_compat

tests/integration/targets/x509_certificate-acme/tasks/main.yml

@@ -109,6 +109,13 @@
     regexp: 'os\.remove\(wellknown_path\)'
     replace: 'pass'
 
+- name: "Monkey-patch acme-tiny: Allow to run with Python 2"
+  replace:
+    path: "{{ remote_tmp_dir }}/acme-tiny"
+    regexp: '#!/usr/bin/env python3'
+    replace: '#!/usr/bin/env python'
+  when: ansible_facts.python.version.major == 2
+
 - name: Create challenges directory
   file:
     path: '{{ remote_tmp_dir }}/challenges'

tests/integration/targets/x509_certificate_info/meta/main.yml

@@ -2,3 +2,4 @@ dependencies:
   - setup_openssl
   - setup_pyopenssl
   - setup_remote_tmp_dir
+  - prepare_jinja2_compat

tests/integration/targets/x509_certificate_info/test_plugins/jinja_compatibility.py

@@ -1,15 +0,0 @@
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-
-def compatibility_in_test(a, b):
-    return a in b
-
-
-class TestModule:
-    ''' Ansible math jinja2 tests '''
-
-    def tests(self):
-        return {
-            'in': compatibility_in_test,
-        }