crypto.math module utils: add some tests, fix quick_is_not_prime() for small primes (#733) (#734)

* Fix quick_is_not_prime() for small primes. Add some tests.

* Fix return value of convert_int_to_bytes(0, 0) on Python 2.

* Add some more test cases.

* Simplify the changelog and point out that these errors only happen for cases not happening in regular use.

(cherry picked from commit 0c62837296)

changelogs/fragments/733-math-prime.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - "crypto.math module utils - change return values for ``quick_is_not_prime()`` for special cases that do not appear when using the collection (https://github.com/ansible-collections/community.crypto/pull/733)."

plugins/module_utils/crypto/math.py

@@ -53,9 +53,18 @@ def quick_is_not_prime(n):
     that we couldn't detect quickly whether it is not prime.
     '''
     if n <= 2:
-        return True
+        return n < 2
     # The constant in the next line is the product of all primes < 200
-    if simple_gcd(n, 7799922041683461553249199106329813876687996789903550945093032474868511536164700810) > 1:
+    prime_product = 7799922041683461553249199106329813876687996789903550945093032474868511536164700810
+    gcd = simple_gcd(n, prime_product)
+    if gcd > 1:
+        if n < 200 and gcd == n:
+            # Explicitly check for all primes < 200
+            return n not in (
+                2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,
+                89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179,
+                181, 191, 193, 197, 199,
+            )
         return True
     # TODO: maybe do some iterations of Miller-Rabin to increase confidence
     # (https://en.wikipedia.org/wiki/Miller%E2%80%93Rabin_primality_test)

tests/unit/plugins/module_utils/crypto/test_math.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2024, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import pytest
+
+from ansible_collections.community.crypto.plugins.module_utils.crypto.math import (
+    binary_exp_mod,
+    simple_gcd,
+    quick_is_not_prime,
+)
+
+
+@pytest.mark.parametrize('f, e, m, result', [
+    (0, 0, 5, 1),
+    (0, 1, 5, 0),
+    (2, 1, 5, 2),
+    (2, 2, 5, 4),
+    (2, 3, 5, 3),
+    (2, 10, 5, 4),
+])
+def test_binary_exp_mod(f, e, m, result):
+    value = binary_exp_mod(f, e, m)
+    print(value)
+    assert value == result
+
+
+@pytest.mark.parametrize('a, b, result', [
+    (0, -123, -123),
+    (0, 123, 123),
+    (-123, 0, -123),
+    (123, 0, 123),
+    (-123, 1, 1),
+    (123, 1, 1),
+    (1, -123, -1),
+    (1, 123, 1),
+    (1024, 10, 2),
+])
+def test_simple_gcd(a, b, result):
+    value = simple_gcd(a, b)
+    print(value)
+    assert value == result
+
+
+@pytest.mark.parametrize('n, result', [
+    (-2, True),
+    (0, True),
+    (1, True),
+    (2, False),
+    (3, False),
+    (4, True),
+    (5, False),
+    (6, True),
+    (7, False),
+    (8, True),
+    (9, True),
+    (10, True),
+    (211, False),  # the smallest prime number >= 200
+])
+def test_quick_is_not_prime(n, result):
+    value = quick_is_not_prime(n)
+    print(value)
+    assert value == result