2014-10-19 05:14:30 +00:00
|
|
|
# (c) 2014, James Tanner <tanner.jc@gmail.com>
|
2016-09-13 14:30:17 +00:00
|
|
|
# (c) 2016, Adrian Likins <alikins@redhat.com>
|
|
|
|
# (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com>
|
2014-10-19 05:14:30 +00:00
|
|
|
#
|
|
|
|
# Ansible is free software: you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
|
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
|
|
# (at your option) any later version.
|
|
|
|
#
|
|
|
|
# Ansible is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
2014-10-29 00:44:21 +00:00
|
|
|
# Make coding more python3-ish
|
|
|
|
from __future__ import (absolute_import, division, print_function)
|
|
|
|
__metaclass__ = type
|
|
|
|
|
2014-10-19 05:14:30 +00:00
|
|
|
import os
|
|
|
|
import shlex
|
|
|
|
import shutil
|
2015-08-26 17:21:20 +00:00
|
|
|
import sys
|
2014-10-19 05:14:30 +00:00
|
|
|
import tempfile
|
2016-01-05 17:04:38 +00:00
|
|
|
import random
|
2014-10-19 05:14:30 +00:00
|
|
|
from io import BytesIO
|
|
|
|
from subprocess import call
|
|
|
|
from hashlib import sha256
|
|
|
|
from binascii import hexlify
|
|
|
|
from binascii import unhexlify
|
2016-09-07 05:54:17 +00:00
|
|
|
from hashlib import md5
|
2016-07-20 10:32:23 +00:00
|
|
|
|
2015-07-11 18:24:00 +00:00
|
|
|
# Note: Only used for loading obsolete VaultAES files. All files are written
|
|
|
|
# using the newer VaultAES256 which does not require md5
|
|
|
|
|
2014-10-19 05:14:30 +00:00
|
|
|
try:
|
|
|
|
from Crypto.Hash import SHA256, HMAC
|
|
|
|
HAS_HASH = True
|
|
|
|
except ImportError:
|
|
|
|
HAS_HASH = False
|
|
|
|
|
|
|
|
# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
|
|
|
|
try:
|
|
|
|
from Crypto.Util import Counter
|
|
|
|
HAS_COUNTER = True
|
|
|
|
except ImportError:
|
|
|
|
HAS_COUNTER = False
|
|
|
|
|
|
|
|
# KDF import fails for 2.0.1, requires >= 2.6.1 from pip
|
|
|
|
try:
|
|
|
|
from Crypto.Protocol.KDF import PBKDF2
|
|
|
|
HAS_PBKDF2 = True
|
|
|
|
except ImportError:
|
|
|
|
HAS_PBKDF2 = False
|
|
|
|
|
|
|
|
# AES IMPORTS
|
|
|
|
try:
|
|
|
|
from Crypto.Cipher import AES as AES
|
2015-04-15 18:08:53 +00:00
|
|
|
HAS_AES = True
|
2014-10-19 05:14:30 +00:00
|
|
|
except ImportError:
|
2015-04-15 18:08:53 +00:00
|
|
|
HAS_AES = False
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
from ansible.errors import AnsibleError
|
2017-03-23 20:35:05 +00:00
|
|
|
from ansible.module_utils.six import PY3, binary_type
|
|
|
|
from ansible.module_utils.six.moves import zip
|
2016-09-07 05:54:17 +00:00
|
|
|
from ansible.module_utils._text import to_bytes, to_text
|
|
|
|
|
|
|
|
try:
|
|
|
|
from __main__ import display
|
|
|
|
except ImportError:
|
|
|
|
from ansible.utils.display import Display
|
|
|
|
display = Display()
|
|
|
|
|
Use PBKDF2HMAC() from cryptography for vault keys.
When stretching the key for vault files, use PBKDF2HMAC() from the
cryptography package instead of pycrypto. This will speed up the opening
of vault files by ~10x.
The problem is here in lib/ansible/utils/vault.py:
hash_function = SHA256
# make two keys and one iv
pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest()
derivedkey = PBKDF2(password, salt, dkLen=(2 * keylength) + ivlength,
count=10000, prf=pbkdf2_prf)
`PBKDF2()` calls a Python callback function (`pbkdf2_pr()`) 10000 times.
If one has several vault files, this will cause excessive start times
with `ansible` or `ansible-playbook` (we experience ~15 second startup
times).
Testing the original implementation in 1.9.2 with a vault file:
In [2]: %timeit v.decrypt(encrypted_data)
1 loops, best of 3: 265 ms per loop
Having a recent OpenSSL version and using the vault.py changes in this commit:
In [2]: %timeit v.decrypt(encrypted_data)
10 loops, best of 3: 23.2 ms per loop
2015-07-22 17:52:42 +00:00
|
|
|
# OpenSSL pbkdf2_hmac
|
|
|
|
HAS_PBKDF2HMAC = False
|
|
|
|
try:
|
|
|
|
from cryptography.hazmat.primitives.hashes import SHA256 as c_SHA256
|
|
|
|
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
|
|
|
|
from cryptography.hazmat.backends import default_backend
|
|
|
|
HAS_PBKDF2HMAC = True
|
|
|
|
except ImportError:
|
|
|
|
pass
|
2016-05-19 18:39:34 +00:00
|
|
|
except Exception as e:
|
2016-10-12 16:58:11 +00:00
|
|
|
display.vvvv("Optional dependency 'cryptography' raised an exception, falling back to 'Crypto'.")
|
2016-07-20 10:32:23 +00:00
|
|
|
import traceback
|
2016-10-12 16:58:11 +00:00
|
|
|
display.vvvv("Traceback from import of cryptography was {0}".format(traceback.format_exc()))
|
Use PBKDF2HMAC() from cryptography for vault keys.
When stretching the key for vault files, use PBKDF2HMAC() from the
cryptography package instead of pycrypto. This will speed up the opening
of vault files by ~10x.
The problem is here in lib/ansible/utils/vault.py:
hash_function = SHA256
# make two keys and one iv
pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest()
derivedkey = PBKDF2(password, salt, dkLen=(2 * keylength) + ivlength,
count=10000, prf=pbkdf2_prf)
`PBKDF2()` calls a Python callback function (`pbkdf2_pr()`) 10000 times.
If one has several vault files, this will cause excessive start times
with `ansible` or `ansible-playbook` (we experience ~15 second startup
times).
Testing the original implementation in 1.9.2 with a vault file:
In [2]: %timeit v.decrypt(encrypted_data)
1 loops, best of 3: 265 ms per loop
Having a recent OpenSSL version and using the vault.py changes in this commit:
In [2]: %timeit v.decrypt(encrypted_data)
10 loops, best of 3: 23.2 ms per loop
2015-07-22 17:52:42 +00:00
|
|
|
|
|
|
|
HAS_ANY_PBKDF2HMAC = HAS_PBKDF2 or HAS_PBKDF2HMAC
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
|
2016-09-07 05:54:17 +00:00
|
|
|
CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform." \
|
|
|
|
" You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto"
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
b_HEADER = b'$ANSIBLE_VAULT'
|
|
|
|
CIPHER_WHITELIST = frozenset((u'AES', u'AES256'))
|
2016-08-24 00:03:11 +00:00
|
|
|
CIPHER_WRITE_WHITELIST = frozenset((u'AES256',))
|
2015-10-16 17:04:37 +00:00
|
|
|
# See also CIPHER_MAPPING at the bottom of the file which maps cipher strings
|
|
|
|
# (used in VaultFile header) to a cipher class
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-04-16 16:53:59 +00:00
|
|
|
|
2015-06-16 13:20:15 +00:00
|
|
|
def check_prereqs():
|
|
|
|
|
Use PBKDF2HMAC() from cryptography for vault keys.
When stretching the key for vault files, use PBKDF2HMAC() from the
cryptography package instead of pycrypto. This will speed up the opening
of vault files by ~10x.
The problem is here in lib/ansible/utils/vault.py:
hash_function = SHA256
# make two keys and one iv
pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest()
derivedkey = PBKDF2(password, salt, dkLen=(2 * keylength) + ivlength,
count=10000, prf=pbkdf2_prf)
`PBKDF2()` calls a Python callback function (`pbkdf2_pr()`) 10000 times.
If one has several vault files, this will cause excessive start times
with `ansible` or `ansible-playbook` (we experience ~15 second startup
times).
Testing the original implementation in 1.9.2 with a vault file:
In [2]: %timeit v.decrypt(encrypted_data)
1 loops, best of 3: 265 ms per loop
Having a recent OpenSSL version and using the vault.py changes in this commit:
In [2]: %timeit v.decrypt(encrypted_data)
10 loops, best of 3: 23.2 ms per loop
2015-07-22 17:52:42 +00:00
|
|
|
if not HAS_AES or not HAS_COUNTER or not HAS_ANY_PBKDF2HMAC or not HAS_HASH:
|
2015-07-11 18:24:00 +00:00
|
|
|
raise AnsibleError(CRYPTO_UPGRADE)
|
2015-06-16 13:20:15 +00:00
|
|
|
|
2016-08-24 00:03:11 +00:00
|
|
|
|
|
|
|
class AnsibleVaultError(AnsibleError):
|
|
|
|
pass
|
|
|
|
|
2016-09-07 05:54:17 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def is_encrypted(data):
|
2016-08-24 00:03:11 +00:00
|
|
|
""" Test if this is vault encrypted data blob
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
:arg data: a byte or text string to test whether it is recognized as vault
|
|
|
|
encrypted data
|
2016-08-24 00:03:11 +00:00
|
|
|
:returns: True if it is recognized. Otherwise, False.
|
|
|
|
"""
|
2016-09-13 14:30:17 +00:00
|
|
|
try:
|
|
|
|
# Make sure we have a byte string and that it only contains ascii
|
|
|
|
# bytes.
|
|
|
|
b_data = to_bytes(to_text(data, encoding='ascii', errors='strict', nonstring='strict'), encoding='ascii', errors='strict')
|
|
|
|
except (UnicodeError, TypeError):
|
|
|
|
# The vault format is pure ascii so if we failed to encode to bytes
|
|
|
|
# via ascii we know that this is not vault data.
|
|
|
|
# Similarly, if it's not a string, it's not vault data
|
|
|
|
return False
|
|
|
|
|
2016-08-24 00:03:11 +00:00
|
|
|
if b_data.startswith(b_HEADER):
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
|
2016-09-07 05:54:17 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def is_encrypted_file(file_obj, start_pos=0, count=-1):
|
2016-08-24 00:03:11 +00:00
|
|
|
"""Test if the contents of a file obj are a vault encrypted data blob.
|
|
|
|
|
|
|
|
:arg file_obj: A file object that will be read from.
|
2016-09-13 14:30:17 +00:00
|
|
|
:kwarg start_pos: A byte offset in the file to start reading the header
|
|
|
|
from. Defaults to 0, the beginning of the file.
|
|
|
|
:kwarg count: Read up to this number of bytes from the file to determine
|
|
|
|
if it looks like encrypted vault data. The default is -1, read to the
|
|
|
|
end of file.
|
|
|
|
:returns: True if the file looks like a vault file. Otherwise, False.
|
2016-08-24 00:03:11 +00:00
|
|
|
"""
|
|
|
|
# read the header and reset the file stream to where it started
|
|
|
|
current_position = file_obj.tell()
|
2016-09-13 14:30:17 +00:00
|
|
|
try:
|
|
|
|
file_obj.seek(start_pos)
|
2017-02-26 23:55:16 +00:00
|
|
|
return is_encrypted(file_obj.read(count))
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
finally:
|
|
|
|
file_obj.seek(current_position)
|
|
|
|
|
2016-08-24 00:03:11 +00:00
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
class VaultLib:
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2017-03-07 20:30:09 +00:00
|
|
|
def __init__(self, b_password):
|
|
|
|
self.b_password = to_bytes(b_password, errors='strict', encoding='utf-8')
|
2014-10-19 05:14:30 +00:00
|
|
|
self.cipher_name = None
|
2015-08-24 22:49:55 +00:00
|
|
|
self.b_version = b'1.1'
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
@staticmethod
|
|
|
|
def is_encrypted(data):
|
2015-08-24 22:49:55 +00:00
|
|
|
""" Test if this is vault encrypted data
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
:arg data: a byte or text string or a python3 to test for whether it is
|
2015-08-24 22:49:55 +00:00
|
|
|
recognized as vault encrypted data
|
|
|
|
:returns: True if it is recognized. Otherwise, False.
|
|
|
|
"""
|
|
|
|
|
2016-08-24 00:03:11 +00:00
|
|
|
# This could in the future, check to see if the data is a vault blob and
|
|
|
|
# is encrypted with a key associated with this vault
|
|
|
|
# instead of just checking the format.
|
2016-09-13 14:30:17 +00:00
|
|
|
display.deprecated(u'vault.VaultLib.is_encrypted is deprecated. Use vault.is_encrypted instead', version='2.4')
|
2016-08-24 00:03:11 +00:00
|
|
|
return is_encrypted(data)
|
2016-07-22 13:06:06 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
@staticmethod
|
|
|
|
def is_encrypted_file(file_obj):
|
|
|
|
display.deprecated(u'vault.VaultLib.is_encrypted_file is deprecated. Use vault.is_encrypted_file instead', version='2.4')
|
2016-08-24 00:03:11 +00:00
|
|
|
return is_encrypted_file(file_obj)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def encrypt(self, plaintext):
|
2015-08-24 22:49:55 +00:00
|
|
|
"""Vault encrypt a piece of data.
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
:arg plaintext: a text or byte string to encrypt.
|
2015-08-24 22:49:55 +00:00
|
|
|
:returns: a utf-8 encoded byte str of encrypted data. The string
|
|
|
|
contains a header identifying this as vault encrypted data and
|
|
|
|
formatted to newline terminated lines of 80 characters. This is
|
|
|
|
suitable for dumping as is to a vault file.
|
2016-08-24 00:03:11 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
If the string passed in is a text string, it will be encoded to UTF-8
|
|
|
|
before encryption.
|
2015-08-24 22:49:55 +00:00
|
|
|
"""
|
2016-09-13 14:30:17 +00:00
|
|
|
b_plaintext = to_bytes(plaintext, errors='surrogate_or_strict')
|
2016-08-24 00:03:11 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
if is_encrypted(b_plaintext):
|
2015-08-26 16:49:54 +00:00
|
|
|
raise AnsibleError("input is already encrypted")
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-08-25 09:24:23 +00:00
|
|
|
if not self.cipher_name or self.cipher_name not in CIPHER_WRITE_WHITELIST:
|
2015-08-24 22:49:55 +00:00
|
|
|
self.cipher_name = u"AES256"
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-10-16 17:04:37 +00:00
|
|
|
try:
|
2017-02-27 04:02:49 +00:00
|
|
|
this_cipher = CIPHER_MAPPING[self.cipher_name]()
|
2015-10-16 17:04:37 +00:00
|
|
|
except KeyError:
|
2015-08-24 22:49:55 +00:00
|
|
|
raise AnsibleError(u"{0} cipher could not be found".format(self.cipher_name))
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
# encrypt data
|
2016-09-13 14:30:17 +00:00
|
|
|
b_ciphertext = this_cipher.encrypt(b_plaintext, self.b_password)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
# format the data for output to the file
|
2016-09-13 14:30:17 +00:00
|
|
|
b_vaulttext = self._format_output(b_ciphertext)
|
|
|
|
return b_vaulttext
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def decrypt(self, vaulttext, filename=None):
|
2015-08-24 22:49:55 +00:00
|
|
|
"""Decrypt a piece of vault encrypted data.
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
:arg vaulttext: a string to decrypt. Since vault encrypted data is an
|
2015-08-24 22:49:55 +00:00
|
|
|
ascii text format this can be either a byte str or unicode string.
|
2016-09-13 14:30:17 +00:00
|
|
|
:kwarg filename: a filename that the data came from. This is only
|
|
|
|
used to make better error messages in case the data cannot be
|
|
|
|
decrypted.
|
2015-08-24 22:49:55 +00:00
|
|
|
:returns: a byte string containing the decrypted data
|
|
|
|
"""
|
2016-09-13 14:30:17 +00:00
|
|
|
b_vaulttext = to_bytes(vaulttext, errors='strict', encoding='utf-8')
|
2015-04-15 18:08:53 +00:00
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
if self.b_password is None:
|
2015-07-11 18:24:00 +00:00
|
|
|
raise AnsibleError("A vault password must be specified to decrypt data")
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
if not is_encrypted(b_vaulttext):
|
2016-08-24 00:03:11 +00:00
|
|
|
msg = "input is not vault encrypted data"
|
2016-06-18 13:30:08 +00:00
|
|
|
if filename:
|
2016-08-24 00:03:11 +00:00
|
|
|
msg += "%s is not a vault encrypted file" % filename
|
2016-06-18 13:30:08 +00:00
|
|
|
raise AnsibleError(msg)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# clean out header
|
2016-09-13 14:30:17 +00:00
|
|
|
b_vaulttext = self._split_header(b_vaulttext)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# create the cipher object
|
2017-02-27 04:02:49 +00:00
|
|
|
if self.cipher_name in CIPHER_WHITELIST:
|
|
|
|
this_cipher = CIPHER_MAPPING[self.cipher_name]()
|
2014-10-19 05:14:30 +00:00
|
|
|
else:
|
2015-08-24 22:49:55 +00:00
|
|
|
raise AnsibleError("{0} cipher could not be found".format(self.cipher_name))
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
# try to unencrypt vaulttext
|
|
|
|
b_plaintext = this_cipher.decrypt(b_vaulttext, self.b_password)
|
|
|
|
if b_plaintext is None:
|
2016-06-18 13:30:08 +00:00
|
|
|
msg = "Decryption failed"
|
|
|
|
if filename:
|
|
|
|
msg += " on %s" % filename
|
|
|
|
raise AnsibleError(msg)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
return b_plaintext
|
2015-08-24 22:49:55 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def _format_output(self, b_ciphertext):
|
2015-08-24 22:49:55 +00:00
|
|
|
""" Add header and format to 80 columns
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
:arg b_vaulttext: the encrypted and hexlified data as a byte string
|
2015-08-24 22:49:55 +00:00
|
|
|
:returns: a byte str that should be dumped into a file. It's
|
|
|
|
formatted to 80 char columns and has the header prepended
|
|
|
|
"""
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
if not self.cipher_name:
|
2015-07-11 18:24:00 +00:00
|
|
|
raise AnsibleError("the cipher must be set before adding a header")
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
header = b';'.join([b_HEADER, self.b_version,
|
2017-05-18 17:41:00 +00:00
|
|
|
to_bytes(self.cipher_name, 'utf-8', errors='strict')])
|
2016-09-13 14:30:17 +00:00
|
|
|
b_vaulttext = [header]
|
|
|
|
b_vaulttext += [b_ciphertext[i:i + 80] for i in range(0, len(b_ciphertext), 80)]
|
|
|
|
b_vaulttext += [b'']
|
|
|
|
b_vaulttext = b'\n'.join(b_vaulttext)
|
2015-08-24 22:49:55 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
return b_vaulttext
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def _split_header(self, b_vaulttext):
|
|
|
|
"""Retrieve information about the Vault and clean the data
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
When data is saved, it has a header prepended and is formatted into 80
|
|
|
|
character lines. This method extracts the information from the header
|
|
|
|
and then removes the header and the inserted newlines. The string returned
|
|
|
|
is suitable for processing by the Cipher classes.
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
:arg b_vaulttext: byte str containing the data from a save file
|
2015-08-24 22:49:55 +00:00
|
|
|
:returns: a byte str suitable for passing to a Cipher class's
|
|
|
|
decrypt() function.
|
|
|
|
"""
|
2014-10-19 05:14:30 +00:00
|
|
|
# used by decrypt
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
b_tmpdata = b_vaulttext.split(b'\n')
|
|
|
|
b_tmpheader = b_tmpdata[0].strip().split(b';')
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
self.b_version = b_tmpheader[1].strip()
|
|
|
|
self.cipher_name = to_text(b_tmpheader[2].strip())
|
|
|
|
b_ciphertext = b''.join(b_tmpdata[1:])
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
return b_ciphertext
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
class VaultEditor:
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2017-03-07 20:30:09 +00:00
|
|
|
def __init__(self, b_password):
|
|
|
|
self.vault = VaultLib(b_password)
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2016-08-24 00:03:11 +00:00
|
|
|
# TODO: mv shred file stuff to it's own class
|
2016-01-05 00:34:45 +00:00
|
|
|
def _shred_file_custom(self, tmp_path):
|
|
|
|
""""Destroy a file, when shred (core-utils) is not available
|
2016-01-13 20:34:12 +00:00
|
|
|
|
|
|
|
Unix `shred' destroys files "so that they can be recovered only with great difficulty with
|
|
|
|
specialised hardware, if at all". It is based on the method from the paper
|
|
|
|
"Secure Deletion of Data from Magnetic and Solid-State Memory",
|
2016-01-05 00:34:45 +00:00
|
|
|
Proceedings of the Sixth USENIX Security Symposium (San Jose, California, July 22-25, 1996).
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2016-01-05 00:34:45 +00:00
|
|
|
We do not go to that length to re-implement shred in Python; instead, overwriting with a block
|
2016-01-13 20:34:12 +00:00
|
|
|
of random data should suffice.
|
|
|
|
|
2016-01-04 17:13:59 +00:00
|
|
|
See https://github.com/ansible/ansible/pull/13700 .
|
|
|
|
"""
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2016-01-05 00:34:45 +00:00
|
|
|
file_len = os.path.getsize(tmp_path)
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2016-09-07 05:54:17 +00:00
|
|
|
if file_len > 0: # avoid work when file was empty
|
2017-05-18 17:41:00 +00:00
|
|
|
max_chunk_len = min(1024 * 1024 * 2, file_len)
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2016-01-21 15:53:02 +00:00
|
|
|
passes = 3
|
2017-05-18 17:41:00 +00:00
|
|
|
with open(tmp_path, "wb") as fh:
|
2016-01-21 15:53:02 +00:00
|
|
|
for _ in range(passes):
|
2017-05-18 17:41:00 +00:00
|
|
|
fh.seek(0, 0)
|
2016-01-21 15:53:02 +00:00
|
|
|
# get a random chunk of data, each pass with other length
|
2017-05-18 17:41:00 +00:00
|
|
|
chunk_len = random.randint(max_chunk_len // 2, max_chunk_len)
|
2016-01-21 15:53:02 +00:00
|
|
|
data = os.urandom(chunk_len)
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2016-01-21 15:53:02 +00:00
|
|
|
for _ in range(0, file_len // chunk_len):
|
|
|
|
fh.write(data)
|
|
|
|
fh.write(data[:file_len % chunk_len])
|
|
|
|
|
2016-09-07 05:54:17 +00:00
|
|
|
assert(fh.tell() == file_len) # FIXME remove this assert once we have unittests to check its accuracy
|
2016-01-21 15:53:02 +00:00
|
|
|
os.fsync(fh)
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2016-01-05 00:34:45 +00:00
|
|
|
def _shred_file(self, tmp_path):
|
|
|
|
"""Securely destroy a decrypted file
|
|
|
|
|
2016-01-13 20:34:12 +00:00
|
|
|
Note standard limitations of GNU shred apply (For flash, overwriting would have no effect
|
2016-01-05 00:34:45 +00:00
|
|
|
due to wear leveling; for other storage systems, the async kernel->filesystem->disk calls never
|
2016-01-13 20:34:12 +00:00
|
|
|
guarantee data hits the disk; etc). Furthermore, if your tmp dirs is on tmpfs (ramdisks),
|
|
|
|
it is a non-issue.
|
|
|
|
|
|
|
|
Nevertheless, some form of overwriting the data (instead of just removing the fs index entry) is
|
|
|
|
a good idea. If shred is not available (e.g. on windows, or no core-utils installed), fall back on
|
2016-01-05 00:34:45 +00:00
|
|
|
a custom shredding method.
|
|
|
|
"""
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2016-01-05 00:34:45 +00:00
|
|
|
if not os.path.isfile(tmp_path):
|
|
|
|
# file is already gone
|
2016-01-13 20:34:12 +00:00
|
|
|
return
|
|
|
|
|
2016-01-05 00:34:45 +00:00
|
|
|
try:
|
|
|
|
r = call(['shred', tmp_path])
|
2016-03-18 12:52:53 +00:00
|
|
|
except (OSError, ValueError):
|
2016-12-11 02:50:09 +00:00
|
|
|
# shred is not available on this system, or some other error occurred.
|
2016-03-18 12:52:53 +00:00
|
|
|
# ValueError caught because OS X El Capitan is raising an
|
|
|
|
# exception big enough to hit a limit in python2-2.7.11 and below.
|
|
|
|
# Symptom is ValueError: insecure pickle when shred is not
|
|
|
|
# installed there.
|
2016-01-05 17:04:38 +00:00
|
|
|
r = 1
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2016-01-05 00:34:45 +00:00
|
|
|
if r != 0:
|
2016-01-13 20:34:12 +00:00
|
|
|
# we could not successfully execute unix shred; therefore, do custom shred.
|
2016-01-05 00:34:45 +00:00
|
|
|
self._shred_file_custom(tmp_path)
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2015-12-30 17:21:34 +00:00
|
|
|
os.remove(tmp_path)
|
2016-01-13 20:34:12 +00:00
|
|
|
|
2015-08-26 13:47:37 +00:00
|
|
|
def _edit_file_helper(self, filename, existing_data=None, force_save=False):
|
2014-10-21 14:33:33 +00:00
|
|
|
|
|
|
|
# Create a tempfile
|
2017-03-27 20:52:18 +00:00
|
|
|
fd, tmp_path = tempfile.mkstemp()
|
|
|
|
os.close(fd)
|
2014-10-21 14:33:33 +00:00
|
|
|
|
2015-12-30 17:21:34 +00:00
|
|
|
try:
|
2017-03-27 20:52:18 +00:00
|
|
|
if existing_data:
|
|
|
|
self.write_data(existing_data, tmp_path, shred=False)
|
|
|
|
|
|
|
|
# drop the user into an editor on the tmp file
|
2015-12-30 17:21:34 +00:00
|
|
|
call(self._editor_shell_command(tmp_path))
|
2016-01-13 20:34:12 +00:00
|
|
|
except:
|
2015-12-30 17:21:34 +00:00
|
|
|
# whatever happens, destroy the decrypted file
|
|
|
|
self._shred_file(tmp_path)
|
2016-01-13 20:34:12 +00:00
|
|
|
raise
|
|
|
|
|
2016-11-04 17:28:44 +00:00
|
|
|
b_tmpdata = self.read_data(tmp_path)
|
2014-10-21 14:33:33 +00:00
|
|
|
|
2014-08-13 12:58:17 +00:00
|
|
|
# Do nothing if the content has not changed
|
2016-11-04 17:28:44 +00:00
|
|
|
if existing_data == b_tmpdata and not force_save:
|
2015-12-30 17:21:34 +00:00
|
|
|
self._shred_file(tmp_path)
|
2014-08-13 12:58:17 +00:00
|
|
|
return
|
|
|
|
|
2014-10-21 14:33:33 +00:00
|
|
|
# encrypt new data and write out to tmp
|
2016-08-24 00:03:11 +00:00
|
|
|
# An existing vaultfile will always be UTF-8,
|
|
|
|
# so decode to unicode here
|
2016-11-04 17:28:44 +00:00
|
|
|
b_ciphertext = self.vault.encrypt(b_tmpdata)
|
|
|
|
self.write_data(b_ciphertext, tmp_path)
|
2014-10-21 14:33:33 +00:00
|
|
|
|
|
|
|
# shuffle tmp file into place
|
2015-08-26 13:47:37 +00:00
|
|
|
self.shuffle_files(tmp_path, filename)
|
2014-10-21 14:33:33 +00:00
|
|
|
|
2017-04-18 17:09:02 +00:00
|
|
|
def _real_path(self, filename):
|
|
|
|
# '-' is special to VaultEditor, dont expand it.
|
|
|
|
if filename == '-':
|
|
|
|
return filename
|
|
|
|
|
|
|
|
real_path = os.path.realpath(filename)
|
|
|
|
return real_path
|
|
|
|
|
2017-02-17 15:12:14 +00:00
|
|
|
def encrypt_bytes(self, b_plaintext):
|
|
|
|
check_prereqs()
|
|
|
|
|
|
|
|
b_ciphertext = self.vault.encrypt(b_plaintext)
|
|
|
|
|
|
|
|
return b_ciphertext
|
|
|
|
|
2015-08-26 17:21:20 +00:00
|
|
|
def encrypt_file(self, filename, output_file=None):
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-06-16 13:20:15 +00:00
|
|
|
check_prereqs()
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-08-24 00:03:11 +00:00
|
|
|
# A file to be encrypted into a vaultfile could be any encoding
|
|
|
|
# so treat the contents as a byte string.
|
2017-02-24 17:35:39 +00:00
|
|
|
|
|
|
|
# follow the symlink
|
2017-04-18 17:09:02 +00:00
|
|
|
filename = self._real_path(filename)
|
2017-02-24 17:35:39 +00:00
|
|
|
|
2016-11-04 17:28:44 +00:00
|
|
|
b_plaintext = self.read_data(filename)
|
|
|
|
b_ciphertext = self.vault.encrypt(b_plaintext)
|
|
|
|
self.write_data(b_ciphertext, output_file or filename)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-08-26 17:21:20 +00:00
|
|
|
def decrypt_file(self, filename, output_file=None):
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-06-16 13:20:15 +00:00
|
|
|
check_prereqs()
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2017-02-24 17:35:39 +00:00
|
|
|
# follow the symlink
|
2017-04-18 17:09:02 +00:00
|
|
|
filename = self._real_path(filename)
|
2017-02-24 17:35:39 +00:00
|
|
|
|
2015-08-26 16:49:54 +00:00
|
|
|
ciphertext = self.read_data(filename)
|
2017-02-24 17:35:39 +00:00
|
|
|
|
2016-02-18 16:56:25 +00:00
|
|
|
try:
|
|
|
|
plaintext = self.vault.decrypt(ciphertext)
|
|
|
|
except AnsibleError as e:
|
2017-05-18 17:41:00 +00:00
|
|
|
raise AnsibleError("%s for %s" % (to_bytes(e), to_bytes(filename)))
|
2016-01-05 00:34:45 +00:00
|
|
|
self.write_data(plaintext, output_file or filename, shred=False)
|
2014-10-21 14:33:33 +00:00
|
|
|
|
2015-08-26 16:49:54 +00:00
|
|
|
def create_file(self, filename):
|
|
|
|
""" create a new encrypted file """
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-06-16 13:20:15 +00:00
|
|
|
check_prereqs()
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-08-26 16:49:54 +00:00
|
|
|
# FIXME: If we can raise an error here, we can probably just make it
|
|
|
|
# behave like edit instead.
|
|
|
|
if os.path.isfile(filename):
|
|
|
|
raise AnsibleError("%s exists, please use 'edit' instead" % filename)
|
2014-10-21 14:33:33 +00:00
|
|
|
|
2015-08-26 16:49:54 +00:00
|
|
|
self._edit_file_helper(filename)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-08-26 13:47:37 +00:00
|
|
|
def edit_file(self, filename):
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-06-16 13:20:15 +00:00
|
|
|
check_prereqs()
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2017-02-24 17:35:39 +00:00
|
|
|
# follow the symlink
|
2017-04-18 17:09:02 +00:00
|
|
|
filename = self._real_path(filename)
|
2017-02-24 17:35:39 +00:00
|
|
|
|
2015-08-26 16:49:54 +00:00
|
|
|
ciphertext = self.read_data(filename)
|
2017-02-24 17:35:39 +00:00
|
|
|
|
2016-02-18 16:56:25 +00:00
|
|
|
try:
|
|
|
|
plaintext = self.vault.decrypt(ciphertext)
|
|
|
|
except AnsibleError as e:
|
2017-05-18 17:41:00 +00:00
|
|
|
raise AnsibleError("%s for %s" % (to_bytes(e), to_bytes(filename)))
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-08-26 15:55:50 +00:00
|
|
|
if self.vault.cipher_name not in CIPHER_WRITE_WHITELIST:
|
2015-08-24 22:49:55 +00:00
|
|
|
# we want to get rid of files encrypted with the AES cipher
|
2015-08-26 16:49:54 +00:00
|
|
|
self._edit_file_helper(filename, existing_data=plaintext, force_save=True)
|
2015-08-24 22:49:55 +00:00
|
|
|
else:
|
2015-08-26 16:49:54 +00:00
|
|
|
self._edit_file_helper(filename, existing_data=plaintext, force_save=False)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-09-30 14:30:34 +00:00
|
|
|
def plaintext(self, filename):
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-06-16 13:20:15 +00:00
|
|
|
check_prereqs()
|
2015-08-26 16:49:54 +00:00
|
|
|
ciphertext = self.read_data(filename)
|
2016-02-18 16:56:25 +00:00
|
|
|
|
|
|
|
try:
|
|
|
|
plaintext = self.vault.decrypt(ciphertext)
|
|
|
|
except AnsibleError as e:
|
2017-05-18 17:41:00 +00:00
|
|
|
raise AnsibleError("%s for %s" % (to_bytes(e), to_bytes(filename)))
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-09-30 14:30:34 +00:00
|
|
|
return plaintext
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2017-03-07 20:30:09 +00:00
|
|
|
def rekey_file(self, filename, b_new_password):
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-06-16 13:20:15 +00:00
|
|
|
check_prereqs()
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2017-02-24 17:35:39 +00:00
|
|
|
# follow the symlink
|
2017-04-18 17:09:02 +00:00
|
|
|
filename = self._real_path(filename)
|
2017-02-24 17:35:39 +00:00
|
|
|
|
2015-10-31 17:56:14 +00:00
|
|
|
prev = os.stat(filename)
|
2015-08-26 16:49:54 +00:00
|
|
|
ciphertext = self.read_data(filename)
|
2017-02-24 17:35:39 +00:00
|
|
|
|
2016-02-18 16:56:25 +00:00
|
|
|
try:
|
|
|
|
plaintext = self.vault.decrypt(ciphertext)
|
|
|
|
except AnsibleError as e:
|
2017-05-18 17:41:00 +00:00
|
|
|
raise AnsibleError("%s for %s" % (to_bytes(e), to_bytes(filename)))
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-10-31 16:56:18 +00:00
|
|
|
# This is more or less an assert, see #18247
|
2017-03-07 20:30:09 +00:00
|
|
|
if b_new_password is None:
|
2016-10-31 16:56:18 +00:00
|
|
|
raise AnsibleError('The value for the new_password to rekey %s with is not valid' % filename)
|
|
|
|
|
2017-03-07 20:30:09 +00:00
|
|
|
new_vault = VaultLib(b_new_password)
|
2015-08-26 16:49:54 +00:00
|
|
|
new_ciphertext = new_vault.encrypt(plaintext)
|
2015-10-31 17:56:14 +00:00
|
|
|
|
2015-08-26 16:49:54 +00:00
|
|
|
self.write_data(new_ciphertext, filename)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-11-03 05:27:48 +00:00
|
|
|
# preserve permissions
|
2015-10-31 17:56:14 +00:00
|
|
|
os.chmod(filename, prev.st_mode)
|
|
|
|
os.chown(filename, prev.st_uid, prev.st_gid)
|
|
|
|
|
2014-10-19 05:14:30 +00:00
|
|
|
def read_data(self, filename):
|
2016-02-18 16:56:25 +00:00
|
|
|
|
2015-08-26 16:49:54 +00:00
|
|
|
try:
|
2015-08-26 17:21:20 +00:00
|
|
|
if filename == '-':
|
2015-08-27 07:07:42 +00:00
|
|
|
data = sys.stdin.read()
|
2015-08-26 17:21:20 +00:00
|
|
|
else:
|
2015-08-27 07:07:42 +00:00
|
|
|
with open(filename, "rb") as fh:
|
|
|
|
data = fh.read()
|
2015-08-26 16:49:54 +00:00
|
|
|
except Exception as e:
|
|
|
|
raise AnsibleError(str(e))
|
|
|
|
|
|
|
|
return data
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-08-24 00:03:11 +00:00
|
|
|
# TODO: add docstrings for arg types since this code is picky about that
|
2016-01-05 00:34:45 +00:00
|
|
|
def write_data(self, data, filename, shred=True):
|
2017-04-24 14:09:03 +00:00
|
|
|
"""Write the data bytes to given path
|
2016-08-24 00:03:11 +00:00
|
|
|
|
2017-04-24 14:09:03 +00:00
|
|
|
This is used to write a byte string to a file or stdout. It is used for
|
|
|
|
writing the results of vault encryption or decryption. It is used for
|
|
|
|
saving the ciphertext after encryption and it is also used for saving the
|
|
|
|
plaintext after decrypting a vault. The type of the 'data' arg should be bytes,
|
|
|
|
since in the plaintext case, the original contents can be of any text encoding
|
|
|
|
or arbitrary binary data.
|
|
|
|
|
|
|
|
When used to write the result of vault encryption, the val of the 'data' arg
|
|
|
|
should be a utf-8 encoded byte string and not a text typ and not a text type..
|
|
|
|
|
|
|
|
When used to write the result of vault decryption, the val of the 'data' arg
|
|
|
|
should be a byte string and not a text type.
|
|
|
|
|
|
|
|
:arg data: the byte string (bytes) data
|
2016-08-24 00:03:11 +00:00
|
|
|
:arg filename: filename to save 'data' to.
|
2017-04-24 14:09:03 +00:00
|
|
|
:arg shred: if shred==True, make sure that the original data is first shredded so that is cannot be recovered.
|
|
|
|
:returns: None
|
2016-01-05 00:34:45 +00:00
|
|
|
"""
|
2016-08-24 00:03:11 +00:00
|
|
|
# FIXME: do we need this now? data_bytes should always be a utf-8 byte string
|
|
|
|
b_file_data = to_bytes(data, errors='strict')
|
|
|
|
|
2017-04-24 14:09:03 +00:00
|
|
|
# get a ref to either sys.stdout.buffer for py3 or plain old sys.stdout for py2
|
|
|
|
# We need sys.stdout.buffer on py3 so we can write bytes to it since the plaintext
|
|
|
|
# of the vaulted object could be anything/binary/etc
|
|
|
|
output = getattr(sys.stdout, 'buffer', sys.stdout)
|
|
|
|
|
2015-08-26 17:21:20 +00:00
|
|
|
if filename == '-':
|
2017-04-24 14:09:03 +00:00
|
|
|
output.write(b_file_data)
|
2015-08-26 17:21:20 +00:00
|
|
|
else:
|
|
|
|
if os.path.isfile(filename):
|
2016-01-05 00:34:45 +00:00
|
|
|
if shred:
|
|
|
|
self._shred_file(filename)
|
|
|
|
else:
|
|
|
|
os.remove(filename)
|
2015-08-27 07:07:42 +00:00
|
|
|
with open(filename, "wb") as fh:
|
2016-08-24 00:03:11 +00:00
|
|
|
fh.write(b_file_data)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
def shuffle_files(self, src, dest):
|
2015-10-31 17:56:14 +00:00
|
|
|
prev = None
|
2014-10-19 05:14:30 +00:00
|
|
|
# overwrite dest with src
|
|
|
|
if os.path.isfile(dest):
|
2015-10-31 17:56:14 +00:00
|
|
|
prev = os.stat(dest)
|
2015-12-30 17:21:34 +00:00
|
|
|
# old file 'dest' was encrypted, no need to _shred_file
|
2014-10-19 05:14:30 +00:00
|
|
|
os.remove(dest)
|
|
|
|
shutil.move(src, dest)
|
|
|
|
|
2015-10-31 17:56:14 +00:00
|
|
|
# reset permissions if needed
|
|
|
|
if prev is not None:
|
2016-08-24 00:03:11 +00:00
|
|
|
# TODO: selinux, ACLs, xattr?
|
2015-10-31 17:56:14 +00:00
|
|
|
os.chmod(dest, prev.st_mode)
|
|
|
|
os.chown(dest, prev.st_uid, prev.st_gid)
|
|
|
|
|
2014-10-19 05:14:30 +00:00
|
|
|
def _editor_shell_command(self, filename):
|
2017-05-18 17:41:00 +00:00
|
|
|
EDITOR = os.environ.get('EDITOR', 'vi')
|
2014-10-19 05:14:30 +00:00
|
|
|
editor = shlex.split(EDITOR)
|
|
|
|
editor.append(filename)
|
|
|
|
|
|
|
|
return editor
|
|
|
|
|
2016-09-07 05:54:17 +00:00
|
|
|
|
2014-10-19 05:14:30 +00:00
|
|
|
########################################
|
|
|
|
# CIPHERS #
|
|
|
|
########################################
|
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
class VaultAES:
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# this version has been obsoleted by the VaultAES256 class
|
|
|
|
# which uses encrypt-then-mac (fixing order) and also improving the KDF used
|
|
|
|
# code remains for upgrade purposes only
|
|
|
|
# http://stackoverflow.com/a/16761459
|
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
# Note: strings in this class should be byte strings by default.
|
|
|
|
|
2014-10-19 05:14:30 +00:00
|
|
|
def __init__(self):
|
|
|
|
if not HAS_AES:
|
2015-07-11 18:24:00 +00:00
|
|
|
raise AnsibleError(CRYPTO_UPGRADE)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def _aes_derive_key_and_iv(self, b_password, b_salt, key_length, iv_length):
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
""" Create a key and an initialization vector """
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
b_d = b_di = b''
|
|
|
|
while len(b_d) < key_length + iv_length:
|
|
|
|
b_text = b''.join([b_di, b_password, b_salt])
|
|
|
|
b_di = to_bytes(md5(b_text).digest(), errors='strict')
|
|
|
|
b_d += b_di
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
b_key = b_d[:key_length]
|
2017-05-18 17:41:00 +00:00
|
|
|
b_iv = b_d[key_length:key_length + iv_length]
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
return b_key, b_iv
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def encrypt(self, b_plaintext, b_password, key_length=32):
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
""" Read plaintext data from in_file and write encrypted to out_file """
|
|
|
|
|
2015-08-27 10:47:13 +00:00
|
|
|
raise AnsibleError("Encryption disabled for deprecated VaultAES class")
|
2015-04-15 18:08:53 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def decrypt(self, b_vaulttext, b_password, key_length=32):
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
""" Decrypt the given data and return it
|
|
|
|
:arg b_data: A byte string containing the encrypted data
|
|
|
|
:arg b_password: A byte string containing the encryption password
|
|
|
|
:arg key_length: Length of the key
|
|
|
|
:returns: A byte string containing the decrypted data
|
|
|
|
"""
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2017-05-18 17:41:00 +00:00
|
|
|
display.deprecated(u'The VaultAES format is insecure and has been '
|
|
|
|
'deprecated since Ansible-1.5. Use vault rekey FILENAME to '
|
|
|
|
'switch to the newer VaultAES256 format', version='2.3')
|
2014-10-19 05:14:30 +00:00
|
|
|
# http://stackoverflow.com/a/14989032
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
b_ciphertext = unhexlify(b_vaulttext)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
in_file = BytesIO(b_ciphertext)
|
2014-10-19 05:14:30 +00:00
|
|
|
in_file.seek(0)
|
|
|
|
out_file = BytesIO()
|
|
|
|
|
|
|
|
bs = AES.block_size
|
2016-09-13 14:30:17 +00:00
|
|
|
b_tmpsalt = in_file.read(bs)
|
|
|
|
b_salt = b_tmpsalt[len(b'Salted__'):]
|
|
|
|
b_key, b_iv = self._aes_derive_key_and_iv(b_password, b_salt, key_length, bs)
|
|
|
|
cipher = AES.new(b_key, AES.MODE_CBC, b_iv)
|
|
|
|
b_next_chunk = b''
|
2014-10-19 05:14:30 +00:00
|
|
|
finished = False
|
|
|
|
|
|
|
|
while not finished:
|
2016-09-13 14:30:17 +00:00
|
|
|
b_chunk, b_next_chunk = b_next_chunk, cipher.decrypt(in_file.read(1024 * bs))
|
|
|
|
if len(b_next_chunk) == 0:
|
2015-06-03 17:24:35 +00:00
|
|
|
if PY3:
|
2016-09-13 14:30:17 +00:00
|
|
|
padding_length = b_chunk[-1]
|
2015-06-03 17:24:35 +00:00
|
|
|
else:
|
2016-09-13 14:30:17 +00:00
|
|
|
padding_length = ord(b_chunk[-1])
|
2015-04-15 18:08:53 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
b_chunk = b_chunk[:-padding_length]
|
2014-10-19 05:14:30 +00:00
|
|
|
finished = True
|
2015-04-16 16:53:59 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
out_file.write(b_chunk)
|
2015-04-16 16:53:59 +00:00
|
|
|
out_file.flush()
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# reset the stream pointer to the beginning
|
|
|
|
out_file.seek(0)
|
2016-09-13 14:30:17 +00:00
|
|
|
b_out_data = out_file.read()
|
2015-04-16 16:53:59 +00:00
|
|
|
out_file.close()
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# split out sha and verify decryption
|
2016-09-13 14:30:17 +00:00
|
|
|
b_split_data = b_out_data.split(b"\n", 1)
|
|
|
|
b_this_sha = b_split_data[0]
|
|
|
|
b_plaintext = b_split_data[1]
|
|
|
|
b_test_sha = to_bytes(sha256(b_plaintext).hexdigest())
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
if b_this_sha != b_test_sha:
|
2015-07-11 18:24:00 +00:00
|
|
|
raise AnsibleError("Decryption failed")
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
return b_plaintext
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
class VaultAES256:
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
"""
|
2015-04-15 18:08:53 +00:00
|
|
|
Vault implementation using AES-CTR with an HMAC-SHA256 authentication code.
|
2014-10-19 05:14:30 +00:00
|
|
|
Keys are derived using PBKDF2
|
|
|
|
"""
|
|
|
|
|
|
|
|
# http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html
|
|
|
|
|
2015-08-24 22:49:55 +00:00
|
|
|
# Note: strings in this class should be byte strings by default.
|
|
|
|
|
2014-10-19 05:14:30 +00:00
|
|
|
def __init__(self):
|
|
|
|
|
2015-06-16 13:20:15 +00:00
|
|
|
check_prereqs()
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
@staticmethod
|
|
|
|
def _create_key(b_password, b_salt, keylength, ivlength):
|
2014-10-19 05:14:30 +00:00
|
|
|
hash_function = SHA256
|
|
|
|
|
|
|
|
# make two keys and one iv
|
2017-05-18 17:41:00 +00:00
|
|
|
def pbkdf2_prf(p, s):
|
|
|
|
return HMAC.new(p, s, hash_function).digest()
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
b_derivedkey = PBKDF2(b_password, b_salt, dkLen=(2 * keylength) + ivlength,
|
2017-05-18 17:41:00 +00:00
|
|
|
count=10000, prf=pbkdf2_prf)
|
2016-09-13 14:30:17 +00:00
|
|
|
return b_derivedkey
|
Use PBKDF2HMAC() from cryptography for vault keys.
When stretching the key for vault files, use PBKDF2HMAC() from the
cryptography package instead of pycrypto. This will speed up the opening
of vault files by ~10x.
The problem is here in lib/ansible/utils/vault.py:
hash_function = SHA256
# make two keys and one iv
pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest()
derivedkey = PBKDF2(password, salt, dkLen=(2 * keylength) + ivlength,
count=10000, prf=pbkdf2_prf)
`PBKDF2()` calls a Python callback function (`pbkdf2_pr()`) 10000 times.
If one has several vault files, this will cause excessive start times
with `ansible` or `ansible-playbook` (we experience ~15 second startup
times).
Testing the original implementation in 1.9.2 with a vault file:
In [2]: %timeit v.decrypt(encrypted_data)
1 loops, best of 3: 265 ms per loop
Having a recent OpenSSL version and using the vault.py changes in this commit:
In [2]: %timeit v.decrypt(encrypted_data)
10 loops, best of 3: 23.2 ms per loop
2015-07-22 17:52:42 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
@classmethod
|
|
|
|
def _gen_key_initctr(cls, b_password, b_salt):
|
Use PBKDF2HMAC() from cryptography for vault keys.
When stretching the key for vault files, use PBKDF2HMAC() from the
cryptography package instead of pycrypto. This will speed up the opening
of vault files by ~10x.
The problem is here in lib/ansible/utils/vault.py:
hash_function = SHA256
# make two keys and one iv
pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest()
derivedkey = PBKDF2(password, salt, dkLen=(2 * keylength) + ivlength,
count=10000, prf=pbkdf2_prf)
`PBKDF2()` calls a Python callback function (`pbkdf2_pr()`) 10000 times.
If one has several vault files, this will cause excessive start times
with `ansible` or `ansible-playbook` (we experience ~15 second startup
times).
Testing the original implementation in 1.9.2 with a vault file:
In [2]: %timeit v.decrypt(encrypted_data)
1 loops, best of 3: 265 ms per loop
Having a recent OpenSSL version and using the vault.py changes in this commit:
In [2]: %timeit v.decrypt(encrypted_data)
10 loops, best of 3: 23.2 ms per loop
2015-07-22 17:52:42 +00:00
|
|
|
# 16 for AES 128, 32 for AES256
|
|
|
|
keylength = 32
|
|
|
|
|
|
|
|
# match the size used for counter.new to avoid extra work
|
|
|
|
ivlength = 16
|
|
|
|
|
|
|
|
if HAS_PBKDF2HMAC:
|
|
|
|
backend = default_backend()
|
|
|
|
kdf = PBKDF2HMAC(
|
|
|
|
algorithm=c_SHA256(),
|
|
|
|
length=2 * keylength + ivlength,
|
2016-09-13 14:30:17 +00:00
|
|
|
salt=b_salt,
|
Use PBKDF2HMAC() from cryptography for vault keys.
When stretching the key for vault files, use PBKDF2HMAC() from the
cryptography package instead of pycrypto. This will speed up the opening
of vault files by ~10x.
The problem is here in lib/ansible/utils/vault.py:
hash_function = SHA256
# make two keys and one iv
pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest()
derivedkey = PBKDF2(password, salt, dkLen=(2 * keylength) + ivlength,
count=10000, prf=pbkdf2_prf)
`PBKDF2()` calls a Python callback function (`pbkdf2_pr()`) 10000 times.
If one has several vault files, this will cause excessive start times
with `ansible` or `ansible-playbook` (we experience ~15 second startup
times).
Testing the original implementation in 1.9.2 with a vault file:
In [2]: %timeit v.decrypt(encrypted_data)
1 loops, best of 3: 265 ms per loop
Having a recent OpenSSL version and using the vault.py changes in this commit:
In [2]: %timeit v.decrypt(encrypted_data)
10 loops, best of 3: 23.2 ms per loop
2015-07-22 17:52:42 +00:00
|
|
|
iterations=10000,
|
|
|
|
backend=backend)
|
2016-09-13 14:30:17 +00:00
|
|
|
b_derivedkey = kdf.derive(b_password)
|
Use PBKDF2HMAC() from cryptography for vault keys.
When stretching the key for vault files, use PBKDF2HMAC() from the
cryptography package instead of pycrypto. This will speed up the opening
of vault files by ~10x.
The problem is here in lib/ansible/utils/vault.py:
hash_function = SHA256
# make two keys and one iv
pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest()
derivedkey = PBKDF2(password, salt, dkLen=(2 * keylength) + ivlength,
count=10000, prf=pbkdf2_prf)
`PBKDF2()` calls a Python callback function (`pbkdf2_pr()`) 10000 times.
If one has several vault files, this will cause excessive start times
with `ansible` or `ansible-playbook` (we experience ~15 second startup
times).
Testing the original implementation in 1.9.2 with a vault file:
In [2]: %timeit v.decrypt(encrypted_data)
1 loops, best of 3: 265 ms per loop
Having a recent OpenSSL version and using the vault.py changes in this commit:
In [2]: %timeit v.decrypt(encrypted_data)
10 loops, best of 3: 23.2 ms per loop
2015-07-22 17:52:42 +00:00
|
|
|
else:
|
2016-09-13 14:30:17 +00:00
|
|
|
b_derivedkey = cls._create_key(b_password, b_salt, keylength, ivlength)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
b_key1 = b_derivedkey[:keylength]
|
|
|
|
b_key2 = b_derivedkey[keylength:(keylength * 2)]
|
|
|
|
b_iv = b_derivedkey[(keylength * 2):(keylength * 2) + ivlength]
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
return b_key1, b_key2, hexlify(b_iv)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def encrypt(self, b_plaintext, b_password):
|
|
|
|
b_salt = os.urandom(32)
|
|
|
|
b_key1, b_key2, b_iv = self._gen_key_initctr(b_password, b_salt)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# PKCS#7 PAD DATA http://tools.ietf.org/html/rfc5652#section-6.3
|
|
|
|
bs = AES.block_size
|
2016-09-13 14:30:17 +00:00
|
|
|
padding_length = (bs - len(b_plaintext) % bs) or bs
|
|
|
|
b_plaintext += to_bytes(padding_length * chr(padding_length), encoding='ascii', errors='strict')
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# COUNTER.new PARAMETERS
|
|
|
|
# 1) nbits (integer) - Length of the counter, in bits.
|
2016-09-13 14:30:17 +00:00
|
|
|
# 2) initial_value (integer) - initial value of the counter. "iv" from _gen_key_initctr
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
ctr = Counter.new(128, initial_value=int(b_iv, 16))
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# AES.new PARAMETERS
|
2016-09-13 14:30:17 +00:00
|
|
|
# 1) AES key, must be either 16, 24, or 32 bytes long -- "key" from _gen_key_initctr
|
2014-10-19 05:14:30 +00:00
|
|
|
# 2) MODE_CTR, is the recommended mode
|
|
|
|
# 3) counter=<CounterObject>
|
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
cipher = AES.new(b_key1, AES.MODE_CTR, counter=ctr)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# ENCRYPT PADDED DATA
|
2016-09-13 14:30:17 +00:00
|
|
|
b_ciphertext = cipher.encrypt(b_plaintext)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# COMBINE SALT, DIGEST AND DATA
|
2016-09-13 14:30:17 +00:00
|
|
|
hmac = HMAC.new(b_key2, b_ciphertext, SHA256)
|
|
|
|
b_vaulttext = b'\n'.join([hexlify(b_salt), to_bytes(hmac.hexdigest()), hexlify(b_ciphertext)])
|
|
|
|
b_vaulttext = hexlify(b_vaulttext)
|
|
|
|
return b_vaulttext
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
def decrypt(self, b_vaulttext, b_password):
|
2014-10-19 05:14:30 +00:00
|
|
|
# SPLIT SALT, DIGEST, AND DATA
|
2016-09-13 14:30:17 +00:00
|
|
|
b_vaulttext = unhexlify(b_vaulttext)
|
|
|
|
b_salt, b_cryptedHmac, b_ciphertext = b_vaulttext.split(b"\n", 2)
|
|
|
|
b_salt = unhexlify(b_salt)
|
|
|
|
b_ciphertext = unhexlify(b_ciphertext)
|
|
|
|
b_key1, b_key2, b_iv = self._gen_key_initctr(b_password, b_salt)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2015-04-15 18:08:53 +00:00
|
|
|
# EXIT EARLY IF DIGEST DOESN'T MATCH
|
2016-09-13 14:30:17 +00:00
|
|
|
hmacDecrypt = HMAC.new(b_key2, b_ciphertext, SHA256)
|
|
|
|
if not self._is_equal(b_cryptedHmac, to_bytes(hmacDecrypt.hexdigest())):
|
2014-10-19 05:14:30 +00:00
|
|
|
return None
|
|
|
|
# SET THE COUNTER AND THE CIPHER
|
2016-09-13 14:30:17 +00:00
|
|
|
ctr = Counter.new(128, initial_value=int(b_iv, 16))
|
|
|
|
cipher = AES.new(b_key1, AES.MODE_CTR, counter=ctr)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# DECRYPT PADDED DATA
|
2016-09-13 14:30:17 +00:00
|
|
|
b_plaintext = cipher.decrypt(b_ciphertext)
|
2014-10-19 05:14:30 +00:00
|
|
|
|
|
|
|
# UNPAD DATA
|
2016-09-13 14:30:17 +00:00
|
|
|
if PY3:
|
|
|
|
padding_length = b_plaintext[-1]
|
|
|
|
else:
|
|
|
|
padding_length = ord(b_plaintext[-1])
|
2015-04-15 18:08:53 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
b_plaintext = b_plaintext[:-padding_length]
|
|
|
|
return b_plaintext
|
2014-10-19 05:14:30 +00:00
|
|
|
|
2016-09-13 14:30:17 +00:00
|
|
|
@staticmethod
|
|
|
|
def _is_equal(b_a, b_b):
|
2015-04-15 18:08:53 +00:00
|
|
|
"""
|
|
|
|
Comparing 2 byte arrrays in constant time
|
|
|
|
to avoid timing attacks.
|
|
|
|
|
|
|
|
It would be nice if there was a library for this but
|
|
|
|
hey.
|
|
|
|
"""
|
2016-09-13 14:30:17 +00:00
|
|
|
if not (isinstance(b_a, binary_type) and isinstance(b_b, binary_type)):
|
|
|
|
raise TypeError('_is_equal can only be used to compare two byte strings')
|
|
|
|
|
2014-10-19 05:14:30 +00:00
|
|
|
# http://codahale.com/a-lesson-in-timing-attacks/
|
2016-09-13 14:30:17 +00:00
|
|
|
if len(b_a) != len(b_b):
|
2014-10-19 05:14:30 +00:00
|
|
|
return False
|
2015-04-15 18:08:53 +00:00
|
|
|
|
2014-10-19 05:14:30 +00:00
|
|
|
result = 0
|
2016-09-13 14:30:17 +00:00
|
|
|
for b_x, b_y in zip(b_a, b_b):
|
2015-06-03 17:24:35 +00:00
|
|
|
if PY3:
|
2016-09-13 14:30:17 +00:00
|
|
|
result |= b_x ^ b_y
|
2015-06-03 17:24:35 +00:00
|
|
|
else:
|
2016-09-13 14:30:17 +00:00
|
|
|
result |= ord(b_x) ^ ord(b_y)
|
2015-04-15 18:08:53 +00:00
|
|
|
return result == 0
|
2015-07-11 18:24:00 +00:00
|
|
|
|
2015-10-16 17:04:37 +00:00
|
|
|
|
|
|
|
# Keys could be made bytes later if the code that gets the data is more
|
|
|
|
# naturally byte-oriented
|
|
|
|
CIPHER_MAPPING = {
|
2016-08-24 00:03:11 +00:00
|
|
|
u'AES': VaultAES,
|
|
|
|
u'AES256': VaultAES256,
|
|
|
|
}
|