Support unix socket with urls/uri (#43560)
* First pass at allowing unix socket with urls/uri. See #42341 * Only insert handler as needed * Fix and add tests * Add HTTPS functionality for unix sockets * Additional test fixes * Create context manager for monkey patching HTTPConnection.connect, de-dupe code, raise better errors * doc * Add a few more tests * Fix __call__ * Remove unused import * Patch HTTPConnection.connect with the functionality we want, instead of duplicating code and disabling * Fix var name * Remove unused var * Add changelog fragment * Update uri docs * Fix rebase indentation issuepull/4420/head
Matt Martz
GitHub
parent
5b28cd65f0
commit
77bfcad33a
|
|
@ -0,0 +1,2 @@
|
|||
minor_changes:
|
||||
- uri/urls - Support unix domain sockets (https://github.com/ansible/ansible/pull/43560)
|
||||
|
|
@ -42,6 +42,8 @@ import sys
|
|||
import tempfile
|
||||
import traceback
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
try:
|
||||
import httplib
|
||||
except ImportError:
|
||||
|
|
@ -362,10 +364,11 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler
|
|||
in place of HTTPSHandler
|
||||
'''
|
||||
|
||||
def __init__(self, client_cert=None, client_key=None, **kwargs):
|
||||
def __init__(self, client_cert=None, client_key=None, unix_socket=None, **kwargs):
|
||||
urllib_request.HTTPSHandler.__init__(self, **kwargs)
|
||||
self.client_cert = client_cert
|
||||
self.client_key = client_key
|
||||
self._unix_socket = unix_socket
|
||||
|
||||
def https_open(self, req):
|
||||
return self.do_open(self._build_https_connection, req)
|
||||
|
|
@ -379,9 +382,69 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler
|
|||
kwargs['context'] = self._context
|
||||
except AttributeError:
|
||||
pass
|
||||
if self._unix_socket:
|
||||
return UnixHTTPSConnection(self._unix_socket)(host, **kwargs)
|
||||
return httplib.HTTPSConnection(host, **kwargs)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def unix_socket_patch_httpconnection_connect():
|
||||
'''Monkey patch ``httplib.HTTPConnection.connect`` to be ``UnixHTTPConnection.connect``
|
||||
so that when calling ``super(UnixHTTPSConnection, self).connect()`` we get the
|
||||
correct behavior of creating self.sock for the unix socket
|
||||
'''
|
||||
_connect = httplib.HTTPConnection.connect
|
||||
httplib.HTTPConnection.connect = UnixHTTPConnection.connect
|
||||
yield
|
||||
httplib.HTTPConnection.connect = _connect
|
||||
|
||||
|
||||
class UnixHTTPSConnection(httplib.HTTPSConnection):
|
||||
def __init__(self, unix_socket):
|
||||
self._unix_socket = unix_socket
|
||||
|
||||
def connect(self):
|
||||
# This method exists simply to ensure we monkeypatch
|
||||
# httplib.HTTPConnection.connect to call UnixHTTPConnection.connect
|
||||
with unix_socket_patch_httpconnection_connect():
|
||||
super(UnixHTTPSConnection, self).connect()
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
|
||||
return self
|
||||
|
||||
|
||||
class UnixHTTPConnection(httplib.HTTPConnection):
|
||||
'''Handles http requests to a unix socket file'''
|
||||
|
||||
def __init__(self, unix_socket):
|
||||
self._unix_socket = unix_socket
|
||||
|
||||
def connect(self):
|
||||
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
self.sock.connect(self._unix_socket)
|
||||
except OSError as e:
|
||||
raise OSError('Invalid Socket File (%s): %s' % (self._unix_socket, e))
|
||||
if self.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
|
||||
self.sock.settimeout(self.timeout)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
httplib.HTTPConnection.__init__(self, *args, **kwargs)
|
||||
return self
|
||||
|
||||
|
||||
class UnixHTTPHandler(urllib_request.HTTPHandler):
|
||||
'''Handler for Unix urls'''
|
||||
|
||||
def __init__(self, unix_socket, **kwargs):
|
||||
urllib_request.HTTPHandler.__init__(self, **kwargs)
|
||||
self._unix_socket = unix_socket
|
||||
|
||||
def http_open(self, req):
|
||||
return self.do_open(UnixHTTPConnection(self._unix_socket), req)
|
||||
|
||||
|
||||
class ParseResultDottedDict(dict):
|
||||
'''
|
||||
A dict that acts similarly to the ParseResult named tuple from urllib
|
||||
|
|
@ -854,7 +917,7 @@ def rfc2822_date_string(timetuple, zone='-0000'):
|
|||
class Request:
|
||||
def __init__(self, headers=None, use_proxy=True, force=False, timeout=10, validate_certs=True,
|
||||
url_username=None, url_password=None, http_agent=None, force_basic_auth=False,
|
||||
follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None):
|
||||
follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None, unix_socket=None):
|
||||
"""This class works somewhat similarly to the ``Session`` class of from requests
|
||||
by defining a cookiejar that an be used across requests as well as cascaded defaults that
|
||||
can apply to repeated requests
|
||||
|
|
@ -875,7 +938,7 @@ class Request:
|
|||
|
||||
self.headers = headers or {}
|
||||
if not isinstance(self.headers, dict):
|
||||
raise ValueError("headers must be a dict")
|
||||
raise ValueError("headers must be a dict: %r" % self.headers)
|
||||
self.use_proxy = use_proxy
|
||||
self.force = force
|
||||
self.timeout = timeout
|
||||
|
|
@ -887,6 +950,7 @@ class Request:
|
|||
self.follow_redirects = follow_redirects
|
||||
self.client_cert = client_cert
|
||||
self.client_key = client_key
|
||||
self.unix_socket = unix_socket
|
||||
if isinstance(cookies, cookiejar.CookieJar):
|
||||
self.cookies = cookies
|
||||
else:
|
||||
|
|
@ -901,7 +965,8 @@ class Request:
|
|||
force=None, last_mod_time=None, timeout=None, validate_certs=None,
|
||||
url_username=None, url_password=None, http_agent=None,
|
||||
force_basic_auth=None, follow_redirects=None,
|
||||
client_cert=None, client_key=None, cookies=None, use_gssapi=False):
|
||||
client_cert=None, client_key=None, cookies=None, use_gssapi=False,
|
||||
unix_socket=None):
|
||||
"""
|
||||
Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
|
||||
|
||||
|
|
@ -936,6 +1001,8 @@ class Request:
|
|||
:kwarg cookies: (optional) CookieJar object to send with the
|
||||
request
|
||||
:kwarg use_gssapi: (optional) Use GSSAPI handler of requests.
|
||||
:kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing
|
||||
connection to the provided url
|
||||
:returns: HTTPResponse
|
||||
"""
|
||||
|
||||
|
|
@ -959,8 +1026,13 @@ class Request:
|
|||
client_cert = self._fallback(client_cert, self.client_cert)
|
||||
client_key = self._fallback(client_key, self.client_key)
|
||||
cookies = self._fallback(cookies, self.cookies)
|
||||
unix_socket = self._fallback(unix_socket, self.unix_socket)
|
||||
|
||||
handlers = []
|
||||
|
||||
if unix_socket:
|
||||
handlers.append(UnixHTTPHandler(unix_socket))
|
||||
|
||||
ssl_handler = maybe_add_ssl_handler(url, validate_certs)
|
||||
if ssl_handler:
|
||||
handlers.append(ssl_handler)
|
||||
|
|
@ -1033,10 +1105,12 @@ class Request:
|
|||
context.check_hostname = False
|
||||
handlers.append(HTTPSClientAuthHandler(client_cert=client_cert,
|
||||
client_key=client_key,
|
||||
context=context))
|
||||
elif client_cert:
|
||||
context=context,
|
||||
unix_socket=unix_socket))
|
||||
elif client_cert or unix_socket:
|
||||
handlers.append(HTTPSClientAuthHandler(client_cert=client_cert,
|
||||
client_key=client_key))
|
||||
client_key=client_key,
|
||||
unix_socket=unix_socket))
|
||||
|
||||
# pre-2.6 versions of python cannot use the custom https
|
||||
# handler, since the socket class is lacking create_connection.
|
||||
|
|
@ -1164,7 +1238,7 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
|
|||
url_username=None, url_password=None, http_agent=None,
|
||||
force_basic_auth=False, follow_redirects='urllib2',
|
||||
client_cert=None, client_key=None, cookies=None,
|
||||
use_gssapi=False):
|
||||
use_gssapi=False, unix_socket=None):
|
||||
'''
|
||||
Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
|
||||
|
||||
|
|
@ -1176,7 +1250,7 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
|
|||
url_username=url_username, url_password=url_password, http_agent=http_agent,
|
||||
force_basic_auth=force_basic_auth, follow_redirects=follow_redirects,
|
||||
client_cert=client_cert, client_key=client_key, cookies=cookies,
|
||||
use_gssapi=use_gssapi)
|
||||
use_gssapi=use_gssapi, unix_socket=unix_socket)
|
||||
|
||||
|
||||
#
|
||||
|
|
@ -1212,7 +1286,7 @@ def url_argument_spec():
|
|||
|
||||
def fetch_url(module, url, data=None, headers=None, method=None,
|
||||
use_proxy=True, force=False, last_mod_time=None, timeout=10,
|
||||
use_gssapi=False):
|
||||
use_gssapi=False, unix_socket=None):
|
||||
"""Sends a request via HTTP(S) or FTP (needs the module as parameter)
|
||||
|
||||
:arg module: The AnsibleModule (used to get username, password etc. (s.b.).
|
||||
|
|
@ -1226,6 +1300,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
|
|||
:kwarg last_mod_time: Default: None
|
||||
:kwarg int timeout: Default: 10
|
||||
:kwarg boolean use_gssapi: Default: False
|
||||
:kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing
|
||||
connection to the provided url
|
||||
|
||||
:returns: A tuple of (**response**, **info**). Use ``response.read()`` to read the data.
|
||||
The **info** contains the 'status' and other meta data. When a HttpError (status > 400)
|
||||
|
|
@ -1275,7 +1351,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
|
|||
validate_certs=validate_certs, url_username=username,
|
||||
url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth,
|
||||
follow_redirects=follow_redirects, client_cert=client_cert,
|
||||
client_key=client_key, cookies=cookies, use_gssapi=use_gssapi)
|
||||
client_key=client_key, cookies=cookies, use_gssapi=use_gssapi,
|
||||
unix_socket=unix_socket)
|
||||
# Lowercase keys, to conform to py2 behavior, so that py3 and py2 are predictable
|
||||
info.update(dict((k.lower(), v) for k, v in r.info().items()))
|
||||
|
||||
|
|
|
|||
|
|
@ -170,6 +170,10 @@ options:
|
|||
- If C(no), it will not use a proxy, even if one is defined in an environment variable on the target hosts.
|
||||
type: bool
|
||||
default: yes
|
||||
unix_socket:
|
||||
description:
|
||||
- Path to Unix domain socket to use for connection
|
||||
version_added: '2.8'
|
||||
notes:
|
||||
- The dependency on httplib2 was removed in Ansible 2.1.
|
||||
- The module returns all the HTTP headers in lower-case.
|
||||
|
|
@ -460,7 +464,7 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout):
|
|||
_, redir_info = fetch_url(module, url, data=body,
|
||||
headers=headers,
|
||||
method=method,
|
||||
timeout=socket_timeout)
|
||||
timeout=socket_timeout, unix_socket=module.params['unix_socket'])
|
||||
# if we are redirected, update the url with the location header,
|
||||
# and update dest with the new url filename
|
||||
if redir_info['status'] in (301, 302, 303, 307):
|
||||
|
|
@ -475,7 +479,8 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout):
|
|||
module.params['follow_redirects'] = follow_redirects
|
||||
|
||||
resp, info = fetch_url(module, url, data=data, headers=headers,
|
||||
method=method, timeout=socket_timeout, **kwargs)
|
||||
method=method, timeout=socket_timeout, unix_socket=module.params['unix_socket'],
|
||||
**kwargs)
|
||||
|
||||
try:
|
||||
content = resp.read()
|
||||
|
|
@ -515,6 +520,7 @@ def main():
|
|||
status_code=dict(type='list', default=[200]),
|
||||
timeout=dict(type='int', default=30),
|
||||
headers=dict(type='dict', default={}),
|
||||
unix_socket=dict(type='path'),
|
||||
)
|
||||
|
||||
module = AnsibleModule(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ __metaclass__ = type
|
|||
import datetime
|
||||
import os
|
||||
|
||||
from ansible.module_utils.urls import Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod
|
||||
from ansible.module_utils.urls import (Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod,
|
||||
UnixHTTPHandler, UnixHTTPSConnection, httplib)
|
||||
from ansible.module_utils.urls import SSLValidationHandler, HTTPSClientAuthHandler, RedirectHandlerFactory
|
||||
|
||||
import pytest
|
||||
|
|
@ -45,6 +46,7 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
|
|||
client_cert='/tmp/client.pem',
|
||||
client_key='/tmp/client.key',
|
||||
cookies=cookies,
|
||||
unix_socket='/foo/bar/baz.sock',
|
||||
)
|
||||
fallback_mock = mocker.spy(request, '_fallback')
|
||||
|
||||
|
|
@ -63,10 +65,11 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
|
|||
call(None, '/tmp/client.pem'), # client_cert
|
||||
call(None, '/tmp/client.key'), # client_key
|
||||
call(None, cookies), # cookies
|
||||
call(None, '/foo/bar/baz.sock'), # unix_socket
|
||||
]
|
||||
fallback_mock.assert_has_calls(calls)
|
||||
|
||||
assert fallback_mock.call_count == 12 # All but headers use fallback
|
||||
assert fallback_mock.call_count == 13 # All but headers use fallback
|
||||
|
||||
args = urlopen_mock.call_args[0]
|
||||
assert args[1] is None # data, this is handled in the Request not urlopen
|
||||
|
|
@ -125,6 +128,39 @@ def test_Request_open_http(urlopen_mock, install_opener_mock):
|
|||
assert len(found_handlers) == 0
|
||||
|
||||
|
||||
def test_Request_open_unix_socket(urlopen_mock, install_opener_mock):
|
||||
r = Request().open('GET', 'http://ansible.com/', unix_socket='/foo/bar/baz.sock')
|
||||
args = urlopen_mock.call_args[0]
|
||||
|
||||
opener = install_opener_mock.call_args[0][0]
|
||||
handlers = opener.handlers
|
||||
|
||||
found_handlers = []
|
||||
for handler in handlers:
|
||||
if isinstance(handler, UnixHTTPHandler):
|
||||
found_handlers.append(handler)
|
||||
|
||||
assert len(found_handlers) == 1
|
||||
|
||||
|
||||
def test_Request_open_https_unix_socket(urlopen_mock, install_opener_mock):
|
||||
r = Request().open('GET', 'https://ansible.com/', unix_socket='/foo/bar/baz.sock')
|
||||
args = urlopen_mock.call_args[0]
|
||||
|
||||
opener = install_opener_mock.call_args[0][0]
|
||||
handlers = opener.handlers
|
||||
|
||||
found_handlers = []
|
||||
for handler in handlers:
|
||||
if isinstance(handler, HTTPSClientAuthHandler):
|
||||
found_handlers.append(handler)
|
||||
|
||||
assert len(found_handlers) == 1
|
||||
|
||||
inst = found_handlers[0]._build_https_connection('foo')
|
||||
assert isinstance(inst, UnixHTTPSConnection)
|
||||
|
||||
|
||||
def test_Request_open_ftp(urlopen_mock, install_opener_mock, mocker):
|
||||
mocker.patch('ansible.module_utils.urls.ParseResultDottedDict.as_list', side_effect=AssertionError)
|
||||
|
||||
|
|
@ -270,6 +306,10 @@ def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock):
|
|||
break
|
||||
|
||||
assert ssl_handler is not None
|
||||
|
||||
inst = ssl_handler._build_https_connection('foo')
|
||||
assert isinstance(inst, httplib.HTTPSConnection)
|
||||
|
||||
context = ssl_handler._context
|
||||
assert context.protocol == ssl.PROTOCOL_SSLv23
|
||||
if ssl.OP_NO_SSLv2:
|
||||
|
|
@ -399,4 +439,5 @@ def test_open_url(urlopen_mock, install_opener_mock, mocker):
|
|||
force=False, last_mod_time=None, timeout=10, validate_certs=True,
|
||||
url_username=None, url_password=None, http_agent=None,
|
||||
force_basic_auth=False, follow_redirects='urllib2',
|
||||
client_cert=None, client_key=None, cookies=None, use_gssapi=False)
|
||||
client_cert=None, client_key=None, cookies=None, use_gssapi=False,
|
||||
unix_socket=None)
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ def test_fetch_url(open_url_mock, fake_ansible_module):
|
|||
open_url_mock.assert_called_once_with('http://ansible.com/', client_cert=None, client_key=None, cookies=kwargs['cookies'], data=None,
|
||||
follow_redirects='urllib2', force=False, force_basic_auth='', headers=None,
|
||||
http_agent='ansible-httpget', last_mod_time=None, method=None, timeout=10, url_password='', url_username='',
|
||||
use_proxy=True, validate_certs=True, use_gssapi=False)
|
||||
use_proxy=True, validate_certs=True, use_gssapi=False, unix_socket=None)
|
||||
|
||||
|
||||
def test_fetch_url_params(open_url_mock, fake_ansible_module):
|
||||
|
|
@ -89,7 +89,7 @@ def test_fetch_url_params(open_url_mock, fake_ansible_module):
|
|||
open_url_mock.assert_called_once_with('http://ansible.com/', client_cert='client.pem', client_key='client.key', cookies=kwargs['cookies'], data=None,
|
||||
follow_redirects='all', force=False, force_basic_auth=True, headers=None,
|
||||
http_agent='ansible-test', last_mod_time=None, method=None, timeout=10, url_password='passwd', url_username='user',
|
||||
use_proxy=True, validate_certs=False, use_gssapi=False)
|
||||
use_proxy=True, validate_certs=False, use_gssapi=False, unix_socket=None)
|
||||
|
||||
|
||||
def test_fetch_url_cookies(mocker, fake_ansible_module):
|
||||
|
|
|
|||
|
|
@ -89,3 +89,11 @@ def test_ParseResultDottedDict():
|
|||
assert parts[0] == dotted_parts.scheme
|
||||
|
||||
assert dotted_parts.as_list() == list(parts)
|
||||
|
||||
|
||||
def test_unix_socket_patch_httpconnection_connect(mocker):
|
||||
unix_conn = mocker.patch.object(urls.UnixHTTPConnection, 'connect')
|
||||
conn = urls.httplib.HTTPConnection('ansible.com')
|
||||
with urls.unix_socket_patch_httpconnection_connect():
|
||||
conn.connect()
|
||||
assert unix_conn.call_count == 1
|
||||
|
|
|
|||
Loading…
Reference in New Issue