Add cli and netconf ability from ansible itself for cloudengine ce modules (#41357)

* for shippable

for shippable

* add cliconf

* add network_cli

* add cliconf and network cli and netconf

* modify bugs of netconf

* add shippable modify

* update shippable

update shippable

lib/ansible/module_utils/network/cloudengine/ce.py

@@ -38,13 +38,11 @@ from ansible.module_utils.network.common.utils import to_list, ComplexList
 from ansible.module_utils.connection import exec_command
 from ansible.module_utils.six import iteritems
 from ansible.module_utils._text import to_native
+from ansible.module_utils.network.common.netconf import NetconfConnection
 
 
 try:
-    from ncclient import manager, xml_
+    from ncclient.xml_ import to_xml
-    from ncclient.operations.rpc import RPCError
-    from ncclient.transport.errors import AuthenticationError
-    from ncclient.operations.errors import TimeoutExpiredError
     HAS_NCCLIENT = True
 except ImportError:
     HAS_NCCLIENT = False
@@ -58,10 +56,11 @@ ce_provider_spec = {
     'port': dict(type='int'),
     'username': dict(fallback=(env_fallback, ['ANSIBLE_NET_USERNAME'])),
     'password': dict(fallback=(env_fallback, ['ANSIBLE_NET_PASSWORD']), no_log=True),
+    'ssh_keyfile': dict(fallback=(env_fallback, ['ANSIBLE_NET_SSH_KEYFILE']), type='path'),
     'use_ssl': dict(type='bool'),
     'validate_certs': dict(type='bool'),
     'timeout': dict(type='int'),
-    'transport': dict(default='cli', choices=['cli']),
+    'transport': dict(default='cli', choices=['cli', 'netconf']),
 }
 ce_argument_spec = {
     'provider': dict(type='dict', options=ce_provider_spec),
@@ -71,19 +70,25 @@ ce_top_spec = {
     'port': dict(removed_in_version=2.9, type='int'),
     'username': dict(removed_in_version=2.9),
     'password': dict(removed_in_version=2.9, no_log=True),
+    'ssh_keyfile': dict(removed_in_version=2.9, type='path'),
     'use_ssl': dict(removed_in_version=2.9, type='bool'),
     'validate_certs': dict(removed_in_version=2.9, type='bool'),
     'timeout': dict(removed_in_version=2.9, type='int'),
-    'transport': dict(removed_in_version=2.9, choices=['cli']),
+    'transport': dict(removed_in_version=2.9, choices=['cli', 'netconf']),
 }
 ce_argument_spec.update(ce_top_spec)
 
 
+def to_string(data):
+    return re.sub(r'<data.+?(/>|>)', r'<data\1', data)
+
+
 def check_args(module, warnings):
     pass
 
 
 def load_params(module):
+    """load_params"""
     provider = module.params.get('provider') or dict()
     for key, value in iteritems(provider):
         if key in ce_argument_spec:
@@ -92,6 +97,7 @@ def load_params(module):
 
 
 def get_connection(module):
+    """get_connection"""
     global _DEVICE_CLI_CONNECTION
     if not _DEVICE_CLI_CONNECTION:
         load_params(module)
@@ -249,6 +255,7 @@ def run_commands(module, commands, check_rc=True):
 
 
 def load_config(module, config):
+    """load_config"""
     conn = get_connection(module)
     return conn.load_config(config)
 
@@ -315,115 +322,11 @@ def merge_nc_xml(xml1, xml2):
     return "\n".join(xml1_list + xml2_list)
 
 
-class Netconf(object):
-    """ Netconf """
-
-    def __init__(self, module):
-
-        self._module = module
-
-        if not HAS_NCCLIENT:
-            self._module.fail_json(msg='Error: The ncclient library is required.')
-
-        try:
-            self.mc = manager.connect(host=module.params["host"], port=module.params["port"],
-                                      username=module.params["username"],
-                                      password=module.params["password"],
-                                      unknown_host_cb=ce_unknown_host_cb,
-                                      allow_agent=False,
-                                      look_for_keys=False,
-                                      hostkey_verify=False,
-                                      device_params={'name': 'huawei'},
-                                      timeout=30)
-        except AuthenticationError:
-            self._module.fail_json(msg='Error: Authentication failed while connecting to device.')
-        except Exception as err:
-            self._module.fail_json(msg='Error: %s' % to_native(err).replace("\r\n", ""),
-                                   exception=traceback.format_exc())
-            raise
-
-    def __del__(self):
-
-        self.mc.close_session()
-
-    def set_config(self, xml_str):
-        """ set_config """
-
-        con_obj = None
-
-        try:
-            con_obj = self.mc.edit_config(target='running', config=xml_str)
-        except RPCError as err:
-            self._module.fail_json(msg='Error: %s' % to_native(err).replace("\r\n", ""))
-
-        return con_obj.xml
-
-    def get_config(self, xml_str):
-        """ get_config """
-
-        con_obj = None
-        try:
-            con_obj = self.mc.get(filter=xml_str)
-        except RPCError as err:
-            self._module.fail_json(msg='Error: %s' % to_native(err).replace("\r\n", ""))
-
-        set_id = get_nc_set_id(con_obj.xml)
-        if not set_id:
-            return con_obj.xml
-
-        # continue to get next
-        xml_str = con_obj.xml
-        while set_id:
-            set_attr = dict()
-            set_attr["set-id"] = str(set_id)
-            xsd_fetch = xml_.new_ele_ns('get-next', "http://www.huawei.com/netconf/capability/base/1.0", set_attr)
-            # get next data
-            try:
-                con_obj_next = self.mc.dispatch(xsd_fetch)
-            except RPCError as err:
-                self._module.fail_json(msg='Error: %s' % to_native(err).replace("\r\n", ""))
-
-            if "<data/>" in con_obj_next.xml:
-                break
-
-            # merge two xml data
-            xml_str = merge_nc_xml(xml_str, con_obj_next.xml)
-            set_id = get_nc_set_id(con_obj_next.xml)
-
-        return xml_str
-
-    def execute_action(self, xml_str):
-        """huawei execute-action"""
-
-        con_obj = None
-
-        try:
-            con_obj = self.mc.action(action=xml_str)
-        except RPCError as err:
-            self._module.fail_json(msg='Error: %s' % to_native(err).replace("\r\n", ""))
-        except TimeoutExpiredError:
-            raise
-
-        return con_obj.xml
-
-    def execute_cli(self, xml_str):
-        """huawei execute-cli"""
-
-        con_obj = None
-
-        try:
-            con_obj = self.mc.cli(command=xml_str)
-        except RPCError as err:
-            self._module.fail_json(msg='Error: %s' % to_native(err).replace("\r\n", ""))
-
-        return con_obj.xml
-
-
 def get_nc_connection(module):
     global _DEVICE_NC_CONNECTION
     if not _DEVICE_NC_CONNECTION:
         load_params(module)
-        conn = Netconf(module)
+        conn = NetconfConnection(module._socket_path)
         _DEVICE_NC_CONNECTION = conn
     return _DEVICE_NC_CONNECTION
 
@@ -432,28 +335,45 @@ def set_nc_config(module, xml_str):
     """ set_config """
 
     conn = get_nc_connection(module)
-    return conn.set_config(xml_str)
+    try:
+        out = conn.edit_config(target='running', config=xml_str, default_operation='merge',
+                               error_option='rollback-on-error')
+    finally:
+        # conn.unlock(target = 'candidate')
+        pass
+    return to_string(to_xml(out))
 
 
 def get_nc_config(module, xml_str):
     """ get_config """
 
     conn = get_nc_connection(module)
-    return conn.get_config(xml_str)
+    if xml_str is not None:
+        response = conn.get(xml_str)
+    else:
+        return None
+
+    return to_string(to_xml(response))
 
 
 def execute_nc_action(module, xml_str):
     """ huawei execute-action """
 
     conn = get_nc_connection(module)
-    return conn.execute_action(xml_str)
+    response = conn.execute_action(xml_str)
+    return to_string(to_xml(response))
 
 
 def execute_nc_cli(module, xml_str):
     """ huawei execute-cli """
 
-    conn = get_nc_connection(module)
+    if xml_str is not None:
-    return conn.execute_cli(xml_str)
+        try:
+            conn = get_nc_connection(module)
+            out = conn.execute_nc_cli(command=xml_str)
+            return to_string(to_xml(out))
+        except Exception as exc:
+            raise Exception(exc)
 
 
 def check_ip_addr(ipaddr):

lib/ansible/plugins/action/ce.py

@@ -36,51 +36,70 @@ except ImportError:
     from ansible.utils.display import Display
     display = Display()
 
+CLI_SUPPORTED_MODULES = ['ce_config', 'ce_command']
+
 
 class ActionModule(_ActionModule):
 
     def run(self, tmp=None, task_vars=None):
         del tmp  # tmp no longer has any effect
 
-        if self._play_context.connection != 'local':
+        socket_path = None
-            return dict(
-                failed=True,
-                msg='invalid connection specified, expected connection=local, '
-                    'got %s' % self._play_context.connection
-            )
 
-        provider = load_provider(ce_provider_spec, self._task.args)
+        if self._play_context.connection == 'local':
-        transport = provider['transport'] or 'cli'
+            provider = load_provider(ce_provider_spec, self._task.args)
+            transport = provider['transport'] or 'cli'
 
-        display.vvvv('connection transport is %s' % transport, self._play_context.remote_addr)
+            display.vvvv('connection transport is %s' % transport, self._play_context.remote_addr)
 
-        if transport == 'cli':
+            if transport == 'cli':
-            pc = copy.deepcopy(self._play_context)
+                pc = copy.deepcopy(self._play_context)
-            pc.connection = 'network_cli'
+                pc.connection = 'network_cli'
-            pc.network_os = 'ce'
+                pc.network_os = 'ce'
-            pc.remote_addr = provider['host'] or self._play_context.remote_addr
+                pc.remote_addr = provider['host'] or self._play_context.remote_addr
-            pc.port = int(provider['port'] or self._play_context.port or 22)
+                pc.port = int(provider['port'] or self._play_context.port or 22)
-            pc.remote_user = provider['username'] or self._play_context.connection_user
+                pc.remote_user = provider['username'] or self._play_context.connection_user
-            pc.password = provider['password'] or self._play_context.password
+                pc.password = provider['password'] or self._play_context.password
-            pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
+                pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
-            self._task.args['provider'] = provider.update(
+                self._task.args['provider'] = provider.update(
-                host=pc.remote_addr,
+                    host=pc.remote_addr,
-                port=pc.port,
+                    port=pc.port,
-                username=pc.remote_user,
+                    username=pc.remote_user,
-                password=pc.password
+                    password=pc.password
-            )
+                )
-            display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
+                if self._task.action in ['ce_netconf'] or self._task.action not in CLI_SUPPORTED_MODULES:
-            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
+                    pc.connection = 'netconf'
+                display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
+                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-            socket_path = connection.run()
+                socket_path = connection.run()
-            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
+                display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-            if not socket_path:
+                if not socket_path:
-                return {'failed': True,
+                    return {'failed': True,
-                        'msg': 'unable to open shell. Please see: ' +
+                            'msg': 'unable to open shell. Please see: ' +
-                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+                                   'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
-            # make sure we are in the right cli context which should be
+                task_vars['ansible_socket'] = socket_path
+                # make sure a transport value is set in args
+                self._task.args['transport'] = transport
+                self._task.args['provider'] = provider
+        elif self._play_context.connection in ('netconf', 'network_cli'):
+            provider = self._task.args.get('provider', {})
+            if any(provider.values()):
+                display.warning('provider is unnessary whene using %s and will be ignored' % self._play_context.connection)
+                del self._task.args['provider']
+
+            if (self._play_context.connection == 'network_cli' and self._task.action not in CLI_SUPPORTED_MODULES) or \
+                    (self._play_context.connection == 'netconf' and self._task.action in CLI_SUPPORTED_MODULES):
+                return {'failed': True, 'msg': "Connection type '%s' is not valid for '%s' module."
+                        % (self._play_context.connection, self._task.action)}
+
+        if (self._play_context.connection == 'local' and transport == 'cli' and self._task.action in CLI_SUPPORTED_MODULES) \
+                or self._play_context.connection == 'network_cli':
+            # make sure we are in the right cli context whitch should be
             # enable mode and not config module
+            if socket_path is None:
+                socket_path = self._connection.socket_path
             conn = Connection(socket_path)
             out = conn.get_prompt()
             while to_text(out, errors='surrogate_then_replace').strip().endswith(']'):
@@ -88,11 +107,5 @@ class ActionModule(_ActionModule):
                 conn.send_command('exit')
                 out = conn.get_prompt()
 
-            task_vars['ansible_socket'] = socket_path
-
-        # make sure a transport value is set in args
-        self._task.args['transport'] = transport
-        self._task.args['provider'] = provider
-
         result = super(ActionModule, self).run(task_vars=task_vars)
         return result

lib/ansible/plugins/cliconf/ce.py

@@ -0,0 +1,93 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+import json
+
+from itertools import chain
+
+from ansible.module_utils._text import to_text
+from ansible.module_utils.network.common.utils import to_list
+from ansible.plugins.cliconf import CliconfBase, enable_mode
+
+
+class Cliconf(CliconfBase):
+
+    def get_device_info(self):
+        device_info = {}
+
+        device_info['network_os'] = 'ce'
+        reply = self.get(b'display version')
+        data = to_text(reply, errors='surrogate_or_strict').strip()
+
+        match = re.search(r'^Huawei.+\n.+\Version\s+(\S+)', data)
+        if match:
+            device_info['network_os_version'] = match.group(1).strip(',')
+
+        match = re.search(r'^Huawei(.+)\n.+\(\S+\s+\S+\)', data, re.M)
+        if match:
+            device_info['network_os_model'] = match.group(1)
+
+        match = re.search(r'HUAWEI\s+(\S+)\s+uptime', data, re.M)
+        if match:
+            device_info['network_os_hostname'] = match.group(1)
+
+        return device_info
+
+    @enable_mode
+    def get_config(self, source='running', format='text', flags=None):
+        if source not in ('running'):
+            return self.invalid_params("fetching configuration from %s is not supported" % source)
+
+        if not flags:
+            flags = []
+
+        cmd = 'display current-configuration'
+
+        return self.send_command(cmd)
+
+    @enable_mode
+    def edit_config(self, command):
+        results = []
+        for cmd in chain(['configure terminal'], to_list(command), ['end']):
+            if isinstance(cmd, dict):
+                command = cmd['command']
+                prompt = cmd['prompt']
+                answer = cmd['answer']
+                newline = cmd.get('newline', True)
+            else:
+                command = cmd
+                prompt = None
+                answer = None
+                newline = True
+
+            results.append(self.send_command(command, prompt, answer, False, newline))
+        return results[1:-1]
+
+    def get(self, command, prompt=None, answer=None, sendonly=False):
+        return self.send_command(command, prompt=prompt, answer=answer, sendonly=sendonly)
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = self.get_base_rpc()
+        result['network_api'] = 'cliconf'
+        result['device_info'] = self.get_device_info()
+        return json.dumps(result)

lib/ansible/plugins/connection/netconf.py

@@ -160,7 +160,8 @@ logging.getLogger('ncclient').setLevel(logging.INFO)
 NETWORK_OS_DEVICE_PARAM_MAP = {
     "nxos": "nexus",
     "ios": "default",
-    "sros": "alu"
+    "sros": "alu",
+    "ce": "huawei"
 }
 
 

lib/ansible/plugins/netconf/ce.py

@@ -0,0 +1,217 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import re
+
+from ansible import constants as C
+from ansible.module_utils._text import to_text, to_bytes
+from ansible.errors import AnsibleConnectionFailure, AnsibleError
+from ansible.plugins.netconf import NetconfBase
+from ansible.plugins.netconf import ensure_connected
+
+try:
+    from ncclient import manager
+    from ncclient.operations import RPCError
+    from ncclient.transport.errors import SSHUnknownHostError
+    from ncclient.xml_ import to_ele, to_xml, new_ele
+except ImportError:
+    raise AnsibleError("ncclient is not installed")
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class Netconf(NetconfBase):
+
+    def get_text(self, ele, tag):
+        try:
+            return to_text(ele.find(tag).text, errors='surrogate_then_replace').strip()
+        except AttributeError:
+            pass
+
+    def get_device_info(self):
+        device_info = dict()
+        device_info['network_os'] = 'ce'
+        ele = new_ele('get-software-information')
+        data = self.execute_rpc(to_xml(ele))
+        reply = to_ele(to_bytes(data, errors='surrogate_or_strict'))
+        sw_info = reply.find('.//software-information')
+
+        device_info['network_os_version'] = self.get_text(sw_info, 'ce-version')
+        device_info['network_os_hostname'] = self.get_text(sw_info, 'host-name')
+        device_info['network_os_model'] = self.get_text(sw_info, 'product-model')
+
+        return device_info
+
+    @ensure_connected
+    def execute_rpc(self, name):
+        """RPC to be execute on remote device
+           :name: Name of rpc in string format"""
+        return self.rpc(name)
+
+    @ensure_connected
+    def load_configuration(self, *args, **kwargs):
+        """Loads given configuration on device
+        :format: Format of configuration (xml, text, set)
+        :action: Action to be performed (merge, replace, override, update)
+        :target: is the name of the configuration datastore being edited
+        :config: is the configuration in string format."""
+        if kwargs.get('config'):
+            kwargs['config'] = to_bytes(kwargs['config'], errors='surrogate_or_strict')
+            if kwargs.get('format', 'xml') == 'xml':
+                kwargs['config'] = to_ele(kwargs['config'])
+
+        try:
+            return self.m.load_configuration(*args, **kwargs).data_xml
+        except RPCError as exc:
+            raise Exception(to_xml(exc.xml))
+
+    def get_capabilities(self):
+        result = dict()
+        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes', 'validate', 'lock', 'unlock', 'copy_copy',
+                                               'execute_rpc', 'load_configuration', 'get_configuration', 'command',
+                                               'reboot', 'halt']
+        result['network_api'] = 'netconf'
+        result['device_info'] = self.get_device_info()
+        result['server_capabilities'] = [c for c in self.m.server_capabilities]
+        result['client_capabilities'] = [c for c in self.m.client_capabilities]
+        result['session_id'] = self.m.session_id
+        return json.dumps(result)
+
+    @staticmethod
+    def guess_network_os(obj):
+
+        try:
+            m = manager.connect(
+                host=obj._play_context.remote_addr,
+                port=obj._play_context.port or 830,
+                username=obj._play_context.remote_user,
+                password=obj._play_context.password,
+                key_filename=obj._play_context.private_key_file,
+                hostkey_verify=C.HOST_KEY_CHECKING,
+                look_for_keys=C.PARAMIKO_LOOK_FOR_KEYS,
+                allow_agent=obj._play_context.allow_agent,
+                timeout=obj._play_context.timeout
+            )
+        except SSHUnknownHostError as exc:
+            raise AnsibleConnectionFailure(str(exc))
+
+        guessed_os = None
+        for c in m.server_capabilities:
+            if re.search('huawei', c):
+                guessed_os = 'ce'
+                break
+
+        m.close_session()
+        return guessed_os
+
+    @ensure_connected
+    def get_configuration(self, *args, **kwargs):
+        """Retrieve all or part of a specified configuration.
+           :format: format in configuration should be retrieved
+           :filter: specifies the portion of the configuration to retrieve
+           (by default entire configuration is retrieved)"""
+        return self.m.get_configuration(*args, **kwargs).data_xml
+
+    @ensure_connected
+    def compare_configuration(self, *args, **kwargs):
+        """Compare configuration
+           :rollback: rollback id"""
+        return self.m.compare_configuration(*args, **kwargs).data_xml
+
+    @ensure_connected
+    def execute_action(self, xml_str):
+        """huawei execute-action"""
+        con_obj = None
+        try:
+            con_obj = self.m.action(action=xml_str)
+        except RPCError as exc:
+            raise Exception(to_xml(exc.xml))
+
+        return con_obj.xml
+
+    @ensure_connected
+    def halt(self):
+        """reboot the device"""
+        return self.m.halt().data_xml
+
+    @ensure_connected
+    def reboot(self):
+        """reboot the device"""
+        return self.m.reboot().data_xml
+
+    @ensure_connected
+    def halt(self):
+        """reboot the device"""
+        return self.m.halt().data_xml
+
+    @ensure_connected
+    def get(self, *args, **kwargs):
+        try:
+            return self.m.get(*args, **kwargs).data_xml
+        except RPCError as exc:
+            raise Exception(to_xml(exc.xml))
+
+    @ensure_connected
+    def get_config(self, *args, **kwargs):
+        try:
+            return self.m.get_config(*args, **kwargs).data_xml
+        except RPCError as exc:
+            raise Exception(to_xml(exc.xml))
+
+    @ensure_connected
+    def edit_config(self, *args, **kwargs):
+        try:
+            return self.m.edit_config(*args, **kwargs).xml
+        except RPCError as exc:
+            raise Exception(to_xml(exc.xml))
+
+    @ensure_connected
+    def execute_nc_cli(self, *args, **kwargs):
+        try:
+            return self.m.cli(*args, **kwargs).xml
+        except RPCError as exc:
+            raise Exception(to_xml(exc.xml))
+
+    @ensure_connected
+    def commit(self, *args, **kwargs):
+        try:
+            return self.m.commit(*args, **kwargs).data_xml
+        except RPCError as exc:
+            raise Exception(to_xml(exc.xml))
+
+    @ensure_connected
+    def validate(self, *args, **kwargs):
+        return self.m.validate(*args, **kwargs).data_xml
+
+    @ensure_connected
+    def discard_changes(self, *args, **kwargs):
+        return self.m.discard_changes(*args, **kwargs).data_xml
+
+    @ensure_connected
+    def execute_rpc(self, name):
+        """RPC to be execute on remote device
+           :name: Name of rpc in string format"""
+        return self.rpc(name)