From 1544dde932f57bee63ca4e9b43b4a95963aa832c Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Fri, 9 Jan 2015 09:37:31 -0600 Subject: [PATCH] Moving over all lookup plugins to v2 --- v2/ansible/executor/play_iterator.py | 3 + v2/ansible/executor/process/worker.py | 16 +- v2/ansible/executor/task_executor.py | 4 +- v2/ansible/parsing/__init__.py | 15 -- v2/ansible/playbook/base.py | 2 +- v2/ansible/playbook/block.py | 7 + v2/ansible/playbook/conditional.py | 2 +- v2/ansible/playbook/role/__init__.py | 7 + v2/ansible/playbook/task.py | 17 +- v2/ansible/plugins/action/debug.py | 2 +- v2/ansible/plugins/action/template.py | 2 +- v2/ansible/plugins/lookup/__init__.py | 10 +- v2/ansible/plugins/lookup/csvfile.py | 79 +++++++ v2/ansible/plugins/lookup/dict.py | 27 +++ v2/ansible/plugins/lookup/dnstxt.py | 68 ++++++ v2/ansible/plugins/lookup/env.py | 34 +++ v2/ansible/plugins/lookup/etcd.py | 75 +++++++ v2/ansible/plugins/lookup/file.py | 58 ++++++ v2/ansible/plugins/lookup/fileglob.py | 32 +++ v2/ansible/plugins/lookup/first_found.py | 191 +++++++++++++++++ v2/ansible/plugins/lookup/flattened.py | 69 ++++++ v2/ansible/plugins/lookup/indexed_items.py | 32 +++ .../plugins/lookup/inventory_hostnames.py | 34 +++ v2/ansible/plugins/lookup/lines.py | 35 ++++ v2/ansible/plugins/lookup/nested.py | 2 +- v2/ansible/plugins/lookup/password.py | 146 +++++++++++++ v2/ansible/plugins/lookup/pipe.py | 49 +++++ v2/ansible/plugins/lookup/random_choice.py | 37 ++++ v2/ansible/plugins/lookup/redis_kv.py | 73 +++++++ v2/ansible/plugins/lookup/sequence.py | 197 ++++++++++++++++++ v2/ansible/plugins/lookup/subelements.py | 59 ++++++ v2/ansible/plugins/lookup/template.py | 43 ++++ v2/ansible/plugins/lookup/together.py | 48 +++++ v2/ansible/plugins/strategies/__init__.py | 2 +- v2/ansible/template/__init__.py | 11 +- v2/ansible/utils/encrypt.py | 46 ++++ v2/ansible/utils/listify.py | 4 +- v2/ansible/vars/__init__.py | 6 +- v2/samples/lookup_file.yml | 5 + v2/samples/lookup_password.yml | 7 + v2/samples/lookup_pipe.py | 4 + v2/samples/lookup_template.yml | 7 + v2/samples/template.j2 | 1 + v2/samples/with_dict.yml | 15 ++ v2/samples/with_env.yml | 5 + v2/samples/with_fileglob.yml | 7 + v2/samples/with_first_found.yml | 10 + v2/samples/with_flattened.yml | 13 ++ v2/samples/with_indexed_items.yml | 11 + v2/samples/with_lines.yml | 6 + v2/samples/with_random_choice.yml | 10 + v2/samples/with_sequence.yml | 13 ++ v2/samples/with_subelements.yml | 18 ++ v2/samples/with_together.yml | 11 + 54 files changed, 1650 insertions(+), 37 deletions(-) create mode 100644 v2/ansible/plugins/lookup/csvfile.py create mode 100644 v2/ansible/plugins/lookup/dict.py create mode 100644 v2/ansible/plugins/lookup/dnstxt.py create mode 100644 v2/ansible/plugins/lookup/env.py create mode 100644 v2/ansible/plugins/lookup/etcd.py create mode 100644 v2/ansible/plugins/lookup/file.py create mode 100644 v2/ansible/plugins/lookup/fileglob.py create mode 100644 v2/ansible/plugins/lookup/first_found.py create mode 100644 v2/ansible/plugins/lookup/flattened.py create mode 100644 v2/ansible/plugins/lookup/indexed_items.py create mode 100644 v2/ansible/plugins/lookup/inventory_hostnames.py create mode 100644 v2/ansible/plugins/lookup/lines.py create mode 100644 v2/ansible/plugins/lookup/password.py create mode 100644 v2/ansible/plugins/lookup/pipe.py create mode 100644 v2/ansible/plugins/lookup/random_choice.py create mode 100644 v2/ansible/plugins/lookup/redis_kv.py create mode 100644 v2/ansible/plugins/lookup/sequence.py create mode 100644 v2/ansible/plugins/lookup/subelements.py create mode 100644 v2/ansible/plugins/lookup/template.py create mode 100644 v2/ansible/plugins/lookup/together.py create mode 100644 v2/ansible/utils/encrypt.py create mode 100644 v2/samples/lookup_file.yml create mode 100644 v2/samples/lookup_password.yml create mode 100644 v2/samples/lookup_pipe.py create mode 100644 v2/samples/lookup_template.yml create mode 100644 v2/samples/template.j2 create mode 100644 v2/samples/with_dict.yml create mode 100644 v2/samples/with_env.yml create mode 100644 v2/samples/with_fileglob.yml create mode 100644 v2/samples/with_first_found.yml create mode 100644 v2/samples/with_flattened.yml create mode 100644 v2/samples/with_indexed_items.yml create mode 100644 v2/samples/with_lines.yml create mode 100644 v2/samples/with_random_choice.yml create mode 100644 v2/samples/with_sequence.yml create mode 100644 v2/samples/with_subelements.yml create mode 100644 v2/samples/with_together.yml diff --git a/v2/ansible/executor/play_iterator.py b/v2/ansible/executor/play_iterator.py index 076d676255..f1d8914f84 100644 --- a/v2/ansible/executor/play_iterator.py +++ b/v2/ansible/executor/play_iterator.py @@ -60,6 +60,7 @@ class PlayState: (rescue/always) ''' + self._parent_iterator = parent_iterator self._run_state = ITERATING_SETUP self._failed_state = FAILED_NONE self._task_list = parent_iterator._play.compile() @@ -104,6 +105,8 @@ class PlayState: if self._gather_facts == 'smart' and not self._host.gathered_facts or boolean(self._gather_facts): self._host.set_gathered_facts(True) task = Task() + # FIXME: this is not the best way to get this... + task.set_loader(self._parent_iterator._play._loader) task.action = 'setup' break elif run_state == ITERATING_TASKS: diff --git a/v2/ansible/executor/process/worker.py b/v2/ansible/executor/process/worker.py index 3419d4ec0a..7831f51ba5 100644 --- a/v2/ansible/executor/process/worker.py +++ b/v2/ansible/executor/process/worker.py @@ -97,9 +97,23 @@ class WorkerProcess(multiprocessing.Process): try: if not self._main_q.empty(): debug("there's work to be done!") - (host, task, job_vars, connection_info) = self._main_q.get(block=False) + (host, task, basedir, job_vars, connection_info) = self._main_q.get(block=False) debug("got a task/handler to work on: %s" % task) + # because the task queue manager starts workers (forks) before the + # playbook is loaded, set the basedir of the loader inherted by + # this fork now so that we can find files correctly + self._loader.set_basedir(basedir) + + # Serializing/deserializing tasks does not preserve the loader attribute, + # since it is passed to the worker during the forking of the process and + # would be wasteful to serialize. So we set it here on the task now, and + # the task handles updating parent/child objects as needed. + task.set_loader(self._loader) + + # apply the given task's information to the connection info, + # which may override some fields already set by the play or + # the options specified on the command line new_connection_info = connection_info.set_task_override(task) # execute the task and build a TaskResult from the result diff --git a/v2/ansible/executor/task_executor.py b/v2/ansible/executor/task_executor.py index 324a4f37ea..e49bc9268f 100644 --- a/v2/ansible/executor/task_executor.py +++ b/v2/ansible/executor/task_executor.py @@ -58,7 +58,7 @@ class TaskExecutor: try: items = self._get_loop_items() - if items: + if items is not None: if len(items) > 0: item_results = self._run_loop(items) res = dict(results=item_results) @@ -84,7 +84,7 @@ class TaskExecutor: items = None if self._task.loop and self._task.loop in lookup_loader: - items = lookup_loader.get(self._task.loop).run(terms=self._task.loop_args, variables=self._job_vars) + items = lookup_loader.get(self._task.loop, loader=self._loader).run(terms=self._task.loop_args, variables=self._job_vars) return items diff --git a/v2/ansible/parsing/__init__.py b/v2/ansible/parsing/__init__.py index 1901a51c8a..233b2ecc00 100644 --- a/v2/ansible/parsing/__init__.py +++ b/v2/ansible/parsing/__init__.py @@ -204,18 +204,3 @@ class DataLoader(): self.set_basedir(cur_basedir) return source2 # which does not exist - #def __getstate__(self): - # data = dict( - # basedir = self._basedir, - # vault_password = self._vault_password, - # FILE_CACHE = self._FILE_CACHE, - # ) - # return data - - #def __setstate__(self, data): - # self._basedir = data.get('basedir', '.') - # self._FILE_CACHE = data.get('FILE_CACHE', dict()) - # self._vault_password = data.get('vault_password', '') - # - # self._vault = VaultLib(password=self._vault_password) - diff --git a/v2/ansible/playbook/base.py b/v2/ansible/playbook/base.py index 785df50032..4f0b726397 100644 --- a/v2/ansible/playbook/base.py +++ b/v2/ansible/playbook/base.py @@ -178,7 +178,7 @@ class Base: if self._loader is not None: basedir = self._loader.get_basedir() - templar = Templar(basedir=basedir, variables=all_vars, fail_on_undefined=fail_on_undefined) + templar = Templar(loader=self._loader, variables=all_vars, fail_on_undefined=fail_on_undefined) for (name, attribute) in iteritems(self._get_base_attributes()): diff --git a/v2/ansible/playbook/block.py b/v2/ansible/playbook/block.py index 9e1e6924e6..e6dd9522de 100644 --- a/v2/ansible/playbook/block.py +++ b/v2/ansible/playbook/block.py @@ -162,3 +162,10 @@ class Block(Base, Conditional, Taggable): return False return super(Block, self).evaluate_tags(only_tags=only_tags, skip_tags=skip_tags) + def set_loader(self, loader): + self._loader = loader + if self._parent_block: + self._parent_block.set_loader(loader) + elif self._role: + self._role.set_loader(loader) + diff --git a/v2/ansible/playbook/conditional.py b/v2/ansible/playbook/conditional.py index bf7eb2dc4d..82a09a3065 100644 --- a/v2/ansible/playbook/conditional.py +++ b/v2/ansible/playbook/conditional.py @@ -45,7 +45,7 @@ class Conditional: False if any of them evaluate as such. ''' - templar = Templar(variables=all_vars) + templar = Templar(loader=self._loader, variables=all_vars) for conditional in self.when: if not self._check_conditional(conditional, templar): return False diff --git a/v2/ansible/playbook/role/__init__.py b/v2/ansible/playbook/role/__init__.py index a175e91333..2a7b409c85 100644 --- a/v2/ansible/playbook/role/__init__.py +++ b/v2/ansible/playbook/role/__init__.py @@ -358,3 +358,10 @@ class Role(Base, Conditional, Taggable): super(Role, self).deserialize(data) + def set_loader(self, loader): + self._loader = loader + for parent in self._parents: + parent.set_loader(loader) + for dep in self.get_direct_dependencies(): + dep.set_loader(loader) + diff --git a/v2/ansible/playbook/task.py b/v2/ansible/playbook/task.py index 17fdf61674..c0f34b5bdd 100644 --- a/v2/ansible/playbook/task.py +++ b/v2/ansible/playbook/task.py @@ -200,7 +200,7 @@ class Task(Base, Conditional, Taggable): super(Task, self).post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined) def _post_validate_loop_args(self, attr, value, all_vars, fail_on_undefined): - return listify_lookup_plugin_terms(value, all_vars) + return listify_lookup_plugin_terms(value, all_vars, loader=self._loader) def get_vars(self): return self.serialize() @@ -283,3 +283,18 @@ class Task(Base, Conditional, Taggable): return False return super(Task, self).evaluate_tags(only_tags=only_tags, skip_tags=skip_tags) + + def set_loader(self, loader): + ''' + Sets the loader on this object and recursively on parent, child objects. + This is used primarily after the Task has been serialized/deserialized, which + does not preserve the loader. + ''' + + self._loader = loader + + if self._block: + self._block.set_loader(loader) + + for dep in self._dep_chain: + dep.set_loader(loader) diff --git a/v2/ansible/plugins/action/debug.py b/v2/ansible/plugins/action/debug.py index aa9035f265..dcee3e6347 100644 --- a/v2/ansible/plugins/action/debug.py +++ b/v2/ansible/plugins/action/debug.py @@ -33,7 +33,7 @@ class ActionModule(ActionBase): result = dict(msg=self._task.args['msg']) # FIXME: move the LOOKUP_REGEX somewhere else elif 'var' in self._task.args: # and not utils.LOOKUP_REGEX.search(self._task.args['var']): - templar = Templar(variables=task_vars) + templar = Templar(loader=self._loader, variables=task_vars) results = templar.template(self._task.args['var'], convert_bare=True) result = dict() result[self._task.args['var']] = results diff --git a/v2/ansible/plugins/action/template.py b/v2/ansible/plugins/action/template.py index 82dbedfc8d..2c8d960ced 100644 --- a/v2/ansible/plugins/action/template.py +++ b/v2/ansible/plugins/action/template.py @@ -77,7 +77,7 @@ class ActionModule(ActionBase): dest = os.path.join(dest, base) # template the source data locally & get ready to transfer - templar = Templar(basedir=self._loader.get_basedir(), variables=task_vars) + templar = Templar(loader=self._loader, variables=task_vars) try: with open(source, 'r') as f: template_data = f.read() diff --git a/v2/ansible/plugins/lookup/__init__.py b/v2/ansible/plugins/lookup/__init__.py index 9417e3830c..8c841c81d2 100644 --- a/v2/ansible/plugins/lookup/__init__.py +++ b/v2/ansible/plugins/lookup/__init__.py @@ -22,8 +22,8 @@ __metaclass__ = type __all__ = ['LookupBase'] class LookupBase: - def __init__(self, **kwargs): - pass + def __init__(self, loader=None, **kwargs): + self._loader = loader def _flatten(self, terms): ret = [] @@ -41,3 +41,9 @@ class LookupBase: results.append(self._flatten([x,y])) return results + def _flatten_hash_to_list(self, terms): + ret = [] + for key in terms: + ret.append({'key': key, 'value': terms[key]}) + return ret + diff --git a/v2/ansible/plugins/lookup/csvfile.py b/v2/ansible/plugins/lookup/csvfile.py new file mode 100644 index 0000000000..87757399ce --- /dev/null +++ b/v2/ansible/plugins/lookup/csvfile.py @@ -0,0 +1,79 @@ +# (c) 2013, Jan-Piet Mens +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import codecs +import csv + +from ansible.errors import * +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def read_csv(self, filename, key, delimiter, dflt=None, col=1): + + try: + f = codecs.open(filename, 'r', encoding='utf-8') + creader = csv.reader(f, delimiter=delimiter) + + for row in creader: + if row[0] == key: + return row[int(col)] + except Exception, e: + raise AnsibleError("csvfile: %s" % str(e)) + + return dflt + + def run(self, terms, variables=None, **kwargs): + + if isinstance(terms, basestring): + terms = [ terms ] + + ret = [] + for term in terms: + params = term.split() + key = params[0] + + paramvals = { + 'file' : 'ansible.csv', + 'default' : None, + 'delimiter' : "TAB", + 'col' : "1", # column to return + } + + # parameters specified? + try: + for param in params[1:]: + name, value = param.split('=') + assert(name in paramvals) + paramvals[name] = value + except (ValueError, AssertionError), e: + raise AnsibleError(e) + + if paramvals['delimiter'] == 'TAB': + paramvals['delimiter'] = "\t" + + path = self._loader.path_dwim(paramvals['file']) + + var = self.read_csv(path, key, paramvals['delimiter'], paramvals['default'], paramvals['col']) + if var is not None: + if type(var) is list: + for v in var: + ret.append(v) + else: + ret.append(var) + return ret diff --git a/v2/ansible/plugins/lookup/dict.py b/v2/ansible/plugins/lookup/dict.py new file mode 100644 index 0000000000..61389df7c2 --- /dev/null +++ b/v2/ansible/plugins/lookup/dict.py @@ -0,0 +1,27 @@ +# (c) 2014, Kent R. Spillner +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def run(self, terms, varibles=None, **kwargs): + + if not isinstance(terms, dict): + raise errors.AnsibleError("with_dict expects a dict") + + return self._flatten_hash_to_list(terms) diff --git a/v2/ansible/plugins/lookup/dnstxt.py b/v2/ansible/plugins/lookup/dnstxt.py new file mode 100644 index 0000000000..7100f8d96d --- /dev/null +++ b/v2/ansible/plugins/lookup/dnstxt.py @@ -0,0 +1,68 @@ +# (c) 2012, Jan-Piet Mens +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os + +HAVE_DNS=False +try: + import dns.resolver + from dns.exception import DNSException + HAVE_DNS=True +except ImportError: + pass + +from ansible.errors import * +from ansible.plugins.lookup import LookupBase + +# ============================================================== +# DNSTXT: DNS TXT records +# +# key=domainname +# TODO: configurable resolver IPs +# -------------------------------------------------------------- + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + if HAVE_DNS == False: + raise AnsibleError("Can't LOOKUP(dnstxt): module dns.resolver is not installed") + + if isinstance(terms, basestring): + terms = [ terms ] + + ret = [] + for term in terms: + domain = term.split()[0] + string = [] + try: + answers = dns.resolver.query(domain, 'TXT') + for rdata in answers: + s = rdata.to_text() + string.append(s[1:-1]) # Strip outside quotes on TXT rdata + + except dns.resolver.NXDOMAIN: + string = 'NXDOMAIN' + except dns.resolver.Timeout: + string = '' + except dns.exception.DNSException, e: + raise AnsibleError("dns.resolver unhandled exception", e) + + ret.append(''.join(string)) + + return ret + diff --git a/v2/ansible/plugins/lookup/env.py b/v2/ansible/plugins/lookup/env.py new file mode 100644 index 0000000000..896f95e13a --- /dev/null +++ b/v2/ansible/plugins/lookup/env.py @@ -0,0 +1,34 @@ +# (c) 2012, Jan-Piet Mens +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os + +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + if isinstance(terms, basestring): + terms = [ terms ] + + ret = [] + for term in terms: + var = term.split()[0] + ret.append(os.getenv(var, '')) + + return ret diff --git a/v2/ansible/plugins/lookup/etcd.py b/v2/ansible/plugins/lookup/etcd.py new file mode 100644 index 0000000000..5b54788985 --- /dev/null +++ b/v2/ansible/plugins/lookup/etcd.py @@ -0,0 +1,75 @@ +# (c) 2013, Jan-Piet Mens +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import urllib2 +try: + import json +except ImportError: + import simplejson as json + +from ansible.plugins.lookup import LookupBase + +# this can be made configurable, not should not use ansible.cfg +ANSIBLE_ETCD_URL = 'http://127.0.0.1:4001' +if os.getenv('ANSIBLE_ETCD_URL') is not None: + ANSIBLE_ETCD_URL = os.environ['ANSIBLE_ETCD_URL'] + +class etcd(): + def __init__(self, url=ANSIBLE_ETCD_URL): + self.url = url + self.baseurl = '%s/v1/keys' % (self.url) + + def get(self, key): + url = "%s/%s" % (self.baseurl, key) + + data = None + value = "" + try: + r = urllib2.urlopen(url) + data = r.read() + except: + return value + + try: + # {"action":"get","key":"/name","value":"Jane Jolie","index":5} + item = json.loads(data) + if 'value' in item: + value = item['value'] + if 'errorCode' in item: + value = "ENOENT" + except: + raise + pass + + return value + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + if isinstance(terms, basestring): + terms = [ terms ] + + etcd = etcd() + + ret = [] + for term in terms: + key = term.split()[0] + value = etcd.get(key) + ret.append(value) + return ret diff --git a/v2/ansible/plugins/lookup/file.py b/v2/ansible/plugins/lookup/file.py new file mode 100644 index 0000000000..add4da7f47 --- /dev/null +++ b/v2/ansible/plugins/lookup/file.py @@ -0,0 +1,58 @@ +# (c) 2012, Daniel Hokka Zakrisson +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import codecs + +from ansible.errors import * +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + if not isinstance(terms, list): + terms = [ terms ] + + ret = [] + for term in terms: + basedir_path = self._loader.path_dwim(term) + relative_path = None + playbook_path = None + + # Special handling of the file lookup, used primarily when the + # lookup is done from a role. If the file isn't found in the + # basedir of the current file, use dwim_relative to look in the + # role/files/ directory, and finally the playbook directory + # itself (which will be relative to the current working dir) + + # FIXME: the original file stuff still needs to be worked out, but the + # playbook_dir stuff should be able to be removed as it should + # be covered by the fact that the loader contains that info + #if '_original_file' in variables: + # relative_path = self._loader.path_dwim_relative(variables['_original_file'], 'files', term, self.basedir, check=False) + #if 'playbook_dir' in variables: + # playbook_path = os.path.join(variables['playbook_dir'], term) + + for path in (basedir_path, relative_path, playbook_path): + if path and os.path.exists(path): + ret.append(codecs.open(path, encoding="utf8").read().rstrip()) + break + else: + raise AnsibleError("could not locate file in lookup: %s" % term) + + return ret diff --git a/v2/ansible/plugins/lookup/fileglob.py b/v2/ansible/plugins/lookup/fileglob.py new file mode 100644 index 0000000000..bde016af9e --- /dev/null +++ b/v2/ansible/plugins/lookup/fileglob.py @@ -0,0 +1,32 @@ +# (c) 2012, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import glob + +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + ret = [] + for term in terms: + dwimmed = self._loader.path_dwim(term) + globbed = glob.glob(dwimmed) + ret.extend(g for g in globbed if os.path.isfile(g)) + return ret diff --git a/v2/ansible/plugins/lookup/first_found.py b/v2/ansible/plugins/lookup/first_found.py new file mode 100644 index 0000000000..439b48ee44 --- /dev/null +++ b/v2/ansible/plugins/lookup/first_found.py @@ -0,0 +1,191 @@ +# (c) 2013, seth vidal red hat, inc +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# take a list of files and (optionally) a list of paths +# return the first existing file found in the paths +# [file1, file2, file3], [path1, path2, path3] +# search order is: +# path1/file1 +# path1/file2 +# path1/file3 +# path2/file1 +# path2/file2 +# path2/file3 +# path3/file1 +# path3/file2 +# path3/file3 + +# first file found with os.path.exists() is returned +# no file matches raises ansibleerror +# EXAMPLES +# - name: copy first existing file found to /some/file +# action: copy src=$item dest=/some/file +# with_first_found: +# - files: foo ${inventory_hostname} bar +# paths: /tmp/production /tmp/staging + +# that will look for files in this order: +# /tmp/production/foo +# ${inventory_hostname} +# bar +# /tmp/staging/foo +# ${inventory_hostname} +# bar + +# - name: copy first existing file found to /some/file +# action: copy src=$item dest=/some/file +# with_first_found: +# - files: /some/place/foo ${inventory_hostname} /some/place/else + +# that will look for files in this order: +# /some/place/foo +# $relative_path/${inventory_hostname} +# /some/place/else + +# example - including tasks: +# tasks: +# - include: $item +# with_first_found: +# - files: generic +# paths: tasks/staging tasks/production +# this will include the tasks in the file generic where it is found first (staging or production) + +# example simple file lists +#tasks: +#- name: first found file +# action: copy src=$item dest=/etc/file.cfg +# with_first_found: +# - files: foo.${inventory_hostname} foo + + +# example skipping if no matched files +# First_found also offers the ability to control whether or not failing +# to find a file returns an error or not +# +#- name: first found file - or skip +# action: copy src=$item dest=/etc/file.cfg +# with_first_found: +# - files: foo.${inventory_hostname} +# skip: true + +# example a role with default configuration and configuration per host +# you can set multiple terms with their own files and paths to look through. +# consider a role that sets some configuration per host falling back on a default config. +# +#- name: some configuration template +# template: src={{ item }} dest=/etc/file.cfg mode=0444 owner=root group=root +# with_first_found: +# - files: +# - ${inventory_hostname}/etc/file.cfg +# paths: +# - ../../../templates.overwrites +# - ../../../templates +# - files: +# - etc/file.cfg +# paths: +# - templates + +# the above will return an empty list if the files cannot be found at all +# if skip is unspecificed or if it is set to false then it will return a list +# error which can be caught bye ignore_errors: true for that action. + +# finally - if you want you can use it, in place to replace first_available_file: +# you simply cannot use the - files, path or skip options. simply replace +# first_available_file with with_first_found and leave the file listing in place +# +# +# - name: with_first_found like first_available_file +# action: copy src=$item dest=/tmp/faftest +# with_first_found: +# - ../files/foo +# - ../files/bar +# - ../files/baz +# ignore_errors: true + + +import os + +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + result = None + anydict = False + skip = False + + for term in terms: + if isinstance(term, dict): + anydict = True + + total_search = [] + if anydict: + for term in terms: + if isinstance(term, dict): + files = term.get('files', []) + paths = term.get('paths', []) + skip = boolean(term.get('skip', False)) + + filelist = files + if isinstance(files, basestring): + files = files.replace(',', ' ') + files = files.replace(';', ' ') + filelist = files.split(' ') + + pathlist = paths + if paths: + if isinstance(paths, basestring): + paths = paths.replace(',', ' ') + paths = paths.replace(':', ' ') + paths = paths.replace(';', ' ') + pathlist = paths.split(' ') + + if not pathlist: + total_search = filelist + else: + for path in pathlist: + for fn in filelist: + f = os.path.join(path, fn) + total_search.append(f) + else: + total_search.append(term) + else: + total_search = terms + + for fn in total_search: + # FIXME: the original file stuff needs to be fixed/implemented + #if variables and '_original_file' in variables: + # # check the templates and vars directories too, + # # if they exist + # for roledir in ('templates', 'vars'): + # path = self._loader.path_dwim(os.path.join(self.basedir, '..', roledir), fn) + # if os.path.exists(path): + # return [path] + + # if none of the above were found, just check the + # current filename against the basedir (this will already + # have ../files from runner, if it's a role task + path = self._loader.path_dwim(fn) + if os.path.exists(path): + return [path] + else: + if skip: + return [] + else: + return [None] + diff --git a/v2/ansible/plugins/lookup/flattened.py b/v2/ansible/plugins/lookup/flattened.py new file mode 100644 index 0000000000..24f1a9ac95 --- /dev/null +++ b/v2/ansible/plugins/lookup/flattened.py @@ -0,0 +1,69 @@ +# (c) 2013, Serge van Ginderachter +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + + +from ansible.errors import * +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms + +class LookupModule(LookupBase): + + def _check_list_of_one_list(self, term): + # make sure term is not a list of one (list of one..) item + # return the final non list item if so + + if isinstance(term,list) and len(term) == 1: + term = term[0] + if isinstance(term,list): + term = self._check_list_of_one_list(term) + + return term + + def _do_flatten(self, terms, variables): + + ret = [] + for term in terms: + term = self._check_list_of_one_list(term) + + if term == 'None' or term == 'null': + # ignore undefined items + break + + if isinstance(term, basestring): + # convert a variable to a list + term2 = listify_lookup_plugin_terms(term, variables, loader=self._loader) + # but avoid converting a plain string to a list of one string + if term2 != [ term ]: + term = term2 + + if isinstance(term, list): + # if it's a list, check recursively for items that are a list + term = self._do_flatten(term, variables) + ret.extend(term) + else: + ret.append(term) + + return ret + + + def run(self, terms, variables, **kwargs): + + if not isinstance(terms, list): + raise AnsibleError("with_flattened expects a list") + + return self._do_flatten(terms, variables) + diff --git a/v2/ansible/plugins/lookup/indexed_items.py b/v2/ansible/plugins/lookup/indexed_items.py new file mode 100644 index 0000000000..1731dc0e84 --- /dev/null +++ b/v2/ansible/plugins/lookup/indexed_items.py @@ -0,0 +1,32 @@ +# (c) 2012, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def __init__(self, basedir=None, **kwargs): + self.basedir = basedir + + def run(self, terms, variables, **kwargs): + + if not isinstance(terms, list): + raise errors.AnsibleError("with_indexed_items expects a list") + + items = self._flatten(terms) + return zip(range(len(items)), items) + diff --git a/v2/ansible/plugins/lookup/inventory_hostnames.py b/v2/ansible/plugins/lookup/inventory_hostnames.py new file mode 100644 index 0000000000..faffe47eb8 --- /dev/null +++ b/v2/ansible/plugins/lookup/inventory_hostnames.py @@ -0,0 +1,34 @@ +# (c) 2012, Michael DeHaan +# (c) 2013, Steven Dossett +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +from ansible.errors import * +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def run(self, terms, inject=None, **kwargs): + if not isinstance(terms, list): + raise AnsibleError("with_inventory_hostnames expects a list") + + # FIXME: the inventory is no longer available this way, so we may have + # to dump the host list into the list of variables and read it back + # in here (or the inventory sources, so we can recreate the list + # of hosts) + #return self._flatten(inventory.Inventory(self.host_list).list_hosts(terms)) + return terms + diff --git a/v2/ansible/plugins/lookup/lines.py b/v2/ansible/plugins/lookup/lines.py new file mode 100644 index 0000000000..507793b18e --- /dev/null +++ b/v2/ansible/plugins/lookup/lines.py @@ -0,0 +1,35 @@ +# (c) 2012, Daniel Hokka Zakrisson +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import subprocess + +from ansible.errors import * +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + ret = [] + for term in terms: + p = subprocess.Popen(term, cwd=self._loader.get_basedir(), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode == 0: + ret.extend(stdout.splitlines()) + else: + raise AnsibleError("lookup_plugin.lines(%s) returned %d" % (term, p.returncode)) + return ret diff --git a/v2/ansible/plugins/lookup/nested.py b/v2/ansible/plugins/lookup/nested.py index 8cafa4a936..0f2d146b47 100644 --- a/v2/ansible/plugins/lookup/nested.py +++ b/v2/ansible/plugins/lookup/nested.py @@ -24,7 +24,7 @@ class LookupModule(LookupBase): def __lookup_variabless(self, terms, variables): results = [] for x in terms: - intermediate = listify_lookup_plugin_terms(x, variables) + intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader) results.append(intermediate) return results diff --git a/v2/ansible/plugins/lookup/password.py b/v2/ansible/plugins/lookup/password.py new file mode 100644 index 0000000000..6e13410e1a --- /dev/null +++ b/v2/ansible/plugins/lookup/password.py @@ -0,0 +1,146 @@ +# (c) 2012, Daniel Hokka Zakrisson +# (c) 2013, Javier Candeira +# (c) 2013, Maykel Moya +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import errno +import string +import random + +from string import ascii_letters, digits + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from ansible.utils.encrypt import do_encrypt + +DEFAULT_LENGTH = 20 + +class LookupModule(LookupBase): + + def random_password(self, length=DEFAULT_LENGTH, chars=C.DEFAULT_PASSWORD_CHARS): + ''' + Return a random password string of length containing only chars. + NOTE: this was moved from the old ansible utils code, as nothing + else appeared to use it. + ''' + + password = [] + while len(password) < length: + new_char = os.urandom(1) + if new_char in chars: + password.append(new_char) + + return ''.join(password) + + def random_salt(self): + salt_chars = ascii_letters + digits + './' + return self.random_password(length=8, chars=salt_chars) + + def run(self, terms, variables, **kwargs): + + ret = [] + + if not isinstance(terms, list): + terms = [ terms ] + + for term in terms: + # you can't have escaped spaces in yor pathname + params = term.split() + relpath = params[0] + + paramvals = { + 'length': DEFAULT_LENGTH, + 'encrypt': None, + 'chars': ['ascii_letters','digits',".,:-_"], + } + + # get non-default parameters if specified + try: + for param in params[1:]: + name, value = param.split('=') + assert(name in paramvals) + if name == 'length': + paramvals[name] = int(value) + elif name == 'chars': + use_chars=[] + if ",," in value: + use_chars.append(',') + use_chars.extend(value.replace(',,',',').split(',')) + paramvals['chars'] = use_chars + else: + paramvals[name] = value + except (ValueError, AssertionError), e: + raise AnsibleError(e) + + length = paramvals['length'] + encrypt = paramvals['encrypt'] + use_chars = paramvals['chars'] + + # get password or create it if file doesn't exist + path = self._loader.path_dwim(relpath) + if not os.path.exists(path): + pathdir = os.path.dirname(path) + if not os.path.isdir(pathdir): + try: + os.makedirs(pathdir, mode=0700) + except OSError, e: + raise AnsibleError("cannot create the path for the password lookup: %s (error was %s)" % (pathdir, str(e))) + + chars = "".join([getattr(string,c,c) for c in use_chars]).replace('"','').replace("'",'') + password = ''.join(random.choice(chars) for _ in range(length)) + + if encrypt is not None: + salt = self.random_salt() + content = '%s salt=%s' % (password, salt) + else: + content = password + with open(path, 'w') as f: + os.chmod(path, 0600) + f.write(content + '\n') + else: + content = open(path).read().rstrip() + sep = content.find(' ') + + if sep >= 0: + password = content[:sep] + salt = content[sep+1:].split('=')[1] + else: + password = content + salt = None + + # crypt requested, add salt if missing + if (encrypt is not None and not salt): + salt = self.random_salt() + content = '%s salt=%s' % (password, salt) + with open(path, 'w') as f: + os.chmod(path, 0600) + f.write(content + '\n') + # crypt not requested, remove salt if present + elif (encrypt is None and salt): + with open(path, 'w') as f: + os.chmod(path, 0600) + f.write(password + '\n') + + if encrypt: + password = do_encrypt(password, encrypt, salt=salt) + + ret.append(password) + + return ret + diff --git a/v2/ansible/plugins/lookup/pipe.py b/v2/ansible/plugins/lookup/pipe.py new file mode 100644 index 0000000000..0a7e5cb31a --- /dev/null +++ b/v2/ansible/plugins/lookup/pipe.py @@ -0,0 +1,49 @@ +# (c) 2012, Daniel Hokka Zakrisson +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import subprocess + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + if isinstance(terms, basestring): + terms = [ terms ] + + ret = [] + for term in terms: + ''' + http://docs.python.org/2/library/subprocess.html#popen-constructor + + The shell argument (which defaults to False) specifies whether to use the + shell as the program to execute. If shell is True, it is recommended to pass + args as a string rather than as a sequence + + https://github.com/ansible/ansible/issues/6550 + ''' + term = str(term) + + p = subprocess.Popen(term, cwd=self._loader.get_basedir(), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode == 0: + ret.append(stdout.decode("utf-8").rstrip()) + else: + raise AnsibleError("lookup_plugin.pipe(%s) returned %d" % (term, p.returncode)) + return ret diff --git a/v2/ansible/plugins/lookup/random_choice.py b/v2/ansible/plugins/lookup/random_choice.py new file mode 100644 index 0000000000..e899a2dbe3 --- /dev/null +++ b/v2/ansible/plugins/lookup/random_choice.py @@ -0,0 +1,37 @@ +# (c) 2013, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import random + +from ansible.plugins.lookup import LookupBase + +# useful for introducing chaos ... or just somewhat reasonably fair selection +# amongst available mirrors +# +# tasks: +# - debug: msg=$item +# with_random_choice: +# - one +# - two +# - three + +class LookupModule(LookupBase): + + def run(self, terms, inject=None, **kwargs): + + return [ random.choice(terms) ] + diff --git a/v2/ansible/plugins/lookup/redis_kv.py b/v2/ansible/plugins/lookup/redis_kv.py new file mode 100644 index 0000000000..08895d4c4e --- /dev/null +++ b/v2/ansible/plugins/lookup/redis_kv.py @@ -0,0 +1,73 @@ +# (c) 2012, Jan-Piet Mens +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import re + +HAVE_REDIS=False +try: + import redis # https://github.com/andymccurdy/redis-py/ + HAVE_REDIS=True +except ImportError: + pass + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + +# ============================================================== +# REDISGET: Obtain value from a GET on a Redis key. Terms +# expected: 0 = URL, 1 = Key +# URL may be empty, in which case redis://localhost:6379 assumed +# -------------------------------------------------------------- + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + if not HAVE_REDIS: + raise AnsibleError("Can't LOOKUP(redis_kv): module redis is not installed") + + if not isinstance(terms, list): + terms = [ terms ] + + ret = [] + for term in terms: + (url,key) = term.split(',') + if url == "": + url = 'redis://localhost:6379' + + # urlsplit on Python 2.6.1 is broken. Hmm. Probably also the reason + # Redis' from_url() doesn't work here. + + p = '(?P[^:]+)://?(?P[^:/ ]+).?(?P[0-9]*).*' + + try: + m = re.search(p, url) + host = m.group('host') + port = int(m.group('port')) + except AttributeError: + raise AnsibleError("Bad URI in redis lookup") + + try: + conn = redis.Redis(host=host, port=port) + res = conn.get(key) + if res is None: + res = "" + ret.append(res) + except: + ret.append("") # connection failed or key not found + return ret diff --git a/v2/ansible/plugins/lookup/sequence.py b/v2/ansible/plugins/lookup/sequence.py new file mode 100644 index 0000000000..5347b90f6a --- /dev/null +++ b/v2/ansible/plugins/lookup/sequence.py @@ -0,0 +1,197 @@ +# (c) 2013, Jayson Vantuyl +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +from re import compile as re_compile, IGNORECASE + +from ansible.errors import * +from ansible.parsing.splitter import parse_kv +from ansible.plugins.lookup import LookupBase + +# shortcut format +NUM = "(0?x?[0-9a-f]+)" +SHORTCUT = re_compile( + "^(" + # Group 0 + NUM + # Group 1: Start + "-)?" + + NUM + # Group 2: End + "(/" + # Group 3 + NUM + # Group 4: Stride + ")?" + + "(:(.+))?$", # Group 5, Group 6: Format String + IGNORECASE +) + + +class LookupModule(LookupBase): + """ + sequence lookup module + + Used to generate some sequence of items. Takes arguments in two forms. + + The simple / shortcut form is: + + [start-]end[/stride][:format] + + As indicated by the brackets: start, stride, and format string are all + optional. The format string is in the style of printf. This can be used + to pad with zeros, format in hexadecimal, etc. All of the numerical values + can be specified in octal (i.e. 0664) or hexadecimal (i.e. 0x3f8). + Negative numbers are not supported. + + Some examples: + + 5 -> ["1","2","3","4","5"] + 5-8 -> ["5", "6", "7", "8"] + 2-10/2 -> ["2", "4", "6", "8", "10"] + 4:host%02d -> ["host01","host02","host03","host04"] + + The standard Ansible key-value form is accepted as well. For example: + + start=5 end=11 stride=2 format=0x%02x -> ["0x05","0x07","0x09","0x0a"] + + This format takes an alternate form of "end" called "count", which counts + some number from the starting value. For example: + + count=5 -> ["1", "2", "3", "4", "5"] + start=0x0f00 count=4 format=%04x -> ["0f00", "0f01", "0f02", "0f03"] + start=0 count=5 stride=2 -> ["0", "2", "4", "6", "8"] + start=1 count=5 stride=2 -> ["1", "3", "5", "7", "9"] + + The count option is mostly useful for avoiding off-by-one errors and errors + calculating the number of entries in a sequence when a stride is specified. + """ + + def reset(self): + """set sensible defaults""" + self.start = 1 + self.count = None + self.end = None + self.stride = 1 + self.format = "%d" + + def parse_kv_args(self, args): + """parse key-value style arguments""" + for arg in ["start", "end", "count", "stride"]: + try: + arg_raw = args.pop(arg, None) + if arg_raw is None: + continue + arg_cooked = int(arg_raw, 0) + setattr(self, arg, arg_cooked) + except ValueError: + raise AnsibleError( + "can't parse arg %s=%r as integer" + % (arg, arg_raw) + ) + if 'format' in args: + self.format = args.pop("format") + if args: + raise AnsibleError( + "unrecognized arguments to with_sequence: %r" + % args.keys() + ) + + def parse_simple_args(self, term): + """parse the shortcut forms, return True/False""" + match = SHORTCUT.match(term) + if not match: + return False + + _, start, end, _, stride, _, format = match.groups() + + if start is not None: + try: + start = int(start, 0) + except ValueError: + raise AnsibleError("can't parse start=%s as integer" % start) + if end is not None: + try: + end = int(end, 0) + except ValueError: + raise AnsibleError("can't parse end=%s as integer" % end) + if stride is not None: + try: + stride = int(stride, 0) + except ValueError: + raise AnsibleError("can't parse stride=%s as integer" % stride) + + if start is not None: + self.start = start + if end is not None: + self.end = end + if stride is not None: + self.stride = stride + if format is not None: + self.format = format + + def sanity_check(self): + if self.count is None and self.end is None: + raise AnsibleError( + "must specify count or end in with_sequence" + ) + elif self.count is not None and self.end is not None: + raise AnsibleError( + "can't specify both count and end in with_sequence" + ) + elif self.count is not None: + # convert count to end + self.end = self.start + self.count * self.stride - 1 + del self.count + if self.end < self.start: + raise AnsibleError("can't count backwards") + if self.format.count('%') != 1: + raise AnsibleError("bad formatting string: %s" % self.format) + + def generate_sequence(self): + numbers = xrange(self.start, self.end + 1, self.stride) + + for i in numbers: + try: + formatted = self.format % i + yield formatted + except (ValueError, TypeError): + raise AnsibleError( + "problem formatting %r with %r" % self.format + ) + + def run(self, terms, variables, **kwargs): + results = [] + + if isinstance(terms, basestring): + terms = [ terms ] + + for term in terms: + try: + self.reset() # clear out things for this iteration + + try: + if not self.parse_simple_args(term): + self.parse_kv_args(parse_kv(term)) + except Exception, e: + raise AnsibleError("unknown error parsing with_sequence arguments: %r" % term) + + self.sanity_check() + + results.extend(self.generate_sequence()) + except AnsibleError: + raise + except Exception: + raise AnsibleError( + "unknown error generating sequence" + ) + + return results diff --git a/v2/ansible/plugins/lookup/subelements.py b/v2/ansible/plugins/lookup/subelements.py new file mode 100644 index 0000000000..93e9e570c4 --- /dev/null +++ b/v2/ansible/plugins/lookup/subelements.py @@ -0,0 +1,59 @@ +# (c) 2013, Serge van Ginderachter +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +from ansible.errors import * +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + terms[0] = listify_lookup_plugin_terms(terms[0], variables, loader=self._loader) + + if not isinstance(terms, list) or not len(terms) == 2: + raise AnsibleError("subelements lookup expects a list of two items, first a dict or a list, and second a string") + + if isinstance(terms[0], dict): # convert to list: + if terms[0].get('skipped',False) != False: + # the registered result was completely skipped + return [] + elementlist = [] + for key in terms[0].iterkeys(): + elementlist.append(terms[0][key]) + else: + elementlist = terms[0] + + subelement = terms[1] + + ret = [] + for item0 in elementlist: + if not isinstance(item0, dict): + raise AnsibleError("subelements lookup expects a dictionary, got '%s'" %item0) + if item0.get('skipped', False) != False: + # this particular item is to be skipped + continue + if not subelement in item0: + raise AnsibleError("could not find '%s' key in iterated item '%s'" % (subelement, item0)) + if not isinstance(item0[subelement], list): + raise AnsibleError("the key %s should point to a list, got '%s'" % (subelement, item0[subelement])) + sublist = item0.pop(subelement, []) + for item1 in sublist: + ret.append((item0, item1)) + + return ret + diff --git a/v2/ansible/plugins/lookup/template.py b/v2/ansible/plugins/lookup/template.py new file mode 100644 index 0000000000..74406f6445 --- /dev/null +++ b/v2/ansible/plugins/lookup/template.py @@ -0,0 +1,43 @@ +# (c) 2012, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from ansible.template import Templar + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + if not isinstance(terms, list): + terms = [ terms ] + + templar = Templar(loader=self._loader, variables=variables) + + ret = [] + for term in terms: + path = self._loader.path_dwim(term) + if os.path.exists(path): + with open(path, 'r') as f: + template_data = f.read() + res = templar.template(template_data, preserve_trailing_newlines=True) + ret.append(res) + else: + raise AnsibleError("the template file %s could not be found for the lookup" % term) + return ret diff --git a/v2/ansible/plugins/lookup/together.py b/v2/ansible/plugins/lookup/together.py new file mode 100644 index 0000000000..aa13cd7e60 --- /dev/null +++ b/v2/ansible/plugins/lookup/together.py @@ -0,0 +1,48 @@ +# (c) 2013, Bradley Young +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +from itertools import izip_longest + +from ansible.errors import * +from ansible.plugins.lookup import LookupBase +from ansible.utils.listify import listify_lookup_plugin_terms + +class LookupModule(LookupBase): + """ + Transpose a list of arrays: + [1, 2, 3], [4, 5, 6] -> [1, 4], [2, 5], [3, 6] + Replace any empty spots in 2nd array with None: + [1, 2], [3] -> [1, 3], [2, None] + """ + + def __lookup_variabless(self, terms, variables): + results = [] + for x in terms: + intermediate = listify_lookup_plugin_terms(x, variables) + results.append(intermediate) + return results + + def run(self, terms, variables=None, **kwargs): + + terms = self.__lookup_variabless(terms, variables) + + my_list = terms[:] + if len(my_list) == 0: + raise errors.AnsibleError("with_together requires at least one element in each list") + + return [self._flatten(x) for x in izip_longest(*my_list, fillvalue=None)] + diff --git a/v2/ansible/plugins/strategies/__init__.py b/v2/ansible/plugins/strategies/__init__.py index 58e76a4c44..49cdca2ccf 100644 --- a/v2/ansible/plugins/strategies/__init__.py +++ b/v2/ansible/plugins/strategies/__init__.py @@ -116,7 +116,7 @@ class StrategyBase: self._cur_worker = 0 self._pending_results += 1 - main_q.put((host, new_task, task_vars, connection_info), block=False) + main_q.put((host, new_task, self._loader.get_basedir(), task_vars, connection_info), block=False) except (EOFError, IOError, AssertionError), e: # most likely an abort debug("got an error while queuing: %s" % e) diff --git a/v2/ansible/template/__init__.py b/v2/ansible/template/__init__.py index 65596d9aa6..1f139b899c 100644 --- a/v2/ansible/template/__init__.py +++ b/v2/ansible/template/__init__.py @@ -41,8 +41,9 @@ class Templar: The main class for templating, with the main entry-point of template(). ''' - def __init__(self, basedir=None, variables=dict(), fail_on_undefined=C.DEFAULT_UNDEFINED_VAR_BEHAVIOR): - self._basedir = basedir + def __init__(self, loader, variables=dict(), fail_on_undefined=C.DEFAULT_UNDEFINED_VAR_BEHAVIOR): + self._loader = loader + self._basedir = loader.get_basedir() self._filters = None self._available_variables = variables @@ -180,15 +181,17 @@ class Templar: return thing if thing is not None else '' def _lookup(self, name, *args, **kwargs): - instance = lookup_loader.get(name.lower(), basedir=kwargs.get('basedir',None)) + instance = lookup_loader.get(name.lower(), loader=self._loader) if instance is not None: # safely catch run failures per #5059 try: - ran = instance.run(*args, inject=self._available_vars, **kwargs) + ran = instance.run(*args, variables=self._available_variables, **kwargs) except AnsibleUndefinedVariable: raise except Exception, e: + if self._fail_on_lookup_errors: + raise ran = None if ran: ran = ",".join(ran) diff --git a/v2/ansible/utils/encrypt.py b/v2/ansible/utils/encrypt.py new file mode 100644 index 0000000000..878b461c86 --- /dev/null +++ b/v2/ansible/utils/encrypt.py @@ -0,0 +1,46 @@ +# (c) 2012-2014, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +PASSLIB_AVAILABLE = False +try: + import passlib.hash + PASSLIB_AVAILABLE = True +except: + pass + +from ansible.errors import AnsibleError + +__all__ = ['do_encrypt'] + +def do_encrypt(result, encrypt, salt_size=None, salt=None): + if PASSLIB_AVAILABLE: + try: + crypt = getattr(passlib.hash, encrypt) + except: + raise AnsibleError("passlib does not support '%s' algorithm" % encrypt) + + if salt_size: + result = crypt.encrypt(result, salt_size=salt_size) + elif salt: + result = crypt.encrypt(result, salt=salt) + else: + result = crypt.encrypt(result) + else: + raise AnsibleError("passlib must be installed to encrypt vars_prompt values") + + return result + diff --git a/v2/ansible/utils/listify.py b/v2/ansible/utils/listify.py index 72b5a08600..bec7326b35 100644 --- a/v2/ansible/utils/listify.py +++ b/v2/ansible/utils/listify.py @@ -30,7 +30,7 @@ __all__ = ['listify_lookup_plugin_terms'] LOOKUP_REGEX = re.compile(r'lookup\s*\(') -def listify_lookup_plugin_terms(terms, variables): +def listify_lookup_plugin_terms(terms, variables, loader): if isinstance(terms, basestring): # someone did: @@ -46,7 +46,7 @@ def listify_lookup_plugin_terms(terms, variables): # if not already a list, get ready to evaluate with Jinja2 # not sure why the "/" is in above code :) try: - templar = Templar(variables=variables) + templar = Templar(loader=loader, variables=variables) new_terms = templar.template("{{ %s }}" % terms) if isinstance(new_terms, basestring) and "{{" in new_terms: pass diff --git a/v2/ansible/vars/__init__.py b/v2/ansible/vars/__init__.py index 49d02121ab..074271c715 100644 --- a/v2/ansible/vars/__init__.py +++ b/v2/ansible/vars/__init__.py @@ -41,8 +41,6 @@ class VariableManager: self._host_vars_files = defaultdict(dict) self._group_vars_files = defaultdict(dict) - self._templar = Templar() - def _get_cache_entry(self, play=None, host=None, task=None): play_id = "NONE" if play: @@ -156,10 +154,10 @@ class VariableManager: if play: all_vars = self._merge_dicts(all_vars, play.get_vars()) + templar = Templar(loader=loader, variables=all_vars) for vars_file in play.get_vars_files(): - self._templar.set_available_variables(all_vars) try: - vars_file = self._templar.template(vars_file) + vars_file = templar.template(vars_file) data = loader.load_from_file(vars_file) all_vars = self._merge_dicts(all_vars, data) except: diff --git a/v2/samples/lookup_file.yml b/v2/samples/lookup_file.yml new file mode 100644 index 0000000000..15cec9d294 --- /dev/null +++ b/v2/samples/lookup_file.yml @@ -0,0 +1,5 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg="the pubkey is {{lookup('file', '~/.ssh/id_rsa.pub')}}" diff --git a/v2/samples/lookup_password.yml b/v2/samples/lookup_password.yml new file mode 100644 index 0000000000..07bc71358b --- /dev/null +++ b/v2/samples/lookup_password.yml @@ -0,0 +1,7 @@ +- hosts: localhost + gather_facts: no + #vars: + # my_password: "{{ lookup('password', '/tmp/test_lookup_password length=15') }}" + tasks: + #- debug: msg="the password is {{my_password}}" + - debug: msg="the password is {{ lookup('password', '/tmp/test_lookup_password length=15') }}" diff --git a/v2/samples/lookup_pipe.py b/v2/samples/lookup_pipe.py new file mode 100644 index 0000000000..4430c76fc5 --- /dev/null +++ b/v2/samples/lookup_pipe.py @@ -0,0 +1,4 @@ +- hosts: localhost + gather_facts: no + tasks: + - debug: msg="the date is {{ lookup('pipe', 'date') }}" diff --git a/v2/samples/lookup_template.yml b/v2/samples/lookup_template.yml new file mode 100644 index 0000000000..8fdd981b9f --- /dev/null +++ b/v2/samples/lookup_template.yml @@ -0,0 +1,7 @@ +- hosts: localhost + gather_facts: no + vars: + my_var: "Bazinga!" + tasks: + - debug: msg="the rendered template is {{ lookup('template', 'template.j2') }}" + diff --git a/v2/samples/template.j2 b/v2/samples/template.j2 new file mode 100644 index 0000000000..b564862cc7 --- /dev/null +++ b/v2/samples/template.j2 @@ -0,0 +1 @@ +the variable is {{my_var}} diff --git a/v2/samples/with_dict.yml b/v2/samples/with_dict.yml new file mode 100644 index 0000000000..59aa3da16e --- /dev/null +++ b/v2/samples/with_dict.yml @@ -0,0 +1,15 @@ +- hosts: localhost + connection: local + gather_facts: no + vars: + users: + alice: + name: Alice Appleworth + telephone: 123-456-7890 + bob: + name: Bob Bananarama + telephone: 987-654-3210 + tasks: + - name: Print phone records + debug: msg="User {{ item.key }} is {{ item.value.name }} ({{ item.value.telephone }})" + with_dict: users diff --git a/v2/samples/with_env.yml b/v2/samples/with_env.yml new file mode 100644 index 0000000000..856df20867 --- /dev/null +++ b/v2/samples/with_env.yml @@ -0,0 +1,5 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg="{{ lookup('env','HOME') }} is an environment variable" diff --git a/v2/samples/with_fileglob.yml b/v2/samples/with_fileglob.yml new file mode 100644 index 0000000000..f955ee2132 --- /dev/null +++ b/v2/samples/with_fileglob.yml @@ -0,0 +1,7 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg="file is {{item}}" + with_fileglob: + - "*.yml" diff --git a/v2/samples/with_first_found.yml b/v2/samples/with_first_found.yml new file mode 100644 index 0000000000..e64b36cb50 --- /dev/null +++ b/v2/samples/with_first_found.yml @@ -0,0 +1,10 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg="file is {{item}}" + with_first_found: + - /etc/foo + - /etc/bar + - /etc/passwd + - /etc/shadow diff --git a/v2/samples/with_flattened.yml b/v2/samples/with_flattened.yml new file mode 100644 index 0000000000..b5d2876ace --- /dev/null +++ b/v2/samples/with_flattened.yml @@ -0,0 +1,13 @@ +- hosts: localhost + connection: local + gather_facts: + vars: + list_a: + - ['foo', 'bar'] + list_b: + - [['bam', 'baz']] + tasks: + - debug: msg="item is {{item}}" + with_flattened: + - list_a + - list_b diff --git a/v2/samples/with_indexed_items.yml b/v2/samples/with_indexed_items.yml new file mode 100644 index 0000000000..de8fdf1888 --- /dev/null +++ b/v2/samples/with_indexed_items.yml @@ -0,0 +1,11 @@ +- hosts: localhost + connection: local + gather_facts: no + vars: + some_list: + - a + - b + - c + tasks: + - debug: msg="at array position {{ item.0 }} there is a value {{ item.1 }}" + with_indexed_items: some_list diff --git a/v2/samples/with_lines.yml b/v2/samples/with_lines.yml new file mode 100644 index 0000000000..ab00491028 --- /dev/null +++ b/v2/samples/with_lines.yml @@ -0,0 +1,6 @@ +- hosts: localhost + gather_facts: no + tasks: + - debug: msg="line is {{item}}" + with_lines: + - "cat /etc/hosts" diff --git a/v2/samples/with_random_choice.yml b/v2/samples/with_random_choice.yml new file mode 100644 index 0000000000..4ad4fc1a35 --- /dev/null +++ b/v2/samples/with_random_choice.yml @@ -0,0 +1,10 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + - debug: msg={{ item }} + with_random_choice: + - "go through the door" + - "drink from the goblet" + - "press the red button" + - "do nothing" diff --git a/v2/samples/with_sequence.yml b/v2/samples/with_sequence.yml new file mode 100644 index 0000000000..f25e9d24b3 --- /dev/null +++ b/v2/samples/with_sequence.yml @@ -0,0 +1,13 @@ +- hosts: localhost + connection: local + gather_facts: no + tasks: + + - debug: msg="name={{ item }} state=present groups=evens" + with_sequence: start=0 end=32 format=testuser%02x + + - debug: msg="dest=/var/stuff/{{ item }} state=directory" + with_sequence: start=4 end=16 stride=2 + + - debug: msg="name=group{{ item }} state=present" + with_sequence: count=4 diff --git a/v2/samples/with_subelements.yml b/v2/samples/with_subelements.yml new file mode 100644 index 0000000000..95d0dda67c --- /dev/null +++ b/v2/samples/with_subelements.yml @@ -0,0 +1,18 @@ +- hosts: localhost + connection: local + gather_facts: no + vars: + users: + - name: alice + authorized: + - /tmp/alice/onekey.pub + - /tmp/alice/twokey.pub + - name: bob + authorized: + - /tmp/bob/id_rsa.pub + + tasks: + - debug: msg="user={{ item.0.name }} key='{{ item.1 }}'" + with_subelements: + - users + - authorized diff --git a/v2/samples/with_together.yml b/v2/samples/with_together.yml new file mode 100644 index 0000000000..073b801033 --- /dev/null +++ b/v2/samples/with_together.yml @@ -0,0 +1,11 @@ +- hosts: localhost + connection: local + gather_facts: no + vars: + alpha: [ 'a', 'b', 'c', 'd' ] + numbers: [ 1, 2, 3, 4 ] + tasks: + - debug: msg="{{ item.0 }} and {{ item.1 }}" + with_together: + - alpha + - numbers