From 41b6a281e17572f886cbae6c585a1541a62657f1 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 2 Dec 2024 21:16:00 +0200 Subject: [PATCH] Add decompress module (#9175) * adds simple implementation of `decompress` module * adds simple test, fixes src and dest arg types * minor refactoring * adds support for common file operations adds integration test for gz decompressing * makes tests parametrized to test all supported compression formats * checks that target file exists * writes to decompressed file now uses atomic_move * adds idempotency for decompression * refactoring, removed classes * adds support for check mode * adds check for destination file. If it exists and it is a directory, the module returns error * refactoring, moves code to a class. Also, simplifies tests (now only tests related to the module core functionality run as parametrized, tests for idempotency and check mode run only for one format) * adds 'remove' parameter that deletes original compressed file after decompression * adds documentation * fixes bug with 'remove' parameter in check mode * makes dest argument not required. Dest filename now can be produced from the src filename * adds dest to output * updates the documentation, adds "RETURN" block * fixes test * adds support for python2 * removes some of the test files that can be generated during testing. Adds copyright header to test files * adds maintainer * apply minor suggestions from code review Co-authored-by: Felix Fontein * fixes code review comments (idempotency issue with non existing src, existing dest and remove=true; fixes the issue and adds test) * refactors the module to use ModuleHelper * refactors lzma dependency manual check to use 'deps.validate' * minor fix * removes registered handlers check * minor refactoring * adds aliases * changes setup for tests * tests: ignores macos and fixes tests for FreeBSD * tests: reverts ignore for macos and fixes issue with centos7 * tests: adds liblzma dependency for python2 * tests: adds backports.lzma * fixes bz2 decompression for python2 * tests: install xz for osx * tests: install xz for osx (2) * fixes code review comments --------- Co-authored-by: Felix Fontein --- .github/BOTMETA.yml | 2 + plugins/modules/decompress.py | 213 ++++++++++++++++++ tests/integration/targets/decompress/aliases | 7 + .../targets/decompress/files/file.txt | 5 + .../targets/decompress/files/second_file.txt | 5 + .../targets/decompress/handlers/main.yml | 9 + .../targets/decompress/meta/main.yml | 7 + .../targets/decompress/tasks/cleanup.yml | 12 + .../targets/decompress/tasks/core.yml | 29 +++ .../targets/decompress/tasks/dest.yml | 51 +++++ .../targets/decompress/tasks/main.yml | 115 ++++++++++ .../targets/decompress/tasks/misc.yml | 74 ++++++ 12 files changed, 529 insertions(+) create mode 100644 plugins/modules/decompress.py create mode 100644 tests/integration/targets/decompress/aliases create mode 100644 tests/integration/targets/decompress/files/file.txt create mode 100644 tests/integration/targets/decompress/files/second_file.txt create mode 100644 tests/integration/targets/decompress/handlers/main.yml create mode 100644 tests/integration/targets/decompress/meta/main.yml create mode 100644 tests/integration/targets/decompress/tasks/cleanup.yml create mode 100644 tests/integration/targets/decompress/tasks/core.yml create mode 100644 tests/integration/targets/decompress/tasks/dest.yml create mode 100644 tests/integration/targets/decompress/tasks/main.yml create mode 100644 tests/integration/targets/decompress/tasks/misc.yml diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 594f01349a..9650fd0ef3 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -506,6 +506,8 @@ files: ignore: skornehl $modules/dconf.py: maintainers: azaghal + $modules/decompress.py: + maintainers: shamilovstas $modules/deploy_helper.py: maintainers: ramondelafuente $modules/dimensiondata_network.py: diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py new file mode 100644 index 0000000000..818213fb0d --- /dev/null +++ b/plugins/modules/decompress.py @@ -0,0 +1,213 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2024, Stanislav Shamilov +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +DOCUMENTATION = r''' +--- +module: decompress +short_description: Decompresses compressed files +version_added: 10.1.0 +description: + - Decompresses compressed files. + - The source (compressed) file and destination (decompressed) files are on the remote host. + - Source file can be deleted after decompression. +extends_documentation_fragment: + - ansible.builtin.files + - community.general.attributes +attributes: + check_mode: + support: full + diff_mode: + support: none +options: + src: + description: + - Remote absolute path for the file to decompress. + type: path + required: true + dest: + description: + - The file name of the destination file where the compressed file will be decompressed. + - If the destination file exists, it will be truncated and overwritten. + - If not specified, the destination filename will be derived from O(src) by removing the compression format + extension. For example, if O(src) is V(/path/to/file.txt.gz) and O(format) is V(gz), O(dest) will be + V(/path/to/file.txt). If the O(src) file does not have an extension for the current O(format), the O(dest) + filename will be made by appending C(_decompressed) to the O(src) filename. For instance, if O(src) is + V(/path/to/file.myextension), the (dest) filename will be V(/path/to/file.myextension_decompressed). + type: path + format: + description: + - The type of compression to use to decompress. + type: str + choices: [ gz, bz2, xz ] + default: gz + remove: + description: + - Remove original compressed file after decompression. + type: bool + default: false +requirements: + - Requires C(lzma) (standard library of Python 3) or L(backports.lzma, https://pypi.org/project/backports.lzma/) (Python 2) if using C(xz) format. +author: + - Stanislav Shamilov (@shamilovstas) +''' + +EXAMPLES = r''' +- name: Decompress file /path/to/file.txt.gz into /path/to/file.txt (gz compression is used by default) + community.general.decompress: + src: /path/to/file.txt.gz + dest: /path/to/file.txt + +- name: Decompress file /path/to/file.txt.gz into /path/to/file.txt + community.general.decompress: + src: /path/to/file.txt.gz + +- name: Decompress file compressed with bzip2 + community.general.decompress: + src: /path/to/file.txt.bz2 + dest: /path/to/file.bz2 + format: bz2 + +- name: Decompress file and delete the compressed file afterwards + community.general.decompress: + src: /path/to/file.txt.gz + dest: /path/to/file.txt + remove: true +''' + +RETURN = r''' +dest: + description: Path to decompressed file + type: str + returned: success + sample: /path/to/file.txt +''' + +import bz2 +import filecmp +import gzip +import os +import shutil +import tempfile + +from ansible.module_utils import six +from ansible_collections.community.general.plugins.module_utils.mh.module_helper import ModuleHelper +from ansible.module_utils.common.text.converters import to_native, to_bytes +from ansible_collections.community.general.plugins.module_utils import deps + +with deps.declare("lzma"): + if six.PY3: + import lzma + else: + from backports import lzma + + +def lzma_decompress(src): + return lzma.open(src, "rb") + + +def bz2_decompress(src): + if six.PY3: + return bz2.open(src, "rb") + else: + return bz2.BZ2File(src, "rb") + + +def gzip_decompress(src): + return gzip.open(src, "rb") + + +def decompress(b_src, b_dest, handler): + with handler(b_src) as src_file: + with open(b_dest, "wb") as dest_file: + shutil.copyfileobj(src_file, dest_file) + + +class Decompress(ModuleHelper): + destination_filename_template = "%s_decompressed" + use_old_vardict = False + output_params = 'dest' + + module = dict( + argument_spec=dict( + src=dict(type='path', required=True), + dest=dict(type='path'), + format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']), + remove=dict(type='bool', default=False) + ), + add_file_common_args=True, + supports_check_mode=True + ) + + def __init_module__(self): + self.handlers = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress} + if self.vars.dest is None: + self.vars.dest = self.get_destination_filename() + deps.validate(self.module) + self.configure() + + def configure(self): + b_dest = to_bytes(self.vars.dest, errors='surrogate_or_strict') + b_src = to_bytes(self.vars.src, errors='surrogate_or_strict') + if not os.path.exists(b_src): + if self.vars.remove and os.path.exists(b_dest): + self.module.exit_json(changed=False) + else: + self.do_raise(msg="Path does not exist: '%s'" % b_src) + if os.path.isdir(b_src): + self.do_raise(msg="Cannot decompress directory '%s'" % b_src) + if os.path.isdir(b_dest): + self.do_raise(msg="Destination is a directory, cannot decompress: '%s'" % b_dest) + + def __run__(self): + b_dest = to_bytes(self.vars.dest, errors='surrogate_or_strict') + b_src = to_bytes(self.vars.src, errors='surrogate_or_strict') + + file_args = self.module.load_file_common_arguments(self.module.params, path=self.vars.dest) + handler = self.handlers[self.vars.format] + try: + tempfd, temppath = tempfile.mkstemp(dir=self.module.tmpdir) + self.module.add_cleanup_file(temppath) + b_temppath = to_bytes(temppath, errors='surrogate_or_strict') + decompress(b_src, b_temppath, handler) + except OSError as e: + self.do_raise(msg="Unable to create temporary file '%s'" % to_native(e)) + + if os.path.exists(b_dest): + self.changed = not filecmp.cmp(b_temppath, b_dest, shallow=False) + else: + self.changed = True + + if self.changed and not self.module.check_mode: + try: + self.module.atomic_move(b_temppath, b_dest) + except OSError: + self.do_raise(msg="Unable to move temporary file '%s' to '%s'" % (b_temppath, self.vars.dest)) + + if self.vars.remove and not self.check_mode: + os.remove(b_src) + self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed) + + def get_destination_filename(self): + src = self.vars.src + fmt_extension = ".%s" % self.vars.format + if src.endswith(fmt_extension) and len(src) > len(fmt_extension): + filename = src[:-len(fmt_extension)] + else: + filename = Decompress.destination_filename_template % src + return filename + + +def main(): + Decompress.execute() + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/decompress/aliases b/tests/integration/targets/decompress/aliases new file mode 100644 index 0000000000..f4049c7dc2 --- /dev/null +++ b/tests/integration/targets/decompress/aliases @@ -0,0 +1,7 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +azp/posix/2 +needs/root +destructive \ No newline at end of file diff --git a/tests/integration/targets/decompress/files/file.txt b/tests/integration/targets/decompress/files/file.txt new file mode 100644 index 0000000000..5d2e0d1458 --- /dev/null +++ b/tests/integration/targets/decompress/files/file.txt @@ -0,0 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +This is sample file \ No newline at end of file diff --git a/tests/integration/targets/decompress/files/second_file.txt b/tests/integration/targets/decompress/files/second_file.txt new file mode 100644 index 0000000000..bd04eca21c --- /dev/null +++ b/tests/integration/targets/decompress/files/second_file.txt @@ -0,0 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +Content of this file must differ from the 'file.txt' \ No newline at end of file diff --git a/tests/integration/targets/decompress/handlers/main.yml b/tests/integration/targets/decompress/handlers/main.yml new file mode 100644 index 0000000000..8c92cc4f81 --- /dev/null +++ b/tests/integration/targets/decompress/handlers/main.yml @@ -0,0 +1,9 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: delete backports.lzma + pip: + name: backports.lzma + state: absent diff --git a/tests/integration/targets/decompress/meta/main.yml b/tests/integration/targets/decompress/meta/main.yml new file mode 100644 index 0000000000..982de6eb03 --- /dev/null +++ b/tests/integration/targets/decompress/meta/main.yml @@ -0,0 +1,7 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +dependencies: + - setup_remote_tmp_dir diff --git a/tests/integration/targets/decompress/tasks/cleanup.yml b/tests/integration/targets/decompress/tasks/cleanup.yml new file mode 100644 index 0000000000..95db42104f --- /dev/null +++ b/tests/integration/targets/decompress/tasks/cleanup.yml @@ -0,0 +1,12 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Delete decompressed files + file: + path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + state: absent + loop: "{{ formats }}" + loop_control: + loop_var: format \ No newline at end of file diff --git a/tests/integration/targets/decompress/tasks/core.yml b/tests/integration/targets/decompress/tasks/core.yml new file mode 100644 index 0000000000..a92ae21b78 --- /dev/null +++ b/tests/integration/targets/decompress/tasks/core.yml @@ -0,0 +1,29 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Set mode for decompressed file ({{ format }} test) + set_fact: + decompressed_mode: "0640" + +- name: Simple decompress ({{ format }} test) + decompress: + src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" + dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + format: "{{ format }}" + mode: "{{ decompressed_mode }}" + register: first_decompression + +- name: Stat decompressed file ({{ format }} test) + stat: + path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + register: decompressed_file_stat + +- name: Check that file was decompressed correctly ({{ format }} test) + assert: + that: + - first_decompression.changed + - decompressed_file_stat.stat.exists + - decompressed_file_stat.stat.mode == decompressed_mode + - orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum diff --git a/tests/integration/targets/decompress/tasks/dest.yml b/tests/integration/targets/decompress/tasks/dest.yml new file mode 100644 index 0000000000..9a7bbe499f --- /dev/null +++ b/tests/integration/targets/decompress/tasks/dest.yml @@ -0,0 +1,51 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Copy a compressed file + copy: + src: "{{ item.orig }}" + dest: "{{ item.new }}" + remote_src: true + loop: + - { orig: "{{ remote_tmp_dir }}/file.txt.gz", new: "{{ remote_tmp_dir }}/dest.txt.gz" } + - { orig: "{{ remote_tmp_dir }}/file.txt.gz", new: "{{ remote_tmp_dir }}/dest" } + +- name: Decompress a file without specifying destination + decompress: + src: "{{ remote_tmp_dir }}/dest.txt.gz" + remove: true + +- name: Decompress a file which lacks extension without specifying destination + decompress: + src: "{{ remote_tmp_dir }}/dest" + remove: true + +- name: Stat result files + stat: + path: "{{ remote_tmp_dir }}/{{ filename }}" + loop: + - dest.txt + - dest_decompressed + loop_control: + loop_var: filename + register: result_files_stat + +- name: Test that file exists + assert: + that: "{{ item.stat.exists }}" + quiet: true + loop: "{{ result_files_stat.results }}" + loop_control: + label: "{{ item.stat.path }}" + +- name: Delete test files + file: + path: "{{ filename }}" + state: absent + loop: + - "dest.txt" + - "dest_decompressed" + loop_control: + loop_var: filename diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml new file mode 100644 index 0000000000..f14f2d5593 --- /dev/null +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -0,0 +1,115 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Copy test files + copy: + src: "files/" + dest: "{{ remote_tmp_dir }}" + +- name: Get original file stat + stat: + path: "{{ remote_tmp_dir }}/file.txt" + register: orig_file_stat + +- name: Set supported formats + set_fact: + formats: + - bz2 + - gz + - xz + +- name: Ensure xz is present to create compressed files (not Debian) + package: + name: + - xz + - bzip2 + state: latest + when: + - ansible_system != 'FreeBSD' + - ansible_os_family != 'Darwin' + - ansible_os_family != 'Debian' + +- name: Ensure xz is present to create compressed files (Debian) + package: + name: xz-utils + state: latest + when: ansible_os_family == 'Debian' + +- name: Install prerequisites for backports.lzma when using python2 (non OSX) + block: + - name: Set liblzma package name depending on the OS + set_fact: + liblzma_dev_package: + Debian: liblzma-dev + RedHat: xz-devel + Suse: xz-devel + - name: Ensure liblzma-dev is present to install backports-lzma + package: + name: "{{ liblzma_dev_package[ansible_os_family] }}" + state: latest + when: ansible_os_family in liblzma_dev_package.keys() + when: + - ansible_python_version.split('.')[0] == '2' + - ansible_os_family != 'Darwin' + +- name: Install prerequisites for backports.lzma when using python2 (OSX) + block: + - name: Find brew binary + command: which brew + register: brew_which + - name: Get owner of brew binary + stat: + path: "{{ brew_which.stdout }}" + register: brew_stat + - name: "Install package" + homebrew: + name: xz + state: present + update_homebrew: false + become: true + become_user: "{{ brew_stat.stat.pw_name }}" + # Newer versions of brew want to compile a package which takes a long time. Do not upgrade homebrew until a + # proper solution can be found + environment: + HOMEBREW_NO_AUTO_UPDATE: "True" + when: + - ansible_os_family == 'Darwin' + +- name: Ensure backports.lzma is present to create test archive (pip) + pip: + name: backports.lzma + state: latest + when: ansible_python_version.split('.')[0] == '2' + notify: + - delete backports.lzma + +- name: Generate compressed files + shell: | + gzip < {{ item }} > {{ item }}.gz + bzip2 < {{ item }} > {{ item }}.bz2 + xz < {{ item }} > {{ item }}.xz + loop: + - "{{ remote_tmp_dir }}/file.txt" + - "{{ remote_tmp_dir }}/second_file.txt" + +# Run tests +- name: Run core tests + block: + - include_tasks: core.yml + loop: "{{ formats }}" + loop_control: + loop_var: format + - import_tasks: cleanup.yml + + +- name: Run idempotency and check mode tests + block: + - import_tasks: misc.yml + - import_tasks: cleanup.yml + +- name: Run tests for destination file + block: + - import_tasks: dest.yml + - import_tasks: cleanup.yml diff --git a/tests/integration/targets/decompress/tasks/misc.yml b/tests/integration/targets/decompress/tasks/misc.yml new file mode 100644 index 0000000000..1514e55030 --- /dev/null +++ b/tests/integration/targets/decompress/tasks/misc.yml @@ -0,0 +1,74 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Decompress with check mode enabled + decompress: + src: "{{ remote_tmp_dir }}/file.txt.gz" + dest: "{{ remote_tmp_dir }}/file_from_gz.txt" + format: gz + check_mode: true + register: decompressed_check_mode + +- name: Decompress second time with check mode enabled + decompress: + src: "{{ remote_tmp_dir }}/file.txt.gz" + dest: "{{ remote_tmp_dir }}/file_from_gz.txt" + format: gz + remove: true + check_mode: true + register: decompressed_check_mode_2 + +- name: Stat original compressed file + stat: + path: "{{ remote_tmp_dir }}/file.txt.gz" + register: original_file + +- name: Stat non-existing file + stat: + path: "{{ remote_tmp_dir }}/file_from_gz.txt" + register: nonexisting_stat + +- name: Check mode test + assert: + that: + - decompressed_check_mode.changed + - decompressed_check_mode_2.changed + - original_file.stat.exists + - not nonexisting_stat.stat.exists + +- name: Copy compressed file + copy: + src: "{{ remote_tmp_dir }}/file.txt.gz" + dest: "{{ remote_tmp_dir }}/file_copied.txt.gz" + remote_src: true + +- name: Decompress, deleting original file + decompress: + src: "{{ remote_tmp_dir }}/file_copied.txt.gz" + dest: "{{ remote_tmp_dir }}/file_copied.txt" + remove: true + +- name: Decompress non existing src + decompress: + src: "{{ remote_tmp_dir }}/file_copied.txt.gz" + dest: "{{ remote_tmp_dir }}/file_copied.txt" + remove: true + register: decompress_non_existing_src + +- name: Stat compressed file + stat: + path: "{{ remote_tmp_dir }}/file_copied.txt.gz" + register: compressed_stat + +- name: Run tests + assert: + that: + - not compressed_stat.stat.exists + - not decompress_non_existing_src.changed + +- name: Delete decompressed file + file: + path: "{{ remote_tmp_dir }}/file_copied.txt" + state: absent