Add decompress module (#9175)

* adds simple implementation of `decompress` module

* adds simple test, fixes src and dest arg types

* minor refactoring

* adds support for common file operations
adds integration test for gz decompressing

* makes tests parametrized to test all supported compression formats

* checks that target file exists

* writes to decompressed file now uses atomic_move

* adds idempotency for decompression

* refactoring, removed classes

* adds support for check mode

* adds check for destination file. If it exists and it is a directory, the module returns error

* refactoring, moves code to a class. Also, simplifies tests (now only tests related to the module core functionality run as parametrized, tests for idempotency and check mode run only for one format)

* adds 'remove' parameter that deletes original compressed file after decompression

* adds documentation

* fixes bug with 'remove' parameter in check mode

* makes dest argument not required. Dest filename now can be produced from the src filename

* adds dest to output

* updates the documentation, adds "RETURN" block

* fixes test

* adds support for python2

* removes some of the test files that can be generated during testing. Adds copyright header to test files

* adds maintainer

* apply minor suggestions from code review

Co-authored-by: Felix Fontein <felix@fontein.de>

* fixes code review comments (idempotency issue with non existing src, existing dest and remove=true; fixes the issue and adds test)

* refactors the module to use ModuleHelper

* refactors lzma dependency manual check to use 'deps.validate'

* minor fix

* removes registered handlers check

* minor refactoring

* adds aliases

* changes setup for tests

* tests: ignores macos and fixes tests for FreeBSD

* tests: reverts ignore for macos and fixes issue with centos7

* tests: adds liblzma dependency for python2

* tests: adds backports.lzma

* fixes bz2 decompression for python2

* tests: install xz for osx

* tests: install xz for osx (2)

* fixes code review comments

---------

Co-authored-by: Felix Fontein <felix@fontein.de>

.github/BOTMETA.yml

@@ -506,6 +506,8 @@ files:
     ignore: skornehl
   $modules/dconf.py:
     maintainers: azaghal
+  $modules/decompress.py:
+    maintainers: shamilovstas
   $modules/deploy_helper.py:
     maintainers: ramondelafuente
   $modules/dimensiondata_network.py:

plugins/modules/decompress.py

@@ -0,0 +1,213 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2024, Stanislav Shamilov <shamilovstas@protonmail.com>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: decompress
+short_description: Decompresses compressed files
+version_added: 10.1.0
+description:
+    - Decompresses compressed files.
+    - The source (compressed) file and destination (decompressed) files are on the remote host.
+    - Source file can be deleted after decompression.
+extends_documentation_fragment:
+    - ansible.builtin.files
+    - community.general.attributes
+attributes:
+  check_mode:
+    support: full
+  diff_mode:
+    support: none
+options:
+  src:
+    description:
+      - Remote absolute path for the file to decompress.
+    type: path
+    required: true
+  dest:
+    description:
+      - The file name of the destination file where the compressed file will be decompressed.
+      - If the destination file exists, it will be truncated and overwritten.
+      - If not specified, the destination filename will be derived from O(src) by removing the compression format
+        extension. For example, if O(src) is V(/path/to/file.txt.gz) and O(format) is V(gz), O(dest) will be
+        V(/path/to/file.txt). If the O(src) file does not have an extension for the current O(format), the O(dest)
+        filename will be made by appending C(_decompressed) to the O(src) filename. For instance, if O(src) is
+        V(/path/to/file.myextension), the (dest) filename will be V(/path/to/file.myextension_decompressed).
+    type: path
+  format:
+    description:
+      - The type of compression to use to decompress.
+    type: str
+    choices: [ gz, bz2, xz ]
+    default: gz
+  remove:
+    description:
+      - Remove original compressed file after decompression.
+    type: bool
+    default: false
+requirements:
+    - Requires C(lzma) (standard library of Python 3) or L(backports.lzma, https://pypi.org/project/backports.lzma/) (Python 2) if using C(xz) format.
+author:
+  - Stanislav Shamilov (@shamilovstas)
+'''
+
+EXAMPLES = r'''
+- name: Decompress file /path/to/file.txt.gz into /path/to/file.txt (gz compression is used by default)
+  community.general.decompress:
+    src: /path/to/file.txt.gz
+    dest: /path/to/file.txt
+
+- name: Decompress file /path/to/file.txt.gz into /path/to/file.txt
+  community.general.decompress:
+    src: /path/to/file.txt.gz
+
+- name: Decompress file compressed with bzip2
+  community.general.decompress:
+    src: /path/to/file.txt.bz2
+    dest: /path/to/file.bz2
+    format: bz2
+
+- name: Decompress file and delete the compressed file afterwards
+  community.general.decompress:
+    src: /path/to/file.txt.gz
+    dest: /path/to/file.txt
+    remove: true
+'''
+
+RETURN = r'''
+dest:
+  description: Path to decompressed file
+  type: str
+  returned: success
+  sample: /path/to/file.txt
+'''
+
+import bz2
+import filecmp
+import gzip
+import os
+import shutil
+import tempfile
+
+from ansible.module_utils import six
+from ansible_collections.community.general.plugins.module_utils.mh.module_helper import ModuleHelper
+from ansible.module_utils.common.text.converters import to_native, to_bytes
+from ansible_collections.community.general.plugins.module_utils import deps
+
+with deps.declare("lzma"):
+    if six.PY3:
+        import lzma
+    else:
+        from backports import lzma
+
+
+def lzma_decompress(src):
+    return lzma.open(src, "rb")
+
+
+def bz2_decompress(src):
+    if six.PY3:
+        return bz2.open(src, "rb")
+    else:
+        return bz2.BZ2File(src, "rb")
+
+
+def gzip_decompress(src):
+    return gzip.open(src, "rb")
+
+
+def decompress(b_src, b_dest, handler):
+    with handler(b_src) as src_file:
+        with open(b_dest, "wb") as dest_file:
+            shutil.copyfileobj(src_file, dest_file)
+
+
+class Decompress(ModuleHelper):
+    destination_filename_template = "%s_decompressed"
+    use_old_vardict = False
+    output_params = 'dest'
+
+    module = dict(
+        argument_spec=dict(
+            src=dict(type='path', required=True),
+            dest=dict(type='path'),
+            format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']),
+            remove=dict(type='bool', default=False)
+        ),
+        add_file_common_args=True,
+        supports_check_mode=True
+    )
+
+    def __init_module__(self):
+        self.handlers = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress}
+        if self.vars.dest is None:
+            self.vars.dest = self.get_destination_filename()
+        deps.validate(self.module)
+        self.configure()
+
+    def configure(self):
+        b_dest = to_bytes(self.vars.dest, errors='surrogate_or_strict')
+        b_src = to_bytes(self.vars.src, errors='surrogate_or_strict')
+        if not os.path.exists(b_src):
+            if self.vars.remove and os.path.exists(b_dest):
+                self.module.exit_json(changed=False)
+            else:
+                self.do_raise(msg="Path does not exist: '%s'" % b_src)
+        if os.path.isdir(b_src):
+            self.do_raise(msg="Cannot decompress directory '%s'" % b_src)
+        if os.path.isdir(b_dest):
+            self.do_raise(msg="Destination is a directory, cannot decompress: '%s'" % b_dest)
+
+    def __run__(self):
+        b_dest = to_bytes(self.vars.dest, errors='surrogate_or_strict')
+        b_src = to_bytes(self.vars.src, errors='surrogate_or_strict')
+
+        file_args = self.module.load_file_common_arguments(self.module.params, path=self.vars.dest)
+        handler = self.handlers[self.vars.format]
+        try:
+            tempfd, temppath = tempfile.mkstemp(dir=self.module.tmpdir)
+            self.module.add_cleanup_file(temppath)
+            b_temppath = to_bytes(temppath, errors='surrogate_or_strict')
+            decompress(b_src, b_temppath, handler)
+        except OSError as e:
+            self.do_raise(msg="Unable to create temporary file '%s'" % to_native(e))
+
+        if os.path.exists(b_dest):
+            self.changed = not filecmp.cmp(b_temppath, b_dest, shallow=False)
+        else:
+            self.changed = True
+
+        if self.changed and not self.module.check_mode:
+            try:
+                self.module.atomic_move(b_temppath, b_dest)
+            except OSError:
+                self.do_raise(msg="Unable to move temporary file '%s' to '%s'" % (b_temppath, self.vars.dest))
+
+        if self.vars.remove and not self.check_mode:
+            os.remove(b_src)
+        self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed)
+
+    def get_destination_filename(self):
+        src = self.vars.src
+        fmt_extension = ".%s" % self.vars.format
+        if src.endswith(fmt_extension) and len(src) > len(fmt_extension):
+            filename = src[:-len(fmt_extension)]
+        else:
+            filename = Decompress.destination_filename_template % src
+        return filename
+
+
+def main():
+    Decompress.execute()
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/decompress/aliases

@@ -0,0 +1,7 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+azp/posix/2
+needs/root
+destructive

tests/integration/targets/decompress/files/file.txt

@@ -0,0 +1,5 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+This is sample file

tests/integration/targets/decompress/files/second_file.txt

@@ -0,0 +1,5 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+Content of this file must differ from the 'file.txt'

tests/integration/targets/decompress/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: delete backports.lzma
+  pip:
+    name: backports.lzma
+    state: absent

tests/integration/targets/decompress/meta/main.yml

@@ -0,0 +1,7 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+dependencies:
+  - setup_remote_tmp_dir

tests/integration/targets/decompress/tasks/cleanup.yml

@@ -0,0 +1,12 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Delete decompressed files
+  file:
+    path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt"
+    state: absent
+  loop: "{{ formats }}"
+  loop_control:
+    loop_var: format

tests/integration/targets/decompress/tasks/core.yml

@@ -0,0 +1,29 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Set mode for decompressed file ({{ format }} test)
+  set_fact:
+    decompressed_mode: "0640"
+
+- name: Simple decompress ({{ format }} test)
+  decompress:
+    src: "{{ remote_tmp_dir }}/file.txt.{{ format }}"
+    dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt"
+    format: "{{ format }}"
+    mode: "{{ decompressed_mode }}"
+  register: first_decompression
+
+- name: Stat decompressed file ({{ format }} test)
+  stat:
+    path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt"
+  register: decompressed_file_stat
+
+- name: Check that file was decompressed correctly ({{ format }} test)
+  assert:
+    that:
+      - first_decompression.changed
+      - decompressed_file_stat.stat.exists
+      - decompressed_file_stat.stat.mode == decompressed_mode
+      - orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum

tests/integration/targets/decompress/tasks/dest.yml

@@ -0,0 +1,51 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Copy a compressed file
+  copy:
+    src: "{{ item.orig }}"
+    dest: "{{ item.new }}"
+    remote_src: true
+  loop:
+    - { orig: "{{ remote_tmp_dir }}/file.txt.gz", new: "{{ remote_tmp_dir }}/dest.txt.gz" }
+    - { orig: "{{ remote_tmp_dir }}/file.txt.gz", new: "{{ remote_tmp_dir }}/dest" }
+
+- name: Decompress a file without specifying destination
+  decompress:
+    src: "{{ remote_tmp_dir }}/dest.txt.gz"
+    remove: true
+
+- name: Decompress a file which lacks extension without specifying destination
+  decompress:
+    src: "{{ remote_tmp_dir }}/dest"
+    remove: true
+
+- name: Stat result files
+  stat:
+    path: "{{ remote_tmp_dir }}/{{ filename }}"
+  loop:
+    - dest.txt
+    - dest_decompressed
+  loop_control:
+    loop_var: filename
+  register: result_files_stat
+
+- name: Test that file exists
+  assert:
+    that: "{{ item.stat.exists }}"
+    quiet: true
+  loop: "{{ result_files_stat.results }}"
+  loop_control:
+    label: "{{ item.stat.path }}"
+
+- name: Delete test files
+  file:
+    path: "{{ filename }}"
+    state: absent
+  loop:
+    - "dest.txt"
+    - "dest_decompressed"
+  loop_control:
+    loop_var: filename

tests/integration/targets/decompress/tasks/main.yml

@@ -0,0 +1,115 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Copy test files
+  copy:
+    src: "files/"
+    dest: "{{ remote_tmp_dir }}"
+
+- name: Get original file stat
+  stat:
+    path: "{{ remote_tmp_dir }}/file.txt"
+  register: orig_file_stat
+
+- name: Set supported formats
+  set_fact:
+    formats:
+      - bz2
+      - gz
+      - xz
+
+- name: Ensure xz is present to create compressed files (not Debian)
+  package:
+    name:
+      - xz
+      - bzip2
+    state: latest
+  when:
+    - ansible_system != 'FreeBSD'
+    - ansible_os_family != 'Darwin'
+    - ansible_os_family != 'Debian'
+
+- name: Ensure xz is present to create compressed files (Debian)
+  package:
+    name: xz-utils
+    state: latest
+  when: ansible_os_family == 'Debian'
+
+- name: Install prerequisites for backports.lzma when using python2 (non OSX)
+  block:
+    - name: Set liblzma package name depending on the OS
+      set_fact:
+        liblzma_dev_package:
+          Debian: liblzma-dev
+          RedHat: xz-devel
+          Suse: xz-devel
+    - name: Ensure liblzma-dev is present to install backports-lzma
+      package:
+        name: "{{ liblzma_dev_package[ansible_os_family] }}"
+        state: latest
+      when: ansible_os_family in liblzma_dev_package.keys()
+  when:
+    - ansible_python_version.split('.')[0] == '2'
+    - ansible_os_family != 'Darwin'
+
+- name: Install prerequisites for backports.lzma when using python2 (OSX)
+  block:
+    - name: Find brew binary
+      command: which brew
+      register: brew_which
+    - name: Get owner of brew binary
+      stat:
+        path: "{{ brew_which.stdout }}"
+      register: brew_stat
+    - name: "Install package"
+      homebrew:
+        name: xz
+        state: present
+        update_homebrew: false
+      become: true
+      become_user: "{{ brew_stat.stat.pw_name }}"
+      # Newer versions of brew want to compile a package which takes a long time. Do not upgrade homebrew until a
+      # proper solution can be found
+      environment:
+        HOMEBREW_NO_AUTO_UPDATE: "True"
+  when:
+    - ansible_os_family == 'Darwin'
+
+- name: Ensure backports.lzma is present to create test archive (pip)
+  pip:
+    name: backports.lzma
+    state: latest
+  when: ansible_python_version.split('.')[0] == '2'
+  notify:
+    - delete backports.lzma
+
+- name: Generate compressed files
+  shell: |
+    gzip < {{ item }} > {{ item }}.gz
+    bzip2 < {{ item }} > {{ item }}.bz2
+    xz < {{ item }} > {{ item }}.xz
+  loop:
+    - "{{ remote_tmp_dir }}/file.txt"
+    - "{{ remote_tmp_dir }}/second_file.txt"
+
+# Run tests
+- name: Run core tests
+  block:
+    - include_tasks: core.yml
+      loop: "{{ formats }}"
+      loop_control:
+        loop_var: format
+    - import_tasks: cleanup.yml
+
+
+- name: Run idempotency and check mode tests
+  block:
+    - import_tasks: misc.yml
+    - import_tasks: cleanup.yml
+
+- name: Run tests for destination file
+  block:
+    - import_tasks: dest.yml
+    - import_tasks: cleanup.yml

tests/integration/targets/decompress/tasks/misc.yml

@@ -0,0 +1,74 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Decompress with check mode enabled
+  decompress:
+    src: "{{ remote_tmp_dir }}/file.txt.gz"
+    dest: "{{ remote_tmp_dir }}/file_from_gz.txt"
+    format: gz
+  check_mode: true
+  register: decompressed_check_mode
+
+- name: Decompress second time with check mode enabled
+  decompress:
+    src: "{{ remote_tmp_dir }}/file.txt.gz"
+    dest: "{{ remote_tmp_dir }}/file_from_gz.txt"
+    format: gz
+    remove: true
+  check_mode: true
+  register: decompressed_check_mode_2
+
+- name: Stat original compressed file
+  stat:
+    path: "{{ remote_tmp_dir }}/file.txt.gz"
+  register: original_file
+
+- name: Stat non-existing file
+  stat:
+    path: "{{ remote_tmp_dir }}/file_from_gz.txt"
+  register: nonexisting_stat
+
+- name: Check mode test
+  assert:
+    that:
+      - decompressed_check_mode.changed
+      - decompressed_check_mode_2.changed
+      - original_file.stat.exists
+      - not nonexisting_stat.stat.exists
+
+- name: Copy compressed file
+  copy:
+    src: "{{ remote_tmp_dir }}/file.txt.gz"
+    dest: "{{ remote_tmp_dir }}/file_copied.txt.gz"
+    remote_src: true
+
+- name: Decompress, deleting original file
+  decompress:
+    src: "{{ remote_tmp_dir }}/file_copied.txt.gz"
+    dest: "{{ remote_tmp_dir }}/file_copied.txt"
+    remove: true
+
+- name: Decompress non existing src
+  decompress:
+    src: "{{ remote_tmp_dir }}/file_copied.txt.gz"
+    dest: "{{ remote_tmp_dir }}/file_copied.txt"
+    remove: true
+  register: decompress_non_existing_src
+
+- name: Stat compressed file
+  stat:
+    path: "{{ remote_tmp_dir }}/file_copied.txt.gz"
+  register: compressed_stat
+
+- name: Run tests
+  assert:
+    that:
+      - not compressed_stat.stat.exists
+      - not decompress_non_existing_src.changed
+
+- name: Delete decompressed file
+  file:
+    path: "{{ remote_tmp_dir }}/file_copied.txt"
+    state: absent