lvm: lvm_pv on RHEL 8, role fixes, Molecule scenario and Python 3.6 unit tests

.github/dependabot.yml

@@ -43,3 +43,17 @@ updates:
       day: 'friday'
       time: '05:00'
       timezone: 'Etc/UTC'
+
+  # Only 2.16: the last ansible-core series that supports Python 3.6 on the managed node, whose
+  # module_utils the RHEL 8 unit tests import (see the requirements.txt there).
+  - package-ecosystem: 'pip'
+    directory: '/.github/unit-tests-py36-ansible-core'
+    schedule:
+      interval: 'weekly'
+      day: 'friday'
+      time: '05:00'
+      timezone: 'Etc/UTC'
+    ignore:
+      - dependency-name: 'ansible-core'
+        versions:
+          - '>= 2.17'

.github/unit-tests-py36-ansible-core/requirements.txt

@@ -0,0 +1,12 @@
+# The ansible-core whose module_utils the managed-node tier of the unit tests imports on Python 3.6,
+# see tox.ini. In production, the controller ships its own module_utils to the managed node, so the
+# modules meet exactly this code on RHEL 8. 2.16 is the last series that supports Python 3.6 on the
+# managed node; Dependabot therefore only proposes 2.16 releases (see .github/dependabot.yml).
+#
+# Only ansible-core itself is installed, with `--no-deps --target`, and put on PYTHONPATH: its
+# controller part and dependencies need Python 3.10, module_utils only need the standard library.
+# That is also why this file is not compiled with pip-compile, which would add the dependencies.
+# The hashes are those of the wheel and the sdist on PyPI.
+ansible-core==2.16.19 \
+    --hash=sha256:5125f26412039ffb99a3a7723618535ab80e6ef38944aa2453997350388f4ef4 \
+    --hash=sha256:d7a32ba0f96f9c6582b7ff159a4a6f0e694662cfc4840497c88fed63bf58cd14

.github/unit-tests-py36/requirements.in

@@ -0,0 +1,9 @@
+# The Python 3.6 venv of the managed-node tier of the unit tests (modules, module_utils), see
+# tox.ini. pytest 6.2.5 is the last release that runs on Python 3.6, so there is nothing to update
+# here until LFOps drops RHEL 8. For the same reason Dependabot does not watch this directory: it
+# resolves with a current Python and would pick releases without Python 3.6 support. Regenerate
+# requirements.txt inside UBI 8 (Python 3.6, pip-tools < 7) with
+# `pip-compile --allow-unsafe --generate-hashes --strip-extras requirements.in`.
+#
+# ansible-core is not installed here, see ../unit-tests-py36-ansible-core.
+pytest==6.2.5

.github/unit-tests-py36/requirements.txt

@@ -0,0 +1,52 @@
+#
+# This file is autogenerated by pip-compile with python 3.6
+# To update, run:
+#
+#    pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt --strip-extras requirements.in
+#
+attrs==22.2.0 \
+    --hash=sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836 \
+    --hash=sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99
+    # via pytest
+importlib-metadata==4.8.3 \
+    --hash=sha256:65a9576a5b2d58ca44d133c42a241905cc45e34d2c06fd5ba2bafa221e5d7b5e \
+    --hash=sha256:766abffff765960fcc18003801f7044eb6755ffae4521c8e8ce8e83b9c9b0668
+    # via
+    #   pluggy
+    #   pytest
+iniconfig==1.1.1 \
+    --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \
+    --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32
+    # via pytest
+packaging==21.3 \
+    --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \
+    --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522
+    # via pytest
+pluggy==1.0.0 \
+    --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \
+    --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3
+    # via pytest
+py==1.11.0 \
+    --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \
+    --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378
+    # via pytest
+pyparsing==3.1.4 \
+    --hash=sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c \
+    --hash=sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032
+    # via packaging
+pytest==6.2.5 \
+    --hash=sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89 \
+    --hash=sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134
+    # via -r requirements.in
+toml==0.10.2 \
+    --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
+    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
+    # via pytest
+typing-extensions==4.1.1 \
+    --hash=sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42 \
+    --hash=sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2
+    # via importlib-metadata
+zipp==3.6.0 \
+    --hash=sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832 \
+    --hash=sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc
+    # via importlib-metadata

.github/workflows/dependency-review.yml

@@ -20,3 +20,16 @@ jobs:
 
       - name: 'Dependency Review'
         uses: 'actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294' # v5.0.0
+        with:
+          # Advisories against the Python 3.6 lockfile in .github/unit-tests-py36, which feeds the
+          # unit tests on the Python 3.6 of RHEL 8. The test runner has to run on the same
+          # interpreter as the modules under test, and the fixed releases need a newer Python, so
+          # there is nothing to update to. The tests run in a throwaway UBI 8 container. Drop these
+          # entries once the py36 lockfile is retired together with RHEL 8 support.
+          # - GHSA-6w46-j5rx-g56g, pytest 6.2.5: other local users abusing the predictable
+          #   /tmp/pytest-of-{user} directories; the container has no other users. Fixed in pytest
+          #   9.0.3, which requires Python 3.10.
+          # - GHSA-jfmj-5v4g-7637, zipp 3.6.0 (via pytest and importlib-metadata): an infinite loop
+          #   on a crafted ZIP file opened with zipp.Path; the tests open no ZIP files. Fixed in
+          #   zipp 3.19.1, which requires Python 3.8.
+          allow-ghsas: 'GHSA-6w46-j5rx-g56g,GHSA-jfmj-5v4g-7637'

.github/workflows/lf-unit-tests.yml

@@ -16,9 +16,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Controller-side plugins (filter, lookup). The managed-node tier
-        # (modules on RHEL 8 / Python 3.6) needs a UBI 8 container and is
-        # scaffolded in tox.ini, not run here yet.
+        # Controller-side plugins (filter, lookup). The module tests run
+        # here as well; their Python 3.6 run is the managed-node job below.
         python-version:
           - '3.9'
           - '3.10'
@@ -60,3 +59,50 @@ jobs:
         # `-f pyXYZ` selects every tox env carrying this Python's factor,
         # e.g. py311 -> py311-ansible215/216/217/218.
         run: 'tox --discover "${{ steps.target-python.outputs.python-path }}" -f "py$(echo "${{ matrix.python-version }}" | tr -d ".")"'
+
+  managed-node-plugins:
+    name: 'Managed-node plugins (RHEL 8, Python 3.6)'
+    runs-on: 'ubuntu-latest'
+    steps:
+      - name: 'Harden the runner (Audit all outbound calls)'
+        uses: 'step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1' # v2.21.1
+        with:
+          egress-policy: 'audit'
+
+      - name: 'Checkout repository'
+        uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # v7.0.1
+
+      # Only to fetch ansible-core below; the tests themselves run on the
+      # Python 3.6 in the container.
+      - name: 'Set up Python'
+        uses: 'actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97' # v7.0.0
+        with:
+          python-version: '3.13'
+
+      # The tests import the module_utils of the controller's ansible-core,
+      # which is what the modules meet on a RHEL 8 host. ansible-core itself
+      # needs Python 3.10, so it is installed here, without dependencies, and
+      # handed to the container read-only.
+      - name: 'Install ansible-core for its module_utils'
+        run: >-
+          pip install --require-hashes --no-deps
+          --target "${RUNNER_TEMP}/ansible-core"
+          --requirement .github/unit-tests-py36-ansible-core/requirements.txt
+
+      # Modules run on the managed node, down to the platform-python 3.6 of
+      # RHEL 8, which no runner ships. Same test run as the py36-target env in
+      # tox.ini, without tox, since tox 4 does not run on Python 3.6.
+      - name: 'Run the module tests on UBI 8'
+        run: |
+          docker run --rm --volume "${PWD}:/src:ro" --workdir /src \
+            --volume "${RUNNER_TEMP}/ansible-core:/opt/ansible-core:ro" \
+            --env PYTHONPATH=/opt/ansible-core \
+            --env PYTEST_ADDOPTS='-p no:cacheprovider' \
+            registry.access.redhat.com/ubi8/ubi \
+            bash -c '
+              set -o errexit
+              dnf --assumeyes --quiet install python3 python3-pip
+              python3 -m venv /tmp/venv
+              /tmp/venv/bin/pip install --require-hashes --requirement .github/unit-tests-py36/requirements.txt
+              /tmp/venv/bin/pytest tests/unit/plugins/modules tests/unit/plugins/module_utils
+            '

CHANGELOG.md

@@ -75,6 +75,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+* **role:lvm**: On Debian and Ubuntu, mounting an LV no longer fails on `restorecon`, which only runs where SELinux is enabled.
+* **role:lvm**: The role installs the tools for XFS and ext2/3/4, so creating the default XFS file system no longer fails on hosts without `mkfs.xfs`, such as those deployed from a Debian cloud image.
+* **role:lvm**: `mount_owner`, `mount_group` and `mount_mode` apply to the mounted file system from the first run on. Until now they only took effect on the second run, which therefore reported a change.
+* **role:lvm**: Removing an LV (`state: absent`) no longer aborts with `'dict object' has no attribute 'size'`.
+* **role:lvm**: Removing a VG (`state: absent`) no longer aborts with `could not find 'pvs' key` when the entry lists no `pvs`.
+* **role:lvm**: Shrinking an LV works. It needs `force: true` in addition to `shrink: true`; with `shrink: true` alone the run aborted.
+* **role:lvm**: The role installs `lvm2`, so it also works on hosts installed without LVM, such as those deployed from a cloud image.
+* **role:lvm**: Creating or resizing a PV no longer fails on RHEL 8 with `SyntaxError: future feature annotations is not defined`.
 * **role:network**: Hosts without `network_connections` or `network_state` no longer run the upstream network role at all, which occasionally hung the play for good after it had finished.
 * **role:chrony**: The role now takes effect on Debian and Ubuntu, where chronyd reads `/etc/chrony/chrony.conf` and ignored the `/etc/chrony.conf` the role deployed. The distribution's DHCP and `sources.d` sources are kept, and without `chrony__ntp_pools` or `chrony__ntp_servers` the distribution's default pools are used. The stale `/etc/chrony.conf` is removed.
 * **role:kernel_settings**: The role no longer aborts on Ubuntu 22.04 with `Verification failed, current system settings differ from the preset profile`. The TuneD release of Ubuntu 22.04 sets two scheduler sysctls the kernel no longer has, and the role now removes them from the profile it builds on.

COMPATIBILITY.md

@@ -95,7 +95,7 @@ Which Ansible role is proven to run on which OS?
 | login                                 |   x    |   x    |   x    |   x    |    x    |     x     |     x     |     x     | Fedora 35+                                   |
 | logrotate                             |   x    |   x    |   x    |   x    |    x    |     x     |     x     |     x     | Fedora                                       |
 | logstash                              |  (x)   |  (x)   |  (x)   |   x    |   (x)   |    (x)    |     x     |    (x)    |                                              |
-| lvm                                   |        |        |  (x)   |  (x)   |    x    |           |           |           |                                              |
+| lvm                                   |   x    |   x    |   x    |   x    |    x    |     x     |     x     |     x     |                                              |
 | lynis                                 |   x    |   x    |   x    |   x    |    x    |     x     |     x     |     x     |                                              |
 | mailto_root                           |   x    |   x    |   x    |   x    |    x    |     x     |     x     |     x     |                                              |
 | mailx                                 |   x    |   x    |   x    |   x    |    x    |     x     |     x     |     x     | Fedora                                       |

CONTRIBUTING.md

@@ -1164,6 +1164,7 @@ Some files under `plugins/modules/` and `plugins/module_utils/` are not authored
 
     * Upstream: <https://github.com/ansible-collections/community.general> (PR [#10070](https://github.com/ansible-collections/community.general/pull/10070), released in community.general 11.0.0).
     * Reason: community.general 11.0.0 requires ansible-core >= 2.18, which LFOps does not yet mandate (RHEL 8 / Python 3.6 still supported).
+    * Local patch: `from __future__ import annotations` is replaced by `from __future__ import absolute_import, division, print_function`, since Python 3.6 does not know the former. Keep this when re-syncing.
     * Drop when: LFOps raises its minimum ansible-core to >= 2.18; switch to `community.general.lvm_pv` and update `roles/lvm` accordingly.
 
 * `plugins/module_utils/gnupg.py` (and its `gnupg.py_LICENSE.txt`)
@@ -1203,7 +1204,7 @@ Unit tests are **mandatory** for every in-house plugin. Any pull request that ad
 * **Two tiers**, because plugins run in different environments:
 
     * Controller plugins (`plugins/filter/`, `plugins/lookup/`) are evaluated on the Ansible controller and only ever see the controller's Python (>= 3.10). They run on the standard CI matrix.
-    * Managed-node plugins (`plugins/modules/`, `plugins/module_utils/`) are executed on the target host and must keep working down to the oldest managed-node Python we maintain (Python 3.6 on RHEL 8). That tier runs inside a RHEL 8 / UBI 8 container; it is scaffolded in `tox.ini` (`[testenv:py36-target]`) and gets enabled once such tests exist.
+    * Managed-node plugins (`plugins/modules/`, `plugins/module_utils/`) are executed on the target host and must keep working down to the oldest managed-node Python we maintain (Python 3.6 on RHEL 8). Their tests run on the controller matrix and additionally inside a RHEL 8 / UBI 8 container (`[testenv:py36-target]` in `tox.ini`), so the test code has to be valid Python 3.6 too. See `tests/README.md`.
 
 * **How to run / verify** (the matrix of Python and ansible-core versions is driven by `tox`; see `tests/README.md` and `tox.ini`):
 
@@ -1214,7 +1215,7 @@ Unit tests are **mandatory** for every in-house plugin. Any pull request that ad
     pytest tests/unit        # against the active interpreter (needs pytest, pyyaml, ansible-core)
     ```
 
-* The `Linuxfabrik: Unit Tests` workflow runs the controller matrix on every push and pull request.
+* The `Linuxfabrik: Unit Tests` workflow runs the controller matrix and the Python 3.6 tier on every push and pull request.
 
 
 ### Testing

extensions/molecule/lvm/converge.yml

@@ -0,0 +1,2 @@
+- name: 'Converge lvm playbook'
+  ansible.builtin.import_playbook: 'linuxfabrik.lfops.lvm'

extensions/molecule/lvm/inventory/group_vars/systems_under_test.yml

@@ -0,0 +1,64 @@
+# Additional disks per VM, so the scenario never touches the boot disk: /dev/vdb, /dev/vdc and
+# /dev/vdd, in this order.
+kvm_vm__additional_disks:
+  - name: 'lvm-b'
+    size: '2G'
+  - name: 'lvm-c'
+    size: '1G'
+  - name: 'lvm-d'
+    size: '1G'
+
+# The target layout. prepare.yml creates the starting layout, see there.
+lvm__lvs__group_var:
+  # extended from 300M, which only fits because the PV below grew. The file system has to grow
+  # with it (resizefs) and is remounted with the new options.
+  - name: 'data'
+    vg: 'molecule'
+    size: '1G'
+    mount_path: '/mnt/molecule-data'
+    mount_opts: 'nodev,noexec,nosuid'
+  # created in this run, mounted on a directory the role creates for a non-root owner. The group
+  # is not "nobody", which is called "nogroup" on Debian and Ubuntu.
+  - name: 'owned'
+    vg: 'molecule'
+    size: '300M'
+    mount_path: '/mnt/molecule-owned'
+    mount_owner: 'nobody'
+    mount_group: 'daemon'
+    mount_mode: 0o750
+  # shrunk from 150M, together with its (unmounted) ext4 file system
+  - name: 'shrink'
+    vg: 'molecule'
+    size: '100M'
+    fstype: 'ext4'
+    shrink: true
+    force: true
+  - name: 'remove'
+    vg: 'multi'
+    state: 'absent'
+    force: true
+  # created with 100%FREE, now the only LV in a VG that got a second PV. 100%VG fills both PVs;
+  # 100%FREE would set it to the size of the space that is free right now (see the README).
+  - name: 'spread'
+    vg: 'multi'
+    size: '100%VG'
+    mount_path: '/mnt/molecule-spread'
+
+lvm__vgs__group_var:
+  # the starting PV only spans the 512 MiB partition. growpart grows the partition to the whole
+  # disk and the PV with it; this is how a VM disk grown on the hypervisor is picked up.
+  - name: 'molecule'
+    pvs:
+      - '/dev/vdb1'
+    growpart: true
+  # removed with the LV that is still in it. Its PV is taken over by the VG below in the same run.
+  - name: 'gone'
+    state: 'absent'
+    force: true
+  # extended by a second PV. growpart is skipped for whole disks.
+  - name: 'multi'
+    pvs:
+      - '/dev/vdc'
+      - '/dev/vdd'
+    pesize: '8'
+    growpart: true

extensions/molecule/lvm/inventory/hosts.yml

@@ -0,0 +1,16 @@
+# yamllint disable rule:empty-values
+
+lfops_lvm:
+  children:
+    systems_under_test:
+
+systems_under_test:
+  hosts:
+    debian12-vm:
+    debian13-vm:
+    rocky8-vm:
+    rocky9-vm:
+    rocky10-vm:
+    ubuntu2204-vm:
+    ubuntu2404-vm:
+    ubuntu2604-vm:

extensions/molecule/lvm/molecule.yml

@@ -0,0 +1,9 @@
+# Molecule scenario marker for the 'lvm' scenario, run with
+# `molecule test --scenario-name lvm`.
+#
+# It overrides the prepare step (see prepare.yml), which partitions the first additional disk and
+# runs the role once with a starting layout. converge then changes that layout into the one in
+# group_vars, so the scenario covers growing, extending, shrinking and removing, not only creating.
+provisioner:
+  playbooks:
+    prepare: '${MOLECULE_SCENARIO_DIRECTORY}/prepare.yml'

extensions/molecule/lvm/prepare.yml

@@ -0,0 +1,65 @@
+# Replaces the shared vm-prepare.yml for this scenario (see molecule.yml), so it imports it first
+# and only adds the precondition on top.
+- name: 'Prepare VMs for Ansible'
+  ansible.builtin.import_playbook: '../playbooks/vm-prepare.yml'
+
+
+# A partition on /dev/vdb that does not fill the disk, so the role has something to grow. sfdisk
+# comes with util-linux and is therefore on every image.
+- name: 'Create a partition that leaves room to grow on /dev/vdb'
+  hosts: 'systems_under_test'
+  gather_facts: false
+
+  tasks:
+
+    - name: 'echo ",512M" | sfdisk --label dos /dev/vdb'
+      ansible.builtin.command:
+        cmd: 'sfdisk --label dos /dev/vdb'
+        stdin: ',512M'
+        creates: '/dev/vdb1'
+
+
+# The starting layout. converge changes it into the one in group_vars, so the scenario covers what
+# the role does to an existing setup (grow, extend, shrink, remove), not only what it creates.
+# Play vars take precedence over the inventory's group_vars.
+- name: 'Create the starting LVM layout'
+  hosts: 'systems_under_test'
+
+  vars:
+    lvm__lvs__group_var:
+      - name: 'data'
+        vg: 'molecule'
+        # mkfs.xfs on RHEL 10 refuses anything below 300 MB
+        size: '300M'
+        mount_path: '/mnt/molecule-data'
+      - name: 'shrink'
+        vg: 'molecule'
+        size: '150M'
+        fstype: 'ext4'
+      - name: 'remove'
+        vg: 'multi'
+        size: '100M'
+        fstype: 'ext4'
+      - name: 'spread'
+        vg: 'multi'
+        size: '100%FREE'
+        mount_path: '/mnt/molecule-spread'
+      - name: 'inside'
+        vg: 'gone'
+        size: '100M'
+        fstype: 'ext4'
+    lvm__vgs__group_var:
+      # no growpart yet, so the PV keeps the size of the 512 MiB partition
+      - name: 'molecule'
+        pvs:
+          - '/dev/vdb1'
+      - name: 'multi'
+        pvs:
+          - '/dev/vdc'
+        pesize: '8'
+      - name: 'gone'
+        pvs:
+          - '/dev/vdd'
+
+  roles:
+    - role: 'linuxfabrik.lfops.lvm'