fix: adapt the UTxO-HD ledger flavors to Cardano node 11.1

.github/workflows/scripts/verify-cardano-db-restoration.sh

@@ -3,12 +3,12 @@
 set -e
 
 if [[ $# -lt 1 ]]; then
-  echo "Usage: $0 --docker-cmd [docker run command string] [--include-ancillary] [--ledger-backend <in-memory|lmdb|legacy>]"
+  echo "Usage: $0 --docker-cmd [docker run command string] [--include-ancillary] [--ledger-backend <in-memory|lmdb|lsm|legacy>]"
   echo ""
   echo "Parameters:"
-  echo "  --docker-cmd <command>            (Required) The 'docker run' command output in the result of a mithril-client CLI download or snapshot converter command."
-  echo "  --include-ancillary               (Optional) Does the ancillary files were included in the restoration."
-  echo "  --ledger-backend <in-memory|lmdb> (Optional) Specify the ledger backend. Default is 'in-memory'. Note: lmdb backend requires --include-ancillary to be set."
+  echo "  --docker-cmd <command>                         (Required) The 'docker run' command output in the result of a mithril-client CLI download or snapshot converter command."
+  echo "  --include-ancillary                            (Optional) Whether the ancillary files were included in the restoration."
+  echo "  --ledger-backend <in-memory|lmdb|lsm|legacy>   (Optional) Specify the ledger backend. Default is 'in-memory'. Note: lmdb and lsm backends require --include-ancillary to be set."
   exit 1
 fi
 
@@ -33,9 +33,17 @@ fi
 echo "Docker command:"
 echo "$DOCKER_CMD"
 
-# Note: ledger conversion to lmdb can only be executed if ancillary files are included
-if [[ ${LEDGER_BACKEND,,} == "lmdb" && "$INCLUDE_ANCILLARY" == "true" ]]; then
-  DOCKER_CMD="${DOCKER_CMD/ ghcr/" -e CARDANO_CONFIG_JSON_MERGE='{\"LedgerDB\":{\"Backend\":\"V1LMDB\"}}' ghcr"}"
+# Note: ledger conversion to an on disk backend can only be executed if ancillary files are included
+if [[ "$INCLUDE_ANCILLARY" == "true" ]]; then
+  case ${LEDGER_BACKEND,,} in
+    lmdb)
+      DOCKER_CMD="${DOCKER_CMD/ ghcr/" -e CARDANO_CONFIG_JSON_MERGE='{\"LedgerDB\":{\"Backend\":\"V1LMDB\"}}' ghcr"}"
+      ;;
+    lsm)
+      # The LSM backend relies on io_uring syscalls which are blocked by the default Docker seccomp profile
+      DOCKER_CMD="${DOCKER_CMD/ ghcr/" -e CARDANO_CONFIG_JSON_MERGE='{\"LedgerDB\":{\"Backend\":\"V2LSM\"}}' --security-opt seccomp=unconfined ghcr"}"
+      ;;
+  esac
 fi
 
 DOCKER_CMD_DETACHED="${DOCKER_CMD/docker run/docker run -d}"

.github/workflows/test-client.yml

@@ -81,6 +81,7 @@ on:
 env:
   NETWORK: ${{ inputs.network }}
   AGGREGATOR_ENDPOINT: ${{ inputs.aggregator_endpoint }}
+  LSM_MINIMUM_CARDANO_NODE_VERSION: "11.1.0"
 
 jobs:
   prepare:
@@ -126,7 +127,7 @@ jobs:
 
           if [[ $CARDANO_DATABASE_V2_CAPABILITY == "true" ]]; then
             echo 'available_cardano_database_backends=["v2"]' >> $GITHUB_OUTPUT
-            echo 'bin-cdb-download-matrix-include=[{"backend":"v2","os":"ubuntu-24.04","ledger_backend":"lmdb","extra_args":"--include-ancillary"}]' >> $GITHUB_OUTPUT
+            echo 'bin-cdb-download-matrix-include=[{"backend":"v2","os":"ubuntu-24.04","ledger_backend":"on-disk","extra_args":"--include-ancillary"}]' >> $GITHUB_OUTPUT
           else
             echo 'available_cardano_database_backends=[]' >> $GITHUB_OUTPUT
             echo 'bin-cdb-download-matrix-include=[]' >> $GITHUB_OUTPUT
@@ -329,6 +330,23 @@ jobs:
           echo "hash=$(./mithril-client --origin-tag CI cardano-db snapshot list --backend ${{ matrix.backend }} --json | jq -r '.[0].hash')" >> $GITHUB_OUTPUT
           echo "cardano_node_version=$(./mithril-client --origin-tag CI cardano-db snapshot list --backend ${{ matrix.backend }} --json | jq -r '.[0].cardano_node_version')" >> $GITHUB_OUTPUT
 
+      - name: Resolve ledger backend
+        id: ledger_backend
+        shell: bash
+        run: |
+          LEDGER_BACKEND="${{ matrix.ledger_backend }}"
+          CARDANO_VERSION="${{ steps.last_snapshot.outputs.cardano_node_version }}"
+          if [[ "$LEDGER_BACKEND" == "on-disk" ]]; then
+            OLDEST_VERSION=$(printf '%s\n%s\n' "$CARDANO_VERSION" "$LSM_MINIMUM_CARDANO_NODE_VERSION" | sort -V | head -n1)
+            if [[ "$OLDEST_VERSION" == "$LSM_MINIMUM_CARDANO_NODE_VERSION" ]]; then
+              LEDGER_BACKEND="lsm"
+            else
+              LEDGER_BACKEND="lmdb"
+            fi
+          fi
+
+          echo "name=$LEDGER_BACKEND" >> $GITHUB_OUTPUT
+
       - name: Cardano Database Snapshot / download & restore latest
         shell: bash
         working-directory: ./bin
@@ -337,19 +355,22 @@ jobs:
                           --backend ${{ matrix.backend }} --download-dir "${{ matrix.backend }}" ${{ matrix.extra_args }} --json \
                           | tee cdb-${{ matrix.backend }}-download-output.json
 
-      - name: Ledger state snapshot conversion from InMemory to ${{ matrix.ledger_backend }}
+      - name: Ledger state snapshot conversion from InMemory to ${{ steps.ledger_backend.outputs.name }}
         # The 'snapshot-converter' binary is not currently supported on Linux ARM64 platforms.
-        if: matrix.os != 'ubuntu-24.04-arm' && matrix.extra_args == '--include-ancillary' && contains(fromJSON('["lmdb", "legacy"]'), matrix.ledger_backend)
+        if: matrix.os != 'ubuntu-24.04-arm' && matrix.extra_args == '--include-ancillary' && contains(fromJSON('["on-disk", "legacy"]'), matrix.ledger_backend)
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         shell: bash
         working-directory: ./bin
         run: |
-          LEDGER_BACKEND="${{ matrix.ledger_backend }}"
+          LEDGER_BACKEND="${{ steps.ledger_backend.outputs.name }}"
           CARDANO_VERSION="${{ steps.last_snapshot.outputs.cardano_node_version }}"
-          if [[ ${LEDGER_BACKEND,,} == "lmdb" ]]; then 
-            UTXO_HD_FLAVOR="LMDB"
-          fi
+          case ${LEDGER_BACKEND,,} in
+            lmdb) UTXO_HD_FLAVOR="LMDB" ;;
+            lsm) UTXO_HD_FLAVOR="LSM" ;;
+            legacy) UTXO_HD_FLAVOR="Legacy" ;;
+            *) echo "Unsupported ledger backend: $LEDGER_BACKEND" >&2; exit 1 ;;
+          esac
 
           ./mithril-client ${{ needs.prepare.outputs.debug_level }} tools utxo-hd snapshot-converter --db-directory ${{ matrix.backend }}/db \
                           --cardano-node-version $CARDANO_VERSION --utxo-hd-flavor $UTXO_HD_FLAVOR --commit --json \
@@ -371,7 +392,7 @@ jobs:
             DOCKER_CMD=$(jq -r ".run_docker_cmd" ./bin/cdb-${{ matrix.backend }}-download-output.json)
           fi
 
-          .github/workflows/scripts/verify-cardano-db-restoration.sh --docker-cmd "$DOCKER_CMD" --ledger-backend ${{ matrix.ledger_backend }} ${{ matrix.extra_args }}
+          .github/workflows/scripts/verify-cardano-db-restoration.sh --docker-cmd "$DOCKER_CMD" --ledger-backend ${{ steps.ledger_backend.outputs.name }} ${{ matrix.extra_args }}
 
       - name: Cardano Database V2 Snapshot / verify tampered and missing immutables from a specific range
         if: matrix.backend == 'v2'
@@ -552,7 +573,8 @@ jobs:
         run: |
           mkdir -p $PWD/data
           chmod -R a+w $PWD/data
-          echo "mithril_client=docker run --rm -e NETWORK=$NETWORK -e GENESIS_VERIFICATION_KEY=$GENESIS_VERIFICATION_KEY -e ANCILLARY_VERIFICATION_KEY=$ANCILLARY_VERIFICATION_KEY -e AGGREGATOR_ENDPOINT=$AGGREGATOR_ENDPOINT -e GITHUB_TOKEN=$GITHUB_TOKEN --name='mithril-client' -v $PWD/data:/app/data ghcr.io/intersectmbo/mithril-client:$MITHRIL_IMAGE_ID"  >> $GITHUB_OUTPUT
+          # The LSM ledger conversion uses io_uring syscalls, which are blocked by the default Docker seccomp profile
+          echo "mithril_client=docker run --rm -e NETWORK=$NETWORK -e GENESIS_VERIFICATION_KEY=$GENESIS_VERIFICATION_KEY -e ANCILLARY_VERIFICATION_KEY=$ANCILLARY_VERIFICATION_KEY -e AGGREGATOR_ENDPOINT=$AGGREGATOR_ENDPOINT -e GITHUB_TOKEN=$GITHUB_TOKEN --name='mithril-client' --security-opt seccomp=unconfined -v $PWD/data:/app/data ghcr.io/intersectmbo/mithril-client:$MITHRIL_IMAGE_ID"  >> $GITHUB_OUTPUT
 
       - name: fetch latest snapshot hash
         id: last_snapshot
@@ -561,15 +583,27 @@ jobs:
           echo "hash=$(${{ steps.command.outputs.mithril_client }} --origin-tag CI cardano-db snapshot list --backend ${{ matrix.backend }} --json | jq -r '.[0].hash')" >> $GITHUB_OUTPUT
           echo "cardano_node_version=$(${{ steps.command.outputs.mithril_client }} --origin-tag CI cardano-db snapshot list --backend ${{ matrix.backend }} --json | jq -r '.[0].cardano_node_version')" >> $GITHUB_OUTPUT
 
+      - name: Resolve UTxO-HD flavor
+        id: utxo_hd_flavor
+        shell: bash
+        run: |
+          CARDANO_VERSION="${{ steps.last_snapshot.outputs.cardano_node_version }}"
+          OLDEST_VERSION=$(printf '%s\n%s\n' "$CARDANO_VERSION" "$LSM_MINIMUM_CARDANO_NODE_VERSION" | sort -V | head -n1)
+          if [[ "$OLDEST_VERSION" == "$LSM_MINIMUM_CARDANO_NODE_VERSION" ]]; then
+            echo "name=LSM" >> $GITHUB_OUTPUT
+          else
+            echo "name=LMDB" >> $GITHUB_OUTPUT
+          fi
+
       - name: Cardano Database Snapshot / download & restore latest
         shell: bash
         run: |
           ${{ steps.command.outputs.mithril_client }} ${{ needs.prepare.outputs.debug_level }} --origin-tag CI cardano-db download ${{ steps.last_snapshot.outputs.hash }} --backend ${{ matrix.backend }} --download-dir "/app/data/${{ matrix.backend }}" ${{ matrix.extra_args }}
 
-      - name: Ledger state snapshot conversion from InMemory to LMDB
+      - name: Ledger state snapshot conversion from InMemory to ${{ steps.utxo_hd_flavor.outputs.name }}
         if: matrix.extra_args == '--include-ancillary' && matrix.backend == 'v2'
         shell: bash
-        run: ${{ steps.command.outputs.mithril_client }} ${{ needs.prepare.outputs.debug_level }} tools utxo-hd snapshot-converter --db-directory /app/data/v2/db --cardano-node-version ${{ steps.last_snapshot.outputs.cardano_node_version }} --utxo-hd-flavor LMDB --commit
+        run: ${{ steps.command.outputs.mithril_client }} ${{ needs.prepare.outputs.debug_level }} tools utxo-hd snapshot-converter --db-directory /app/data/v2/db --cardano-node-version ${{ steps.last_snapshot.outputs.cardano_node_version }} --utxo-hd-flavor ${{ steps.utxo_hd_flavor.outputs.name }} --commit
 
       - name: Cardano Database V2 Snapshot / verify immutables
         if: matrix.backend == 'v2'

docs/website/root/manual/getting-started/bootstrap-cardano-node.md

@@ -207,10 +207,12 @@ Then, create a shell function for the Mithril client:
 
 ```bash
 mithril_client () {
-  docker run --rm -e GENESIS_VERIFICATION_KEY=$GENESIS_VERIFICATION_KEY -e AGGREGATOR_ENDPOINT=$AGGREGATOR_ENDPOINT --name='mithril-client' -v $(pwd):/app/data -w /app/data -u $(id -u) ghcr.io/intersectmbo/mithril-client:$MITHRIL_IMAGE_ID $@
+  docker run --rm -e GENESIS_VERIFICATION_KEY=$GENESIS_VERIFICATION_KEY -e AGGREGATOR_ENDPOINT=$AGGREGATOR_ENDPOINT --name='mithril-client' --security-opt seccomp=unconfined -v $(pwd):/app/data -w /app/data -u $(id -u) ghcr.io/intersectmbo/mithril-client:$MITHRIL_IMAGE_ID "$@"
 }
 ```
 
+The `--security-opt seccomp=unconfined` option is required by the `LSM` ledger state conversion, which uses io_uring syscalls blocked by the default Docker seccomp profile.
+
 You can now use the `mithril_client` function:
 
 ```bash
@@ -520,20 +522,30 @@ Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by runni
 
 ### Step 5 (optional): Convert the ledger state snapshot to another flavor
 
-After restoring a snapshot with the `--include-ancillary` option, the ledger state is in the `InMemory` format. You can convert it to another UTxO-HD flavor (e.g., `LMDB` or `Legacy`) using the Mithril client `tools utxo-hd snapshot-converter` command.
+After restoring a snapshot with the `--include-ancillary` option, the ledger state is in the `InMemory` format. You can convert it to another UTxO-HD flavor using the Mithril client `tools utxo-hd snapshot-converter` command.
+
+Each flavor can only be read by a range of Cardano node versions:
 
-To do so, run the following command:
+| Flavor   | Cardano node versions running the converted ledger state |
+| -------- | -------------------------------------------------------- |
+| `LSM`    | `10.7.0` and above                                       |
+| `LMDB`   | `11.0.1` and below                                       |
+| `Legacy` | `10.3.1` and below                                       |
+
+To convert the ledger state to the `LSM` flavor, run the following command:
 
 ```
-mithril-client tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor LMDB
+mithril-client tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor LSM
 ```
 
-Or, to convert it to the `Legacy` flavor:
+Or, to convert it to the `LMDB` flavor, which is useful to run a Cardano node older than the one used by the aggregator:
 
 ```
-mithril-client tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor Legacy
+mithril-client tools utxo-hd snapshot-converter --db-directory db --cardano-node-version 11.0.1 --utxo-hd-flavor LMDB
 ```
 
+The `LMDB` backend was dropped in Cardano node `11.1.0`. Requesting that flavor with `11.1.0` or upper fails, so pass a version able to run it.
+
 Use the `--commit` option to replace the current ledger state with the converted snapshot.
 
 You can also replace `latest` with a specific Cardano node version tag which will be used to download the corresponding Cardano node distribution and extract the `snapshot-converter` binary tool.

docs/website/versioned_docs/version-maintained/manual/getting-started/bootstrap-cardano-node.md

@@ -207,10 +207,12 @@ Then, create a shell function for the Mithril client:
 
 ```bash
 mithril_client () {
-  docker run --rm -e GENESIS_VERIFICATION_KEY=$GENESIS_VERIFICATION_KEY -e AGGREGATOR_ENDPOINT=$AGGREGATOR_ENDPOINT --name='mithril-client' -v $(pwd):/app/data -w /app/data -u $(id -u) ghcr.io/intersectmbo/mithril-client:$MITHRIL_IMAGE_ID $@
+  docker run --rm -e GENESIS_VERIFICATION_KEY=$GENESIS_VERIFICATION_KEY -e AGGREGATOR_ENDPOINT=$AGGREGATOR_ENDPOINT --name='mithril-client' --security-opt seccomp=unconfined -v $(pwd):/app/data -w /app/data -u $(id -u) ghcr.io/intersectmbo/mithril-client:$MITHRIL_IMAGE_ID "$@"
 }
 ```
 
+The `--security-opt seccomp=unconfined` option is required by the `LSM` ledger state conversion, which uses io_uring syscalls blocked by the default Docker seccomp profile.
+
 You can now use the `mithril_client` function:
 
 ```bash
@@ -520,20 +522,30 @@ Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by runni
 
 ### Step 5 (optional): Convert the ledger state snapshot to another flavor
 
-After restoring a snapshot with the `--include-ancillary` option, the ledger state is in the `InMemory` format. You can convert it to another UTxO-HD flavor (e.g., `LMDB` or `Legacy`) using the Mithril client `tools utxo-hd snapshot-converter` command.
+After restoring a snapshot with the `--include-ancillary` option, the ledger state is in the `InMemory` format. You can convert it to another UTxO-HD flavor using the Mithril client `tools utxo-hd snapshot-converter` command.
+
+Each flavor can only be read by a range of Cardano node versions:
 
-To do so, run the following command:
+| Flavor   | Cardano node versions running the converted ledger state |
+| -------- | -------------------------------------------------------- |
+| `LSM`    | `10.7.0` and above                                       |
+| `LMDB`   | `11.0.1` and below                                       |
+| `Legacy` | `10.3.1` and below                                       |
+
+To convert the ledger state to the `LSM` flavor, run the following command:
 
 ```
-mithril-client tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor LMDB
+mithril-client tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor LSM
 ```
 
-Or, to convert it to the `Legacy` flavor:
+Or, to convert it to the `LMDB` flavor, which is useful to run a Cardano node older than the one used by the aggregator:
 
 ```
-mithril-client tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor Legacy
+mithril-client tools utxo-hd snapshot-converter --db-directory db --cardano-node-version 11.0.1 --utxo-hd-flavor LMDB
 ```
 
+The `LMDB` backend was dropped in Cardano node `11.1.0`. Requesting that flavor with `11.1.0` or upper fails, so pass a version able to run it.
+
 Use the `--commit` option to replace the current ledger state with the converted snapshot.
 
 You can also replace `latest` with a specific Cardano node version tag which will be used to download the corresponding Cardano node distribution and extract the `snapshot-converter` binary tool.

mithril-client-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client-cli"
-version = "0.13.22"
+version = "0.13.23"
 description = "A Mithril Client"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-client-cli/src/commands/cardano_db/shared_steps.rs

@@ -10,8 +10,8 @@ use mithril_client::{
 };
 
 use crate::utils::{
-    CARDANO_NODE_V10_6_2, CARDANO_NODE_V10_7_0, CardanoDbUtils, LedgerFormat, ProgressPrinter,
-    is_version_equal_or_upper,
+    CardanoDbUtils, LedgerFormat, ProgressPrinter, is_version_at_least_10_6_2_or_latest,
+    is_version_at_least_10_7_0_or_latest, is_version_at_least_11_1_0_or_latest,
 };
 
 pub struct ComputeCardanoDatabaseMessageOptions {
@@ -197,10 +197,14 @@ pub fn log_download_information(
                     json["snapshot_converter_cmd_to_lsm"] =
                         serde_json::Value::String(snapshot_converter_cmd("LSM"));
                 }
-                json["snapshot_converter_cmd_to_lmdb"] =
-                    serde_json::Value::String(snapshot_converter_cmd("LMDB"));
-                json["snapshot_converter_cmd_to_legacy"] =
-                    serde_json::Value::String(snapshot_converter_cmd("Legacy"));
+                if !is_version_at_least_11_1_0_or_latest(cardano_node_version) {
+                    json["snapshot_converter_cmd_to_lmdb"] =
+                        serde_json::Value::String(snapshot_converter_cmd("LMDB"));
+                }
+                if !is_version_at_least_10_6_2_or_latest(cardano_node_version) {
+                    json["snapshot_converter_cmd_to_legacy"] =
+                        serde_json::Value::String(snapshot_converter_cmd("Legacy"));
+                }
             }
         }
 
@@ -235,13 +239,15 @@ pub fn log_download_information(
                     );
                 }
 
-                println!(
-                    r###"Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by running the command:
+                if !is_version_at_least_11_1_0_or_latest(cardano_node_version) {
+                    println!(
+                        r###"Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by running the command:
 
     {}
     "###,
-                    snapshot_converter_cmd("LMDB"),
-                );
+                        snapshot_converter_cmd("LMDB"),
+                    );
+                }
 
                 if !is_version_at_least_10_6_2_or_latest(cardano_node_version) {
                     println!(
@@ -259,14 +265,6 @@ pub fn log_download_information(
     Ok(())
 }
 
-pub fn is_version_at_least_10_7_0_or_latest(version: &str) -> bool {
-    is_version_equal_or_upper(version, CARDANO_NODE_V10_7_0)
-}
-
-pub fn is_version_at_least_10_6_2_or_latest(version: &str) -> bool {
-    is_version_equal_or_upper(version, CARDANO_NODE_V10_6_2)
-}
-
 #[cfg(test)]
 mod tests {
     use mithril_client::{