test: support multiple responses per AT fixture

.github/workflows/vendor-fixtures.yml

@@ -38,6 +38,8 @@ jobs:
           sh -n application/qmodem/tests/test_core_cmds.sh
           bash -n application/qmodem/tests/test_collect_encryption.sh
           bash -n application/qmodem/tests/test_fixture_profiles.sh
+          bash -n application/qmodem/tests/test_fixture_response_queue.sh
+          bash -n application/qmodem/tests/test_import_testcases.sh
           sh -n application/qmodem/tests/test_vendor_cmds_boundary.sh
           bash -n application/qmodem/tests/test_fixture_collection_bytes.sh
           bash -n application/qmodem/tests/test_recognition_collection.sh
@@ -51,6 +53,8 @@ jobs:
           application/qmodem/tests/test_recognition_collection.sh
           application/qmodem/tests/test_recognition_fixtures.sh
           application/qmodem/tests/test_fixture_profiles.sh
+          application/qmodem/tests/test_fixture_response_queue.sh
+          application/qmodem/tests/test_import_testcases.sh
           QMODEM_TEST_SEAL_BIN=/tmp/qmodem-seal \
             application/qmodem/tests/test_collect_encryption.sh
       - name: Replay vendor fixtures

application/qmodem/files/etc/config/qmodem

@@ -16,3 +16,4 @@ config main 'main'
 	option usage_stats_nvram_interval '300'
 	#collect AT command fixtures to /tmp/qmodem/testcases for qmodem_collect pack
 	option testcase_collect '0'
+	option testcase_scenario 'default'

application/qmodem/files/usr/sbin/qmodem_collect

@@ -88,12 +88,14 @@ resolve_recognition_fixture()
 
 cmd_status()
 {
-    local seal_bin recipient_id
+    local seal_bin recipient_id scenario
     if collect_enabled; then
         echo "testcase_collect: on"
     else
         echo "testcase_collect: off (enable: uci set qmodem.main.testcase_collect=1 && uci commit qmodem)"
     fi
+    scenario=$(uci -q get qmodem.main.testcase_scenario 2>/dev/null)
+    echo "testcase_scenario: ${scenario:-default}"
     if [ -d "$COLLECT_DIR" ]; then
         echo "collect dir: $COLLECT_DIR ($(find "$COLLECT_DIR" -name '*.json' 2>/dev/null | wc -l) fixture files)"
     else
@@ -135,7 +137,7 @@ cmd_pack()
     }
     local work out_file file count timestamp inner_file seal_work seal_bin
     local rel resolved target_rel expected_vendor expected_platform expected_model prepared
-    local recipient recipient_id seal_output review_key wrapped_key_hex
+    local recipient recipient_id seal_output review_key wrapped_key_hex response_count response_index update_file
     work=$(mktemp -d) || exit 1
     PACK_WORK="$work"
     count=0
@@ -148,10 +150,20 @@ cmd_pack()
         expected_platform=$(printf '%s' "$resolved" | cut -f3)
         expected_model=$(printf '%s' "$resolved" | cut -f4)
         prepared="$file"
-        if [ -n "$expected_vendor" ]; then
+        if [ "${target_rel#*/expected/}" = "$target_rel" ]; then
             prepared="$work/.fixture.$count.json"
-            jq --arg vendor "$expected_vendor" --arg platform "$expected_platform" --arg model "$expected_model" \
-                '.expected_identity={vendor:$vendor, platform:$platform, model:$model}' "$file" > "$prepared" || exit 1
+            jq --arg vendor "$expected_vendor" --arg platform "$expected_platform" --arg model "$expected_model" '
+                if has("responses") then .
+                else .responses=[{response_hex:(.response_hex // ""), response:.response,
+                                  rc:(.rc // 0), timestamp:(.timestamp // "")}]
+                  | del(.response_hex, .response, .rc, .timestamp)
+                end
+                | .responses |= map(if .response == null then del(.response) else . end)
+                | .responses |= to_entries | .responses |= map(
+                    .value + {scenario:(.value.scenario // "default"), sequence:(.value.sequence // .key),
+                              capture_sequence:(.value.capture_sequence // .value.sequence // .key)})
+                | if $vendor != "" then .expected_identity={vendor:$vendor, platform:$platform, model:$model}
+                  else . end' "$file" > "$prepared" || exit 1
         fi
         mkdir -p "$work/$(dirname "$target_rel")"
         if [ "$raw" = "1" ]; then
@@ -168,28 +180,33 @@ cmd_pack()
             cmd_masked=$(jq -r '.command // ""' "$prepared" | mask_digits)
             raw_response="$work/.response.raw"
             sanitized_response="$work/.response.sanitized"
-            resp_hex=$(jq -r '.response_hex // empty' "$prepared")
-            if [ -n "$resp_hex" ]; then
-                printf '%s' "$resp_hex" | xxd -r -p > "$raw_response" || {
+            jq --arg c "$cmd_masked" '.command=$c | .sanitized=true' "$prepared" > "$work/$target_rel" || exit 1
+            response_count=$(jq '.responses | length' "$prepared")
+            response_index=0
+            while [ "$response_index" -lt "$response_count" ]; do
+                resp_hex=$(jq -r --argjson i "$response_index" '.responses[$i].response_hex // empty' "$prepared")
+                if [ -n "$resp_hex" ]; then
+                    printf '%s' "$resp_hex" | xxd -r -p > "$raw_response" || {
+                        rm -rf "$work"
+                        echo "invalid response_hex in fixture: $file response[$response_index]" >&2
+                        exit 1
+                    }
+                else
+                    jq -j --argjson i "$response_index" '.responses[$i].response // ""' "$prepared" > "$raw_response"
+                fi
+                mask_digits < "$raw_response" > "$sanitized_response"
+                resp_hex=$(xxd -p "$sanitized_response" | tr -d '\n') || {
                     rm -rf "$work"
-                    echo "invalid response_hex in fixture: $file" >&2
+                    echo "failed to encode fixture response: $file response[$response_index]" >&2
                     exit 1
                 }
-            else
-                jq -j '.response // ""' "$prepared" > "$raw_response"
-            fi
-            mask_digits < "$raw_response" > "$sanitized_response"
-            resp_hex=$(xxd -p "$sanitized_response" | tr -d '\n') || {
-                rm -rf "$work"
-                echo "failed to encode fixture response: $file" >&2
-                exit 1
-            }
-            jq --arg c "$cmd_masked" --arg h "$resp_hex" \
-                '.command=$c | .response_hex=$h | del(.response) | .sanitized=true' "$prepared" > "$work/$target_rel" || {
-                rm -rf "$work"
-                echo "failed to sanitize fixture: $file" >&2
-                exit 1
-            }
+                update_file="$work/.sanitized.$count.json"
+                jq --argjson i "$response_index" --arg h "$resp_hex" \
+                    '.responses[$i].response_hex=$h | del(.responses[$i].response)' \
+                    "$work/$target_rel" > "$update_file" || exit 1
+                mv "$update_file" "$work/$target_rel"
+                response_index=$((response_index + 1))
+            done
             rm -f "$raw_response" "$sanitized_response"
         fi
         [ "$prepared" = "$file" ] || rm -f "$prepared"

application/qmodem/files/usr/share/qmodem/modem_ctrl.sh

@@ -332,7 +332,7 @@ case $method in
 esac
 #optionally keep a golden snapshot of the final JSON for fixture tests
 if qmodem_testcase_collect_enabled; then
-    expected_dir="$(qmodem_testcase_profile_dir)/expected"
+    expected_dir="$(qmodem_testcase_profile_dir)/expected/$(qmodem_testcase_scenario)"
     mkdir -p "$expected_dir" 2>/dev/null
     json_dump | tee "$expected_dir/$method.json"
 else

application/qmodem/files/usr/share/qmodem/modem_util.sh

@@ -31,6 +31,13 @@ qmodem_testcase_path_segment()
   printf '%s' "$slug"
 }
 
+qmodem_testcase_scenario()
+{
+  local scenario="${QMODEM_TESTCASE_SCENARIO:-}"
+  [ -n "$scenario" ] || scenario=$(uci -q get qmodem.main.testcase_scenario 2>/dev/null)
+  qmodem_testcase_path_segment "${scenario:-default}" default
+}
+
 qmodem_testcase_profile_dir()
 {
   local collect_dir="${QMODEM_COLLECT_DIR:-/tmp/qmodem/testcases}"
@@ -61,7 +68,7 @@ qmodem_record_testcase_file()
   local vendor_name="${vendor:-${manufacturer:-core}}"
   local platform_name="${platform:-unknown}"
   local model_name="${QMODEM_TESTCASE_MODEL:-}" dir phase=vendor
-  local slug hash file
+  local slug hash file update_file timestamp scenario lock_dir attempts
   [ -n "$model_name" ] || model_name=$(uci -q get "qmodem.${config_section:-}.name" 2>/dev/null)
   [ -n "$model_name" ] || model_name="unknown"
   if [ "$vendor_name" = "core" ] || [ "$vendor_name" = "unknown" ] || [ "$model_name" = "unknown" ]; then
@@ -73,6 +80,43 @@ qmodem_record_testcase_file()
   hash=$(printf '%s' "$atcmd" | md5sum | cut -c1-8)
   file="${dir}/${slug}-${hash}.json"
   response_hex=$(xxd -p "$response_file" | tr -d '\n') || return 0
+  timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+  scenario=$(qmodem_testcase_scenario)
+  lock_dir="${file}.lock"
+  attempts=0
+  while ! mkdir "$lock_dir" 2>/dev/null; do
+    attempts=$((attempts + 1))
+    [ "$attempts" -lt 100 ] || return 0
+    sleep 0.05
+  done
+  update_file=$(mktemp "${dir}/.fixture.XXXXXX") || {
+    rmdir "$lock_dir" 2>/dev/null
+    return 0
+  }
+  if [ -f "$file" ]; then
+    jq --arg response_hex "$response_hex" --argjson rc "$rc" --arg timestamp "$timestamp" \
+      --arg scenario "$scenario" '
+      if has("responses") then .
+      else .responses=[{response_hex:(.response_hex // ""), rc:(.rc // 0),
+                        timestamp:(.timestamp // ""), scenario:"default", sequence:0}]
+        | del(.response_hex, .response, .rc, .timestamp)
+      end
+      | .responses |= map(if .response == null then del(.response) else . end)
+      | .responses |= to_entries | .responses |= map(
+          .value + {scenario:(.value.scenario // "default"), sequence:(.value.sequence // .key),
+                    capture_sequence:(.value.capture_sequence // .value.sequence // .key)})
+      | .responses |= map(.value // .)
+      | ([.responses[] | select(.scenario == $scenario) | .sequence] | max // -1) as $last
+      | .responses += [{response_hex:$response_hex, rc:$rc, timestamp:$timestamp,
+                        scenario:$scenario, sequence:($last + 1), capture_sequence:($last + 1)}]' "$file" > "$update_file" 2>/dev/null || {
+          rm -f "$update_file"
+          rmdir "$lock_dir" 2>/dev/null
+          return 0
+        }
+    mv "$update_file" "$file" 2>/dev/null || rm -f "$update_file"
+    rmdir "$lock_dir" 2>/dev/null
+    return 0
+  fi
   jq -n \
     --arg vendor "$vendor_name" \
     --arg platform "$platform_name" \
@@ -81,13 +125,16 @@ qmodem_record_testcase_file()
     --arg response_hex "$response_hex" \
     --arg tool "$tool" \
     --arg phase "$phase" \
+    --arg scenario "$scenario" \
     --arg config_section "${config_section:-unknown}" \
     --argjson rc "$rc" \
-    --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+    --arg timestamp "$timestamp" \
     '{vendor:$vendor, platform:$platform, model:$model, phase:$phase,
-      config_section:$config_section, command:$command,
-      response_hex:$response_hex, tool:$tool, rc:$rc, timestamp:$timestamp}' \
-    > "$file" 2>/dev/null || rm -f "$file"
+      config_section:$config_section, command:$command, tool:$tool,
+      responses:[{response_hex:$response_hex, rc:$rc, timestamp:$timestamp,
+                  scenario:$scenario, sequence:0, capture_sequence:0}]}' \
+    > "$update_file" 2>/dev/null && mv "$update_file" "$file" 2>/dev/null || rm -f "$update_file"
+  rmdir "$lock_dir" 2>/dev/null
 }
 
 at()

application/qmodem/tests/lib/at_fixture.sh

@@ -1,5 +1,5 @@
 # Replay recorded fixtures instead of touching hardware.
-# Requires FIXTURE_LOOKUP: a dir with <md5(command)>.response / .rc files.
+# Requires FIXTURE_LOOKUP: ordered responses grouped by command hash and scenario.
 
 at() { _fixture_send at "$@"; }
 fastat() { _fixture_send fastat "$@"; }
@@ -8,12 +8,19 @@ _fixture_send()
 {
     _fs_cmd=$3
     _fs_h=$(printf '%s' "$_fs_cmd" | md5sum | cut -c1-8)
-    if [ -f "$FIXTURE_LOOKUP/$_fs_h.response" ]; then
-        cat "$FIXTURE_LOOKUP/$_fs_h.response"
+    _fs_cursor="$FIXTURE_LOOKUP/cursors/$_fs_h"
+    _fs_index=0
+    [ ! -f "$_fs_cursor" ] || _fs_index=$(cat "$_fs_cursor")
+    _fs_variant="$FIXTURE_LOOKUP/$_fs_h.responses/${FIXTURE_SCENARIO:-default}/$_fs_index"
+    if [ -f "$_fs_variant.response" ]; then
+        printf '%s\n' "$_fs_h" >> "$FIXTURE_LOOKUP/hits.log"
+        printf '%s\n' "$((_fs_index + 1))" > "$_fs_cursor"
+        cat "$_fs_variant.response"
         _fs_rc=0
-        [ -f "$FIXTURE_LOOKUP/$_fs_h.rc" ] && _fs_rc=$(cat "$FIXTURE_LOOKUP/$_fs_h.rc")
+        [ -f "$_fs_variant.rc" ] && _fs_rc=$(cat "$_fs_variant.rc")
         return "$_fs_rc"
     fi
-    echo "$_fs_cmd" >> "$FIXTURE_LOOKUP/misses.log"
-    return 0
+    printf '%s\t%s\t%s\n' "${FIXTURE_SCENARIO:-default}" "$_fs_index" "$_fs_cmd" \
+        >> "$FIXTURE_LOOKUP/misses.log"
+    return 127
 }

application/qmodem/tests/test_collect_encryption.sh

@@ -20,9 +20,11 @@ trap cleanup EXIT
 profile=quectel/qualcomm/rm500q-ae-9f94df3c
 mkdir -p "$test_dir/fixtures/$profile"
 response_hex=$(printf 'AT+CGSN\r\n861234567890123\r\nOK\r\n\r\n' | xxd -p | tr -d '\n')
-jq -n --arg h "$response_hex" \
+response_hex_2=$(printf 'AT+CGSN\r\n869876543210987\r\nOK\r\n\r\n' | xxd -p | tr -d '\n')
+jq -n --arg h "$response_hex" --arg h2 "$response_hex_2" \
     '{vendor:"quectel",platform:"qualcomm",model:"RM500Q-AE",
-      command:"AT+CGSN",response_hex:$h,tool:"at",rc:0}' \
+      command:"AT+CGSN",tool:"at",
+      responses:[{response_hex:$h,rc:0},{response_hex:$h2,rc:0}]}' \
     > "$test_dir/fixtures/$profile/cgsn.json"
 
 plain_output=$(QMODEM_COLLECT_DIR="$test_dir/fixtures" \
@@ -66,8 +68,11 @@ owner_review=$(printf '%s\n' "$token" | "$seal_bin" review-key --token-stdin \
 printf '%s\n' "$review_key" | "$seal_bin" decrypt --review-key --token-stdin \
     --input "$encrypted_file" --output "$test_dir/review.tar.gz"
 tar -xOzf "$test_dir/review.tar.gz" "./$profile/cgsn.json" \
-    | jq -r '.response_hex' | xxd -r -p \
+    | jq -r '.responses[0].response_hex' | xxd -r -p \
     | cmp - <(printf 'AT+CGSN\r\n860000000000023\r\nOK\r\n\r\n')
+tar -xOzf "$test_dir/review.tar.gz" "./$profile/cgsn.json" \
+    | jq -r '.responses[1].response_hex' | xxd -r -p \
+    | cmp - <(printf 'AT+CGSN\r\n860000000000087\r\nOK\r\n\r\n')
 
 forced_output=$(QMODEM_COLLECT_DIR="$test_dir/fixtures" QMODEM_SEAL_BIN="$seal_bin" \
     "$collect_bin" pack --unencrypted 2>&1)

application/qmodem/tests/test_fixture_collection_bytes.sh

@@ -40,9 +40,9 @@ rc=$?
 [ "$rc" -eq 7 ]
 cmp "$expected" "$actual"
 fixture=$(find "$QMODEM_COLLECT_DIR/quectel" -name '*BYTECHECK*.json')
-fixture_hex_decode "$(jq -r '.response_hex' "$fixture")" > "$recorded"
+fixture_hex_decode "$(jq -r '.responses[0].response_hex' "$fixture")" > "$recorded"
 cmp "$expected" "$recorded"
-[ "$(jq -r '.rc' "$fixture")" -eq 7 ]
+[ "$(jq -r '.responses[0].rc' "$fixture")" -eq 7 ]
 [ "$(jq -r '.platform' "$fixture")" = qualcomm ]
 [ "$(jq -r '.model' "$fixture")" = RM500Q-AE ]
 case "$fixture" in */quectel/qualcomm/rm500q-ae-9f94df3c/*) ;; *) exit 1 ;; esac
@@ -53,8 +53,41 @@ rc=$?
 [ "$rc" -eq 9 ]
 cmp "$expected" "$actual"
 fixture=$(find "$QMODEM_COLLECT_DIR/quectel" -name '*FASTBYTECHECK*.json')
-fixture_hex_decode "$(jq -r '.response_hex' "$fixture")" > "$recorded"
+fixture_hex_decode "$(jq -r '.responses[0].response_hex' "$fixture")" > "$recorded"
 cmp "$expected" "$recorded"
-[ "$(jq -r '.rc' "$fixture")" -eq 9 ]
+[ "$(jq -r '.responses[0].rc' "$fixture")" -eq 9 ]
+[ "$(jq -r '.responses[0].scenario' "$fixture")" = default ]
+[ "$(jq -r '.responses[0].sequence' "$fixture")" -eq 0 ]
+[ "$(jq -r '.responses[0].capture_sequence' "$fixture")" -eq 0 ]
+
+variant=ONE
+tom_modem()
+{
+    printf '%s\r\nOK\r\n' "$variant"
+}
+at /dev/ttyUSB2 'AT+VARIANT' >/dev/null
+variant=TWO
+at /dev/ttyUSB2 'AT+VARIANT' >/dev/null
+at /dev/ttyUSB2 'AT+VARIANT' >/dev/null
+QMODEM_TESTCASE_SCENARIO=sim-absent
+export QMODEM_TESTCASE_SCENARIO
+at /dev/ttyUSB2 'AT+VARIANT' >/dev/null
+fixture=$(find "$QMODEM_COLLECT_DIR/quectel" -name '*VARIANT*.json')
+[ "$(jq '.responses | length' "$fixture")" -eq 4 ]
+[ "$(jq -r '.responses[0].response_hex' "$fixture")" != "$(jq -r '.responses[1].response_hex' "$fixture")" ]
+[ "$(jq -r '[.responses[] | select(.scenario == "default") | .sequence] | join(" ")' "$fixture")" = '0 1 2' ]
+[ "$(jq -r '[.responses[] | select(.scenario == "sim-absent") | .sequence] | join(" ")' "$fixture")" = '0' ]
+
+QMODEM_TESTCASE_SCENARIO=parallel
+export QMODEM_TESTCASE_SCENARIO
+for n in 0 1 2 3 4 5 6 7; do
+    response_file="$QMODEM_COLLECT_DIR/parallel-$n.response"
+    printf 'RESPONSE-%s\r\n' "$n" > "$response_file"
+    (qmodem_record_testcase_file at 'AT+PARALLEL' "$response_file" 0) &
+done
+wait
+fixture=$(find "$QMODEM_COLLECT_DIR/quectel" -name '*PARALLEL*.json')
+[ "$(jq '.responses | length' "$fixture")" -eq 8 ]
+[ "$(jq -r '[.responses[].sequence] | sort | join(" ")' "$fixture")" = '0 1 2 3 4 5 6 7' ]
 
 echo 'fixture collection byte tests passed'

application/qmodem/tests/test_fixture_profiles.sh

@@ -8,20 +8,33 @@ trap 'rm -rf "$test_root"' EXIT
 
 write_fixture()
 {
-    local platform=$1 model=$2 model_dir=$3 response=$4 dir hex
+    local platform=$1 model=$2 model_dir=$3 response=$4 response2=${5:-} dir hex hex2
     dir="$test_root/testcases/quectel/$platform/$model_dir"
     mkdir -p "$dir"
     hex=$(printf '%s' "$response" | xxd -p | tr -d '\n')
-    jq -n --arg platform "$platform" --arg model "$model" --arg hex "$hex" \
-        '{vendor:"quectel",platform:$platform,model:$model,command:"AT+CGSN",
-          response_hex:$hex,tool:"at",rc:0}' > "$dir/AT_CGSN.json"
+    hex2=$(printf '%s' "$response2" | xxd -p | tr -d '\n')
+    jq -n --arg platform "$platform" --arg model "$model" --arg hex "$hex" --arg hex2 "$hex2" \
+        '{vendor:"quectel",platform:$platform,model:$model,command:"AT+CGSN",tool:"at",
+          responses:([{response_hex:$hex,rc:0,timestamp:"test",scenario:"default",sequence:0,capture_sequence:0}] +
+                     (if $hex2 == "" then [] else [{response_hex:$hex2,rc:0,timestamp:"test",scenario:"roaming",sequence:0,capture_sequence:0}] end))}' \
+        > "$dir/AT_CGSN.json"
 }
 
-write_fixture qualcomm RM500Q-AE rm500q-ae-9f94df3c $'QUALCOMM\r\nOK\r\n'
+write_fixture qualcomm RM500Q-AE rm500q-ae-9f94df3c \
+    $'AT+CGSN\r\r\n860000000000012\r\n\r\nOK\r\n' \
+    $'AT+CGSN\r\r\n860000000000013\r\n\r\nOK\r\n'
 write_fixture unisoc UDX710 udx710-861f4136 $'UNISOC\r\nOK\r\n'
+mkdir -p "$test_root/testcases/quectel/qualcomm/rm500q-ae-9f94df3c/expected/default"
+mkdir -p "$test_root/testcases/quectel/qualcomm/rm500q-ae-9f94df3c/expected/roaming"
+jq -n '{result:{},imei:"860000000000012"}' \
+    > "$test_root/testcases/quectel/qualcomm/rm500q-ae-9f94df3c/expected/default/get_imei.json"
+jq -n '{result:{},imei:"860000000000013"}' \
+    > "$test_root/testcases/quectel/qualcomm/rm500q-ae-9f94df3c/expected/roaming/get_imei.json"
 
 output=$(QMODEM_TEST_REPO_ROOT="$test_root" "$PACKAGE_DIR/tests/test_vendor_fixtures.sh")
-printf '%s\n' "$output" | grep -q 'quectel/qualcomm/rm500q-ae-9f94df3c: replay passed'
-printf '%s\n' "$output" | grep -q 'quectel/unisoc/udx710-861f4136: replay passed'
+printf '%s\n' "$output" | grep -q 'quectel/qualcomm/rm500q-ae-9f94df3c/default: replay passed'
+printf '%s\n' "$output" | grep -q 'qualcomm/rm500q-ae-9f94df3c/default: get_imei matches golden snapshot'
+printf '%s\n' "$output" | grep -q 'qualcomm/rm500q-ae-9f94df3c/roaming: get_imei matches golden snapshot'
+printf '%s\n' "$output" | grep -q 'quectel/unisoc/udx710-861f4136/default: replay passed'
 
 echo 'fixture profile isolation tests passed'