fix(relayer): harden sponsored registration and balance alerts

apps/app/src/components/AdminWalletBalances.tsx

@@ -79,8 +79,8 @@ export function AdminWalletBalances({ adminKey, onInvalidKey }: AdminWalletBalan
             <thead>
               <tr>
                 <th scope="col">Address</th>
-                <th scope="col" style={{ textAlign: 'right' }}>SUI</th>
-                <th scope="col" style={{ textAlign: 'right' }}>WAL</th>
+                <th scope="col" style={{ textAlign: 'right' }}>SUI (spendable)</th>
+                <th scope="col" style={{ textAlign: 'right' }}>WAL (spendable)</th>
                 <th scope="col">Status</th>
               </tr>
             </thead>
@@ -96,11 +96,25 @@ export function AdminWalletBalances({ adminKey, onInvalidKey }: AdminWalletBalan
                   <td title={wallet.address} className="admin-table-monospace">
                     {abbreviateAddress(wallet.address)}
                   </td>
-                  <td style={{ textAlign: 'right' }} className="admin-table-monospace" title={`${wallet.suiBalance} mist`}>
+                  <td
+                    style={{ textAlign: 'right' }}
+                    className="admin-table-monospace"
+                    title={`spendable ${wallet.suiBalance} mist / total ${wallet.suiTotal} mist`}
+                  >
                     {formatBalance(wallet.suiBalance, 'SUI')}
+                    {wallet.suiTotal !== wallet.suiBalance ? (
+                      <div className="admin-balance-total">total {formatBalance(wallet.suiTotal, 'SUI')}</div>
+                    ) : null}
                   </td>
-                  <td style={{ textAlign: 'right' }} className="admin-table-monospace" title={`${wallet.walBalance} frost`}>
+                  <td
+                    style={{ textAlign: 'right' }}
+                    className="admin-table-monospace"
+                    title={`spendable ${wallet.walBalance} frost / total ${wallet.walTotal} frost`}
+                  >
                     {formatBalance(wallet.walBalance, 'WAL')}
+                    {wallet.walTotal !== wallet.walBalance ? (
+                      <div className="admin-balance-total">total {formatBalance(wallet.walTotal, 'WAL')}</div>
+                    ) : null}
                   </td>
                   <td>
                     <span className={`admin-status-badge admin-status-badge--${wallet.status}`}>

apps/app/src/utils/admin-api.ts

@@ -4,6 +4,8 @@ export interface WalletBalance {
   address: string
   suiBalance: bigint
   walBalance: bigint
+  suiTotal: bigint
+  walTotal: bigint
   status: 'healthy' | 'warning' | 'critical'
 }
 
@@ -109,6 +111,8 @@ interface RawWalletBalance {
   address: string
   sui: string
   wal: string
+  sui_total?: string
+  wal_total?: string
   status: string
 }
 
@@ -167,6 +171,8 @@ export async function fetchAdminWallets(
       address: wallet.address,
       suiBalance: BigInt(wallet.sui || '0'),
       walBalance: BigInt(wallet.wal || '0'),
+      suiTotal: BigInt(wallet.sui_total || wallet.sui || '0'),
+      walTotal: BigInt(wallet.wal_total || wallet.wal || '0'),
       status: toBadgeStatus(wallet.status),
     })),
     sponsorWallet: {

docs/reference/environment-variables.md

@@ -129,13 +129,19 @@ These are not all enforced at boot, but most real deployments need them.
 | `SEAL_THRESHOLD` | `min(2, total configured weight)` | Required configured server weight for SEAL encrypt/decrypt |
 | `ENOKI_API_KEY` | none | Optional Enoki key for sponsored sidecar transactions |
 | `ENOKI_NETWORK` | `mainnet` | Network used for Enoki-sponsored flows |
-| `ENOKI_FALLBACK_TO_DIRECT_SIGN` | `false` | If true, sidecar pays gas directly with the server wallet when Enoki sponsorship fails or is not configured |
+| `DURABLE_ENOKI_REGISTER_ENABLED` | `false` | Enables Enoki sponsorship for durable Walrus registration. Deploy all replicas with this off first, then enable it after old replicas drain |
+| `ENOKI_FALLBACK_TO_DIRECT_SIGN` | `false` | If true, rebuildable Enoki flows may pay gas directly with the server wallet. Durable registration fails closed after `DURABLE_ENOKI_REGISTER_ENABLED=true` |
 | `ENOKI_TRANSIENT_MAX_ATTEMPTS` | `2` | Attempts for sidecar-level retries of transient Enoki failures (`429`, `5xx`, network errors) before failing the wallet job |
 | `ENOKI_TRANSIENT_BASE_DELAY_MS` | `5000` | Base delay for transient Enoki retries when the response does not include `Retry-After` or a retry hint |
 | `ENOKI_TRANSIENT_MAX_DELAY_MS` | `30000` | Maximum delay for one transient Enoki retry, including parsed retry hints such as “try again in 30 seconds” |
 | `ENOKI_INVALIDATED_MAX_ATTEMPTS` | `4` | Attempts for rebuildable sponsored transactions invalidated by Enoki `expired` responses or short Sui object visibility lag before failing the wallet job |
 | `ENOKI_INVALIDATED_BASE_DELAY_MS` | `1000` | Base delay for retrying rebuildable sponsored transactions after Enoki invalidation |
 | `ENOKI_INVALIDATED_MAX_DELAY_MS` | `8000` | Maximum delay for one rebuildable sponsored transaction invalidation retry |
+| `BALANCE_MONITOR_INTERVAL_SECS` | `900` | How often the relayer polls uploader and sponsor address balances for low-balance Slack alerts |
+| `WALLET_BALANCE_LOW_THRESHOLD_WAL` | `50000000000` | Uploader WAL address-balance threshold in FROST (50 WAL). Alerts independently of SUI |
+| `WALLET_BALANCE_LOW_THRESHOLD_SUI` | `5000000000` | Uploader SUI address-balance threshold in MIST (5 SUI). Load-bearing during phase 1, when durable register pays gas from the uploader wallet |
+| `SPONSOR_BALANCE_LOW_THRESHOLD_SUI` | `5000000000` | Sponsor wallet SUI address-balance threshold in MIST (5 SUI) |
+| `WALLET_BALANCE_LOW_ALERT_DEDUP_SECS` | `43200` | Dedup window for wallet low-balance Slack alerts, per `(network, wallet type, token, address)` |
 | `MEMWAL_RELAYER_URL` | `http://127.0.0.1:$PORT` | Relayer URL passed from the Rust server to the sidecar for MCP tool calls |
 | `MCP_MAX_TOTAL_SESSIONS` | `1000` | Maximum active MCP sessions across SSE and Streamable HTTP transports |
 | `MCP_MAX_SESSIONS_PER_IP` | `16` | Maximum active MCP sessions from one source IP |
@@ -145,7 +151,8 @@ These are not all enforced at boot, but most real deployments need them.
 ### Notes
 
 - If both `SERVER_SUI_PRIVATE_KEYS` and `SERVER_SUI_PRIVATE_KEY` are set, the key pool takes priority for uploads. Upload jobs use the pool in round-robin order.
-- Keep `ENOKI_FALLBACK_TO_DIRECT_SIGN=false` in production if the server wallet should not pay gas when sponsorship is missing, expired, or rejected.
+- Roll out durable registration sponsorship in two phases: first deploy all replicas with `DURABLE_ENOKI_REGISTER_ENABLED=false`; after old replicas have drained, set it to `true`. This prevents mixed-version workers from rejecting persisted sponsored journals. During phase 1, if `ENOKI_API_KEY` is set, durable register **direct-signs and pays gas from the uploader wallet** — the new SUI address-balance alert is load-bearing for that window. After phase 2, drain every replica before rolling the gate back to `false`; otherwise old replicas can 409 sponsored journals as `INVALID_PREPARED_REGISTER_TRANSACTION`.
+- Keep `ENOKI_FALLBACK_TO_DIRECT_SIGN=false` in production if the server wallet should not pay gas for rebuildable Enoki flows when sponsorship is missing, expired, or rejected. Durable registration does not fall back after its rollout gate is enabled.
 - `OPENAI_API_KEY` and `OPENAI_API_BASE` control the embedding and fact-extraction provider used by `remember`, `recall`, `analyze`, `ask`, and restore re-indexing.
 - `WALRUS_AGGREGATOR_URLS` is only used after the Redis ciphertext cache misses. Put low-latency cache/proxy endpoints first after the primary and keep 404/5xx cache TTLs short in your proxy.
 - `WALRUS_SKIP_CONSISTENCY_CHECK=true` should only be used for trusted blobs written by the relayer. Restore keeps consistency checks enabled for on-chain-discovered blobs.

docs/relayer/runbook-gas-pool.md

@@ -28,7 +28,7 @@ questions:
   - "What causes Enoki dry_run_failed errors with balance split on MemWal?"
   - "How do I consolidate SUI gas coins on relayer pool wallets?"
 answer: >-
-  Gas-pool exhaustion occurs when relayer pool wallets have no single SUI coin large enough for Enoki-sponsored Walrus transactions, typically due to coin fragmentation. The fix is to consolidate fragmented coins using sui client merge-coin or pay-all-sui, or top up the wallet with additional SUI so the largest coin exceeds the sponsored budget.
+  Gas ownership depends on DURABLE_ENOKI_REGISTER_ENABLED. Phase 1 (flag false, current default) direct-signs durable register from the uploader wallet address balance — top up that uploader, not an Enoki pool wallet. Phase 2 (flag true) is Enoki-sponsored: exhaustion means a pool wallet has no single SUI coin large enough, typically from fragmentation; consolidate with sui client merge-coin or pay-all-sui, or top up the pool wallet so the largest coin exceeds the sponsored budget.
 ---
 
 When to use this: a **gas-pool alert** fires (the SUI gas pool maintenance
@@ -38,11 +38,20 @@ alert), or relayer logs show wallet jobs aborting with
 
 ## What it means
 
-The relayer sponsors Walrus register transactions through Enoki. Enoki's dry-run
-splits a SUI gas coin on the selected pool wallet to cover the sponsored budget.
-When that wallet has **no single SUI coin large enough** (its gas is fragmented
-into many small coins, or it is low on SUI), Sui aborts in `0x2::balance::split`
-with `ENotEnough`.
+Gas ownership depends on the durable-register rollout gate:
+
+- **Phase 1** (`DURABLE_ENOKI_REGISTER_ENABLED=false`, the current staging/prod
+  default): durable register **direct-signs**. The **uploader wallet** pays SUI
+  gas from its address balance. Top up that wallet; do not send SUI to an Enoki
+  pool wallet expecting it to sponsor this path.
+- **Phase 2** (`DURABLE_ENOKI_REGISTER_ENABLED=true`): durable register is
+  Enoki-sponsored. Enoki's dry-run splits a SUI gas coin on the selected pool
+  wallet. When that wallet has **no single SUI coin large enough** (fragmented
+  or low), Sui aborts in `0x2::balance::split` with `ENotEnough`.
+
+The rest of this runbook is the phase-2 / Enoki-sponsored path. Phase-1
+starvation is a low **uploader** SUI address balance — see the wallet-balance
+alert, not this gas-pool alert.
 
 The relayer now classifies this as `GasPoolExhausted` and **aborts the job
 immediately** instead of retrying across the whole pool (which would just
@@ -102,8 +111,12 @@ sui client pay-all-sui --input-coins <COIN_ID_1> <COIN_ID_2> ... --recipient <PO
 
 ### Top up
 
-Send SUI to the starved pool wallets so the largest coin comfortably exceeds
-the sponsored budget with headroom.
+**Phase 2 only.** Send SUI to the starved Enoki pool wallets so the largest
+coin comfortably exceeds the sponsored budget with headroom.
+
+**Phase 1:** send SUI to the **uploader** wallet address that fired the
+low-balance alert (`WALLET_BALANCE_LOW_THRESHOLD_SUI`). Pool-wallet top-ups do
+not pay durable-register gas while the gate is off.
 
 ## Verify
 

services/server/.env.example

@@ -159,6 +159,9 @@ MEMWAL_REGISTRY_ID=0x...
 # Enoki Sponsored Transactions (sidecar)
 ENOKI_API_KEY=
 ENOKI_NETWORK=mainnet
+# Two-phase rollout: deploy every replica with this false, then set true only
+# after old replicas have drained so persisted sponsored journals are compatible.
+DURABLE_ENOKI_REGISTER_ENABLED=false
 # Keep false in production so missing/expired/rejected sponsorship retries
 # through Apalis instead of making the server wallet pay gas directly.
 ENOKI_FALLBACK_TO_DIRECT_SIGN=false

services/server/scripts/__tests__/sidecar-query-helpers.test.ts

@@ -28,6 +28,7 @@ import {
 } from "../sidecar/config.js";
 import {
     assertAddressBalanceRegisterTransaction,
+    assertSponsoredRegisterTransactionKind,
     createdBlobObjectIdFromTransaction,
     durableRegisterDirectSigningAllowed,
     executePreparedRegisterTransaction,
@@ -132,9 +133,9 @@ test("legacy uploads preserve fallback behavior while using address balances", (
     });
 });
 
-test("durable register never direct-signs when Enoki is configured", () => {
+test("durable register sponsorship uses an explicit two-phase rollout gate", () => {
     assert.equal(durableRegisterDirectSigningAllowed(false, false), true);
-    assert.equal(durableRegisterDirectSigningAllowed(true, false), false);
+    assert.equal(durableRegisterDirectSigningAllowed(true, false), true);
     assert.equal(durableRegisterDirectSigningAllowed(true, true), false);
 });
 
@@ -166,15 +167,30 @@ test("sponsored registration keeps WAL on the sender while assigning gas to the
     const bytes = await transaction.build();
     const signed = await signer.signTransaction(bytes);
     const digest = TransactionDataBuilder.getDigestFromBytes(bytes);
+    const sponsorDigest = `sponsor-${digest}`;
     const prepared: PreparedRegisterTransaction = {
         transactionBytes: signed.bytes,
         signature: signed.signature,
         digest,
-        sponsorDigest: digest,
+        sponsorDigest,
     };
 
     const validated = await validatePreparedRegisterTransaction(prepared, signer.toSuiAddress());
-    assert.equal(validated.sponsorDigest, digest);
+    assert.equal(validated.sponsorDigest, sponsorDigest);
+    const expectedKind = TransactionDataBuilder.fromBytes(bytes).build({ onlyTransactionKind: true });
+    assert.doesNotThrow(() => assertSponsoredRegisterTransactionKind(
+        TransactionDataBuilder.fromBytes(bytes),
+        expectedKind,
+    ));
+    const tamperedKind = expectedKind.slice();
+    tamperedKind[tamperedKind.length - 1] ^= 1;
+    assert.throws(
+        () => assertSponsoredRegisterTransactionKind(
+            TransactionDataBuilder.fromBytes(bytes),
+            tamperedKind,
+        ),
+        /kind differs/,
+    );
 
     let submitted = false;
     let directExecutions = 0;
@@ -195,15 +211,92 @@ test("sponsored registration keeps WAL on the sender while assigning gas to the
         () => {},
         async () => 1n,
         false,
-        async (sponsorDigest, signature) => {
-            assert.equal(sponsorDigest, digest);
+        async (executedSponsorDigest, signature) => {
+            assert.equal(executedSponsorDigest, sponsorDigest);
             assert.equal(signature, signed.signature);
             submitted = true;
             return { digest };
         },
     );
     assert.equal(result, finalized);
     assert.equal(directExecutions, 0);
+
+    submitted = false;
+    await assert.rejects(
+        executePreparedRegisterTransaction(
+            validated,
+            client,
+            () => {},
+            async () => 1n,
+            false,
+            async () => {
+                throw new Error('Enoki API error (400): {"errors":[{"code":"expired"}]}');
+            },
+            1,
+        ),
+        (error: unknown) => error instanceof NoSideEffectError && /rebuild sponsorship/.test(error.message),
+    );
+
+    await assert.rejects(
+        executePreparedRegisterTransaction(
+            validated,
+            client,
+            () => {},
+            async () => 1n,
+            false,
+            async () => {
+                throw new Error('Enoki API error (400): {"errors":[{"code":"not_found"}]}');
+            },
+            1,
+        ),
+        (error: unknown) => error instanceof Error
+            && (error as { code?: string }).code === "UNAVAILABLE"
+            && /ambiguous/.test(error.message),
+    );
+
+    let lookups = 0;
+    const retryClient = {
+        async getTransaction() {
+            lookups += 1;
+            // 1 = pre-execute probe. 2 = first expired-path miss. 3 = hit.
+            // A helper that only looks up once would stop after #2 and fail.
+            if (lookups < 3) throw Object.assign(new Error("not found"), { code: "NOT_FOUND" });
+            return finalized;
+        },
+        async executeTransaction() {
+            throw new Error("sponsored journal must not execute directly");
+        },
+    };
+    const recovered = await executePreparedRegisterTransaction(
+        validated,
+        retryClient,
+        () => {},
+        async () => 1n,
+        false,
+        async () => {
+            throw new Error('Enoki API error (400): {"errors":[{"code":"expired"}]}');
+        },
+        2,
+    );
+    assert.equal(recovered, finalized);
+    assert.equal(lookups, 3);
+
+    await assert.rejects(
+        executePreparedRegisterTransaction(
+            validated,
+            client,
+            () => {},
+            async () => 1n,
+            true,
+            async () => {
+                throw new Error('Enoki API error (400): {"errors":[{"code":"expired"}]}');
+            },
+            1,
+        ),
+        (error: unknown) => error instanceof Error
+            && (error as { code?: string }).code === "UNAVAILABLE"
+            && /ambiguous/.test(error.message),
+    );
 });
 
 test("prepared registration rejects tampered digest, signature, and wallet", async () => {

services/server/scripts/__tests__/sidecar-writer-mode.test.ts

@@ -150,8 +150,11 @@ test("writer mode exposes only durable writer and observability routes", async (
             perWallet: [
                 {
                     address: writerKey.toSuiAddress(),
+                    walletIndex: 0,
                     suiMist: "1230000000",
+                    suiAddressBalanceMist: "1200000000",
                     walFrost: "4560000000",
+                    walAddressBalanceFrost: "4500000000",
                 },
             ],
         });

services/server/scripts/sidecar/clients.ts

@@ -243,7 +243,14 @@ export type WalletBalanceSnapshot = {
     walletWalAddressBalanceFrost: string;
     walletWalCoinBalanceFrost: string;
     walletWalAddressFundedCount: number;
-    perWallet: Array<{ address: string; suiMist: string; walFrost: string }>;
+    perWallet: Array<{
+        address: string;
+        walletIndex: number;
+        suiMist: string;
+        suiAddressBalanceMist: string;
+        walFrost: string;
+        walAddressBalanceFrost: string;
+    }>;
 };
 
 const BALANCE_RPC_TIMEOUT_MS = 1_500;
@@ -300,7 +307,7 @@ async function loadWalletBalanceSnapshot(owners: string[]): Promise<WalletBalanc
     let suiAddressFundedCount = 0;
     let walAddressFundedCount = 0;
     const suiType = normalizeStructTag(SUI_TYPE);
-    const perWallet: Array<{ address: string; suiMist: string; walFrost: string }> = [];
+    const perWallet: WalletBalanceSnapshot["perWallet"] = [];
     balancesByOwner.forEach((balances, index) => {
         let ownerSuiAddressBalance = 0n;
         let ownerWalAddressBalance = 0n;
@@ -332,8 +339,11 @@ async function loadWalletBalanceSnapshot(owners: string[]): Promise<WalletBalanc
         if (ownerWalAddressBalance > 0n) walAddressFundedCount += 1;
         perWallet.push({
             address: owners[index],
+            walletIndex: index,
             suiMist: ownerSuiTotal.toString(),
+            suiAddressBalanceMist: ownerSuiAddressBalance.toString(),
             walFrost: ownerWalTotal.toString(),
+            walAddressBalanceFrost: ownerWalAddressBalance.toString(),
         });
     });
     return {

services/server/scripts/sidecar/config.ts

@@ -249,6 +249,10 @@ export const ENOKI_NETWORK = (process.env.ENOKI_NETWORK || process.env.SUI_NETWO
     | "mainnet"
     | "testnet"
     | "devnet";
+// Roll out durable-register sponsorship in two phases: deploy this code with
+// the gate off so old and new replicas both understand persisted journals,
+// then enable it after the old replica set has drained.
+export const DURABLE_ENOKI_REGISTER_ENABLED = parseBooleanEnv("DURABLE_ENOKI_REGISTER_ENABLED", false);
 export const ENOKI_FALLBACK_TO_DIRECT_SIGN = (() => {
     const raw = (process.env.ENOKI_FALLBACK_TO_DIRECT_SIGN || "false").trim().toLowerCase();
     return raw !== "0" && raw !== "false" && raw !== "no";