vtxo: retract an unroll whose transaction the chain no longer has

api-spec/openapi/swagger/arkwallet/v1/bitcoin_wallet.openapi.json

@@ -1712,6 +1712,18 @@
           },
           "confirmed": {
             "type": "boolean"
+          },
+          "dropped": {
+            "type": "boolean",
+            "description": "dropped is the wallet's judgement that this transaction will not confirm:\nsuperseded, unknown to the backend, or unconfirmed and no longer held by\nthe node. It is computed here because only the wallet can see all three.\n\nFalse whenever the wallet cannot tell, including on an older wallet that\nnever sets it, so a caller acting on this can never act on a live\ntransaction."
+          },
+          "notFound": {
+            "type": "boolean",
+            "description": "not_found separates a transaction the backend has no record of from one it\nknows and sees unconfirmed. Both answer confirmed = false, which is why the\ndistinction needs its own field.\n\nAdded after the other fields, and phrased so false is the safe reading: an\nolder wallet never sets it, and a caller then sees \"not known to be\nmissing\" rather than \"missing\"."
+          },
+          "replacedBy": {
+            "type": "string",
+            "description": "replaced_by names the transaction that superseded this one. It is the\nbackend's only positive statement that a transaction will not confirm: a\nreplaced transaction keeps answering confirmed = false and not_found =\nfalse forever, so neither of those can stand in for it.\n\nEmpty when the transaction was not replaced, which is also what an older\nwallet returns, so the safe reading is the zero value here too."
           }
         }
       },

api-spec/protobuf/arkwallet/v1/bitcoin_wallet.proto

@@ -223,6 +223,30 @@ message IsTransactionConfirmedResponse {
   bool confirmed = 1;
   int64 blocknumber = 2;
   int64 blocktime = 3;
+  // not_found separates a transaction the backend has no record of from one it
+  // knows and sees unconfirmed. Both answer confirmed = false, which is why the
+  // distinction needs its own field.
+  //
+  // Added after the other fields, and phrased so false is the safe reading: an
+  // older wallet never sets it, and a caller then sees "not known to be
+  // missing" rather than "missing".
+  bool not_found = 4;
+  // replaced_by names the transaction that superseded this one. It is the
+  // backend's only positive statement that a transaction will not confirm: a
+  // replaced transaction keeps answering confirmed = false and not_found =
+  // false forever, so neither of those can stand in for it.
+  //
+  // Empty when the transaction was not replaced, which is also what an older
+  // wallet returns, so the safe reading is the zero value here too.
+  string replaced_by = 5;
+  // dropped is the wallet's judgement that this transaction will not confirm:
+  // superseded, unknown to the backend, or unconfirmed and no longer held by
+  // the node. It is computed here because only the wallet can see all three.
+  //
+  // False whenever the wallet cannot tell, including on an older wallet that
+  // never sets it, so a caller acting on this can never act on a live
+  // transaction.
+  bool dropped = 6;
 }
 
 message GetOutpointStatusRequest {

internal/core/application/indexer_test.go

@@ -1232,6 +1232,11 @@ func (m *mockVtxoRepoForIndexer) SpendVtxos(
 	return nil
 }
 
+func (m *mockVtxoRepoForIndexer) UnmarkVtxosUnrolled(
+	ctx context.Context, outpoints []domain.Outpoint,
+) error {
+	return nil
+}
 func (m *mockVtxoRepoForIndexer) UnrollVtxos(
 	ctx context.Context,
 	outpoints []domain.Outpoint,

internal/core/application/mocks_test.go

@@ -74,6 +74,12 @@ func (m *mockedVtxoRepo) GetVtxoPubKeysByCommitmentTxids(
 	return nil, args.Error(1)
 }
 
+func (m *mockedVtxoRepo) UnmarkVtxosUnrolled(
+	ctx context.Context, outpoints []domain.Outpoint,
+) error {
+	return m.Called(ctx, outpoints).Error(0)
+}
+
 func (m *mockedVtxoRepo) MarkVtxosOnchainSpent(
 	ctx context.Context, spentBy map[domain.Outpoint]string,
 ) error {
@@ -207,12 +213,17 @@ type mockedScanner struct {
 	mu        sync.Mutex
 
 	// onchain-spend fixtures, read by the reconcile tests
-	spendCh    chan []ports.Spend
-	spends     []ports.Spend
-	spendsErr  error
-	spendsFrom []*time.Time
-	unspent    map[domain.Outpoint]struct{}
-	unspentErr error
+	// dropped maps a txid to whether the backend says it will not confirm; a
+	// txid absent from the map reads as still live, the safe default.
+	dropped      map[string]bool
+	droppedErr   error
+	droppedCalls []string
+	spendCh      chan []ports.Spend
+	spends       []ports.Spend
+	spendsErr    error
+	spendsFrom   []*time.Time
+	unspent      map[domain.Outpoint]struct{}
+	unspentErr   error
 }
 
 func (m *mockedScanner) WatchScripts(
@@ -265,6 +276,23 @@ func (m *mockedScanner) RescanUtxos(_ context.Context, _ []wire.OutPoint) error
 	return nil
 }
 
+func (m *mockedScanner) IsTransactionDropped(_ context.Context, txid string) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.droppedCalls = append(m.droppedCalls, txid)
+	if m.droppedErr != nil {
+		return false, m.droppedErr
+	}
+	return m.dropped[txid], nil
+}
+
+// DroppedCalls returns every txid IsTransactionDropped was asked about.
+func (m *mockedScanner) DroppedCalls() []string {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return append([]string(nil), m.droppedCalls...)
+}
+
 func (m *mockedScanner) GetSpendNotificationChannel(
 	_ context.Context,
 ) <-chan []ports.Spend {

internal/core/application/onchain_spend.go

@@ -166,6 +166,10 @@ func (s *service) reconcileOnchainSpendsOnce(ctx context.Context, from *time.Tim
 	}
 
 	s.retractStaleOnchainSpends(ctx, recorded)
+
+	// Same candidate set, the other direction: a spend that never confirmed is
+	// retracted above, an unroll that never confirmed is retracted here.
+	s.retractStaleUnrolls(ctx, candidates)
 }
 
 // retractStaleOnchainSpends undoes spends whose transaction is gone.

internal/core/application/service.go

@@ -58,6 +58,12 @@ type service struct {
 	// unrolled vtxos
 	onchainSpendReconcileInterval time.Duration
 
+	// consecutive reconcile passes that found a vtxo's materialising tx unknown
+	// to the chain backend, keyed by outpoint. In memory on purpose: losing it
+	// on restart only delays a retraction, which is the safe direction.
+	unrollObservations   map[domain.Outpoint]int
+	unrollObservationsMu sync.Mutex
+
 	operatorPrvkey *btcec.PrivateKey
 	operatorPubkey *btcec.PublicKey
 
@@ -168,6 +174,7 @@ func NewService(
 		feeManager:               feeManager,
 
 		onchainSpendReconcileInterval: onchainSpendReconcileInterval,
+		unrollObservations:            make(map[domain.Outpoint]int),
 	}
 	svc.sweeper.onSweepCheckpoint = svc.propagateTransactionEvent
 	return svc, nil

internal/core/application/sweeper_test.go

@@ -370,6 +370,9 @@ func (m *mockWalletService) IsTransactionConfirmed(
 ) (bool, *ports.BlockTimestamp, error) {
 	return false, nil, nil
 }
+func (m *mockWalletService) IsTransactionDropped(ctx context.Context, txid string) (bool, error) {
+	return false, nil
+}
 func (m *mockWalletService) RescanUtxos(ctx context.Context, outpoints []wire.OutPoint) error {
 	return nil
 }
@@ -438,6 +441,11 @@ func (m *mockVtxoRepository) SpendVtxos(
 ) error {
 	return nil
 }
+func (m *mockVtxoRepository) UnmarkVtxosUnrolled(
+	ctx context.Context, outpoints []domain.Outpoint,
+) error {
+	return nil
+}
 func (m *mockVtxoRepository) UnrollVtxos(ctx context.Context, outpoints []domain.Outpoint) error {
 	return nil
 }

internal/core/application/unroll_retraction.go

@@ -0,0 +1,117 @@
+package application
+
+import (
+	"context"
+
+	"github.com/arkade-os/arkd/internal/core/domain"
+	log "github.com/sirupsen/logrus"
+)
+
+// unrollRetractionObservations is how many consecutive reconcile passes must
+// find a vtxo's materialising tx dropped by the chain backend before its unroll
+// is retracted. One pass is not enough: a transaction that has been broadcast
+// but has not reached our node yet reads exactly like one that is gone, and
+// retracting a live unroll would hand its owner back a vtxo whose output exists
+// on chain. Several passes apart make that reading stable rather than a race
+// with propagation.
+const unrollRetractionObservations = 3
+
+// retractStaleUnrolls clears the unrolled mark on vtxos whose materialising
+// transaction the chain backend no longer has any record of.
+//
+// A vtxo is marked unrolled the moment its outpoint appears on chain, before any
+// confirmation, which is deliberate: the mark is protective and blocks the vtxo
+// from being spent inside the Ark while its unroll is in flight. Nothing ever
+// cleared it, so an unroll that is evicted or replaced and never mines leaves
+// the vtxo wrongly unrolled forever, unspendable by its owner and unsweepable by
+// the operator.
+//
+// Retraction runs on one piece of positive evidence: the node itself no longer
+// holds the transaction that would materialise the vtxo, and has not mined it.
+//
+// The wallet's unspent set is deliberately not consulted. It reads as the
+// stronger signal and was used here at first, but it is built from the chain
+// backend's own index, which keeps an unconfirmed transaction long after the
+// node has dropped it. Verified on a live stack: with the unroll evicted, the
+// node reported no such output while the backend still listed the outpoint as
+// unspent, so consulting it vetoed exactly the retraction this exists to make.
+// The node's answer already accounts for confirmation, so nothing is lost.
+func (s *service) retractStaleUnrolls(ctx context.Context, candidates []domain.Vtxo) {
+	if len(candidates) == 0 {
+		s.forgetUnrollObservations(nil)
+		return
+	}
+
+	seen := make(map[domain.Outpoint]struct{}, len(candidates))
+	stale := make([]domain.Outpoint, 0)
+	for _, vtxo := range candidates {
+		seen[vtxo.Outpoint] = struct{}{}
+
+		dropped, err := s.scanner.IsTransactionDropped(ctx, vtxo.Txid)
+		if err != nil {
+			// Not evidence of anything. Leave the count untouched so a flapping
+			// backend cannot accumulate its way to a retraction.
+			log.WithError(err).Warnf(
+				"unroll retraction: failed to look up tx %s, leaving vtxo %s alone",
+				vtxo.Txid, vtxo.Outpoint,
+			)
+			continue
+		}
+
+		if s.recordUnrollObservation(vtxo.Outpoint, !dropped) >= unrollRetractionObservations {
+			stale = append(stale, vtxo.Outpoint)
+		}
+	}
+
+	// Anything no longer a candidate has been resolved by another path, so its
+	// count must not survive to be counted against a future unroll.
+	s.forgetUnrollObservations(seen)
+
+	if len(stale) == 0 {
+		return
+	}
+
+	if err := s.repoManager.Vtxos().UnmarkVtxosUnrolled(ctx, stale); err != nil {
+		log.WithError(err).Warn("unroll retraction: failed to retract unrolls")
+		return
+	}
+
+	for _, outpoint := range stale {
+		s.recordUnrollObservation(outpoint, true)
+		log.Debugf(
+			"vtxo %s unroll retracted, its tx is no longer held by the node", outpoint,
+		)
+	}
+}
+
+// recordUnrollObservation advances or clears the consecutive count for an
+// outpoint and returns the count after the update.
+func (s *service) recordUnrollObservation(outpoint domain.Outpoint, known bool) int {
+	s.unrollObservationsMu.Lock()
+	defer s.unrollObservationsMu.Unlock()
+
+	if known {
+		delete(s.unrollObservations, outpoint)
+		return 0
+	}
+	// Built on demand so a zero-value service counts correctly rather than
+	// panicking on a nil map.
+	if s.unrollObservations == nil {
+		s.unrollObservations = make(map[domain.Outpoint]int)
+	}
+	s.unrollObservations[outpoint]++
+	return s.unrollObservations[outpoint]
+}
+
+// forgetUnrollObservations drops counts for outpoints outside the given set,
+// or all of them when the set is nil.
+func (s *service) forgetUnrollObservations(keep map[domain.Outpoint]struct{}) {
+	s.unrollObservationsMu.Lock()
+	defer s.unrollObservationsMu.Unlock()
+
+	for outpoint := range s.unrollObservations {
+		if _, ok := keep[outpoint]; !ok {
+			delete(s.unrollObservations, outpoint)
+		}
+	}
+}