diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 380bad5adc..3177549f82 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -121,6 +121,7 @@ 'wallet.py', 'wallet-encryption.py', 'wallet-hd.py', + 'wallet-internalchain.py', 'wallet-dump.py', 'walletbackup.py', # 'wallet-accounts.py', diff --git a/qa/rpc-tests/wallet-dump.py b/qa/rpc-tests/wallet-dump.py index d495330d8f..a45018997f 100755 --- a/qa/rpc-tests/wallet-dump.py +++ b/qa/rpc-tests/wallet-dump.py @@ -18,6 +18,7 @@ def read_dump(file_name, addrs, hd_master_addr_old): found_addr_chg = 0 found_addr_rsv = 0 found_addr_sigma = 0 + found_addr_internal = 0 hd_master_addr_ret = None for line in inputfile: # only read non comment lines @@ -60,7 +61,10 @@ def read_dump(file_name, addrs, hd_master_addr_old): elif keytype == "sigma=1": found_addr_sigma += 1 break - return found_addr, found_addr_chg, found_addr_rsv, found_addr_sigma, hd_master_addr_ret + elif keytype == "internal=1": + found_addr_internal += 1 + break + return found_addr, found_addr_chg, found_addr_rsv, found_addr_sigma, found_addr_internal, hd_master_addr_ret class WalletDumpTest(BitcoinTestFramework): @@ -104,13 +108,14 @@ def run_test (self): self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump", key) assert key, 'Import wallet did not raise exception when was called first time without one-time code.' - found_addr, found_addr_chg, found_addr_rsv, found_addr_sigma, hd_master_addr_unenc = \ + found_addr, found_addr_chg, found_addr_rsv, found_addr_sigma, found_addr_internal, hd_master_addr_unenc = \ read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, None) assert_equal(found_addr, test_addr_count) # all keys must be in the dump assert_equal(found_addr_chg, 50) # 50 block were mined assert_equal(found_addr_sigma, hdmint_key_count) # hdmint keys assert_equal(found_addr_rsv, 90 + 1) # keypool size (TODO: fix off-by-one) + assert_equal(found_addr_internal, 90) # internal chain lookahead, which follows the keypool size #encrypt wallet, restart, unlock and dump self.nodes[0].encryptwallet('test') @@ -127,7 +132,7 @@ def run_test (self): self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.encrypted.dump", key) assert key, 'Import wallet did not raise exception when was called first time without one-time code.' - found_addr, found_addr_chg, found_addr_rsv, found_addr_sigma, hd_master_addr_enc = \ + found_addr, found_addr_chg, found_addr_rsv, found_addr_sigma, found_addr_internal, hd_master_addr_enc = \ read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, hd_master_addr_unenc) assert_equal(found_addr, test_addr_count) @@ -135,6 +140,7 @@ def run_test (self): # Wallet encryption doesn't change master key anymore; sigma key count unchanged after Lelantus strip (0). assert_equal(found_addr_sigma, hdmint_key_count) assert_equal(found_addr_rsv, 90 + 1) # keypool size (TODO: fix off-by-one) + assert_equal(found_addr_internal, 90) # internal chain lookahead, which follows the keypool size if __name__ == '__main__': WalletDumpTest().main () diff --git a/qa/rpc-tests/wallet-internalchain.py b/qa/rpc-tests/wallet-internalchain.py new file mode 100755 index 0000000000..a80b363ab2 --- /dev/null +++ b/qa/rpc-tests/wallet-internalchain.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 The Firo Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Tests that the wallet watches the bip44 internal chain, m/44'/1'/0'/1/. + +Firo takes both its addresses and its change off the external chain and keeps doing so, but a +seed is not only ever used here. A wallet that follows bip44 more closely leaves its change on +the internal chain, and once that seed comes back, those coins have to be seen and spendable. +""" + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + connect_nodes_bi, + start_node, + start_nodes, + stop_node, + sync_blocks, +) +from test_framework.test_helper import get_dumpwallet_otp + +INTERNAL = "m/44'/1'/0'/1/" +EXTERNAL = "m/44'/1'/0'/0/" +LOOKAHEAD = 20 + + +class WalletInternalChainTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 3 + # node0 mines. node1 holds the wallet under test and comes up without a lookahead, the way + # a wallet that predates the internal chain being watched does. node2 is restored from the + # same seed and stands in for the other bip44 wallet the coins are received through. + self.node_args = [ + ['-usemnemonic=1'], + ['-usemnemonic=1', '-keypool=0'], + ['-usemnemonic=1', '-keypool=%d' % LOOKAHEAD], + ] + + def setup_network(self, split=False): + # node2 is started later, once node1's seed is known. + self.nodes = start_nodes(2, self.options.tmpdir, self.node_args[:2]) + connect_nodes_bi(self.nodes, 0, 1) + self.is_network_split = False + self.sync_all() + + def dump_wallet(self, node, name): + """Returns the wallet's mnemonic and a keypath -> address map of every key it holds.""" + path = os.path.join(self.options.tmpdir, name) + try: + self.nodes[node].dumpwallet(path) + except Exception as ex: + # The first dump a node is asked for answers with a code to repeat the call with + self.nodes[node].dumpwallet(path, get_dumpwallet_otp(ex.error['message'])) + + mnemonic = None + keys = {} + with open(path, encoding='utf8') as dump: + for line in dump: + if line.startswith('# mnemonic: '): + mnemonic = line[len('# mnemonic: '):].strip() + continue + fields = [field for field in line.split() if '=' in field] + fields = dict(field.split('=', 1) for field in fields) + if 'hdKeypath' in fields and 'addr' in fields: + keys[fields['hdKeypath']] = fields['addr'] + return mnemonic, keys + + def internal_chain(self, node, name): + """The child index -> address map of the internal chain the wallet holds.""" + _, keys = self.dump_wallet(node, name) + return {int(path[len(INTERNAL):]): addr for path, addr in keys.items() if path.startswith(INTERNAL)} + + def run_test(self): + self.nodes[0].generate(101) + self.sync_all() + + mnemonic, keys = self.dump_wallet(1, 'node1.dump') + assert mnemonic, 'the wallet under test has to hold a mnemonic' + # No lookahead was asked for, so this wallet has no internal chain at all yet + assert_equal([path for path in keys if path.startswith(INTERNAL)], []) + + # Bring the other wallet up on the same seed and read the internal chain off it + self.nodes.append(start_node(2, self.options.tmpdir, self.node_args[2] + ['-mnemonic=' + mnemonic])) + connect_nodes_bi(self.nodes, 0, 2) + sync_blocks(self.nodes) + + internal = self.internal_chain(2, 'node2.dump') + assert_equal(sorted(internal), list(range(LOOKAHEAD))) + + # Pay it, the way another bip44 wallet leaves change there + self.nodes[0].sendtoaddress(internal[0], 10) + self.nodes[0].sendtoaddress(internal[LOOKAHEAD - 1], 5) + self.nodes[0].generate(1) + sync_blocks(self.nodes) + + # The wallet holding those keys sees the coins, and using the last of them moves the + # lookahead up past it, the way a used keypool key tops the keypool up + assert_equal(self.nodes[2].getbalance(), 15) + assert_equal(max(self.internal_chain(2, 'node2-used.dump')), 2 * LOOKAHEAD - 1) + + # The wallet under test holds no internal chain, so the same coins go unnoticed + assert_equal(self.nodes[1].getbalance(), 0) + + # Giving it a lookahead derives the chain and has it rescan for what it missed + stop_node(self.nodes[1], 1) + self.nodes[1] = start_node(1, self.options.tmpdir, self.node_args[2]) + connect_nodes_bi(self.nodes, 0, 1) + sync_blocks(self.nodes) + + assert_equal(self.nodes[1].getbalance(), 15) + restored = self.internal_chain(1, 'node1-restored.dump') + assert_equal(restored[0], internal[0]) + assert_equal(max(restored), 2 * LOOKAHEAD - 1) + + # Addresses are still handed out off the external chain, nothing comes off the internal one + for _ in range(5): + addr = self.nodes[1].getnewaddress() + assert self.nodes[1].validateaddress(addr)['hdkeypath'].startswith(EXTERNAL) + + # What was received on the internal chain is spendable, and its change goes back to the + # external chain, where this wallet keeps all of its own outputs + txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 14) + self.nodes[0].generate(1) + sync_blocks(self.nodes) + + change = [out['scriptPubKey']['addresses'][0] for out in self.nodes[1].getrawtransaction(txid, 1)['vout']] + change = [addr for addr in change if self.nodes[1].validateaddress(addr)['ismine']] + assert_equal(len(change), 1) + assert self.nodes[1].validateaddress(change[0])['hdkeypath'].startswith(EXTERNAL) + fee = self.nodes[1].gettransaction(txid)['fee'] # negative + assert_equal(self.nodes[1].getbalance(), 15 - 14 + fee) + + # The rescan is asked for once, not on every start from here on + stop_node(self.nodes[1], 1) + self.nodes[1] = start_node(1, self.options.tmpdir, self.node_args[2]) + connect_nodes_bi(self.nodes, 0, 1) + sync_blocks(self.nodes) + + log = os.path.join(self.options.tmpdir, 'node1', 'regtest', 'debug.log') + with open(log, encoding='utf8') as debug_log: + rescans = [line for line in debug_log if 'transactions on the bip44 internal chain' in line] + assert_equal(len(rescans), 1) + + +if __name__ == '__main__': + WalletInternalChainTest().main() diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 6d2ad1871f..26d9e75b8e 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -522,6 +522,8 @@ UniValue importwallet(const JSONRPCRequest& request) fLabel = false; if (!masterKeyID.IsNull() && vstr[nStr] == "sigma=1") fLabel = false; + if (!masterKeyID.IsNull() && vstr[nStr] == "internal=1") + fLabel = false; if (vstr[nStr] == "reserve=1") fLabel = false; if (boost::algorithm::starts_with(vstr[nStr], "label=")) { @@ -793,6 +795,8 @@ UniValue dumpwallet(const JSONRPCRequest& request) file << "inactivehdmaster=1"; } else if (!masterKeyID.IsNull() && pwallet->mapKeyMetadata[keyid].nChange.first == 2) { file << "sigma=1"; + } else if (!masterKeyID.IsNull() && pwallet->mapKeyMetadata[keyid].nChange.first == BIP44_INTERNAL_INDEX) { + file << "internal=1"; } else { file << "change=1"; } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 704ec75121..117c6e4f31 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -347,12 +347,140 @@ void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret) throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); } +namespace { + +/* The keypath shared by every key on the bip44 internal chain, up to the child index. */ +std::string InternalChainKeypathPrefix() +{ + uint32_t nIndex = Params().GetConsensus().IsMain() ? BIP44_FIRO_INDEX : BIP44_TEST_INDEX; + return "m/44'/" + std::to_string(nIndex) + "'/0'/" + std::to_string(BIP44_INTERNAL_INDEX) + "/"; +} + +/* How many unused keys are kept ahead on the internal chain. How far this wallet looks ahead on a + * chain is what the keypool setting says, so it decides this too. */ +uint32_t InternalChainLookahead() +{ + return uint32_t(std::max(GetArg("-keypool", int64_t(DEFAULT_KEYPOOL_SIZE)), int64_t(0))); +} + +std::string const internalChainRescanKey = "internal_chain_rescan"; + +} + +void CWallet::RegisterInternalChainKey(const CTxDestination& dest, const CKeyMetadata& metadata) +{ + const CKeyID* keyid = boost::get(&dest); + if (!keyid) + return; + + std::string const prefix = InternalChainKeypathPrefix(); + if (metadata.hdKeypath.compare(0, prefix.size(), prefix) != 0) + return; + + uint32_t nChild; + if (!ParseUInt32(metadata.hdKeypath.substr(prefix.size()), &nChild)) + return; + + m_internal_chain_keys[*keyid] = nChild; +} + +bool CWallet::TopUpInternalChain(uint32_t nSize) +{ + LOCK(cs_wallet); + + if (!IsHDEnabled() || hdChain.nVersion < CHDChain::VERSION_WITH_BIP44) + return false; + + if (nSize == 0) + nSize = InternalChainLookahead(); + // Change is a normal, non-hardened chain, so it ends where the hardened range begins. + nSize = std::min(nSize, BIP32_HARDENED_KEY_LIMIT); + + uint32_t nCounter = hdChain.nExternalChainCounters[BIP44_INTERNAL_INDEX]; + if (nSize <= nCounter) + return false; + + if (IsLocked()) { + LogPrintf("%s: Deriving internal chain keys failed (locked wallet)\n", __func__); + return false; + } + + std::string const prefix = InternalChainKeypathPrefix(); + // These keys are as old as the seed they come from, so they must not move the wallet birthday + // and make a later rescan start after the point where they may have been paid. + int64_t const nCreateTime = nTimeFirstKey ? nTimeFirstKey : GetTime(); + + for (uint32_t nChild = nCounter; nChild < nSize; ++nChild) { + CKey secret; + CPubKey pubkey = GetKeyFromKeypath(BIP44_INTERNAL_INDEX, nChild, secret); + CKeyID keyid = pubkey.GetID(); + + if (!HaveKey(keyid)) { + CKeyMetadata metadata(nCreateTime); + metadata.hdKeypath = prefix + std::to_string(nChild); + metadata.hdMasterKeyID = hdChain.masterKeyID; + metadata.nChange = Component(BIP44_INTERNAL_INDEX, false); + metadata.nChild = Component(nChild, false); + mapKeyMetadata[keyid] = metadata; + + if (!AddKeyPubKey(secret, pubkey)) + throw std::runtime_error(std::string(__func__) + ": AddKey failed"); + } + + m_internal_chain_keys[keyid] = nChild; + } + + hdChain.nExternalChainCounters[BIP44_INTERNAL_INDEX] = nSize; + if (!CWalletDB(strWalletFile).WriteHDChain(hdChain)) + throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); + + LogPrintf("%s: derived internal chain keys %u-%u\n", __func__, nCounter, nSize - 1); + return true; +} + +void CWallet::MarkInternalChainKeyUsed(const CKeyID& keyid) +{ + AssertLockHeld(cs_wallet); + + std::map::const_iterator mi = m_internal_chain_keys.find(keyid); + if (mi == m_internal_chain_keys.end()) + return; + + uint64_t nSize = uint64_t(mi->second) + InternalChainLookahead() + 1; + TopUpInternalChain(uint32_t(std::min(nSize, uint64_t(BIP32_HARDENED_KEY_LIMIT)))); +} + +bool CWallet::IsInternalChainRescanPending() const +{ + LOCK(cs_wallet); + std::multimap::const_iterator iter = mapCustomKeyValues.find(internalChainRescanKey); + return iter != mapCustomKeyValues.end() && iter->second == "1"; +} + +void CWallet::SetInternalChainRescanPending(bool fPending) +{ + CWalletDB walletDb(strWalletFile); + { + LOCK(cs_wallet); + mapCustomKeyValues.erase(internalChainRescanKey); + if (fPending) + mapCustomKeyValues.insert(std::make_pair(internalChainRescanKey, "1")); + } + walletDb.EraseKV(internalChainRescanKey); + if (fPending) + walletDb.WriteKV(internalChainRescanKey, "1"); +} + bool CWallet::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) { AssertLockHeld(cs_wallet); // mapKeyMetadata if (!CCryptoKeyStore::AddKeyPubKey(secret, pubkey)) return false; + std::map::const_iterator mi = mapKeyMetadata.find(pubkey.GetID()); + if (mi != mapKeyMetadata.end()) + RegisterInternalChainKey(pubkey.GetID(), mi->second); + // check if we need to remove from watch-only CScript script; script = GetScriptForDestination(pubkey.GetID()); @@ -398,6 +526,7 @@ bool CWallet::LoadKeyMetadata(const CTxDestination& keyID, const CKeyMetadata &m AssertLockHeld(cs_wallet); // mapKeyMetadata UpdateTimeFirstKey(meta.nCreateTime); mapKeyMetadata[keyID] = meta; + RegisterInternalChainKey(keyID, meta); return true; } @@ -495,6 +624,15 @@ bool CWallet::Unlock(const SecureString &strWalletPassphrase, const bool& fFirst continue; // try another master key if (CCryptoKeyStore::Unlock(vMasterKey, fFirstUnlock)) { fUnlockRequested.store(false); + // An encrypted wallet is locked while it starts up, so this is the first chance to + // derive the internal chain. What was received on it earlier is found by the rescan + // the flag asks for on the next start; unlocking is not a good time for one. + try { + if (TopUpInternalChain() && !mnemonicContainer.IsNull()) + SetInternalChainRescanPending(true); + } catch (std::exception const & e) { + LogPrintf("%s: Deriving internal chain keys failed: %s\n", __func__, e.what()); + } return true; } } @@ -1354,6 +1492,9 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex LogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); } } + // Internal chain keys are watched rather than handed out, so they are not in + // the keypool and are kept ahead of the highest index seen on chain instead. + MarkInternalChainKeyUsed(keyid); } } @@ -5160,6 +5301,12 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); + // Watch the bip44 internal chain, see CWallet::TopUpInternalChain. On a wallet that was used + // before this was done, whatever it was paid there went unnoticed, so the chain has to be + // rescanned once. A wallet being restored is rescanned anyway, and a new one has no history. + if (walletInstance->TopUpInternalChain() && !fFirstRun && !walletInstance->mnemonicContainer.IsNull()) + walletInstance->SetInternalChainRescanPending(true); + CBlockIndex *pindexRescan = chainActive.Tip(); if (GetBoolArg("-rescan", false)) pindexRescan = chainActive.Genesis(); @@ -5172,6 +5319,19 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) else pindexRescan = chainActive.Genesis(); } + if (walletInstance->IsInternalChainRescanPending()) { + if (fPruneMode) { + // There is nothing to scan behind the pruned data, and a node that cannot serve the + // rescan must not be kept from starting by it. + InitWarning(_("Coins received on the bip44 internal chain before now cannot be found on a pruned node. Reindex to look for them.")); + walletInstance->SetInternalChainRescanPending(false); + } else { + // ScanForWalletTransactions walks this forward to the wallet birthday, which no key on + // the internal chain is older than. + LogPrintf("Rescanning for transactions on the bip44 internal chain\n"); + pindexRescan = chainActive.Genesis(); + } + } if (chainActive.Tip() && chainActive.Tip() != pindexRescan) { //We can't rescan beyond non-pruned blocks, stop and throw an error @@ -5191,7 +5351,9 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) if (!(GetBoolArg("-newwallet", false))) {uiInterface.InitMessage(_("Rescanning..."));} nStart = GetTimeMillis(); - walletInstance->ScanForWalletTransactions(pindexRescan, true, fRecoverMnemonic); + // A scan that was interrupted returns nothing, and is asked for again on the next start. + if (walletInstance->ScanForWalletTransactions(pindexRescan, true, fRecoverMnemonic)) + walletInstance->SetInternalChainRescanPending(false); if (!(GetBoolArg("-newwallet", false))) {LogPrintf(" rescan %15dms\n", GetTimeMillis() - nStart);} walletInstance->SetBestChain(chainActive.GetLocator()); CWalletDB::IncrementUpdateCounter(); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 1d2cb2b718..f7b5c78c4c 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -97,6 +97,10 @@ const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; const uint32_t BIP44_INDEX = 0x2C; const uint32_t BIP44_TEST_INDEX = 0x1; // https://github.com/satoshilabs/slips/blob/master/slip-0044.md#registered-coin-types const uint32_t BIP44_FIRO_INDEX = 0x88; // https://github.com/satoshilabs/slips/blob/master/slip-0044.md#registered-coin-types +/* The bip44 internal (change) chain. Firo takes both its addresses and its change from the + * external chain, so nothing is ever generated here, but wallets that follow bip44 more + * closely do put change on it, and a wallet restored from such a seed has to see it. */ +const uint32_t BIP44_INTERNAL_INDEX = 0x1; const uint32_t BIP44_MINT_INDEX = 0x2; const uint32_t BIP44_MINT_VALUE_INDEX = 0x5; @@ -710,6 +714,14 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface std::map m_pool_key_to_index; + /* Child index of every key this wallet holds on the bip44 internal chain. Those keys are + * watched and spendable but never handed out, so unlike the keypool they are not indexed + * by a pool entry. */ + std::map m_internal_chain_keys; + + /* Records the key in the map above if its keypath places it on the internal chain. */ + void RegisterInternalChainKey(const CTxDestination& dest, const CKeyMetadata& metadata); + int64_t nTimeFirstKey; std::shared_ptr bip47wallet; @@ -1123,6 +1135,28 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface void MarkReserveKeysAsUsed(int64_t keypool_id); const std::map& GetAllReserveKeys() const { return m_pool_key_to_index; } + /** + * Derives keys on the bip44 internal chain, m/44'/'/0'/1/, and adds them to the + * wallet. They are watched so that coins another bip44 wallet sent there are seen and + * stay spendable; they are kept out of the keypool, so this wallet keeps taking both its + * addresses and its change from the external chain. + * @param[in] nSize How many keys the chain should hold, 0 for the default lookahead. + * @return whether any key was added. + */ + bool TopUpInternalChain(uint32_t nSize = 0); + /** + * Extends the internal chain past keyid when it turns out to have been used, the way a + * used keypool key tops the keypool up. No-op for a key that is not on that chain. + */ + void MarkInternalChainKeyUsed(const CKeyID& keyid); + /** + * Whether the internal chain was derived after the wallet had already been used, which + * leaves the chain to be rescanned for what was received on it. Kept in the wallet, so an + * interrupted rescan is resumed on the next start. + */ + bool IsInternalChainRescanPending() const; + void SetInternalChainRescanPending(bool fPending); + spark::FullViewKey GetSparkViewKey(); std::string GetSparkViewKeyStr();