BEEFY session keys and worker

README.md

@@ -288,6 +288,7 @@ Chain specifications are located in `/res/` directory.
 | AURA seed | `AURA_SEED_FILE=/path/to/seed` | - | Path to AURA consensus seed file |
 | GRANDPA seed | `GRANDPA_SEED_FILE=/path/to/seed` | - | Path to GRANDPA finality seed file |
 | Cross-chain seed | `CROSS_CHAIN_SEED_FILE=/path/to/seed` | - | Path to cross-chain seed file |
+| BEEFY seed | `BEEFY_SEED_FILE=/path/to/seed` | - | Path to BEEFY seed file (ECDSA, like cross-chain) |
 | Chain spec | `CHAIN=dev` | `--chain dev` | Network to connect to |
 | Base path | `BASE_PATH=/tmp/node-1` | `--base-path /tmp/node-1` | Data directory |
 | Validator mode | `VALIDATOR=true` | `--validator` | Run as validator (true/1/TRUE) |
@@ -312,35 +313,35 @@ CFG_PRESET=dev AURA_SEED_FILE=/tmp/alice-seed GRANDPA_SEED_FILE=/tmp/alice-seed
 
 ```shell
 echo "//Alice" > /tmp/alice-seed && echo "0000000000000000000000000000000000000000000000000000000000000001" > /tmp/alice-key && \
-CFG_PRESET=dev AURA_SEED_FILE=/tmp/alice-seed GRANDPA_SEED_FILE=/tmp/alice-seed CROSS_CHAIN_SEED_FILE=/tmp/alice-seed \
+CFG_PRESET=dev AURA_SEED_FILE=/tmp/alice-seed GRANDPA_SEED_FILE=/tmp/alice-seed CROSS_CHAIN_SEED_FILE=/tmp/alice-seed BEEFY_SEED_FILE=/tmp/alice-seed \
   NODE_KEY_FILE=/tmp/alice-key BASE_PATH=/tmp/node-1 CHAIN=dev VALIDATOR=true ./target/release/midnight-node --port 30333
 
 echo "//Bob" > /tmp/bob-seed && echo "0000000000000000000000000000000000000000000000000000000000000002" > /tmp/bob-key && \
-CFG_PRESET=dev AURA_SEED_FILE=/tmp/bob-seed GRANDPA_SEED_FILE=/tmp/bob-seed CROSS_CHAIN_SEED_FILE=/tmp/bob-seed \
+CFG_PRESET=dev AURA_SEED_FILE=/tmp/bob-seed GRANDPA_SEED_FILE=/tmp/bob-seed CROSS_CHAIN_SEED_FILE=/tmp/bob-seed BEEFY_SEED_FILE=/tmp/bob-seed \
   NODE_KEY_FILE=/tmp/bob-key BASE_PATH=/tmp/node-2 CHAIN=dev VALIDATOR=true \
   BOOTNODES="/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp" \
   ./target/release/midnight-node --port 30334
 
 echo "//Charlie" > /tmp/charlie-seed && echo "0000000000000000000000000000000000000000000000000000000000000003" > /tmp/charlie-key && \
-CFG_PRESET=dev AURA_SEED_FILE=/tmp/charlie-seed GRANDPA_SEED_FILE=/tmp/charlie-seed CROSS_CHAIN_SEED_FILE=/tmp/charlie-seed \
+CFG_PRESET=dev AURA_SEED_FILE=/tmp/charlie-seed GRANDPA_SEED_FILE=/tmp/charlie-seed CROSS_CHAIN_SEED_FILE=/tmp/charlie-seed BEEFY_SEED_FILE=/tmp/charlie-seed \
   NODE_KEY_FILE=/tmp/charlie-key BASE_PATH=/tmp/node-3 CHAIN=dev VALIDATOR=true \
   BOOTNODES="/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp" \
   ./target/release/midnight-node --port 30335
 
 echo "//Dave" > /tmp/dave-seed && echo "0000000000000000000000000000000000000000000000000000000000000004" > /tmp/dave-key && \
-CFG_PRESET=dev AURA_SEED_FILE=/tmp/dave-seed GRANDPA_SEED_FILE=/tmp/dave-seed CROSS_CHAIN_SEED_FILE=/tmp/dave-seed \
+CFG_PRESET=dev AURA_SEED_FILE=/tmp/dave-seed GRANDPA_SEED_FILE=/tmp/dave-seed CROSS_CHAIN_SEED_FILE=/tmp/dave-seed BEEFY_SEED_FILE=/tmp/dave-seed \
   NODE_KEY_FILE=/tmp/dave-key BASE_PATH=/tmp/node-4 CHAIN=dev VALIDATOR=true \
   BOOTNODES="/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp" \
   ./target/release/midnight-node --port 30336
 
 echo "//Eve" > /tmp/eve-seed && echo "0000000000000000000000000000000000000000000000000000000000000005" > /tmp/eve-key && \
-CFG_PRESET=dev AURA_SEED_FILE=/tmp/eve-seed GRANDPA_SEED_FILE=/tmp/eve-seed CROSS_CHAIN_SEED_FILE=/tmp/eve-seed \
+CFG_PRESET=dev AURA_SEED_FILE=/tmp/eve-seed GRANDPA_SEED_FILE=/tmp/eve-seed CROSS_CHAIN_SEED_FILE=/tmp/eve-seed BEEFY_SEED_FILE=/tmp/eve-seed \
   NODE_KEY_FILE=/tmp/eve-key BASE_PATH=/tmp/node-5 CHAIN=dev VALIDATOR=true \
   BOOTNODES="/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp" \
   ./target/release/midnight-node --port 30337
 
 echo "//Ferdie" > /tmp/ferdie-seed && echo "0000000000000000000000000000000000000000000000000000000000000006" > /tmp/ferdie-key && \
-CFG_PRESET=dev AURA_SEED_FILE=/tmp/ferdie-seed GRANDPA_SEED_FILE=/tmp/ferdie-seed CROSS_CHAIN_SEED_FILE=/tmp/ferdie-seed \
+CFG_PRESET=dev AURA_SEED_FILE=/tmp/ferdie-seed GRANDPA_SEED_FILE=/tmp/ferdie-seed CROSS_CHAIN_SEED_FILE=/tmp/ferdie-seed BEEFY_SEED_FILE=/tmp/ferdie-seed \
   NODE_KEY_FILE=/tmp/ferdie-key BASE_PATH=/tmp/node-6 CHAIN=dev VALIDATOR=true \
   BOOTNODES="/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp" \
   ./target/release/midnight-node --port 30338

changes/node/added/beefy-key-provisioning.md

@@ -0,0 +1,23 @@
+#node #keystore #beefy
+
+# Provide BEEFY keys via seed file, and add them to the local-env validators
+
+Added `BEEFY_SEED_FILE`, alongside the existing `AURA_SEED_FILE`,
+`BABE_SEED_FILE`, `GRANDPA_SEED_FILE` and `CROSS_CHAIN_SEED_FILE`. The seed is
+inserted into the keystore under the `beef` key type as ECDSA, like the
+cross-chain key.
+
+Validators need a BEEFY key in their keystore once beefy is part of
+`SessionKeys`; the key is looked up by `(key type, public key)`, so a key
+inserted only under another key type does not satisfy a BEEFY request even when
+the bytes are identical.
+
+The local-env validators (nodes 2 to 5) now carry one. The committee registers
+each validator's cross-chain key as its beefy key — `beefy_pub_key` equals
+`sidechain_pub_key` in `res/local/permissioned-candidates-config.json` — so the
+existing cross-chain key material is reused rather than new keys generated:
+keystore entries for the keystore-mounted nodes, and `seeds/beefy.seed` plus
+`BEEFY_SEED_FILE` for the seed-file node.
+
+PR: https://github.com/midnightntwrk/midnight-node/pull/2084
+Issue: https://github.com/midnightntwrk/midnight-node/issues/1742

changes/node/changed/beefy-genesis-authorities-from-session.md

@@ -0,0 +1,27 @@
+#node #genesis #beefy
+
+# Let session seed the BEEFY genesis authorities
+
+Building genesis panicked once beefy became a session key:
+
+```
+GenesisBuilder_build_state -> pallet_session::GenesisConfig::build
+  -> <(_, _, _) as SessionHandler>::on_genesis_session -> unwrap_failed
+```
+
+`pallet_beefy` (pallet index 21) initialized its authorities from the chain
+spec's `BeefyConfig::authorities`, and then `pallet_session` (index 30)
+initialized them a second time via the now three-element `SessionHandler`.
+`pallet_beefy::initialize` returns `Err` when the authority list is already
+populated, and `on_genesis_session` turns that into
+`.expect("Authorities vec too big")` — a misleading message, since nothing is
+too big.
+
+The chain spec now leaves `beefy: Default::default()` (empty authorities,
+`genesis_block: Some(1)`), matching `aura`, `babe` and `grandpa` beside it, and
+lets `pallet_session` seed the BEEFY authorities from the committee's session
+keys. Note this changes genesis storage, and therefore the genesis hash, so it
+applies to chains created from this version onward.
+
+PR: https://github.com/midnightntwrk/midnight-node/pull/2084
+Issue: https://github.com/midnightntwrk/midnight-node/issues/1742

changes/node/changed/committee-membership-storage-version-decode.md

@@ -0,0 +1,19 @@
+#node #committee-selection #beefy
+
+# Decode the committee by pallet storage version in the membership watcher
+
+The committee-membership watcher called the `get_current_committee` runtime
+API, which decodes committee members with the node binary's own `SessionKeys`.
+Between updating the nodes and running the runtime upgrade, that meant decoding
+pre-beefy (aura + grandpa) committee bytes as the three-key shape: the decode
+falls short and yields the empty default, so the watcher logged every validator
+as not being in a zero-size committee.
+
+It now reads `CurrentCommittee` and `pallet-session-validator-management`'s
+on-chain storage version straight from state and lets the runtime pick the
+matching shape (`committee_keys_migrated` / `decode_current_committee`),
+upgrading legacy-shaped members on the fly. Storage version `>= 2` means the
+committee carries beefy keys; absent or lower means it does not.
+
+PR: https://github.com/midnightntwrk/midnight-node/pull/2084
+Issue: https://github.com/midnightntwrk/midnight-node/issues/1742

changes/runtime/changed/add-beefy-session-key.md

@@ -0,0 +1,20 @@
+#runtime #session-keys #beefy
+
+# Add the BEEFY session key
+
+`opaque::SessionKeys` gains `beefy`, translated by the
+`AddBeefyToSessionKeysMigration` cutover, which gives each validator its own
+cross-chain key as its beefy key. Both are ECDSA and the committee registers
+them as equal (`beefy_pub_key == sidechain_pub_key`), so the migrated
+authority set is one the validators actually hold secrets for — a derived
+placeholder would leave BEEFY unable to reach a quorum until the next
+committee rotation.
+
+Because BEEFY is now a session key, `pallet_session`'s genesis initializes
+the BEEFY authorities from the committee, so the chain spec no longer sets
+`BeefyConfig::authorities` itself — see the separate genesis change.
+
+The runtime version moves to `002_001_001`.
+
+PR: https://github.com/midnightntwrk/midnight-node/pull/2084
+Issue: https://github.com/midnightntwrk/midnight-node/issues/1742

changes/runtime/changed/combined-session-migration.md

@@ -0,0 +1,12 @@
+#runtime #migration #committee-selection
+
+# Combine committee v1-to-v2 and session-key migrations in the runtime
+
+This chain moves `pallet-session-validator-management` from v1 to v2 in a single
+upgrade, which spans two changes the toolkit ships as separate migrations: the
+`AuthorityKeys` shape change (`migrations::authority_keys`) and v2's new
+`QueuedCommittee` (`migrations::v2`). They cannot be wired one after the other,
+so a custom migration that combines both is added.
+
+PR: https://github.com/midnightntwrk/midnight-node/pull/2084
+Issue: https://github.com/midnightntwrk/midnight-node/issues/1742

docs/openrpc.json

@@ -2,7 +2,7 @@
   "openrpc": "1.4.0",
   "info": {
     "title": "Midnight Node JSON-RPC API",
-    "version": "2.1.0",
+    "version": "2.1.1",
     "description": "JSON-RPC API for the Midnight privacy blockchain node. Custom methods provide access to the privacy ledger, governance parameters, and peer management. Standard Substrate methods are also listed."
   },
   "methods": [

local-environment/src/networks/local-env/configurations/midnight-nodes/midnight-node-2/keystore/626565660390084fdbf27d2b79d26a4f13f0ccd982cb755a661969143c37cbc49ef5b91f27

@@ -0,0 +1 @@
+"//Bob"

local-environment/src/networks/local-env/configurations/midnight-nodes/midnight-node-3/seeds/beefy.seed

@@ -0,0 +1 @@
+0xf8d74108dbe199c4a6e4ef457046db37c325ba3f709b14cabfa1885663e4c589

local-environment/src/networks/local-env/configurations/midnight-nodes/midnight-node-4/keystore/62656566039799ff93d184146deacaa455dade51b13ed16f23cdad11d1ad6af20103391180

@@ -0,0 +1 @@
+"volume wire damp response scene tape truck easily enjoy shaft face message"

local-environment/src/networks/local-env/configurations/midnight-nodes/midnight-node-5/keystore/626565660364b1f01f6e803be10abc6dd6fe08ced61cf3eaaef2dbdc72b4e774c4b6a564af

@@ -0,0 +1 @@
+"victory horse scatter empty silly live royal differ friend nurse poverty benefit"

local-environment/src/networks/local-env/docker-compose.yml

@@ -417,6 +417,7 @@ services:
       AURA_SEED_FILE: "/seeds/aura.seed"
       CROSS_CHAIN_SEED_FILE: "/seeds/cross_chain.seed"
       GRANDPA_SEED_FILE: "/seeds/grandpa.seed"
+      BEEFY_SEED_FILE: "/seeds/beefy.seed"
     entrypoint: ["/bin/bash", "/entrypoint.sh"]
     labels:
       io.midnight.role: validator

metadata/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "midnight-node-metadata"
-version = "2.1.0"
+version = "2.1.1"
 edition = "2024"
 build = "build.rs"
 license-file.workspace = true

metadata/src/lib.rs

@@ -17,4 +17,7 @@ pub mod midnight_metadata_2_0_0 {}
 #[subxt::subxt(runtime_metadata_path = "static/midnight_metadata_2.1.0.scale")]
 pub mod midnight_metadata_2_1_0 {}
 
-pub use midnight_metadata_2_1_0 as midnight_metadata_latest;
+#[subxt::subxt(runtime_metadata_path = "static/midnight_metadata_2.1.1.scale")]
+pub mod midnight_metadata_2_1_1 {}
+
+pub use midnight_metadata_2_1_1 as midnight_metadata_latest;

node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "midnight-node"
-version = "2.1.0"
+version = "2.1.1"
 description = "Midnight blockchain node"
 authors = ["Substrate DevHub <https://github.com/substrate-developer-hub>"]
 homepage = "https://substrate.io/"

node/src/cfg/midnight_cfg/mod.rs

@@ -57,6 +57,11 @@ pub struct MidnightCfg {
 	/// Docs: https://paritytech.github.io/polkadot-sdk/master/sp_core/crypto/struct.AddressUri.html#structfield.phrase
 	pub cross_chain_seed_file: Option<String>,
 
+	/// Path to file containing a secret string to use as the BEEFY seed (32 bytes)
+	/// Seed should be either a Phrase, hexadecimal string, or ss58-compatible string.
+	/// Docs: https://paritytech.github.io/polkadot-sdk/master/sp_core/crypto/struct.AddressUri.html#structfield.phrase
+	pub beefy_seed_file: Option<String>,
+
 	/// Mock ariadne parameters
 	pub use_main_chain_follower_mock: bool,
 	/// Required if use_main_chain_follower_mock is true

node/src/chain_spec/mod.rs

@@ -18,7 +18,7 @@ use serde_valid::Validate as _;
 use midnight_node_ledger_helpers::BlockContext;
 
 use midnight_node_runtime::{
-	AccountId, BeefyConfig, Block, BridgeConfig, C2MBridgeConfig, CNightObservationCall,
+	AccountId, Block, BridgeConfig, C2MBridgeConfig, CNightObservationCall,
 	CNightObservationConfig, CouncilConfig, CouncilMembershipConfig, CrossChainPublic,
 	FederatedAuthorityObservationConfig, MidnightCall, MidnightConfig, MidnightSystemCall,
 	RuntimeCall, RuntimeGenesisConfig, SessionCommitteeManagementConfig, SidechainConfig,
@@ -31,12 +31,13 @@ use midnight_primitives_cnight_observation::ObservedUtxos;
 use sc_chain_spec::{ChainSpecExtension, GenericChainSpec};
 use sidechain_domain::{AssetName, MainchainAddress, McTxHash};
 use sp_consensus_aura::sr25519::AuthorityId as AuraId;
+use sp_consensus_beefy::ecdsa_crypto::AuthorityId as BeefyId;
 use sp_consensus_grandpa::AuthorityId as GrandpaId;
 use sp_core::{Encode, H256, Pair, Public};
 use sp_partner_chains_bridge::{
 	MainChainScripts as BridgeMainChainScripts, SubminimalTransfersConfig,
 };
-use sp_runtime::traits::{IdentifyAccount, One, Verify};
+use sp_runtime::traits::{IdentifyAccount, Verify};
 use std::{fmt, str::FromStr};
 
 /// Parse asset name from config - accepts either hex-encoded string or plain UTF-8 string.
@@ -112,6 +113,7 @@ pub fn authority_keys_from_seed(s: &str) -> AuthorityKeys {
 		session: SessionKeys {
 			aura: get_from_seed::<AuraId>(s),
 			grandpa: get_from_seed::<GrandpaId>(s),
+			beefy: get_from_seed::<BeefyId>(s),
 		},
 		cross_chain: get_from_seed::<CrossChainPublic>(s),
 	}
@@ -243,6 +245,7 @@ fn genesis_config<T: MidnightNetwork>(genesis: T) -> Result<serde_json::Value, C
 			session: SessionKeys {
 				aura: keys.aura_pubkey.into(),
 				grandpa: keys.grandpa_pubkey.into(),
+				beefy: keys.beefy_pubkey.into(),
 			},
 			cross_chain: keys.crosschain_pubkey.into(),
 		})
@@ -257,14 +260,7 @@ fn genesis_config<T: MidnightNetwork>(genesis: T) -> Result<serde_json::Value, C
 		system: Default::default(),
 		aura: Default::default(),
 		babe: Default::default(),
-		beefy: BeefyConfig {
-			authorities: genesis
-				.initial_authorities()
-				.iter()
-				.map(|v| v.beefy_pubkey.into())
-				.collect(),
-			genesis_block: Some(One::one()),
-		},
+		beefy: Default::default(),
 		grandpa: Default::default(),
 		midnight: MidnightConfig {
 			_config: Default::default(),

node/src/command.rs

@@ -49,7 +49,8 @@ use sidechain_domain::mainchain_epoch::MainchainEpochConfig;
 use sp_core::{
 	ByteArray, Pair,
 	crypto::key_types::{
-		AURA as AURA_KEY_TYPE, BABE as BABE_KEY_TYPE, GRANDPA as GRANDPA_KEY_TYPE,
+		AURA as AURA_KEY_TYPE, BABE as BABE_KEY_TYPE, BEEFY as BEEFY_KEY_TYPE,
+		GRANDPA as GRANDPA_KEY_TYPE,
 	},
 	offchain::KeyTypeId,
 };
@@ -277,6 +278,19 @@ fn run_node(cfg: Cfg) -> sc_cli::Result<()> {
 		log::info!("CROSS_CHAIN pubkey: {}", &keypair.public())
 	}
 
+	if let Some(seed_file) = &cfg.midnight_cfg.beefy_seed_file {
+		let seed = std::fs::read_to_string(seed_file).map_err(|e| {
+			sc_cli::Error::Input(format!(
+				"error when reading BEEFY seed file at {seed_file}. Error: {e}"
+			))
+		})?;
+		let seed = seed.trim();
+		let (keypair, _) = sp_core::ecdsa::Pair::from_string_with_seed(seed, None)
+			.map_err(|e| sc_cli::Error::Input(format!("Invalid BEEFY seed: {e}")))?;
+		keystore.insert(BEEFY_KEY_TYPE, seed, &keypair.public().to_raw_vec()).unwrap();
+		log::info!("BEEFY pubkey: {}", &keypair.public())
+	}
+
 	// Hold the database backend handle outside the tokio runtime so we can
 	// explicitly drop it after all async tasks have finished.  Without this,
 	// the backend's Arc may be leaked inside aborted tokio tasks during