diff --git a/nix/devShells/default.nix b/nix/devShells/default.nix index f249c96..cfff2fb 100644 --- a/nix/devShells/default.nix +++ b/nix/devShells/default.nix @@ -20,6 +20,8 @@ python3Packages.flake8 nix + sshpass + socat ]; }; }; diff --git a/nix/packages/navi.nix b/nix/packages/navi.nix index 8455506..fb94754 100644 --- a/nix/packages/navi.nix +++ b/nix/packages/navi.nix @@ -6,6 +6,8 @@ nix-eval-jobs, makeBinaryWrapper, nixos-anywhere, + sshpass, + socat, }: let @@ -66,7 +68,7 @@ in craneLib.buildPackage (commonArgs // { ''} wrapProgram $out/bin/navi \ - --prefix PATH : ${lib.makeBinPath [ nixos-anywhere ]} + --prefix PATH : ${lib.makeBinPath [ nixos-anywhere sshpass socat ]} ''; passthru = { diff --git a/src/command/install.rs b/src/command/install.rs index 12c44a5..ea328e6 100644 --- a/src/command/install.rs +++ b/src/command/install.rs @@ -111,14 +111,17 @@ pub async fn run(hive: Hive, opts: Opts) -> NaviResult<()> { } })?; - // We only care about Terranix/Terraform provisioners for IP lookup - if prov_config.kind != crate::nix::ProvisionerType::Terranix { - tracing::info!( - "Skipping provisioner '{}' (type {:?}) as it is not Terraform-based.", - prov_name, - prov_config.kind - ); - continue; + // Only Terranix and BareMetal provisioners support installation + match prov_config.kind { + crate::nix::ProvisionerType::Terranix | crate::nix::ProvisionerType::BareMetal => {} + _ => { + tracing::info!( + "Skipping provisioner '{}' (type {:?}) — installation not supported.", + prov_name, + prov_config.kind + ); + continue; + } } // Check if nixos-anywhere is enabled for this provisioner @@ -147,8 +150,8 @@ pub async fn run(hive: Hive, opts: Opts) -> NaviResult<()> { let facts_dir = Path::new(&meta.facts.dir_name).join(&prov_name); - // 3a. Handle Reinstall Logic (Infrastructure Recreation) - if opts.reinstall { + // 3a. Handle Reinstall Logic (Infrastructure Recreation) — Terranix only + if opts.reinstall && prov_config.kind == crate::nix::ProvisionerType::Terranix { tracing::info!("Reinstall requested. Checking for resources to destroy and recreate..."); // Reconstruct the workspace path @@ -162,7 +165,7 @@ pub async fn run(hive: Hive, opts: Opts) -> NaviResult<()> { tracing::info!("Found resource for node {}: {}", node_name, addr); tracing::warn!("RECREATING infrastructure for node {}", node_name); - eprintln!("\nTargeting Terraform resource: {}", addr); + eprintln!("\n[33mTargeting Terraform resource: {}[0m", addr); if confirm_action(&format!("Are you sure you want to destroy and recreate this resource for node '{}'?", node_name))? { match executor.replace_resource(&addr).await { Ok(_) => { @@ -190,6 +193,8 @@ pub async fn run(hive: Hive, opts: Opts) -> NaviResult<()> { } else { tracing::warn!("Provisioner workspace not found at {:?}. Cannot perform reinstall actions.", work_dir); } + } else if opts.reinstall && prov_config.kind == crate::nix::ProvisionerType::BareMetal { + tracing::info!("Reinstall requested for bare-metal provisioner '{}'. No infrastructure to recreate.", prov_name); } // 4. Load Outputs (Cache or Live) @@ -228,6 +233,7 @@ pub async fn run(hive: Hive, opts: Opts) -> NaviResult<()> { &outputs_json, opts.unlock, Some(final_nodes), + None, ) .await?; } diff --git a/src/command/provision.rs b/src/command/provision.rs index 9865b16..0f693b6 100644 --- a/src/command/provision.rs +++ b/src/command/provision.rs @@ -20,7 +20,7 @@ pub struct Opts { pub list: bool, /// Explicitly select a provisioner to run - #[arg(conflicts_with = "on", conflicts_with = "exclude")] + #[arg(conflicts_with = "on")] pub provisioner: Option, /// Destroy and recreate the infrastructure @@ -39,6 +39,15 @@ pub struct Opts { #[arg(long)] pub skip_install: bool, + /// IP address for bare-metal provisioning (skips interactive prompt) + #[arg(long)] + pub ip: Option, + + /// Password for initial SSH connection to the installer (e.g. for bare-metal + /// hosts that haven't been keyed yet). Passed to nixos-anywhere via --env-password. + #[arg(long)] + pub initial_password: Option, + #[command(flatten)] pub node_filter: NodeFilterOpts, } @@ -148,6 +157,9 @@ async fn run_provisioner( ProvisionerType::Terranix => { run_terranix_provisioner(hive, name, config, targets, opts, meta).await } + ProvisionerType::BareMetal => { + run_bare_metal_provisioner(hive, name, config, targets, opts, meta).await + } } } @@ -363,6 +375,133 @@ async fn run_terranix_provisioner( &output_json, opts.unlock, Some(relevant_nodes), + None, + ) + .await?; + } + } + } else { + tracing::info!("Skipping install step as requested."); + } + + Ok(()) +} + +/// Bare-metal provisioner: resolves node IPs interactively or via `--ip`, +/// writes them as facts, then optionally runs nixos-anywhere. +async fn run_bare_metal_provisioner( + hive: &Hive, + name: &str, + config: &ProvisionerConfig, + targets: &HashMap, + opts: &Opts, + meta: &MetaConfig, +) -> NaviResult<()> { + let facts_dir = Path::new(&meta.facts.dir_name).join(name); + + // Load existing facts if present + let mut existing_outputs: serde_json::Value = if facts_dir.join("outputs.json").exists() { + let content = tokio::fs::read_to_string(facts_dir.join("outputs.json")) + .await + .map_err(|e| NaviError::IoContext { + error: e, + context: format!("reading existing facts from {:?}", facts_dir), + })?; + serde_json::from_str(&content).unwrap_or_else(|_| serde_json::json!({})) + } else { + serde_json::json!({}) + }; + + // Determine relevant nodes for this provisioner + let relevant_nodes: Vec<(&NodeName, &TargetNode)> = targets + .iter() + .filter(|(_, target)| target.config.provisioner.as_deref() == Some(name)) + .collect(); + + if relevant_nodes.is_empty() { + tracing::warn!("No nodes assigned to bare-metal provisioner '{}'.", name); + return Ok(()); + } + + // Resolve IP for each node + for (node_name, _target) in &relevant_nodes { + let ip_key = format!("{}_ip", node_name.as_str().replace('-', "_")); + + // Check if IP already exists in facts + let existing_ip = existing_outputs + .get(&ip_key) + .and_then(|v| v.get("value")) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + let ip = if opts.reprovision || existing_ip.is_none() { + // Need to get the IP: from --ip flag or interactively + if let Some(ip) = &opts.ip { + tracing::info!( + "[bare-metal] Using provided IP for {}: {}", + node_name.as_str(), + ip + ); + ip.clone() + } else if let Some(existing) = &existing_ip { + if !opts.reprovision { + tracing::info!( + "[bare-metal] Using existing IP for {}: {}", + node_name.as_str(), + existing + ); + existing.clone() + } else { + // Reprovision: prompt with existing as hint + prompt_for_ip(node_name.as_str(), Some(existing))? + } + } else { + // No existing IP, no flag: prompt + prompt_for_ip(node_name.as_str(), None)? + } + } else { + let ip = existing_ip.unwrap(); + tracing::info!( + "[bare-metal] Using existing IP for {}: {}", + node_name.as_str(), + ip + ); + ip + }; + + // Write/update the fact for this node + existing_outputs[&ip_key] = serde_json::json!({ + "sensitive": false, + "type": "string", + "value": ip + }); + } + + // Persist facts + if meta.facts.enable { + write_bare_metal_facts(&facts_dir, &existing_outputs)?; + tracing::info!("Facts saved to {:?}", facts_dir); + } + + // Handle NixOS Anywhere + if !opts.skip_install { + if let Some(na_config) = &config.nixos_anywhere { + if na_config.enable { + tracing::info!("Running nixos-anywhere for bare-metal nodes..."); + + let node_names: Vec<&str> = relevant_nodes + .iter() + .map(|(n, _)| n.as_str()) + .collect(); + + crate::nix::nixos_anywhere::run( + hive, + targets, + na_config, + &existing_outputs, + opts.unlock, + Some(node_names), + opts.initial_password.as_deref(), ) .await?; } @@ -374,6 +513,68 @@ async fn run_terranix_provisioner( Ok(()) } +/// Prompts the user for an IP address interactively. +fn prompt_for_ip(node_name: &str, existing: Option<&str>) -> NaviResult { + if let Some(existing) = existing { + eprint!("Enter IP address for {} [{}]: ", node_name, existing); + } else { + eprint!("Enter IP address for {}: ", node_name); + } + + let mut input = String::new(); + std::io::stdin() + .read_line(&mut input) + .map_err(|e| NaviError::IoError { error: e })?; + + let trimmed = input.trim(); + if trimmed.is_empty() { + if let Some(existing) = existing { + Ok(existing.to_string()) + } else { + Err(NaviError::DeploymentError { + message: format!("No IP address provided for node '{}'", node_name), + }) + } + } else { + Ok(trimmed.to_string()) + } +} + +/// Writes bare-metal facts (IP mappings) in the same format as Terraform outputs. +pub fn write_bare_metal_facts( + facts_dir: &Path, + outputs: &serde_json::Value, +) -> NaviResult<()> { + // Create directory + std::fs::create_dir_all(facts_dir).map_err(|e| NaviError::IoContext { + error: e, + context: format!("creating facts directory {:?}", facts_dir), + })?; + + // Write outputs.json + let json_path = facts_dir.join("outputs.json"); + let json_content = + serde_json::to_string_pretty(outputs).expect("Failed to serialize outputs"); + std::fs::write(&json_path, json_content).map_err(|e| NaviError::IoContext { + error: e, + context: format!("writing facts to {:?}", json_path), + })?; + + // Write default.nix (same format as Terraform facts) + let nix_path = facts_dir.join("default.nix"); + let nix_content = r#"let + raw = builtins.fromJSON (builtins.readFile ./outputs.json); +in + builtins.mapAttrs (n: v: v.value) raw +"#; + std::fs::write(&nix_path, nix_content).map_err(|e| NaviError::IoContext { + error: e, + context: format!("writing nix facts to {:?}", nix_path), + })?; + + Ok(()) +} + async fn inject_registrant_providers( work_dir: &Path, registrants: &crate::nix::RegistrantsConfig, diff --git a/src/nix/hive/options.nix b/src/nix/hive/options.nix index 9aa612f..b6875c6 100644 --- a/src/nix/hive/options.nix +++ b/src/nix/hive/options.nix @@ -242,6 +242,22 @@ rec { type = types.nullOr types.str; default = null; }; + forceHwLink = lib.mkOption { + description = '' + Force SSH connections through physical network interfaces (e.g. + enp*, wlp*), bypassing overlay networks such as Tailscale. + + When enabled, Navi will detect an appropriate physical interface + and route SSH traffic through it using socat's bindtodevice. + This is useful for bare-metal hosts on a LAN that should be + reached directly rather than through a VPN tunnel. + + Note: This setting applies to regular deployment connections. + For initrd/unlock connections, use deployment.unlock.forceHwLink. + ''; + type = types.bool; + default = false; + }; providers = lib.mkOption { description = '' Cloud provider settings. diff --git a/src/nix/host/ssh.rs b/src/nix/host/ssh.rs index c76bd6c..e7db469 100644 --- a/src/nix/host/ssh.rs +++ b/src/nix/host/ssh.rs @@ -43,6 +43,12 @@ pub struct Ssh { provider: Provider, + /// Force connections through physical interfaces, bypassing overlay networks. + force_hw_link: bool, + + /// Explicit list of allowed interfaces for hw-link binding. + hw_link_interfaces: Option>, + job: Option, } @@ -390,6 +396,8 @@ impl Ssh { extra_ssh_options: Vec::new(), use_nix3_copy: false, provider: Provider::Ssh, + force_hw_link: false, + hw_link_interfaces: None, job: None, } } @@ -426,10 +434,97 @@ impl Ssh { self.use_nix3_copy = enable; } + pub fn set_force_hw_link(&mut self, enable: bool) { + self.force_hw_link = enable; + } + + pub fn set_hw_link_interfaces(&mut self, interfaces: Option>) { + self.hw_link_interfaces = interfaces; + } + pub fn upcast(self) -> Box { Box::new(self) } + /// Applies hardware-link interface binding to this SSH host's options. + /// + /// Detects an appropriate physical interface (e.g. `enp*`, `wlp*`) and adds + /// a `ProxyCommand` using `socat` with `bindtodevice` to bypass overlay + /// networks like Tailscale. + /// + /// If `explicit_interfaces` is provided, only those interface prefixes are + /// allowed. Otherwise, the default prefixes `enp` and `wlp` are used. + pub fn apply_hw_link_binding(&mut self, explicit_interfaces: Option<&[String]>) { + let target_host = self.host.clone(); + + let allowed_patterns: Vec = if let Some(explicit) = explicit_interfaces { + explicit.to_vec() + } else { + vec!["enp".to_string(), "wlp".to_string()] + }; + + let matches = + |iface: &str| -> bool { allowed_patterns.iter().any(|p| iface.starts_with(p)) }; + + // Try route lookup + let best_interface = std::process::Command::new("ip") + .args(&["route", "get", &target_host, "table", "main"]) + .output() + .ok() + .and_then(|output| { + let s = String::from_utf8_lossy(&output.stdout); + s.split_whitespace() + .skip_while(|&part| part != "dev") + .nth(1) + .map(|s| s.to_string()) + }); + + let selected_interface = if let Some(iface) = best_interface.filter(|i| matches(i)) { + Some(iface) + } else { + tracing::warn!("Route lookup failed or returned disallowed interface. Scanning available interfaces..."); + std::process::Command::new("ip") + .args(&["-o", "link", "show", "up"]) + .output() + .ok() + .and_then(|output| { + String::from_utf8_lossy(&output.stdout) + .lines() + .find_map(|line| { + // Only consider interfaces with LOWER_UP (physical link present), + // not just administratively UP (which includes wifi with no carrier) + if !line.contains("LOWER_UP") { + return None; + } + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() >= 2 { + let name = parts[1].trim_end_matches(':'); + if matches(name) { + return Some(name.to_string()); + } + } + None + }) + }) + }; + + if let Some(iface) = selected_interface { + tracing::info!("Binding connection to physical interface: {}", iface); + self.extra_ssh_options.push("-o".to_string()); + self.extra_ssh_options.push(format!( + "ProxyCommand=sudo socat - TCP:%h:%p,bindtodevice={}", + iface + )); + } else { + tracing::error!( + "Failed to find allowed physical interface! Blocking connection." + ); + self.extra_ssh_options.push("-o".to_string()); + self.extra_ssh_options + .push("ProxyCommand=false".to_string()); + } + } + /// Prepares this host instance for connecting to initrd (e.g. for unlocking). pub fn configure_for_initrd(&mut self, config: &DiskUnlockConfig) { // Override connection details for initrd @@ -446,69 +541,7 @@ impl Ssh { // Handle interface binding if config.force_hw_link || config.interfaces.is_some() { - let target_host = self.host.clone(); - - let allowed_patterns = if let Some(explicit) = &config.interfaces { - explicit.clone() - } else { - vec!["enp".to_string(), "wlp".to_string()] - }; - - let matches = - |iface: &str| -> bool { allowed_patterns.iter().any(|p| iface.starts_with(p)) }; - - // Try route lookup - let best_interface = std::process::Command::new("ip") - .args(&["route", "get", &target_host, "table", "main"]) - .output() - .ok() - .and_then(|output| { - let s = String::from_utf8_lossy(&output.stdout); - s.split_whitespace() - .skip_while(|&part| part != "dev") - .nth(1) - .map(|s| s.to_string()) - }); - - let selected_interface = if let Some(iface) = best_interface.filter(|i| matches(i)) { - Some(iface) - } else { - tracing::warn!("Route lookup failed or returned disallowed interface. Scanning available interfaces..."); - std::process::Command::new("ip") - .args(&["-o", "link", "show", "up"]) - .output() - .ok() - .and_then(|output| { - String::from_utf8_lossy(&output.stdout) - .lines() - .find_map(|line| { - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() >= 2 { - let name = parts[1].trim_end_matches(':'); - if matches(name) { - return Some(name.to_string()); - } - } - None - }) - }) - }; - - if let Some(iface) = selected_interface { - tracing::info!("Binding unlock connection to interface: {}", iface); - self.extra_ssh_options.push("-o".to_string()); - self.extra_ssh_options.push(format!( - "ProxyCommand=sudo socat - TCP:%h:%p,bindtodevice={}", - iface - )); - } else { - tracing::error!( - "Failed to find allowed interface for unlock! Blocking connection." - ); - self.extra_ssh_options.push("-o".to_string()); - self.extra_ssh_options - .push("ProxyCommand=false".to_string()); - } + self.apply_hw_link_binding(config.interfaces.as_deref()); } self.extra_ssh_options.extend(config.ssh_options.clone()); @@ -789,6 +822,22 @@ impl Ssh { // TODO: Allow configuation of SSH parameters let mut options = self.extra_ssh_options.clone(); + + // Apply hardware-link binding if configured and not already set + // (configure_for_initrd handles its own binding, so we check for ProxyCommand) + if self.force_hw_link { + let already_has_proxy = options.windows(2).any(|w| { + w[0] == "-o" && w[1].starts_with("ProxyCommand") + }); + if !already_has_proxy { + let mut binding_host = self.clone(); + binding_host.force_hw_link = false; // prevent recursion + binding_host.extra_ssh_options.clear(); + binding_host.apply_hw_link_binding(self.hw_link_interfaces.as_deref()); + options.extend(binding_host.extra_ssh_options); + } + } + options.extend( [ "-o", diff --git a/src/nix/mod.rs b/src/nix/mod.rs index 6927efa..04f9a31 100644 --- a/src/nix/mod.rs +++ b/src/nix/mod.rs @@ -118,6 +118,11 @@ pub struct NodeConfig { #[serde(rename = "provisioner")] pub provisioner: Option, + /// Force SSH connections through physical network interfaces, bypassing + /// overlay networks like Tailscale. + #[serde(rename = "forceHwLink", default)] + pub force_hw_link: bool, + #[validate(custom(function = "validate_keys"))] #[serde(default)] pub keys: HashMap, @@ -138,6 +143,7 @@ impl Default for NodeConfig { privilege_escalation_command: Vec::new(), extra_ssh_options: Vec::new(), provisioner: None, + force_hw_link: false, keys: HashMap::new(), } } @@ -326,6 +332,7 @@ pub enum ProvisionerType { Command, FlakeApp, Terranix, + BareMetal, } /// Nix CLI flags. @@ -454,6 +461,7 @@ impl NodeConfig { host.set_privilege_escalation_command(self.privilege_escalation_command.clone()); host.set_extra_ssh_options(self.extra_ssh_options.clone()); host.set_provider(self.get_provider()); + host.set_force_hw_link(self.force_hw_link); if let Some(target_port) = self.target_port { host.set_port(target_port); diff --git a/src/nix/nixos_anywhere.rs b/src/nix/nixos_anywhere.rs index ecfd04c..1c513a6 100644 --- a/src/nix/nixos_anywhere.rs +++ b/src/nix/nixos_anywhere.rs @@ -20,6 +20,7 @@ pub async fn run( tf_outputs: &serde_json::Value, unlock_after_install: bool, subset_nodes: Option>, + initial_password: Option<&str>, ) -> NaviResult<()> { // Determine which nodes to process. If subset_nodes is provided, use it, otherwise use all targets. // However, we still filter by the provisioner in the caller usually, but here we just iterate @@ -90,6 +91,30 @@ pub async fn run( // Add strict checking options for security, unless disabled let ssh_opts = ssh_helper.ssh_options(); + // When using password auth via sshpass, filter out BatchMode=yes + // which prevents password prompts. This affects both the connectivity + // check and the args passed to nixos-anywhere (and its ssh-copy-id). + let ssh_opts = if initial_password.is_some() { + let mut filtered = Vec::new(); + let mut iter = ssh_opts.iter(); + while let Some(arg) = iter.next() { + if arg == "-o" { + if let Some(val) = iter.next() { + if val.starts_with("BatchMode=") { + continue; + } + filtered.push(arg.clone()); + filtered.push(val.clone()); + } + } else { + filtered.push(arg.clone()); + } + } + filtered + } else { + ssh_opts + }; + // Determine user to connect as let node_target_user = targets .iter() @@ -106,7 +131,7 @@ pub async fn run( // Wait for connectivity before launching nixos-anywhere // This is crucial for IAP tunnels which take time to establish/propagate - if let Err(e) = wait_for_connectivity(&target_str, &ssh_opts).await { + if let Err(e) = wait_for_connectivity(&target_str, &ssh_opts, initial_password).await { tracing::error!("Skipping node {} due to connectivity/auth issues: {}", node, e); failed.push((node.to_string(), e.to_string())); continue; @@ -123,6 +148,12 @@ pub async fn run( let mut na_cmd = Command::new("nixos-anywhere"); na_cmd.arg("--flake").arg(&flake_ref).arg(&target_str); + // If an initial password is provided, use sshpass via --env-password + if let Some(password) = initial_password { + na_cmd.env("SSHPASS", password); + na_cmd.arg("--env-password"); + } + if na_config.download_kexec_locally { match get_node_system(hive, node).await { Ok(system) => { @@ -282,7 +313,7 @@ fn populate_ssh_args(cmd: &mut Command, ssh_opts: &[String]) { } } -async fn wait_for_connectivity(target: &str, ssh_opts: &[String]) -> NaviResult<()> { +async fn wait_for_connectivity(target: &str, ssh_opts: &[String], initial_password: Option<&str>) -> NaviResult<()> { tracing::info!("Waiting for connectivity to {}...", target); // Increase timeout to 10 minutes (600s) to accommodate slow IAP tunnel propagation @@ -309,10 +340,27 @@ async fn wait_for_connectivity(target: &str, ssh_opts: &[String]) -> NaviResult< }); } - let mut cmd = Command::new("ssh"); + // If initial_password is set, use sshpass for the connectivity check + let mut cmd = if let Some(password) = initial_password { + let mut c = Command::new("sshpass"); + c.env("SSHPASS", password); + c.arg("-e").arg("ssh"); + c + } else { + Command::new("ssh") + }; - // Pass standard SSH opts directly. - cmd.args(ssh_opts); + // Pass standard SSH opts (already filtered if initial_password is set) + cmd.args(ssh_opts.iter()); + + // When using password auth, force it and limit identities to avoid + // the "too many authentication failures" problem + if initial_password.is_some() { + cmd.args([ + "-o", "PreferredAuthentications=password", + "-o", "IdentitiesOnly=yes", + ]); + } // Add robust options for checking // We use exit 0 so it's a no-op on the server side @@ -324,6 +372,8 @@ async fn wait_for_connectivity(target: &str, ssh_opts: &[String]) -> NaviResult< target, "exit", "0" ]); + tracing::debug!("SSH connectivity check: {:?}", cmd.as_std()); + // We capture output to check for specific IAP errors let mut exec = CommandExecution::new(cmd); exec.set_quiet(true); diff --git a/src/nix/node_filter.rs b/src/nix/node_filter.rs index 7994a7e..e3d314c 100644 --- a/src/nix/node_filter.rs +++ b/src/nix/node_filter.rs @@ -247,6 +247,7 @@ mod tests { privilege_escalation_command: vec![], extra_ssh_options: vec![], provisioner: None, + force_hw_link: false, keys: HashMap::new(), providers: Default::default(), unlock: Default::default(),