diff --git a/Cargo.lock b/Cargo.lock index c91c65c79..3f19aade1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3185,6 +3185,7 @@ dependencies = [ "async-trait", "base32", "chrono", + "futures", "nexus-common", "opentelemetry 0.31.0", "pubky", diff --git a/examples/watcher/watcher-config.toml b/examples/watcher/watcher-config.toml index e309bf9ae..e105bb565 100644 --- a/examples/watcher/watcher-config.toml +++ b/examples/watcher/watcher-config.toml @@ -1,7 +1,9 @@ testnet = false homeserver = "8um71us3fyw6h8wbcxb5ar3rwusy1a6u49956ikzojg3gcwd1dty" events_limit = 50 +key_based_events_limit = 50 watcher_sleep = 5000 +hs_resolver_sleep = 10000 # Initial backoff duration (in seconds) after the first failure of a homeserver initial_backoff_secs = 60 # Maximum backoff duration (in seconds) for a failing homeserver @@ -18,6 +20,21 @@ moderated_tags = [ "il_adult_nu_sex_act", ] +# Event retry configuration +[retry] +# Transient error retry limit before dead-letter +max_retries = 10 +# Safety net for homeservers that disappear silently +max_dependency_retries = 50 +# Base for exponential backoff on transient retries (seconds) +initial_backoff_secs = 10 +# Backoff ceiling for transient retries (seconds) +max_backoff_secs = 3600 +# Base for MissingDependency polling backoff (seconds) +initial_missing_dep_backoff_secs = 60 +# Backoff ceiling for MissingDependency (seconds) +max_missing_dep_backoff_secs = 3600 + [stack] # Logging, options: error, warn, info, debug and trace log_level = "debug" diff --git a/nexus-common/default.config.toml b/nexus-common/default.config.toml index ffa15a96d..dff35d724 100644 --- a/nexus-common/default.config.toml +++ b/nexus-common/default.config.toml @@ -14,11 +14,16 @@ testnet = false testnet_host = "localhost" # Synonym homeserver pubky homeserver = "8um71us3fyw6h8wbcxb5ar3rwusy1a6u49956ikzojg3gcwd1dty" -# Maximum number of events to fetch per run from each homeserver +# Maximum number of events to fetch per run from the default homeserver (max: 1000) events_limit = 50 +# Maximum events per user per run for key-based (non-default) homeservers (max: 100) +key_based_events_limit = 50 # Maximum number of monitored homeservers. If set to 1, only the default homeserver is monitored. monitored_homeservers_limit = 50 watcher_sleep = 5000 +hs_resolver_sleep = 10000 +# Minimum time (ms) before a user's homeserver mapping is considered stale and therefore eligible to be re-resolved (default: 1 hour) +hs_resolver_ttl = 3600000 # Initial backoff duration (in seconds) after the first failure of a homeserver initial_backoff_secs = 60 # Maximum backoff duration (in seconds) for a failing homeserver @@ -35,6 +40,20 @@ moderated_tags = [ "il_adult_nu_sex_act", ] +# Event retry configuration +[watcher.retry] +# Transient error retry limit before dead-letter +max_retries = 10 +# Safety net for homeservers that disappear silently +max_dependency_retries = 50 +# Base for exponential backoff on transient retries (seconds) +initial_backoff_secs = 10 +# Backoff ceiling for transient retries (seconds) +max_backoff_secs = 3600 +# Base for MissingDependency polling backoff (seconds) +initial_missing_dep_backoff_secs = 60 +# Backoff ceiling for MissingDependency (seconds) +max_missing_dep_backoff_secs = 3600 [stack] # Logging, options: error, warn, info, debug and trace diff --git a/nexus-common/src/config/daemon.rs b/nexus-common/src/config/daemon.rs index 809088a0c..42eab7642 100644 --- a/nexus-common/src/config/daemon.rs +++ b/nexus-common/src/config/daemon.rs @@ -84,7 +84,9 @@ mod tests { PubkyId::try_from("8um71us3fyw6h8wbcxb5ar3rwusy1a6u49956ikzojg3gcwd1dty").unwrap() ); assert_eq!(c.watcher.events_limit, 50); + assert_eq!(c.watcher.key_based_events_limit, 50); assert_eq!(c.watcher.watcher_sleep, 5_000); + assert_eq!(c.watcher.hs_resolver_sleep, 10_000); assert_eq!( c.watcher.moderation_id, PubkyId::try_from(DEFAULT_MODERATION_ID).unwrap() diff --git a/nexus-common/src/config/mod.rs b/nexus-common/src/config/mod.rs index 7f241e01c..1d030055f 100644 --- a/nexus-common/src/config/mod.rs +++ b/nexus-common/src/config/mod.rs @@ -32,13 +32,15 @@ mod api; mod daemon; pub mod file; mod stack; -mod watcher; +pub mod watcher; pub use api::ApiConfig; pub use daemon::DaemonConfig; pub use stack::{default_stack, OtlpConfig, StackConfig}; -pub use watcher::WatcherConfig; -pub use watcher::{DEFAULT_INITIAL_BACKOFF_SECS, DEFAULT_MAX_BACKOFF_SECS}; +pub use watcher::{ + EventRetryConfig, WatcherConfig, DEFAULT_HS_RESOLVER_TTL, DEFAULT_INITIAL_BACKOFF_SECS, + DEFAULT_MAX_BACKOFF_SECS, MAX_EVENTS_LIMIT, MAX_KEY_BASED_EVENTS_LIMIT, +}; use crate::file::validate_and_expand_path; diff --git a/nexus-common/src/config/watcher.rs b/nexus-common/src/config/watcher.rs index 651d5f69c..207aa7051 100644 --- a/nexus-common/src/config/watcher.rs +++ b/nexus-common/src/config/watcher.rs @@ -1,8 +1,10 @@ +use crate::models::event::EventProcessorError; + use super::file::ConfigLoader; use super::{default_stack, DaemonConfig, StackConfig}; use async_trait::async_trait; use pubky_app_specs::PubkyId; -use serde::{Deserialize, Serialize}; +use serde::{de::Error, Deserialize, Deserializer, Serialize}; use std::fmt::Debug; pub const TESTNET: bool = false; @@ -10,15 +12,40 @@ pub const DEFAULT_TESTNET_HOST: &str = "localhost"; // Testnet homeserver key pub const HOMESERVER_PUBKY: &str = "8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo"; /// Default for [WatcherConfig::events_limit] -pub const DEFAULT_EVENTS_LIMIT: u32 = 1_000; +pub const DEFAULT_EVENTS_LIMIT: u16 = 1_000; +/// Default for [WatcherConfig::key_based_events_limit] +pub const DEFAULT_KEY_BASED_EVENTS_LIMIT: u16 = 50; +/// Upper bound for [WatcherConfig::events_limit] +pub const MAX_EVENTS_LIMIT: u16 = 1_000; +/// Upper bound for [WatcherConfig::key_based_events_limit] +pub const MAX_KEY_BASED_EVENTS_LIMIT: u16 = 100; /// Default for [WatcherConfig::monitored_homeservers_limit] pub const DEFAULT_MONITORED_HOMESERVERS_LIMIT: usize = 50; /// Default for [WatcherConfig::watcher_sleep] pub const DEFAULT_WATCHER_SLEEP: u64 = 5_000; +/// Default for [WatcherConfig::hs_resolver_sleep] +pub const DEFAULT_HS_RESOLVER_SLEEP: u64 = 10_000; +/// Default for [WatcherConfig::hs_resolver_ttl]: 1 hour in milliseconds +pub const DEFAULT_HS_RESOLVER_TTL: u64 = 3_600_000; /// Default for [WatcherConfig::initial_backoff_secs] pub const DEFAULT_INITIAL_BACKOFF_SECS: u64 = 60; /// Default for [WatcherConfig::max_backoff_secs] pub const DEFAULT_MAX_BACKOFF_SECS: u64 = 3_600; + +// Retry configuration defaults +/// Default for [EventRetryConfig::max_retries] +pub const DEFAULT_MAX_RETRIES: u32 = 10; +/// Default for [EventRetryConfig::max_dependency_retries] +pub const DEFAULT_MAX_DEPENDENCY_RETRIES: u32 = 50; +/// Default for [EventRetryConfig::initial_backoff_secs] (transient errors) +pub const DEFAULT_INITIAL_TRANSIENT_BACKOFF_SECS: u64 = 10; +/// Default for [EventRetryConfig::max_backoff_secs] (transient errors) +pub const DEFAULT_MAX_TRANSIENT_BACKOFF_SECS: u64 = 3_600; +/// Default for [EventRetryConfig::initial_missing_dep_backoff_secs] +pub const DEFAULT_INITIAL_MISSING_DEP_BACKOFF_SECS: u64 = 60; +/// Default for [EventRetryConfig::max_missing_dep_backoff_secs] +pub const DEFAULT_MAX_MISSING_DEP_BACKOFF_SECS: u64 = 3_600; + // Default moderation service key (test user key, overridden by config.toml value) pub const DEFAULT_MODERATION_ID: &str = "uo7jgkykft4885n8cruizwy6khw71mnu5pq3ay9i8pw1ymcn85ko"; // Moderation service key @@ -31,19 +58,89 @@ pub const MODERATED_TAGS: [&str; 6] = [ "il_adult_nu_sex_act", ]; +/// Retry configuration settings +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +#[serde(default)] +pub struct EventRetryConfig { + /// Transient error retry limit before dead-letter + pub max_retries: u32, + /// Safety net for homeservers that disappear silently (no DEL events, content just gone) + pub max_dependency_retries: u32, + /// Base for exponential backoff on transient retries (seconds) + pub initial_backoff_secs: u64, + /// Backoff ceiling for transient retries (seconds) + pub max_backoff_secs: u64, + /// Base for MissingDependency polling backoff (seconds) + pub initial_missing_dep_backoff_secs: u64, + /// Backoff ceiling for MissingDependency (seconds) + pub max_missing_dep_backoff_secs: u64, +} + +impl Default for EventRetryConfig { + fn default() -> Self { + Self { + max_retries: DEFAULT_MAX_RETRIES, + max_dependency_retries: DEFAULT_MAX_DEPENDENCY_RETRIES, + initial_backoff_secs: DEFAULT_INITIAL_TRANSIENT_BACKOFF_SECS, + max_backoff_secs: DEFAULT_MAX_TRANSIENT_BACKOFF_SECS, + initial_missing_dep_backoff_secs: DEFAULT_INITIAL_MISSING_DEP_BACKOFF_SECS, + max_missing_dep_backoff_secs: DEFAULT_MAX_MISSING_DEP_BACKOFF_SECS, + } + } +} + +impl EventRetryConfig { + /// Returns (initial_backoff, max_backoff) values, in seconds, for the given error + pub fn get_backoff_params(&self, error: &EventProcessorError) -> (u64, u64) { + let initial = match error.is_missing_dependency() { + true => self.initial_missing_dep_backoff_secs, + false => self.initial_backoff_secs, + }; + let max = match error.is_missing_dependency() { + true => self.max_missing_dep_backoff_secs, + false => self.max_backoff_secs, + }; + (initial, max) + } +} + /// Configuration settings for the Nexus Watcher service #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] pub struct WatcherConfig { pub testnet: bool, pub testnet_host: String, + /// Default homeserver. Other homeservers may be ingested in addition, but this one is prioritized. pub homeserver: PubkyId, - /// Maximum number of events to fetch per run from each homeserver - pub events_limit: u32, + + /// Maximum number of events to fetch per run from the default homeserver. + /// Must not exceed [MAX_EVENTS_LIMIT]. + #[serde(deserialize_with = "deserialize_events_limit")] + pub events_limit: u16, + + /// Maximum events per user per run for key-based (non-default) homeservers. + /// Must not exceed [MAX_KEY_BASED_EVENTS_LIMIT]. + #[serde( + default = "default_key_based_events_limit", + deserialize_with = "deserialize_key_based_events_limit" + )] + pub key_based_events_limit: u16, + /// Maximum number of monitored homeservers pub monitored_homeservers_limit: usize, + /// Sleep between every full run (over all monitored homeservers), in milliseconds pub watcher_sleep: u64, + + /// Sleep between every run of the user HS resolver periodic task, in milliseconds + #[serde(default = "default_hs_resolver_sleep")] + pub hs_resolver_sleep: u64, + + /// Minimum time (ms) before a user's homeserver mapping is re-resolved. + /// Users whose `HOSTED_BY.resolved_at` is newer than this TTL are skipped. + #[serde(default = "default_hs_resolver_ttl")] + pub hs_resolver_ttl: u64, + /// Initial backoff duration (in seconds) after the first failure of a homeserver #[serde(default = "default_initial_backoff_secs")] pub initial_backoff_secs: u64, @@ -52,6 +149,11 @@ pub struct WatcherConfig { pub max_backoff_secs: u64, #[serde(default = "default_stack")] pub stack: StackConfig, + + // Retry configuration + #[serde(default)] + pub retry: EventRetryConfig, + // Moderation pub moderation_id: PubkyId, pub moderated_tags: Vec, @@ -72,16 +174,69 @@ impl Default for WatcherConfig { testnet_host: DEFAULT_TESTNET_HOST.to_string(), homeserver, events_limit: DEFAULT_EVENTS_LIMIT, + key_based_events_limit: DEFAULT_KEY_BASED_EVENTS_LIMIT, monitored_homeservers_limit: DEFAULT_MONITORED_HOMESERVERS_LIMIT, watcher_sleep: DEFAULT_WATCHER_SLEEP, + hs_resolver_sleep: DEFAULT_HS_RESOLVER_SLEEP, + hs_resolver_ttl: DEFAULT_HS_RESOLVER_TTL, initial_backoff_secs: DEFAULT_INITIAL_BACKOFF_SECS, max_backoff_secs: DEFAULT_MAX_BACKOFF_SECS, + retry: EventRetryConfig::default(), moderation_id, moderated_tags: MODERATED_TAGS.iter().map(|s| s.to_string()).collect(), } } } +fn default_hs_resolver_sleep() -> u64 { + DEFAULT_HS_RESOLVER_SLEEP +} + +fn default_key_based_events_limit() -> u16 { + DEFAULT_KEY_BASED_EVENTS_LIMIT +} + +fn deserialize_events_limit<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let val = u16::deserialize(deserializer)?; + if val == 0 { + return Err(D::Error::custom("events_limit must be at least 1")); + } + + if val > MAX_EVENTS_LIMIT { + let err_msg = format!("events_limit ({val}) exceeds max ({MAX_EVENTS_LIMIT})"); + Err(D::Error::custom(err_msg)) + } else { + Ok(val) + } +} + +fn deserialize_key_based_events_limit<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let val = u16::deserialize(deserializer)?; + if val == 0 { + return Err(D::Error::custom( + "key_based_events_limit must be at least 1", + )); + } + + if val > MAX_KEY_BASED_EVENTS_LIMIT { + let err_msg = + format!("key_based_events_limit ({val}) exceeds max ({MAX_KEY_BASED_EVENTS_LIMIT})"); + Err(D::Error::custom(err_msg)) + } else { + Ok(val) + } +} + +fn default_hs_resolver_ttl() -> u64 { + DEFAULT_HS_RESOLVER_TTL +} + /// Converts a [`DaemonConfig`] into an [`WatcherConfig`], extracting only the Watcher-related settings /// and the shared application stack impl From for WatcherConfig { diff --git a/nexus-common/src/db/connectors/pubky.rs b/nexus-common/src/db/connectors/pubky.rs index c3b56f96d..a7313552e 100644 --- a/nexus-common/src/db/connectors/pubky.rs +++ b/nexus-common/src/db/connectors/pubky.rs @@ -1,4 +1,4 @@ -use pubky::{Pubky, PubkyHttpClient}; +use pubky::{Pubky, PubkyHttpClient, StatusCode}; use serde::{Deserialize, Serialize}; use std::sync::Arc; use thiserror::Error; @@ -12,8 +12,84 @@ pub enum PubkyClientError { #[error("PubkyClient not initialized")] NotInitialized, - #[error("Client initialization error: {0}")] - ClientError(String), + #[error("404: {message}")] + NotFound404 { message: String }, + + #[error("Server error (5xx): {message}")] + ServerError5xx { message: String }, + + #[error("Request failed (is_transport: {is_transport}): {message}")] + RequestFailed { is_transport: bool, message: String }, + + #[error("Pkarr failed: {message}")] + PkarrFailed { message: String }, + + #[error("Authentication failed: {message}")] + AuthenticationFailed { message: String }, + + #[error("Build failed: {message}")] + BuildFailed { message: String }, + + #[error("Parse failed: {message}")] + ParseFailed { message: String }, +} + +impl From for PubkyClientError { + fn from(err: pubky::Error) -> Self { + match err { + pubky::Error::Request(req_err) => match req_err { + pubky::errors::RequestError::Server { status, message } => { + if status == StatusCode::NOT_FOUND { + Self::NotFound404 { message } + } else if status.is_server_error() { + Self::ServerError5xx { message } + } else { + Self::RequestFailed { + is_transport: false, + message, + } + } + } + pubky::errors::RequestError::Transport(err) => Self::RequestFailed { + is_transport: true, + message: err.to_string(), + }, + pubky::errors::RequestError::Validation { message } => Self::RequestFailed { + is_transport: false, + message, + }, + pubky::errors::RequestError::DecodeJson { message } => Self::RequestFailed { + is_transport: false, + message, + }, + }, + pubky::Error::Pkarr(pkarr_err) => Self::PkarrFailed { + message: pkarr_err.to_string(), + }, + pubky::Error::Authentication(auth_err) => Self::AuthenticationFailed { + message: auth_err.to_string(), + }, + pubky::Error::Build(build_err) => Self::BuildFailed { + message: build_err.to_string(), + }, + pubky::Error::Parse(parse_err) => Self::ParseFailed { + message: parse_err.to_string(), + }, + } + } +} + +impl PubkyClientError { + /// Returns true if this error is transient and worth retrying + pub fn is_retryable(&self) -> bool { + matches!( + self, + Self::NotInitialized + | Self::ServerError5xx { .. } + | Self::RequestFailed { .. } + | Self::PkarrFailed { .. } + ) + } } pub struct PubkyConnector; @@ -40,7 +116,7 @@ impl PubkyConnector { .build(), None => PubkyHttpClient::new(), } - .map_err(|e| PubkyClientError::ClientError(e.to_string()))?; + .map_err(|e| PubkyClientError::from(pubky::Error::from(e)))?; Ok(Arc::new(Pubky::with_client(client))) }) .await diff --git a/nexus-common/src/db/graph/error.rs b/nexus-common/src/db/graph/error.rs index 1974c540a..2195d134f 100644 --- a/nexus-common/src/db/graph/error.rs +++ b/nexus-common/src/db/graph/error.rs @@ -34,6 +34,17 @@ pub enum GraphError { Generic(String), } +impl GraphError { + #[allow(clippy::match_like_matches_macro)] + pub fn is_infrastructure_err(&self) -> bool { + match self { + GraphError::ConnectionNotInitialized => true, + GraphError::QueryFailed(_) => true, + _ => false, + } + } +} + impl From for GraphError { fn from(e: neo4rs::DeError) -> Self { GraphError::DeserializationFailed(Box::new(e)) diff --git a/nexus-common/src/db/graph/queries/del.rs b/nexus-common/src/db/graph/queries/del.rs index 323552763..edbf5e27c 100644 --- a/nexus-common/src/db/graph/queries/del.rs +++ b/nexus-common/src/db/graph/queries/del.rs @@ -106,6 +106,16 @@ pub fn delete_tag(user_id: &str, tag_id: &str, app: Option<&str>) -> Query { query } +/// Removes the `HOSTED_BY` relationship from a user, if one exists. +pub fn remove_user_homeserver(user_id: &str) -> Query { + Query::new( + "remove_user_homeserver", + "MATCH (u:User {id: $user_id})-[r:HOSTED_BY]->(:Homeserver) + DELETE r;", + ) + .param("user_id", user_id.to_string()) +} + /// Deletes a file node and all its relationships /// # Arguments /// * `owner_id` - The unique identifier of the user who owns the file diff --git a/nexus-common/src/db/graph/queries/get.rs b/nexus-common/src/db/graph/queries/get.rs index 473ff7cb1..523d133c1 100644 --- a/nexus-common/src/db/graph/queries/get.rs +++ b/nexus-common/src/db/graph/queries/get.rs @@ -385,14 +385,48 @@ pub fn get_homeserver_by_id(id: &str) -> Query { .param("id", id) } -/// Retrieves all homeserver IDs -pub fn get_all_homeservers() -> Query { +/// Retrieves all homeserver IDs that have at least one active user +/// (incoming `HOSTED_BY` relationships from `User` nodes). +/// +/// The results are sorted by the number of active users in descending order. +/// Returns a single `homeservers_list` column containing the collected IDs. +pub fn get_all_homeservers_with_active_users() -> Query { + Query::new( + "get_all_homeservers_with_active_users", + "MATCH (u:User)-[:HOSTED_BY]->(hs:Homeserver) + WHERE u.name <> '[DELETED]' + WITH hs.id AS id, count(u) AS active_users + ORDER BY active_users DESC + RETURN collect(id) AS homeservers_list", + ) +} + +/// Retrieves user IDs whose homeserver mapping is stale +/// (`resolved_at` is older than `ttl_ms`) or missing (no `HOSTED_BY` edge). +pub fn get_users_needing_hs_resolution(ttl_ms: u64) -> Query { + Query::new( + "get_users_needing_hs_resolution", + "MATCH (u:User) + WHERE u.name <> '[DELETED]' + OPTIONAL MATCH (u)-[r:HOSTED_BY]->(:Homeserver) + WITH u, r + WHERE r IS NULL + OR r.resolved_at IS NULL + OR r.resolved_at < (timestamp() - $ttl_ms) + RETURN collect(u.id) AS user_ids", + ) + .param("ttl_ms", ttl_ms as i64) +} + +/// Retrieves all user IDs hosted on a given homeserver. +pub fn get_users_by_homeserver(hs_id: &str) -> Query { Query::new( - "get_all_homeservers", - "MATCH (hs:Homeserver) - WITH collect(hs.id) AS homeservers_list - RETURN homeservers_list", + "get_users_by_homeserver", + "MATCH (u:User)-[:HOSTED_BY]->(:Homeserver {id: $hs_id}) + WHERE u.name <> '[DELETED]' + RETURN collect(u.id) AS user_ids", ) + .param("hs_id", hs_id.to_string()) } /// Retrieve tags for a user within the viewer's trusted network diff --git a/nexus-common/src/db/graph/queries/put.rs b/nexus-common/src/db/graph/queries/put.rs index 2f29c55df..82e915f91 100644 --- a/nexus-common/src/db/graph/queries/put.rs +++ b/nexus-common/src/db/graph/queries/put.rs @@ -366,3 +366,29 @@ pub fn create_homeserver(homeserver_id: &str) -> Query { ) .param("id", homeserver_id) } + +/// Sets the `HOSTED_BY` relationship between a user and a homeserver. +/// +/// If the user is already on the target homeserver, refreshes `resolved_at`. +/// If the user is on a different homeserver, replaces the old relationship. +/// MERGEs the target homeserver node if it doesn't exist yet. +pub fn set_user_homeserver(user_id: &str, homeserver_id: &str) -> Query { + Query::new( + "set_user_homeserver", + "MATCH (u:User {id: $user_id}) + + // Remove existing HOSTED_BY only if homeserver changed + OPTIONAL MATCH (u)-[old:HOSTED_BY]->(old_hs:Homeserver) + WHERE old_hs.id <> $hs_id + DELETE old + + WITH u + + // Ensure target homeserver and relationship exist + MERGE (hs:Homeserver {id: $hs_id}) + MERGE (u)-[r:HOSTED_BY]->(hs) + SET r.resolved_at = timestamp()", + ) + .param("user_id", user_id.to_string()) + .param("hs_id", homeserver_id.to_string()) +} diff --git a/nexus-common/src/db/kv/error.rs b/nexus-common/src/db/kv/error.rs index 05e4ff5d1..024d48fc3 100644 --- a/nexus-common/src/db/kv/error.rs +++ b/nexus-common/src/db/kv/error.rs @@ -27,6 +27,18 @@ pub enum RedisError { InvalidInput(String), } +impl RedisError { + #[allow(clippy::match_like_matches_macro)] + pub fn is_infrastructure_err(&self) -> bool { + match self { + RedisError::ConnectionNotInitialized => true, + RedisError::ConnectionPoolError(_) => true, + RedisError::IoError(_) => true, + _ => false, + } + } +} + impl From for RedisError { fn from(e: redis::RedisError) -> Self { if e.is_connection_refusal() || e.is_timeout() || e.is_io_error() { diff --git a/nexus-common/src/lib.rs b/nexus-common/src/lib.rs index f7b9e0276..6ffed13c9 100644 --- a/nexus-common/src/lib.rs +++ b/nexus-common/src/lib.rs @@ -13,13 +13,14 @@ //! //! This crate forms the foundation for other Nexus services, ensuring consistency and reuse across the backend. -mod config; +pub mod config; pub mod db; mod macros; pub mod media; pub mod models; mod stack; pub mod types; +pub mod universal_tag; pub mod utils; pub use config::*; diff --git a/nexus-common/src/models/event/errors.rs b/nexus-common/src/models/event/errors.rs index cfe622251..eea83a7b3 100644 --- a/nexus-common/src/models/event/errors.rs +++ b/nexus-common/src/models/event/errors.rs @@ -1,24 +1,24 @@ +use std::fmt::Display; + use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::{ - db::{kv::RedisError, GraphError}, - models::error::ModelError, -}; +use crate::db::{kv::RedisError, GraphError, PubkyClientError}; +use crate::models::error::ModelError; #[derive(Error, Debug, Clone, Serialize, Deserialize)] pub enum EventProcessorError { /// Failed to execute query in the graph database - #[error("GraphQueryFailed: {0}")] - GraphQueryFailed(String), + #[error("GraphQueryFailed (is_infrastructure_err: {0}): {1}")] + GraphQueryFailed(bool, String), /// The event could not be indexed due to missing graph dependencies #[error("MissingDependency: Could not be indexed")] MissingDependency { dependency: Vec }, /// Failed to complete indexing due to a Redis operation error - #[error("IndexOperationFailed: Indexing incomplete due to Redis error - {0}")] - IndexOperationFailed(String), + #[error("IndexOperationFailed (is_infrastructure_err: {0}): Indexing incomplete due to Redis error: {1}")] + IndexOperationFailed(bool, String), /// The event appears to be unindexed. Verify the event in the retry queue #[error("SkipIndexing: The PUT event appears to be unindexed, so we cannot delete an object that doesn't exist")] @@ -28,9 +28,16 @@ pub enum EventProcessorError { #[error("InvalidEventLine: {0}")] InvalidEventLine(String), + #[error("HS returned an event for different user than expected: hs_id={hs_id}, expected={expected_user_id}, received={event_user_id}")] + UserIdMismatch { + hs_id: String, + expected_user_id: String, + event_user_id: String, + }, + /// The Pubky client could not resolve the pubky #[error("PubkyClientError: {0}")] - PubkyClientError(#[from] crate::db::PubkyClientError), + PubkyClientError(#[from] PubkyClientError), #[error("MediaProcessor: {0}")] MediaProcessorError(String), @@ -50,10 +57,12 @@ impl From for EventProcessorError { fn from(e: ModelError) -> Self { match e { ModelError::GraphOperationFailed(source) => { - EventProcessorError::GraphQueryFailed(source.to_string()) + let is_infrastructure_err = source.is_infrastructure_err(); + EventProcessorError::GraphQueryFailed(is_infrastructure_err, source.to_string()) } ModelError::KvOperationFailed(source) => { - EventProcessorError::IndexOperationFailed(source.to_string()) + let is_infrastructure_err = source.is_infrastructure_err(); + EventProcessorError::IndexOperationFailed(is_infrastructure_err, source.to_string()) } ModelError::MediaProcessorError(source) => { EventProcessorError::MediaProcessorError(source.to_string()) @@ -68,7 +77,7 @@ impl From for EventProcessorError { impl From for EventProcessorError { fn from(e: pubky::Error) -> Self { - EventProcessorError::client_error(e.to_string()) + EventProcessorError::PubkyClientError(PubkyClientError::from(e)) } } @@ -80,13 +89,15 @@ impl From for EventProcessorError { impl From for EventProcessorError { fn from(e: RedisError) -> Self { - EventProcessorError::IndexOperationFailed(e.to_string()) + let is_infrastructure_err = e.is_infrastructure_err(); + EventProcessorError::IndexOperationFailed(is_infrastructure_err, e.to_string()) } } impl From for EventProcessorError { fn from(e: GraphError) -> Self { - EventProcessorError::GraphQueryFailed(e.to_string()) + let is_infrastructure_err = e.is_infrastructure_err(); + EventProcessorError::GraphQueryFailed(is_infrastructure_err, e.to_string()) } } @@ -98,18 +109,69 @@ impl EventProcessorError { } pub fn client_error(message: String) -> Self { - Self::PubkyClientError(crate::db::PubkyClientError::ClientError(message)) + Self::PubkyClientError(PubkyClientError::RequestFailed { + is_transport: false, + message, + }) } - pub fn static_save_failed(source: impl std::fmt::Display) -> Self { - Self::StaticSaveFailed(source.to_string()) + pub fn client_error_404(message: String) -> Self { + Self::PubkyClientError(PubkyClientError::NotFound404 { message }) } - pub fn graph_query_failed(source: impl std::fmt::Display) -> Self { - Self::GraphQueryFailed(source.to_string()) + pub fn static_save_failed(source: impl Display) -> Self { + Self::StaticSaveFailed(source.to_string()) } - pub fn generic(source: impl std::fmt::Display) -> Self { + pub fn generic(source: impl Display) -> Self { Self::Generic(source.to_string()) } + + pub fn internal_error(source: impl Display) -> Self { + Self::InternalError(source.to_string()) + } + + /// Returns whether or not this is an infrastructure error. + /// + /// These are the kinds of errors that are expected to be thrown again, + /// if the event processor caller continues processing other events. + #[allow(clippy::match_like_matches_macro)] + pub fn is_infrastructure(&self) -> bool { + match self { + Self::GraphQueryFailed(true, _) => true, + Self::IndexOperationFailed(true, _) => true, + Self::PubkyClientError(err) => match err { + PubkyClientError::NotFound404 { .. } + | PubkyClientError::ServerError5xx { .. } + | PubkyClientError::NotInitialized + | PubkyClientError::PkarrFailed { .. } + | PubkyClientError::AuthenticationFailed { .. } + | PubkyClientError::BuildFailed { .. } + | PubkyClientError::ParseFailed { .. } => false, + PubkyClientError::RequestFailed { is_transport, .. } => *is_transport, + }, + _ => false, + } + } + + /// Returns whether this error is transient and worth queuing for retry. + /// + /// Default is **retryable**: when in doubt we enqueue rather than drop, since + /// `max_retries` bounds the waste on a misclassified deterministic error, + /// while silently dropping a misclassified transient error loses data outright. + /// Only variants we know to be deterministic at conversion time opt out. + pub fn is_retryable(&self) -> bool { + match self { + Self::PubkyClientError(err) => err.is_retryable(), + Self::InvalidEventLine(_) => false, + Self::SkipIndexing => false, + Self::UserIdMismatch { .. } => false, + _ => true, + } + } + + /// Returns whether this error is a missing dependency + pub fn is_missing_dependency(&self) -> bool { + matches!(self, Self::MissingDependency { .. }) + } } diff --git a/nexus-common/src/models/event/mod.rs b/nexus-common/src/models/event/mod.rs index 88e8dda62..0b0c76ea4 100644 --- a/nexus-common/src/models/event/mod.rs +++ b/nexus-common/src/models/event/mod.rs @@ -1,10 +1,14 @@ mod errors; -use crate::db::{kv::RedisResult, RedisOps}; -use pubky_app_specs::{ParsedUri, Resource}; +use crate::{ + db::{kv::RedisResult, RedisOps}, + universal_tag::homeserver_parsed_uri::HomeserverParsedUri, +}; +use pubky::Event as StreamEvent; +use pubky_app_specs::Resource; use serde::{Deserialize, Serialize}; use std::{fmt, path::PathBuf}; -use tracing::{debug, error}; +use tracing::{debug, error, warn}; pub use errors::EventProcessorError; @@ -14,6 +18,15 @@ pub enum EventType { Del, } +impl From for EventType { + fn from(value: pubky::EventType) -> Self { + match value { + pubky::EventType::Put { .. } => Self::Put, + pubky::EventType::Delete => Self::Del, + } + } +} + /// Result of parsing an event line from a homeserver. #[allow(clippy::large_enum_variant)] #[derive(Debug)] @@ -61,7 +74,7 @@ pub struct Event { pub event_type: EventType, /// Parsed representation of [`Self::uri`]. - pub parsed_uri: ParsedUri, + pub parsed_uri: HomeserverParsedUri, /// Local files directory on Nexus used for file-backed events. pub files_path: PathBuf, @@ -79,9 +92,7 @@ impl AsRef<[String]> for Event { } impl Event { - /// Parse event based on event line returned by homeservers' /events endpoint. - /// - line - event line string - /// - files_path - path to the directory where files are stored on nexus + /// Parse event from a line returned by the homeserver's `/events` endpoint. pub fn parse_event( line: &str, files_path: PathBuf, @@ -102,31 +113,60 @@ impl Event { ))), }?; - // Validate and parse the URI using pubky-app-specs let uri = parts[1].to_string(); - let parsed_uri = match ParsedUri::try_from(uri.as_str()) { - Ok(parsed) => parsed, - Err(e) => { - let reason = e.to_string(); - return Ok(ParseResult::unrecognized_uri(event_type, uri, reason)); - } - }; + let event_line = line.to_string(); - match parsed_uri.resource { - // Unknown resource - Resource::Unknown => { - return Err(EventProcessorError::InvalidEventLine(format!( - "Unknown resource in URI: {uri}" - ))) - } - // Known resources not handled by Nexus - Resource::LastRead | Resource::Feed(_) | Resource::Blob(_) => { - return Ok(ParseResult::Skipped) + Self::parse_event_parts(event_type, uri, event_line, files_path) + } + + /// Constructs a nexus [`Event`] directly from a [`StreamEvent`], avoiding + /// the string round-trip through [`Self::parse_event`]. + pub fn from_stream_event( + stream_event: &StreamEvent, + files_path: PathBuf, + ) -> Result, EventProcessorError> { + let event_type: EventType = stream_event.event_type.clone().into(); + + let uri = stream_event.resource.to_pubky_url(); + debug!("New stream event: {event_type} {uri}"); + + let event_line = format!("{event_type} {uri}"); + match Self::parse_event_parts(event_type, uri, event_line, files_path)? { + ParseResult::Parsed(event) => Ok(Some(event)), + ParseResult::Skipped => Ok(None), + ParseResult::UnrecognizedUri { reason, .. } => { + warn!("Unrecognized event URI: {reason}"); + Ok(None) } - _ => (), + } + } + + fn parse_event_parts( + event_type: EventType, + uri: String, + event_line: String, + files_path: PathBuf, + ) -> Result { + // Validate and parse the URI using HomeserverParsedUri. This handles both + // standard pubky-app-specs URIs and universal tag URIs from other apps. + let parsed_uri = match HomeserverParsedUri::try_from(uri.as_str()) { + Ok(parsed) => parsed, + Err(e) => return Ok(ParseResult::unrecognized_uri(event_type, uri, e)), }; - let event_line = line.to_string(); + if let HomeserverParsedUri::AppSpec { resource, .. } = &parsed_uri { + match resource { + Resource::Unknown => { + return Err(EventProcessorError::InvalidEventLine(format!( + "Unknown resource in URI: {uri}" + ))) + } + Resource::LastRead | Resource::Feed(_) | Resource::Blob(_) => { + return Ok(ParseResult::Skipped) + } + _ => (), + } + } Ok(ParseResult::Parsed(Event { uri, diff --git a/nexus-common/src/models/homeserver.rs b/nexus-common/src/models/homeserver.rs index 8d32bed27..5e6fe23b3 100644 --- a/nexus-common/src/models/homeserver.rs +++ b/nexus-common/src/models/homeserver.rs @@ -3,14 +3,10 @@ use crate::db::fetch_key_from_graph; use crate::db::kv::RedisError; use crate::db::kv::RedisResult; use crate::db::queries; -use crate::db::GraphError; use crate::db::GraphResult; -use crate::db::{PubkyConnector, RedisOps}; +use crate::db::RedisOps; use crate::models::error::ModelError; use crate::models::error::ModelResult; -use crate::models::user::UserDetails; - -use pubky_app_specs::ParsedUri; use pubky_app_specs::PubkyId; use serde::{Deserialize, Serialize}; use tracing::info; @@ -106,63 +102,15 @@ impl Homeserver { Ok(()) } - /// Retrieves all homeservers from the graph. + /// Returns all HS IDs with at least one active user, sorted by user count descending. /// /// # Returns - /// A list of all known homeserver IDs. - /// - /// # Errors - /// Throws an error if no homeservers are found. - pub async fn get_all_from_graph() -> GraphResult> { - let query = queries::get::get_all_homeservers(); + /// A list of active homeserver IDs. + pub async fn get_all_active_from_graph() -> GraphResult> { + let query = queries::get::get_all_homeservers_with_active_users(); let maybe_hs_ids = fetch_key_from_graph(query, "homeservers_list").await?; let hs_ids: Vec = maybe_hs_ids.unwrap_or_default(); - - match hs_ids.is_empty() { - true => Err(GraphError::Generic("No homeservers found in graph".into())), - false => Ok(hs_ids), - } - } - - /// If a referenced post is hosted on a new, unknown homeserver, this method triggers ingestion of that homeserver. - /// - /// ### Arguments - /// - /// - `referenced_post_uri`: The parent post (if current post is a reply to it), or a reposted post (if current post is a Repost) - pub async fn maybe_ingest_for_post(referenced_post_uri: &ParsedUri) -> ModelResult<()> { - Self::maybe_ingest_for_user(&referenced_post_uri.user_id).await - } - - /// If a referenced user is using a new, unknown homeserver, this method triggers ingestion of that homeserver. - /// - /// ### Arguments - /// - /// - `referenced_user_id`: The `PubkyId` of the referenced user - #[tracing::instrument(name = "homeserver.ingest", skip_all)] - pub async fn maybe_ingest_for_user(referenced_user_id: &PubkyId) -> ModelResult<()> { - let pubky = PubkyConnector::get().map_err(ModelError::from_generic)?; - - if UserDetails::get_by_id(referenced_user_id.as_ref()) - .await? - .is_some() - { - tracing::debug!( - "Skipping homeserver ingestion: author {referenced_user_id} already known" - ); - return Ok(()); - } - - let ref_post_author_pk = referenced_user_id.to_public_key(); - let Some(ref_post_author_hs) = pubky.get_homeserver_of(&ref_post_author_pk).await else { - tracing::warn!("Skipping homeserver ingestion: author {ref_post_author_pk} has no published homeserver"); - return Ok(()); - }; - - let hs_pk = PubkyId::from(ref_post_author_hs); - Self::persist_if_unknown(hs_pk.clone()) - .await - .inspect(|_| tracing::info!("Ingested homeserver {hs_pk}")) - .inspect_err(|e| tracing::error!("Failed to ingest homeserver {hs_pk}: {e}")) + Ok(hs_ids) } } @@ -180,7 +128,7 @@ mod tests { StackManager::setup(&StackConfig::default()).await?; let keys = Keypair::random(); - let id = PubkyId::try_from(&keys.public_key().to_z32())?; + let id = PubkyId::from(keys.public_key()); let hs = Homeserver::new(id.clone()); hs.put_to_graph() @@ -203,7 +151,7 @@ mod tests { StackManager::setup(&StackConfig::default()).await?; let keys = Keypair::random(); - let id = PubkyId::try_from(&keys.public_key().to_z32())?; + let id = PubkyId::from(keys.public_key()); let hs = Homeserver::new(id.clone()); hs.put_to_index() diff --git a/nexus-common/src/models/post/details.rs b/nexus-common/src/models/post/details.rs index baec073a9..8035b8edf 100644 --- a/nexus-common/src/models/post/details.rs +++ b/nexus-common/src/models/post/details.rs @@ -4,8 +4,9 @@ use crate::db::{ execute_graph_operation, fetch_row_from_graph, queries, GraphResult, OperationOutcome, RedisOps, }; use crate::models::error::ModelResult; +use crate::models::user::UserDetails; use chrono::Utc; -use pubky_app_specs::{post_uri_builder, PubkyAppPost, PubkyAppPostKind, PubkyId}; +use pubky_app_specs::{post_uri_builder, ParsedUri, PubkyAppPost, PubkyAppPostKind, PubkyId}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; @@ -167,60 +168,19 @@ impl PostDetails { Ok(()) } + /// If a referenced post is authored by a new, unknown user, this method triggers ingestion of that user. + /// + /// ### Arguments + /// + /// - `referenced_post_uri`: The parent post (if current post is a reply to it), or a reposted post (if current post is a Repost) + pub async fn maybe_ingest_author_of_post(referenced_post_uri: &ParsedUri) -> ModelResult<()> { + let ref_post_author_id = referenced_post_uri.user_id.as_str(); + + UserDetails::maybe_ingest_user(ref_post_author_id).await + } + /// Determines whether or not a given [PostDetails] is different than (e.g. may be an edit of) this post. pub fn is_different_than(&self, other: &PostDetails) -> bool { self.content != other.content || self.attachments != other.attachments } } - -#[cfg(test)] -mod tests { - use super::*; - use pubky_app_specs::PubkyAppPostKind; - - #[tokio_shared_rt::test(shared)] - async fn test_is_different_than() { - // Create a base PostDetails - let base_post = PostDetails { - content: "Original content".into(), - id: "post1".into(), - indexed_at: 123456789, - author: "author1".into(), - kind: PubkyAppPostKind::Short, - uri: "uri1".into(), - attachments: Some(vec!["image1.jpg".into(), "image2.jpg".into()]), - }; - - // Test with same content and attachments - let same_post = base_post.clone(); - assert!(!base_post.is_different_than(&same_post)); - - // Test with same attachments but different order - let different_order_attachments_post = PostDetails { - attachments: Some(vec!["image2.jpg".into(), "image1.jpg".into()]), - ..base_post.clone() - }; - assert!(base_post.is_different_than(&different_order_attachments_post)); - - // Test with different content - let different_content_post = PostDetails { - content: "Updated content".to_string(), - ..base_post.clone() - }; - assert!(base_post.is_different_than(&different_content_post)); - - // Test with different attachments - let different_attachments_post = PostDetails { - attachments: Some(vec!["image3.jpg".to_string()]), - ..base_post.clone() - }; - assert!(base_post.is_different_than(&different_attachments_post)); - - // Test with no attachments - let no_attachments_post = PostDetails { - attachments: None, - ..base_post.clone() - }; - assert!(base_post.is_different_than(&no_attachments_post)); - } -} diff --git a/nexus-common/src/models/resource/mod.rs b/nexus-common/src/models/resource/mod.rs index 862faac6e..f8c83a71f 100644 --- a/nexus-common/src/models/resource/mod.rs +++ b/nexus-common/src/models/resource/mod.rs @@ -2,147 +2,9 @@ pub mod stream; pub mod tag; pub mod view; -use pubky_app_specs::{ParsedUri, Resource}; use serde::{Deserialize, Serialize}; -use url::Url; use utoipa::ToSchema; -// --------------------------------------------------------------------------- -// URI Normalization (universal_tags_specs.md Section 5) -// --------------------------------------------------------------------------- - -/// Normalizes a URI for deterministic Resource identification. -/// Returns `(normalized_uri, scheme)`. -/// -/// Rules: -/// - Lowercase scheme and host -/// - Strip default ports (80 for http, 443 for https) -/// - Strip fragments and userinfo -/// - Preserve path and query as-is -/// - Fallback for non-standard schemes that fail URL parsing -pub fn normalize_uri(uri: &str) -> Result<(String, String), String> { - match Url::parse(uri) { - Ok(parsed) => { - let scheme = parsed.scheme().to_string(); - let normalized = normalize_parsed_url(&parsed); - Ok((normalized, scheme)) - } - Err(_) => { - // Fallback for non-standard schemes (e.g., nostr:note1abc...) - // that may not parse as URLs (no // authority) - if let Some(colon_pos) = uri.find(':') { - let scheme = &uri[..colon_pos]; - // Validate scheme: RFC 3986 §3.1 — ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) - let mut scheme_bytes = scheme.bytes(); - let starts_with_alpha = - scheme_bytes.next().is_some_and(|b| b.is_ascii_alphabetic()); - let remainder_is_valid = scheme_bytes - .all(|b| b.is_ascii_alphanumeric() || b == b'+' || b == b'-' || b == b'.'); - if !starts_with_alpha || !remainder_is_valid { - return Err(format!("Invalid URI scheme: {uri}")); - } - let scheme_lower = scheme.to_ascii_lowercase(); - let remainder = &uri[colon_pos + 1..]; - Ok((format!("{scheme_lower}:{remainder}"), scheme_lower)) - } else { - Err(format!("Invalid URI: {uri}")) - } - } - } -} - -fn normalize_parsed_url(parsed: &Url) -> String { - let scheme = parsed.scheme(); // already lowercase per url crate - - // For non-hierarchical schemes (no authority, e.g. nostr:note1abc), - // the url crate parses them as "cannot-be-a-base" URLs. - // Return scheme + ":" + opaque path (no //) - if parsed.cannot_be_a_base() { - // Strip fragment from the opaque form - let full = parsed.as_str(); - let without_fragment = match full.find('#') { - Some(pos) => &full[..pos], - None => full, - }; - return without_fragment.to_string(); - } - - let host = parsed - .host_str() - .map(|h| h.to_ascii_lowercase()) - .unwrap_or_default(); - - // Strip default ports - let port = match (parsed.port(), scheme) { - (Some(80), "http") => None, - (Some(443), "https") => None, - (other, _) => other, - }; - - let path = parsed.path(); - let query = parsed.query(); - // Fragment and userinfo are discarded - - let mut result = format!("{scheme}://{host}"); - if let Some(p) = port { - result.push_str(&format!(":{p}")); - } - result.push_str(path); - if let Some(q) = query { - result.push('?'); - result.push_str(q); - } - result -} - -// --------------------------------------------------------------------------- -// Resource ID (universal_tags_specs.md Section 6) -// --------------------------------------------------------------------------- - -/// Generates a deterministic 32-char hex Resource ID from a normalized URI. -/// -/// `resource_id = hex(BLAKE3(normalized_uri)[0..16])` -pub fn resource_id(normalized_uri: &str) -> String { - let hash = blake3::hash(normalized_uri.as_bytes()); - hex::encode(&hash.as_bytes()[..16]) -} - -// --------------------------------------------------------------------------- -// URI Classification (universal_tags_specs.md Section 7) -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, PartialEq)] -pub enum UriCategory { - /// pubky:// URI matching a known schema (posts, users) - InternalKnown, - /// pubky:// URI NOT matching any known schema (e.g., eventky events) - InternalUnknown, - /// Non-pubky:// URI (https://, nostr:, ipfs://, etc.) - External, -} - -/// Classifies a tag's target URI into one of three categories. -/// Scheme check is case-insensitive per RFC 3986. -pub fn classify_uri(uri: &str) -> UriCategory { - let is_pubky = uri - .get(..8) - .is_some_and(|s| s.eq_ignore_ascii_case("pubky://")); - if is_pubky { - match ParsedUri::try_from(uri) { - Ok(parsed) if matches!(parsed.resource, Resource::Post(_) | Resource::User) => { - UriCategory::InternalKnown - } - _ => UriCategory::InternalUnknown, - } - } else { - UriCategory::External - } -} - -// --------------------------------------------------------------------------- -// ResourceDetails -// --------------------------------------------------------------------------- - #[derive(Debug, Clone, Serialize, Deserialize, Default, ToSchema)] pub struct ResourceDetails { pub id: String, @@ -164,130 +26,3 @@ impl ResourceDetails { })) } } - -#[cfg(test)] -mod tests { - use super::*; - - // -- normalize_uri test vectors from spec Section 5 -- - - #[test] - fn test_normalize_https_with_default_port_and_fragment() { - let (uri, scheme) = normalize_uri("HTTPS://Example.COM:443/path?q=1#frag").unwrap(); - assert_eq!(uri, "https://example.com/path?q=1"); - assert_eq!(scheme, "https"); - } - - #[test] - fn test_normalize_root_trailing_slash() { - let (uri, _) = normalize_uri("https://example.com").unwrap(); - assert_eq!(uri, "https://example.com/"); - } - - #[test] - fn test_normalize_non_default_port() { - let (uri, _) = normalize_uri("http://example.com:8080/path").unwrap(); - assert_eq!(uri, "http://example.com:8080/path"); - } - - #[test] - fn test_normalize_pubky_uri() { - let input = "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/eventky.app/events/E001"; - let (uri, scheme) = normalize_uri(input).unwrap(); - assert_eq!(uri, input); - assert_eq!(scheme, "pubky"); - } - - #[test] - fn test_normalize_query_order_preserved() { - let (uri, _) = normalize_uri("HTTPS://Example.COM/path?b=2&a=1").unwrap(); - assert_eq!(uri, "https://example.com/path?b=2&a=1"); - } - - #[test] - fn test_normalize_strip_userinfo() { - let (uri, _) = normalize_uri("https://user:pass@example.com/path").unwrap(); - assert_eq!(uri, "https://example.com/path"); - } - - #[test] - fn test_normalize_nostr_fallback() { - let (uri, scheme) = normalize_uri("nostr:note1abc123...").unwrap(); - assert_eq!(uri, "nostr:note1abc123..."); - assert_eq!(scheme, "nostr"); - } - - #[test] - fn test_normalize_http_default_port() { - let (uri, _) = normalize_uri("HTTP://Example.COM:80/").unwrap(); - assert_eq!(uri, "http://example.com/"); - } - - #[test] - fn test_normalize_rejects_malformed_scheme() { - // "ht tps" has a space — invalid per RFC 3986 §3.1 - assert!(normalize_uri("ht tps://example.com").is_err()); - } - - #[test] - fn test_normalize_rejects_scheme_starting_with_digit() { - // RFC 3986 §3.1 requires the first scheme character to be alphabetic. - assert!(normalize_uri("1ttp://example.com").is_err()); - } - - #[test] - fn test_normalize_rejects_no_colon() { - assert!(normalize_uri("justtext").is_err()); - } - - // -- resource_id tests -- - - #[test] - fn test_resource_id_deterministic() { - let uri = "https://example.com/path?q=1"; - let id1 = resource_id(uri); - let id2 = resource_id(uri); - assert_eq!(id1, id2); - assert_eq!(id1.len(), 32); - } - - #[test] - fn test_resource_id_different_uris() { - let id1 = resource_id("https://example.com/a"); - let id2 = resource_id("https://example.com/b"); - assert_ne!(id1, id2); - } - - // -- classify_uri tests -- - - #[test] - fn test_classify_external_https() { - assert_eq!( - classify_uri("https://example.com/article"), - UriCategory::External - ); - } - - #[test] - fn test_classify_external_nostr() { - assert_eq!(classify_uri("nostr:note1abc123"), UriCategory::External); - } - - #[test] - fn test_classify_internal_unknown_eventky() { - // eventky URI is pubky:// but not a recognized pubky.app resource - assert_eq!( - classify_uri("pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/eventky.app/events/E001"), - UriCategory::InternalUnknown - ); - } - - #[test] - fn test_classify_uppercase_pubky_scheme() { - // RFC 3986: schemes are case-insensitive - assert_eq!( - classify_uri("PUBKY://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/eventky.app/events/E001"), - UriCategory::InternalUnknown - ); - } -} diff --git a/nexus-common/src/models/user/details.rs b/nexus-common/src/models/user/details.rs index 00d8bf383..8f5d80d1e 100644 --- a/nexus-common/src/models/user/details.rs +++ b/nexus-common/src/models/user/details.rs @@ -1,16 +1,25 @@ use super::UserSearch; use crate::db::graph::Query; use crate::db::kv::RedisResult; -use crate::db::{exec_single_row, queries, GraphResult, RedisOps}; -use crate::models::error::ModelResult; +use crate::db::{exec_single_row, queries, GraphResult, PubkyConnector, RedisOps}; +use crate::models::error::{ModelError, ModelResult}; use crate::models::traits::Collection; use async_trait::async_trait; use chrono::Utc; +use pubky::PublicKey; use pubky_app_specs::{PubkyAppUser, PubkyAppUserLink, PubkyId}; use serde::{Deserialize, Deserializer, Serialize}; use serde_json; use utoipa::ToSchema; +pub const USER_HS_CURSOR: [&str; 2] = ["Users", "Homeservers"]; + +/// Builds the Redis key path for per-user homeserver cursor storage: +/// `["Users", "Homeservers", ]`. +pub fn user_hs_cursor_key(user_id: &str) -> [&str; 3] { + [USER_HS_CURSOR[0], USER_HS_CURSOR[1], user_id] +} + #[async_trait] impl RedisOps for UserDetails {} @@ -90,6 +99,20 @@ impl UserDetails { Ok(details_collection.into_iter().flatten().next()) } + /// Creates a minimal `UserDetails` with only the public key. + /// All profile fields (bio, links, status, image) default to `None`. + pub fn from_pubky(user_id: PubkyId) -> Self { + UserDetails { + name: user_id.to_string(), + id: user_id.clone(), + indexed_at: Utc::now().timestamp_millis(), + bio: None, + links: None, + status: None, + image: None, + } + } + pub fn from_homeserver(homeserver_user: PubkyAppUser, user_id: &PubkyId) -> Self { UserDetails { name: homeserver_user.name, @@ -110,6 +133,48 @@ impl UserDetails { Ok(()) } + + /// If a referenced user is unknown, not ingested in the graph yet, resolves their homeserver + /// and persists the user node in the graph. + #[tracing::instrument(name = "user.ingest", skip_all)] + pub async fn maybe_ingest_user(user_id: &str) -> ModelResult<()> { + if Self::get_by_id(user_id).await?.is_some() { + tracing::debug!("Skipping user ingestion: {user_id} already known"); + return Ok(()); + } + + let pubky = PubkyConnector::get().map_err(ModelError::from_generic)?; + + let user_pk = user_id + .parse::() + .map_err(ModelError::from_generic)?; + + let Some(hs_pk) = pubky.get_homeserver_of(&user_pk).await else { + tracing::warn!( + "Skipping user ingestion: {user_id} has no published homeserver or it's a homeserver pubky" + ); + return Ok(()); + }; + + let pubky_id = PubkyId::from(user_pk); + let user_details = Self::from_pubky(pubky_id); + + let hs_id = &hs_pk.into_inner().to_z32(); + + // Do not add to index, as this would affect the timeline of events for this user. + // Only create stub graph node for HS-resolver to store user-HS mapping. + user_details + .put_to_graph() + .await + .inspect(|_| tracing::info!("Ingested user {user_id} from homeserver {hs_id}")) + .inspect_err(|e| tracing::error!("Failed to ingest user {user_id}: {e}"))?; + + // Store the start point of the homeserver cursor + let key = user_hs_cursor_key(user_id); + Self::put_index_sorted_set(&key, &[(0.0, hs_id)], None, None).await?; + + Ok(()) + } } #[cfg(test)] diff --git a/nexus-common/src/models/user/mod.rs b/nexus-common/src/models/user/mod.rs index f4a366273..190a64025 100644 --- a/nexus-common/src/models/user/mod.rs +++ b/nexus-common/src/models/user/mod.rs @@ -9,7 +9,7 @@ mod tags; mod view; pub use counts::UserCounts; -pub use details::UserDetails; +pub use details::{user_hs_cursor_key, UserDetails, USER_HS_CURSOR}; pub use influencers::Influencers; pub use relationship::Relationship; pub use search::{UserSearch, USER_NAME_KEY_PARTS}; diff --git a/nexus-watcher/src/events/handlers/universal_tag.rs b/nexus-common/src/universal_tag/app_tag_info.rs similarity index 52% rename from nexus-watcher/src/events/handlers/universal_tag.rs rename to nexus-common/src/universal_tag/app_tag_info.rs index 866afa22b..d13253465 100644 --- a/nexus-watcher/src/events/handlers/universal_tag.rs +++ b/nexus-common/src/universal_tag/app_tag_info.rs @@ -1,9 +1,4 @@ -use nexus_common::db::PubkyConnector; -use nexus_common::models::event::{EventProcessorError, EventType}; -use pubky_app_specs::{PubkyAppTag, PubkyId}; -use tracing::debug; - -use super::tag; +use pubky_app_specs::PubkyId; /// Info extracted from a universal tag path: `pubky:///pub//tags/` pub struct AppTagInfo { @@ -13,66 +8,6 @@ pub struct AppTagInfo { pub uri: String, } -/// Second-chance handler for possible universal-tag events. -/// -/// Called when `Event::parse_event()` returns `UnrecognizedUri`. -/// -/// Returns `None` if the URI isn't an app-specific tag path. -/// Returns `Some(Ok(()))` on success or `Some(Err(...))` on processing failure. -pub async fn try_handle( - event_type: &EventType, - uri: &str, -) -> Option> { - let info = try_parse_app_tag_path(uri)?; - - debug!( - "Universal tag event: {} {} (app={})", - event_type, info.uri, info.app - ); - - Some(match event_type { - EventType::Put => handle_put(info).await, - EventType::Del => handle_del(info).await, - }) -} - -async fn handle_put(info: AppTagInfo) -> Result<(), EventProcessorError> { - // Fetch the tag blob from the homeserver - let pubky = PubkyConnector::get()?; - let response = pubky.public_storage().get(&info.uri).await?; - - if !response.status().is_success() { - let status = response.status(); - let body = response - .text() - .await - .unwrap_or_else(|_| "".to_string()); - return Err(EventProcessorError::client_error(format!( - "Fetch universal tag failed {}: HTTP {status} - {body}", - info.uri - ))); - } - - let blob = response - .bytes() - .await - .map_err(|e| EventProcessorError::client_error(e.to_string()))?; - - // Deserialize as PubkyAppTag — if it's not a valid tag, this fails cleanly - let app_tag: PubkyAppTag = serde_json::from_slice(&blob).map_err(|e| { - EventProcessorError::generic(format!( - "Failed to deserialize universal tag at {}: {e}", - info.uri - )) - })?; - - tag::sync_put_resource(app_tag, info.user_id, info.tag_id, info.app).await -} - -async fn handle_del(info: AppTagInfo) -> Result<(), EventProcessorError> { - tag::del(info.user_id, info.tag_id).await -} - /// Try to parse a URI as an app-specific tag path. /// /// Matches: `pubky:///pub//tags/` @@ -80,31 +15,26 @@ async fn handle_del(info: AppTagInfo) -> Result<(), EventProcessorError> { /// - Not a pubky:// URI /// - Not a */tags/* path /// - App is "pubky.app" (handled by the standard event flow) -fn try_parse_app_tag_path(uri: &str) -> Option { +/// - App or tag_id contains slashes (invalid segments) +pub fn try_parse_app_tag_path(uri: &str) -> Option { // Case-insensitive scheme check per RFC 3986 (safe UTF-8 access) - let rest = match uri.get(..8) { - Some(prefix) if prefix.eq_ignore_ascii_case("pubky://") => &uri[8..], - _ => return None, - }; + let rest = to_ascii_lower_prefix(uri, "pubky://")?; // Split: /pub//tags/ - let slash_pos = rest.find('/')?; - let user_id_str = &rest[..slash_pos]; - let path = &rest[slash_pos..]; // starts with / - - // Expected: /pub//tags/ - let path = path.strip_prefix("/pub/")?; + let (user_id_str, rest) = rest.split_once('/')?; + let rest = rest.strip_prefix("pub/")?; // Split on /tags/ - let tags_pos = path.find("/tags/")?; - let app = &path[..tags_pos]; - let tag_id = &path[tags_pos + 6..]; // skip "/tags/" + let (app, tag_id) = rest.split_once("/tags/")?; // Skip if app is pubky.app — those go through the standard flow if app == "pubky.app" { return None; } + // Strip query string (?...) or fragment (#...) from tag_id + let tag_id = tag_id.find(['?', '#']).map_or(tag_id, |pos| &tag_id[..pos]); + // Validate: app must be a single path segment, tag_id must not contain slashes if app.is_empty() || app.contains('/') || tag_id.is_empty() || tag_id.contains('/') { return None; @@ -126,15 +56,28 @@ fn try_parse_app_tag_path(uri: &str) -> Option { }) } +/// Strip a case-insensitive prefix from a string. +fn to_ascii_lower_prefix<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { + if s.len() < prefix.len() { + return None; + } + if s[..prefix.len()].eq_ignore_ascii_case(prefix) { + Some(&s[prefix.len()..]) + } else { + None + } +} + #[cfg(test)] mod tests { use super::*; + const BASE_URI: &str = + "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/mapky/tags/ABC123"; + #[test] fn test_try_parse_app_tag_path_mapky() { - let info = try_parse_app_tag_path( - "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/mapky/tags/ABC123", - ); + let info = try_parse_app_tag_path(BASE_URI); assert!(info.is_some()); let info = info.unwrap(); assert_eq!(info.app, "mapky"); @@ -189,4 +132,71 @@ mod tests { ); assert!(info.is_some(), "Should handle mixed-case Pubky:// scheme"); } + + #[test] + fn test_try_parse_app_tag_path_slash_in_app_returns_none() { + assert!(try_parse_app_tag_path( + "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/my/app/tags/ABC" + ) + .is_none()); + } + + #[test] + fn test_try_parse_app_tag_path_slash_in_tag_returns_none() { + assert!(try_parse_app_tag_path( + "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/mapky/tags/ABC/DEF" + ) + .is_none()); + } + + #[test] + fn test_try_parse_app_tag_path_empty_app_returns_none() { + assert!(try_parse_app_tag_path( + "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub//tags/ABC" + ) + .is_none()); + } + + #[test] + fn test_try_parse_app_tag_path_empty_tag_returns_none() { + assert!(try_parse_app_tag_path( + "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/mapky/tags/" + ) + .is_none()); + } + + #[test] + fn test_try_parse_app_tag_path_query_string_stripped() { + let info = try_parse_app_tag_path( + "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/mapky/tags/ABC123?foo=bar", + ); + assert!(info.is_some(), "Should accept URI with query string"); + assert_eq!( + info.unwrap().tag_id, + "ABC123", + "tag_id must not include query string" + ); + } + + #[test] + fn test_try_parse_app_tag_path_fragment_stripped() { + let info = try_parse_app_tag_path( + "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/mapky/tags/ABC123#section", + ); + assert!(info.is_some(), "Should accept URI with fragment"); + assert_eq!( + info.unwrap().tag_id, + "ABC123", + "tag_id must not include fragment" + ); + } + + #[test] + fn test_try_parse_app_tag_path_empty_tag_after_query_returns_none() { + // tag_id becomes empty after stripping the query string + assert!(try_parse_app_tag_path( + "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/mapky/tags/?foo=bar" + ) + .is_none()); + } } diff --git a/nexus-common/src/universal_tag/homeserver_parsed_uri.rs b/nexus-common/src/universal_tag/homeserver_parsed_uri.rs new file mode 100644 index 000000000..93b7358e7 --- /dev/null +++ b/nexus-common/src/universal_tag/homeserver_parsed_uri.rs @@ -0,0 +1,215 @@ +use pubky_app_specs::{ParsedUri, PubkyId, Resource}; +use serde::{Deserialize, Serialize}; +use std::convert::{From, TryFrom}; + +use super::app_tag_info::try_parse_app_tag_path; + +/// Parsed URI representation that can handle both: +/// 1. Standard pubky-app-specs URIs (pubky:///pub/pubky.app/...) +/// 2. Universal tag URIs from other apps (pubky:///pub//tags/) +/// +/// This is needed because homeservers may return events from applications other than +/// pubky.app (e.g., eventky.app, mapky), and ParsedUri from pubky-app-specs strictly +/// requires the app path to be "pubky.app". +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum HomeserverParsedUri { + /// Standard pubky-app-specs ParsedUri (pubky.app path) + AppSpec { + user_id: PubkyId, + resource: Resource, + }, + /// Universal tag URI from a different app. + /// Format: pubky:///pub//tags/ + UniversalTag { + user_id: PubkyId, + app: String, + resource: Resource, + tag_id: String, + }, +} + +impl HomeserverParsedUri { + /// Returns the user ID from the parsed URI. + pub fn user_id(&self) -> &PubkyId { + match self { + HomeserverParsedUri::AppSpec { user_id, .. } => user_id, + HomeserverParsedUri::UniversalTag { user_id, .. } => user_id, + } + } + + /// Returns the resource from the parsed URI. + pub fn resource(&self) -> &Resource { + match self { + HomeserverParsedUri::AppSpec { resource, .. } => resource, + HomeserverParsedUri::UniversalTag { resource, .. } => resource, + } + } + + /// Returns the app name, if available. + /// Returns "pubky.app" for AppSpec variants. + pub fn app(&self) -> &str { + match self { + HomeserverParsedUri::AppSpec { .. } => "pubky.app", + HomeserverParsedUri::UniversalTag { app, .. } => app.as_str(), + } + } + + /// Returns the tag ID, if this is a UniversalTag with a tag resource. + pub fn tag_id(&self) -> Option<&str> { + match self { + HomeserverParsedUri::AppSpec { .. } => None, + HomeserverParsedUri::UniversalTag { tag_id, .. } => Some(tag_id.as_str()), + } + } +} + +impl From for HomeserverParsedUri { + fn from(parsed: ParsedUri) -> Self { + // ParsedUri from pubky-app-specs is always a pubky.app path + HomeserverParsedUri::AppSpec { + user_id: parsed.user_id, + resource: parsed.resource, + } + } +} + +impl TryFrom<&str> for HomeserverParsedUri { + type Error = String; + + fn try_from(uri: &str) -> Result { + // First, try parsing as a standard pubky-app-specs ParsedUri (pubky.app path). + // This handles URL validation, scheme checking, user_id extraction, and resource parsing + // for pubky.app URIs in one call. + if let Ok(parsed_uri) = ParsedUri::try_from(uri) { + return Ok(HomeserverParsedUri::AppSpec { + user_id: parsed_uri.user_id, + resource: parsed_uri.resource, + }); + } + + // If ParsedUri::try_from failed, the URI might be from a different app. + // Try parsing as a universal tag URI: pubky:///pub//tags/ + if let Some(info) = try_parse_app_tag_path(uri) { + return Ok(HomeserverParsedUri::UniversalTag { + user_id: info.user_id, + app: info.app, + resource: Resource::Tag(info.tag_id.clone()), + tag_id: info.tag_id, + }); + } + + Err(format!( + "URI is not a recognized pubky-app-specs path or universal tag path: {uri}" + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_standard_post_uri() { + let user_id = "operrr8wsbpr3ue9d4qj41ge1kcc6r7fdiy6o3ugjrrhi4y77rdo"; + let post_id = "0032SSN7Q4EVG"; + let uri = format!("pubky://{user_id}/pub/pubky.app/posts/{post_id}"); + let parsed = HomeserverParsedUri::try_from(uri.as_str()).expect("Failed to parse post URI"); + + assert!(matches!(parsed, HomeserverParsedUri::AppSpec { .. })); + assert_eq!(parsed.resource(), &Resource::Post(post_id.to_string())); + assert_eq!(parsed.app(), "pubky.app"); + } + + #[test] + fn test_parse_standard_tag_uri() { + let user_id = "operrr8wsbpr3ue9d4qj41ge1kcc6r7fdiy6o3ugjrrhi4y77rdo"; + let tag_id = "8Z8CWH8NVYQY39ZEBFGKQWWEKG"; + let uri = format!("pubky://{user_id}/pub/pubky.app/tags/{tag_id}"); + let parsed = HomeserverParsedUri::try_from(uri.as_str()).expect("Failed to parse tag URI"); + + assert!(matches!(parsed, HomeserverParsedUri::AppSpec { .. })); + assert_eq!(parsed.resource(), &Resource::Tag(tag_id.to_string())); + assert_eq!(parsed.app(), "pubky.app"); + } + + #[test] + fn test_parse_universal_tag_uri_mapky() { + let user_id = "operrr8wsbpr3ue9d4qj41ge1kcc6r7fdiy6o3ugjrrhi4y77rdo"; + let tag_id = "ABC123"; + let uri = format!("pubky://{user_id}/pub/mapky/tags/{tag_id}"); + let parsed = + HomeserverParsedUri::try_from(uri.as_str()).expect("Failed to parse mapky tag URI"); + + assert!(matches!(parsed, HomeserverParsedUri::UniversalTag { .. })); + assert_eq!(parsed.user_id(), &PubkyId::try_from(user_id).unwrap()); + assert_eq!(parsed.app(), "mapky"); + assert_eq!(parsed.resource(), &Resource::Tag(tag_id.to_string())); + assert_eq!(parsed.tag_id(), Some("ABC123")); + } + + #[test] + fn test_parse_universal_tag_uri_eventky() { + let user_id = "operrr8wsbpr3ue9d4qj41ge1kcc6r7fdiy6o3ugjrrhi4y77rdo"; + let tag_id = "XYZ789"; + let uri = format!("pubky://{user_id}/pub/eventky.app/tags/{tag_id}"); + let parsed = + HomeserverParsedUri::try_from(uri.as_str()).expect("Failed to parse eventky tag URI"); + + assert!(matches!(parsed, HomeserverParsedUri::UniversalTag { .. })); + assert_eq!(parsed.user_id(), &PubkyId::try_from(user_id).unwrap()); + assert_eq!(parsed.app(), "eventky.app"); + assert_eq!(parsed.resource(), &Resource::Tag(tag_id.to_string())); + assert_eq!(parsed.tag_id(), Some("XYZ789")); + } + + #[test] + fn test_reject_universal_non_tag_path() { + let user_id = "operrr8wsbpr3ue9d4qj41ge1kcc6r7fdiy6o3ugjrrhi4y77rdo"; + let uri = format!("pubky://{user_id}/pub/eventky.app/posts/123"); + let result = HomeserverParsedUri::try_from(uri.as_str()); + assert!(result.is_err(), "Should reject non-tag universal paths"); + } + + #[test] + fn test_reject_non_pubky_scheme() { + let result = HomeserverParsedUri::try_from("https://example.com/pub/pubky.app/"); + assert!(result.is_err(), "Should reject non-pubky scheme"); + } + + #[test] + fn test_reject_missing_user_id() { + let result = HomeserverParsedUri::try_from("pubky:///pub/pubky.app/"); + assert!(result.is_err(), "Should reject missing user ID"); + } + + #[test] + fn test_uppercase_scheme() { + let user_id = "operrr8wsbpr3ue9d4qj41ge1kcc6r7fdiy6o3ugjrrhi4y77rdo"; + let tag_id = "ABC123"; + let uri = format!("PUBKY://{user_id}/pub/mapky/tags/{tag_id}"); + let result = HomeserverParsedUri::try_from(uri.as_str()); + assert!(result.is_ok()); + } + + #[test] + fn test_universal_tag_uri_with_query_string() { + let user_id = "operrr8wsbpr3ue9d4qj41ge1kcc6r7fdiy6o3ugjrrhi4y77rdo"; + let uri = format!("pubky://{user_id}/pub/mapky/tags/ABC123?foo=bar"); + let parsed = HomeserverParsedUri::try_from(uri.as_str()) + .expect("Should accept universal tag URI with query string"); + assert!(matches!(parsed, HomeserverParsedUri::UniversalTag { .. })); + assert_eq!(parsed.resource(), &Resource::Tag("ABC123".to_string())); + assert_eq!(parsed.tag_id(), Some("ABC123")); + } + + #[test] + fn test_universal_tag_uri_with_fragment() { + let user_id = "operrr8wsbpr3ue9d4qj41ge1kcc6r7fdiy6o3ugjrrhi4y77rdo"; + let uri = format!("pubky://{user_id}/pub/mapky/tags/ABC123#section"); + let parsed = HomeserverParsedUri::try_from(uri.as_str()) + .expect("Should accept universal tag URI with fragment"); + assert!(matches!(parsed, HomeserverParsedUri::UniversalTag { .. })); + assert_eq!(parsed.resource(), &Resource::Tag("ABC123".to_string())); + assert_eq!(parsed.tag_id(), Some("ABC123")); + } +} diff --git a/nexus-common/src/universal_tag/mod.rs b/nexus-common/src/universal_tag/mod.rs new file mode 100644 index 000000000..a6fff1c26 --- /dev/null +++ b/nexus-common/src/universal_tag/mod.rs @@ -0,0 +1,3 @@ +pub mod app_tag_info; +pub mod homeserver_parsed_uri; +pub mod normalize; diff --git a/nexus-common/src/universal_tag/normalize.rs b/nexus-common/src/universal_tag/normalize.rs new file mode 100644 index 000000000..cf279fe22 --- /dev/null +++ b/nexus-common/src/universal_tag/normalize.rs @@ -0,0 +1,242 @@ +use pubky_app_specs::{ParsedUri, Resource}; +use url::Url; + +/// Normalizes a URI for deterministic Resource identification. +/// Returns `(normalized_uri, scheme)`. +/// +/// Rules: +/// - Lowercase scheme and host +/// - Strip default ports (80 for http, 443 for https) +/// - Strip fragments and userinfo +/// - Preserve path and query as-is +/// - Fallback for non-standard schemes that fail URL parsing +pub fn normalize_uri(uri: &str) -> Result<(String, String), String> { + match Url::parse(uri) { + Ok(parsed) => { + let scheme = parsed.scheme().to_string(); + let normalized = normalize_parsed_url(&parsed); + Ok((normalized, scheme)) + } + Err(_) => { + // Fallback for non-standard schemes (e.g., nostr:note1abc...) + // that may not parse as URLs (no // authority) + if let Some(colon_pos) = uri.find(':') { + let scheme = &uri[..colon_pos]; + // Validate scheme: RFC 3986 §3.1 — ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) + if scheme.is_empty() + || !scheme + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'+' || b == b'-' || b == b'.') + { + return Err(format!("Invalid URI scheme: {uri}")); + } + let scheme_lower = scheme.to_ascii_lowercase(); + let remainder = &uri[colon_pos + 1..]; + Ok((format!("{scheme_lower}:{remainder}"), scheme_lower)) + } else { + Err(format!("Invalid URI: {uri}")) + } + } + } +} + +fn normalize_parsed_url(parsed: &Url) -> String { + let scheme = parsed.scheme(); // already lowercase per url crate + + // For non-hierarchical schemes (no authority, e.g. nostr:note1abc), + // the url crate parses them as "cannot-be-a-base" URLs. + // Return scheme + ":" + opaque path (no //) + if parsed.cannot_be_a_base() { + // Strip fragment from the opaque form + let full = parsed.as_str(); + let without_fragment = match full.find('#') { + Some(pos) => &full[..pos], + None => full, + }; + return without_fragment.to_string(); + } + + let host = parsed + .host_str() + .map(|h| h.to_ascii_lowercase()) + .unwrap_or_default(); + + // Strip default ports + let port = match (parsed.port(), scheme) { + (Some(80), "http") => None, + (Some(443), "https") => None, + (other, _) => other, + }; + + let path = parsed.path(); + let query = parsed.query(); + // Fragment and userinfo are discarded + + let mut result = format!("{scheme}://{host}"); + if let Some(p) = port { + result.push_str(&format!(":{p}")); + } + result.push_str(path); + if let Some(q) = query { + result.push('?'); + result.push_str(q); + } + result +} + +#[derive(Debug, Clone, PartialEq)] +pub enum UriCategory { + /// pubky:// URI matching a known schema (posts, users) + InternalKnown, + /// pubky:// URI NOT matching any known schema (e.g., eventky events) + InternalUnknown, + /// Non-pubky:// URI (https://, nostr:, ipfs://, etc.) + External, +} + +/// Classifies a tag's target URI into one of three categories. +/// Scheme check is case-insensitive per RFC 3986. +pub fn classify_uri(uri: &str) -> UriCategory { + let is_pubky = uri + .get(..8) + .is_some_and(|s| s.eq_ignore_ascii_case("pubky://")); + if is_pubky { + match ParsedUri::try_from(uri) { + Ok(parsed) if matches!(parsed.resource, Resource::Post(_) | Resource::User) => { + UriCategory::InternalKnown + } + _ => UriCategory::InternalUnknown, + } + } else { + UriCategory::External + } +} + +/// Generates a deterministic 32-char hex Resource ID from a normalized URI. +/// +/// `resource_id = hex(BLAKE3(normalized_uri)[0..16])` +pub fn resource_id(normalized_uri: &str) -> String { + let hash = blake3::hash(normalized_uri.as_bytes()); + hex::encode(&hash.as_bytes()[..16]) +} + +#[cfg(test)] +mod tests { + use super::*; + + // -- normalize_uri test vectors from spec Section 5 -- + + #[test] + fn test_normalize_https_with_default_port_and_fragment() { + let (uri, scheme) = normalize_uri("HTTPS://Example.COM:443/path?q=1#frag").unwrap(); + assert_eq!(uri, "https://example.com/path?q=1"); + assert_eq!(scheme, "https"); + } + + #[test] + fn test_normalize_root_trailing_slash() { + let (uri, _) = normalize_uri("https://example.com").unwrap(); + assert_eq!(uri, "https://example.com/"); + } + + #[test] + fn test_normalize_non_default_port() { + let (uri, _) = normalize_uri("http://example.com:8080/path").unwrap(); + assert_eq!(uri, "http://example.com:8080/path"); + } + + #[test] + fn test_normalize_pubky_uri() { + let input = "pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/eventky.app/events/E001"; + let (uri, scheme) = normalize_uri(input).unwrap(); + assert_eq!(uri, input); + assert_eq!(scheme, "pubky"); + } + + #[test] + fn test_normalize_query_order_preserved() { + let (uri, _) = normalize_uri("HTTPS://Example.COM/path?b=2&a=1").unwrap(); + assert_eq!(uri, "https://example.com/path?b=2&a=1"); + } + + #[test] + fn test_normalize_strip_userinfo() { + let (uri, _) = normalize_uri("https://user:pass@example.com/path").unwrap(); + assert_eq!(uri, "https://example.com/path"); + } + + #[test] + fn test_normalize_nostr_fallback() { + let (uri, scheme) = normalize_uri("nostr:note1abc123...").unwrap(); + assert_eq!(uri, "nostr:note1abc123..."); + assert_eq!(scheme, "nostr"); + } + + #[test] + fn test_normalize_http_default_port() { + let (uri, _) = normalize_uri("HTTP://Example.COM:80/").unwrap(); + assert_eq!(uri, "http://example.com/"); + } + + #[test] + fn test_normalize_rejects_malformed_scheme() { + // "ht tps" has a space — invalid per RFC 3986 §3.1 + assert!(normalize_uri("ht tps://example.com").is_err()); + } + + #[test] + fn test_normalize_rejects_no_colon() { + assert!(normalize_uri("justtext").is_err()); + } + + // -- resource_id tests -- + + #[test] + fn test_resource_id_deterministic() { + let uri = "https://example.com/path?q=1"; + let id1 = resource_id(uri); + let id2 = resource_id(uri); + assert_eq!(id1, id2); + assert_eq!(id1.len(), 32); + } + + #[test] + fn test_resource_id_different_uris() { + let id1 = resource_id("https://example.com/a"); + let id2 = resource_id("https://example.com/b"); + assert_ne!(id1, id2); + } + + // -- classify_uri tests -- + + #[test] + fn test_classify_external_https() { + assert_eq!( + classify_uri("https://example.com/article"), + UriCategory::External + ); + } + + #[test] + fn test_classify_external_nostr() { + assert_eq!(classify_uri("nostr:note1abc123"), UriCategory::External); + } + + #[test] + fn test_classify_internal_unknown_eventky() { + // eventky URI is pubky:// but not a recognized pubky.app resource + assert_eq!( + classify_uri("pubky://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/eventky.app/events/E001"), + UriCategory::InternalUnknown + ); + } + + #[test] + fn test_classify_uppercase_pubky_scheme() { + // RFC 3986: schemes are case-insensitive + assert_eq!( + classify_uri("PUBKY://8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo/pub/eventky.app/events/E001"), + UriCategory::InternalUnknown + ); + } +} diff --git a/nexus-watcher/Cargo.toml b/nexus-watcher/Cargo.toml index 45430db2e..ada34e8ed 100644 --- a/nexus-watcher/Cargo.toml +++ b/nexus-watcher/Cargo.toml @@ -10,10 +10,11 @@ license = "MIT" [dependencies] async-trait = { workspace = true } chrono = { workspace = true } +nexus-common = { path = "../nexus-common" } opentelemetry = { workspace = true } pubky = { workspace = true } pubky-app-specs = { workspace = true } -nexus-common = { version = "0.4.1", path = "../nexus-common" } +futures = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } diff --git a/nexus-watcher/src/events/handlers/follow.rs b/nexus-watcher/src/events/handlers/follow.rs index 810c53b99..0349a30a7 100644 --- a/nexus-watcher/src/events/handlers/follow.rs +++ b/nexus-watcher/src/events/handlers/follow.rs @@ -1,12 +1,11 @@ -use crate::events::retry::event::RetryEvent; use crate::events::EventProcessorError; use nexus_common::db::kv::JsonAction; use nexus_common::db::OperationOutcome; use nexus_common::models::follow::{Followers, Following, Friends, UserFollows}; -use nexus_common::models::homeserver::Homeserver; use nexus_common::models::notification::Notification; use nexus_common::models::user::UserCounts; +use nexus_common::models::user::UserDetails; use pubky_app_specs::PubkyId; use tracing::debug; @@ -36,12 +35,15 @@ pub async fn sync_put( return Ok(()); } OperationOutcome::MissingDependency => { - if let Err(e) = Homeserver::maybe_ingest_for_user(&followee_id).await { - tracing::error!("Failed to ingest homeserver: {e}"); + if let Err(e) = UserDetails::maybe_ingest_user(followee_id.as_str()).await { + tracing::error!("Failed to ingest user: {e}"); } - let key = RetryEvent::generate_index_key_from_uri(&followee_id.to_uri()); - let dependency = vec![key]; + let followee_uri = followee_id + .to_uri() + .try_to_uri_str() + .map_err(EventProcessorError::generic)?; + let dependency = vec![followee_uri]; return Err(EventProcessorError::MissingDependency { dependency }); } // The relationship did not exist, create all related indexes diff --git a/nexus-watcher/src/events/handlers/mod.rs b/nexus-watcher/src/events/handlers/mod.rs index e00c19b81..fb4407298 100644 --- a/nexus-watcher/src/events/handlers/mod.rs +++ b/nexus-watcher/src/events/handlers/mod.rs @@ -3,6 +3,5 @@ pub mod file; pub mod follow; pub mod post; pub mod tag; -pub mod universal_tag; pub mod user; pub mod utils; diff --git a/nexus-watcher/src/events/handlers/post.rs b/nexus-watcher/src/events/handlers/post.rs index 2b54bd824..25a0be45a 100644 --- a/nexus-watcher/src/events/handlers/post.rs +++ b/nexus-watcher/src/events/handlers/post.rs @@ -1,15 +1,13 @@ -use crate::events::retry::event::RetryEvent; use crate::events::EventProcessorError; use nexus_common::db::queries::get::post_is_safe_to_delete; use nexus_common::db::{exec_single_row, execute_graph_operation, OperationOutcome}; use nexus_common::db::{queries, RedisOps}; -use nexus_common::models::homeserver::Homeserver; use nexus_common::models::notification::{Notification, PostChangedSource, PostChangedType}; use nexus_common::models::post::{ PostCounts, PostDetails, PostRelationships, PostStream, POST_TOTAL_ENGAGEMENT_KEY_PARTS, }; -use nexus_common::models::user::UserCounts; +use nexus_common::models::user::{UserCounts, UserDetails}; use pubky_app_specs::{ post_uri_builder, ParsedUri, PubkyAppPost, PubkyAppPostKind, PubkyId, Resource, }; @@ -37,24 +35,31 @@ pub async fn sync_put( OperationOutcome::MissingDependency => { let mut dependency_event_keys = Vec::new(); if let Some(replied_to_uri) = &post_relationships.replied { - let reply_dependency = RetryEvent::generate_index_key_from_uri(replied_to_uri); - dependency_event_keys.push(reply_dependency); + let replied_uri_str = replied_to_uri + .try_to_uri_str() + .map_err(EventProcessorError::generic)?; + dependency_event_keys.push(replied_uri_str); - if let Err(e) = Homeserver::maybe_ingest_for_post(replied_to_uri).await { - tracing::error!("Failed to ingest homeserver: {e}"); + if let Err(e) = PostDetails::maybe_ingest_author_of_post(replied_to_uri).await { + tracing::error!("Failed to ingest user: {e}"); } } if let Some(reposted_uri) = &post_relationships.reposted { - let reply_dependency = RetryEvent::generate_index_key_from_uri(reposted_uri); - dependency_event_keys.push(reply_dependency); + let reposted_uri_str = reposted_uri + .try_to_uri_str() + .map_err(EventProcessorError::generic)?; + dependency_event_keys.push(reposted_uri_str); - if let Err(e) = Homeserver::maybe_ingest_for_post(reposted_uri).await { - tracing::error!("Failed to ingest homeserver: {e}"); + if let Err(e) = PostDetails::maybe_ingest_author_of_post(reposted_uri).await { + tracing::error!("Failed to ingest user: {e}"); } } if dependency_event_keys.is_empty() { - let key = RetryEvent::generate_index_key_from_uri(&author_id.to_uri()); - dependency_event_keys.push(key); + let author_uri = author_id + .to_uri() + .try_to_uri_str() + .map_err(EventProcessorError::generic)?; + dependency_event_keys.push(author_uri); } return Err(EventProcessorError::missing_dependencies( dependency_event_keys, @@ -94,9 +99,9 @@ pub async fn sync_put( // We only consider the first mentioned (tagged) user, to mitigate DoS attacks against Nexus // whereby posts with many (inexistent) tagged PKs can cause Nexus to spend a lot of time trying to resolve them - if let Some(mentioned_user_id) = post_relationships.mentioned.first() { - if let Err(e) = Homeserver::maybe_ingest_for_user(mentioned_user_id).await { - tracing::error!("Failed to ingest homeserver: {e}"); + if let Some(mentioned_user_id) = &post_relationships.mentioned.first() { + if let Err(e) = UserDetails::maybe_ingest_user(mentioned_user_id).await { + tracing::error!("Failed to ingest user: {e}"); } } @@ -365,9 +370,7 @@ async fn put_mentioned_relationships_for_prefix( for pubky_id in find_mentioned_ids(content, prefix) { // Create the MENTIONED relationship in the graph let query = queries::put::create_mention_relationship(author_id, post_id, &pubky_id); - exec_single_row(query) - .await - .map_err(EventProcessorError::graph_query_failed)?; + exec_single_row(query).await?; let maybe_mentioned_id = Notification::new_mention(author_id, &pubky_id, post_id).await?; if let Some(mentioned_user_id) = maybe_mentioned_id { @@ -419,10 +422,7 @@ pub async fn del(author_id: PubkyId, post_id: String) -> Result<(), EventProcess // If there is none other relationship (OperationOutcome::CreatedOrDeleted), we delete from graph and redis. // But if there is any (OperationOutcome::Updated), then we simply update the post with keyword content [DELETED]. // A deleted post is a post whose content is EXACTLY `"[DELETED]"` - match execute_graph_operation(query) - .await - .map_err(EventProcessorError::graph_query_failed)? - { + match execute_graph_operation(query).await? { OperationOutcome::CreatedOrDeleted => sync_del(author_id, post_id).await?, OperationOutcome::Updated => { let existing_relationships = PostRelationships::get_by_id(&author_id, &post_id).await?; diff --git a/nexus-watcher/src/events/handlers/tag.rs b/nexus-watcher/src/events/handlers/tag.rs index 3eb16b4cd..3dff96e06 100644 --- a/nexus-watcher/src/events/handlers/tag.rs +++ b/nexus-watcher/src/events/handlers/tag.rs @@ -1,22 +1,23 @@ -use crate::events::retry::event::RetryEvent; use crate::events::EventProcessorError; - use chrono::Utc; use nexus_common::db::kv::{RedisResult, ScoreAction}; use nexus_common::db::{fetch_row_from_graph, queries, OperationOutcome, RedisOps}; -use nexus_common::models::homeserver::Homeserver; use nexus_common::models::notification::Notification; use nexus_common::models::post::search::PostsByTagSearch; +use nexus_common::models::post::PostDetails; use nexus_common::models::post::{PostCounts, PostStream}; use nexus_common::models::resource::stream::ResourceStream; use nexus_common::models::resource::tag::TagResource; -use nexus_common::models::resource::{classify_uri, normalize_uri, resource_id, UriCategory}; use nexus_common::models::tag::post::TagPost; use nexus_common::models::tag::search::TagSearch; use nexus_common::models::tag::traits::{TagCollection, TaggersCollection}; use nexus_common::models::tag::user::TagUser; use nexus_common::models::user::UserCounts; +use nexus_common::models::user::UserDetails; use nexus_common::types::Pagination; +use nexus_common::universal_tag::normalize::{ + classify_uri, normalize_uri, resource_id, UriCategory, +}; use pubky_app_specs::{post_uri_builder, ParsedUri, PubkyAppTag, PubkyId, Resource}; use tracing::debug; @@ -61,13 +62,9 @@ pub async fn sync_put_resource( tag_id: String, app: String, ) -> Result<(), EventProcessorError> { - debug!( - "Indexing resource tag: {} -> {} (app={})", - tagger_id, tag_id, app - ); + debug!("Indexing resource tag: {tagger_id} -> {tag_id} (app={app})",); - let category = classify_uri(&tag.uri); - match category { + match classify_uri(&tag.uri) { UriCategory::InternalKnown => { // The tagged URI is a known Post/User — delegate to existing flow sync_put(tag, tagger_id, tag_id).await @@ -227,14 +224,15 @@ async fn put_sync_post( Ok(()) } OperationOutcome::MissingDependency => { - // Ensure that dependencies follow the same format as the RetryManager keys - let dependency = vec![format!("{author_id}:posts:{post_id}")]; if let Ok(referenced_post_uri) = ParsedUri::try_from(post_uri) { - if let Err(e) = Homeserver::maybe_ingest_for_post(&referenced_post_uri).await { - tracing::error!("Failed to ingest homeserver: {e}"); + if let Err(e) = PostDetails::maybe_ingest_author_of_post(&referenced_post_uri).await + { + tracing::error!("Failed to ingest user: {e}"); } } - Err(EventProcessorError::MissingDependency { dependency }) + Err(EventProcessorError::MissingDependency { + dependency: vec![post_uri.to_owned()], + }) } OperationOutcome::CreatedOrDeleted => { // SAVE TO INDEXES @@ -345,12 +343,15 @@ async fn put_sync_user( Ok(()) } OperationOutcome::MissingDependency => { - if let Err(e) = Homeserver::maybe_ingest_for_user(&tagged_user_id).await { - tracing::error!("Failed to ingest homeserver: {e}"); + if let Err(e) = UserDetails::maybe_ingest_user(tagged_user_id.as_str()).await { + tracing::error!("Failed to ingest user: {e}"); } - let key = RetryEvent::generate_index_key_from_uri(&tagged_user_id.to_uri()); - let dependency = vec![key]; + let tagged_uri = tagged_user_id + .to_uri() + .try_to_uri_str() + .map_err(EventProcessorError::generic)?; + let dependency = vec![tagged_uri]; Err(EventProcessorError::MissingDependency { dependency }) } OperationOutcome::CreatedOrDeleted => { diff --git a/nexus-watcher/src/events/handlers/user.rs b/nexus-watcher/src/events/handlers/user.rs index 747eb0831..bd74f0907 100644 --- a/nexus-watcher/src/events/handlers/user.rs +++ b/nexus-watcher/src/events/handlers/user.rs @@ -60,10 +60,7 @@ pub async fn del(user_id: PubkyId) -> Result<(), EventProcessorError> { // 3. But if there is any relationship (OperationOutcome::Updated), then we simply update the user with empty profile // and keyword username [DELETED]. // A deleted user is a user whose profile is empty and has username `"[DELETED]"` - match execute_graph_operation(query) - .await - .map_err(EventProcessorError::graph_query_failed)? - { + match execute_graph_operation(query).await? { OperationOutcome::CreatedOrDeleted => { // 1. UserSearch reads UserDetails — must run before UserDetails Redis is removed UserSearch::delete(&user_id).await?; @@ -81,9 +78,7 @@ pub async fn del(user_id: PubkyId) -> Result<(), EventProcessorError> { indexing_results.1?; // 3. Graph deletion LAST - exec_single_row(queries::del::delete_user(&user_id)) - .await - .map_err(EventProcessorError::graph_query_failed)?; + exec_single_row(queries::del::delete_user(&user_id)).await?; } OperationOutcome::Updated => { let deleted_user = PubkyAppUser { diff --git a/nexus-watcher/src/events/mod.rs b/nexus-watcher/src/events/mod.rs index a3ab5353f..4ec80d379 100644 --- a/nexus-watcher/src/events/mod.rs +++ b/nexus-watcher/src/events/mod.rs @@ -1,5 +1,6 @@ use nexus_common::db::PubkyConnector; use nexus_common::models::event::{Event, EventProcessorError, EventType}; +use nexus_common::universal_tag::homeserver_parsed_uri::HomeserverParsedUri; use pubky_app_specs::{PubkyAppObject, Resource}; use std::sync::Arc; use tracing::debug; @@ -10,14 +11,37 @@ pub mod retry; pub use moderation::Moderation; -pub async fn handle(event: &Event, moderation: Arc) -> Result<(), EventProcessorError> { - match event.event_type { - EventType::Put => handle_put_event(event, moderation).await, - EventType::Del => handle_del_event(event).await, - }?; +/// Trait for handling events. +/// +/// This trait abstracts event handling logic to allow for flexible implementations, +/// including mocked versions for testing. +#[async_trait::async_trait] +pub trait EventHandler: Send + Sync { + async fn handle(&self, event: &Event) -> Result<(), EventProcessorError>; +} - event.store_event().await?; - Ok(()) +/// Default implementation of `EventHandler` that uses the actual event handling logic. +pub struct DefaultEventHandler { + moderation: Arc, +} + +impl DefaultEventHandler { + pub fn new(moderation: Arc) -> Self { + Self { moderation } + } +} + +#[async_trait::async_trait] +impl EventHandler for DefaultEventHandler { + async fn handle(&self, event: &Event) -> Result<(), EventProcessorError> { + match event.event_type { + EventType::Put => handle_put_event(event, self.moderation.clone()).await, + EventType::Del => handle_del_event(event).await, + }?; + + event.store_event().await?; + Ok(()) + } } pub async fn handle_put_event( @@ -29,31 +53,17 @@ pub async fn handle_put_event( let pubky = PubkyConnector::get()?; let response = pubky.public_storage().get(&event.uri).await?; - if !response.status().is_success() { - let status = response.status(); - let body = response - .text() - .await - .unwrap_or_else(|_| "".to_string()); - - let err_msg = format!( - "Fetch resource failed {}: HTTP {status} - {body}", - event.uri - ); - return Err(EventProcessorError::client_error(err_msg))?; - } - let blob = response .bytes() .await .map_err(|e| EventProcessorError::client_error(e.to_string()))?; - let resource = event.parsed_uri.resource.clone(); + let resource = event.parsed_uri.resource().clone(); // Use the new importer from pubky-app-specs let pubky_object = PubkyAppObject::from_resource(&resource, &blob).map_err(EventProcessorError::generic)?; - let user_id = event.parsed_uri.user_id.clone(); + let user_id = event.parsed_uri.user_id().clone(); match (pubky_object, resource) { (PubkyAppObject::User(user), Resource::User) => { handlers::user::sync_put(user, user_id).await? @@ -71,10 +81,19 @@ pub async fn handle_put_event( handlers::bookmark::sync_put(user_id, bookmark, bookmark_id).await? } (PubkyAppObject::Tag(tag), Resource::Tag(tag_id)) => { - if moderation.should_delete(&tag, user_id.clone()).await { - Moderation::apply_moderation(tag, event.files_path.clone()).await? + if moderation.should_delete(&tag, user_id.clone()) { + moderation + .apply_moderation(tag, event.files_path.clone()) + .await? } else { - handlers::tag::sync_put(tag, user_id, tag_id).await? + // Route universal tag events (non-pubky.app apps) to sync_put_resource + // which handles Resource nodes for InternalUnknown/InternalUnknown URIs. + if let HomeserverParsedUri::UniversalTag { app, .. } = &event.parsed_uri { + handlers::tag::sync_put_resource(tag, user_id, tag_id.to_string(), app.clone()) + .await? + } else { + handlers::tag::sync_put(tag, user_id, tag_id.to_string()).await? + } } } (PubkyAppObject::File(file), Resource::File(file_id)) => { @@ -96,8 +115,8 @@ pub async fn handle_put_event( pub async fn handle_del_event(event: &Event) -> Result<(), EventProcessorError> { debug!("Handling DEL event for URI: {}", event.uri); - let user_id = event.parsed_uri.user_id.clone(); - match &event.parsed_uri.resource { + let user_id = event.parsed_uri.user_id().clone(); + match event.parsed_uri.resource() { Resource::User => handlers::user::del(user_id).await?, Resource::Post(post_id) => handlers::post::del(user_id, post_id.clone()).await?, Resource::Follow(followee_id) => { diff --git a/nexus-watcher/src/events/moderation.rs b/nexus-watcher/src/events/moderation.rs index 46d319562..998256036 100644 --- a/nexus-watcher/src/events/moderation.rs +++ b/nexus-watcher/src/events/moderation.rs @@ -1,7 +1,9 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::events::handlers; use nexus_common::models::event::EventProcessorError; +use nexus_common::WatcherConfig; use pubky_app_specs::{ParsedUri, PubkyAppTag, PubkyId, Resource}; use tracing::info; @@ -13,12 +15,28 @@ pub struct Moderation { } impl Moderation { - pub async fn should_delete(&self, tag: &PubkyAppTag, tagger_id: PubkyId) -> bool { + pub fn from_config(config: &WatcherConfig) -> Arc { + Arc::new(Self { + id: config.moderation_id.clone(), + tags: config.moderated_tags.clone(), + }) + } + + /// Check if a tag should trigger deletion of the tagged content. + /// + /// Returns `true` if the tag was applied by the moderator and matches + /// a moderated tag label. + pub fn should_delete(&self, tag: &PubkyAppTag, tagger_id: PubkyId) -> bool { tagger_id == self.id && self.tags.contains(&tag.label) } + /// Apply moderation by deleting the tagged resource. + /// + /// Parses the embedded URI in the moderator tag and deletes the corresponding + /// resource (post, tag, user, or file). #[tracing::instrument(name = "moderation.apply", skip_all)] pub async fn apply_moderation( + &self, moderator_tag: PubkyAppTag, files_path: PathBuf, ) -> Result<(), EventProcessorError> { @@ -26,38 +44,27 @@ impl Moderation { let parsed_uri = ParsedUri::try_from(moderator_tag.uri.as_str()) .map_err(EventProcessorError::generic)?; let user_id = parsed_uri.user_id; + let label = moderator_tag.label; match parsed_uri.resource { Resource::Post(post_id) => { // Delete the post and return the result - info!( - "Moderation tag '{}' detected. Deleting post {}:{}", - moderator_tag.label, user_id, post_id - ); + info!("Moderation tag '{label}' detected. Deleting post {user_id}:{post_id}"); handlers::post::sync_del(user_id, post_id).await } Resource::Tag(tag_id) => { // Delete the tag and return the result - info!( - "Moderation tag '{}' detected. Deleting tag {}:{}", - moderator_tag.label, user_id, tag_id - ); + info!("Moderation tag '{label}' detected. Deleting tag {user_id}:{tag_id}"); handlers::tag::del(user_id, tag_id).await } Resource::User => { // Delete the user profile and return the result - info!( - "Moderation tag '{}' detected. Deleting user profile {}", - moderator_tag.label, user_id - ); + info!("Moderation tag '{label}' detected. Deleting user profile {user_id}"); handlers::user::del(user_id).await } Resource::File(file_id) => { // Delete the file and return the result - info!( - "Moderation tag '{}' detected. Deleting file {}:{}", - moderator_tag.label, user_id, file_id - ); + info!("Moderation tag '{label}' detected. Deleting file {user_id}:{file_id}"); handlers::file::del(&user_id, file_id, files_path).await } _ => Ok(()), diff --git a/nexus-watcher/src/events/retry/event.rs b/nexus-watcher/src/events/retry/event.rs index b367c5b57..e05609fb3 100644 --- a/nexus-watcher/src/events/retry/event.rs +++ b/nexus-watcher/src/events/retry/event.rs @@ -1,16 +1,15 @@ use async_trait::async_trait; -use chrono::Utc; -use nexus_common::db::kv::RedisResult; -use pubky_app_specs::ParsedUri; +use nexus_common::db::kv::{RedisResult, SortOrder}; +use nexus_common::models::event::EventType; use serde::{Deserialize, Serialize}; use nexus_common::db::RedisOps; use crate::events::EventProcessorError; -pub const RETRY_MANAGER_PREFIX: &str = "RetryManager"; -pub const RETRY_MANAGER_EVENTS_INDEX: [&str; 1] = ["events"]; -pub const RETRY_MANAGER_STATE_INDEX: [&str; 1] = ["state"]; +const RETRY_MANAGER_PREFIX: &str = "RetryManager"; +const RETRY_MANAGER_EVENTS_INDEX: [&str; 1] = ["events"]; +const RETRY_MANAGER_STATE_INDEX: [&str; 1] = ["state"]; /// Represents an event in the retry queue and it is used to manage events that have failed /// to process and need to be retried @@ -18,9 +17,12 @@ pub const RETRY_MANAGER_STATE_INDEX: [&str; 1] = ["state"]; pub struct RetryEvent { /// Retry attempts made for this event pub retry_count: u32, - /// The type of error that caused the event to fail - /// This determines how the event should be processed during the retry process - pub error_type: EventProcessorError, + /// The type of event - needed to reconstruct the event on retry + pub event_type: EventType, + /// Original URI - blob is re-fetched on retry + pub event_uri: String, + /// Unix ms - when to next attempt (exponential backoff) + pub next_retry_at: i64, } #[async_trait] @@ -31,86 +33,127 @@ impl RedisOps for RetryEvent { } impl RetryEvent { - pub fn new(error_type: EventProcessorError) -> Self { + /// Creates a new RetryEvent + pub fn new(event_type: EventType, event_uri: String, next_retry_at: i64) -> Self { Self { retry_count: 0, - error_type, - } - } - - /// It processes a homeserver URI and extracts specific components to form a index key - /// in the format `"{pubkyId}:{repository_model}:{event_id}"` - /// # Parameters - /// - `event_uri`: A string slice representing the event URI to be processed - pub fn generate_index_key(event_uri: &str) -> Option { - let parsed_uri = match ParsedUri::try_from(event_uri) { - Ok(parsed_uri) => parsed_uri, - Err(_) => return None, - }; - - let user_id = parsed_uri.user_id; - let key = match parsed_uri.resource.id() { - Some(id) => format!("{}:{}:{}", user_id, parsed_uri.resource, id), - None => format!("{}:{}", user_id, parsed_uri.resource), - }; - - Some(key) - } - - pub fn generate_index_key_from_uri(event_uri: &ParsedUri) -> String { - let user_id = &event_uri.user_id; - let event_resource = &event_uri.resource; - - match event_uri.resource.id() { - Some(id) => format!("{user_id}:{event_resource}:{id}"), - None => format!("{user_id}:{event_resource}"), + event_type, + event_uri, + next_retry_at, } } /// Stores an event in both a sorted set and a JSON index in Redis. - /// It adds an event line to a Redis sorted set with a timestamp-based score - /// and also stores the event details in a separate JSON index for retrieval. + /// The sorted set uses next_retry_at as the score for efficient retrieval of ready events. /// # Arguments - /// * `event_line` - A `String` representing the event line to be indexed. + /// * `index_key` - the index key (used as member in sorted set and JSON key) #[tracing::instrument(name = "retry.index.write", skip_all)] - pub async fn put_to_index(&self, event_line: String) -> RedisResult<()> { + pub async fn put_to_index(&self, index_key: &str) -> RedisResult<()> { + // Add to sorted set with next_retry_at as score Self::put_index_sorted_set( &RETRY_MANAGER_EVENTS_INDEX, - // NOTE: Don't know if we should use now timestamp or the event timestamp - &[(Utc::now().timestamp_millis() as f64, &event_line)], + &[(self.next_retry_at as f64, index_key)], Some(RETRY_MANAGER_PREFIX), None, ) .await?; - let index = &[RETRY_MANAGER_STATE_INDEX, [&event_line]].concat(); + // Store full RetryEvent struct in JSON + let index = &[RETRY_MANAGER_STATE_INDEX[0], index_key]; self.put_index_json(index, None, None).await?; Ok(()) } - /// Checks if a specific event exists in the Redis sorted set - /// # Arguments - /// * `event_index` - A `&str` representing the event index to check - pub async fn check_uri(event_index: &str) -> Result, EventProcessorError> { + /// Checks if a specific event exists in the Redis sorted set. + /// + /// Only used by integration tests (`nexus-watcher/tests/`); kept `pub` because + /// those tests compile against this crate as an external consumer. + pub async fn check_uri(index_key: &str) -> RedisResult { Self::check_sorted_set_member( Some(RETRY_MANAGER_PREFIX), &RETRY_MANAGER_EVENTS_INDEX, - &[event_index], + &[index_key], ) .await - .map_err(|e| { - EventProcessorError::InternalError(format!( - "Could not check uri for event: {event_index}, reason {e}" - )) - }) + .map(|rank| rank.is_some()) } /// Retrieves an event from the JSON index in Redis based on its index - /// # Arguments - /// * `event_index` - A `&str` representing the event index to retrieve - pub async fn get_from_index(event_index: &str) -> RedisResult> { - let index: &Vec<&str> = &[RETRY_MANAGER_STATE_INDEX, [event_index]].concat(); + #[tracing::instrument(name = "retry.index.get", skip_all)] + pub async fn get_from_index(index_key: &str) -> RedisResult> { + let index = &[RETRY_MANAGER_STATE_INDEX[0], index_key]; Self::try_from_index_json(index, None).await } + + /// Batched variant of [`Self::get_from_index`] backed by a single `JSON.MGET`. + /// + /// Results are returned positionally: element `i` corresponds to `index_keys[i]`, + /// with `None` for keys whose JSON state is missing (tombstones). + #[tracing::instrument(name = "retry.index.get_multiple", skip_all)] + pub async fn get_multiple_from_index(index_keys: &[&str]) -> RedisResult>> { + let key_parts: Vec<[&str; 2]> = index_keys + .iter() + .map(|k| [RETRY_MANAGER_STATE_INDEX[0], *k]) + .collect(); + let key_parts_refs: Vec<&[&str]> = key_parts.iter().map(|p| p.as_slice()).collect(); + Self::try_from_index_multiple_json(&key_parts_refs).await + } + + /// Removes an event from the retry queue (both sorted set and JSON state) + #[tracing::instrument(name = "retry.index.remove", skip_all)] + pub async fn remove_from_index(index_key: &str) -> RedisResult<()> { + // Remove from sorted set + Self::remove_from_index_sorted_set( + Some(RETRY_MANAGER_PREFIX), + &RETRY_MANAGER_EVENTS_INDEX, + &[index_key], + ) + .await?; + + // Remove JSON state + let index = &[RETRY_MANAGER_STATE_INDEX[0], index_key]; + Self::remove_from_index_multiple_json(&[index.as_slice()]).await?; + + Ok(()) + } + + /// Removes multiple sorted-set index entries without touching JSON state. + /// + /// Used for tombstone cleanup in the retry store: the JSON state is already + /// missing, so a single batched ZREM reconciles the index. + #[tracing::instrument(name = "retry.index.remove_stale", skip_all)] + pub async fn remove_stale_index_entries(index_keys: &[&str]) -> RedisResult<()> { + Self::remove_from_index_sorted_set( + Some(RETRY_MANAGER_PREFIX), + &RETRY_MANAGER_EVENTS_INDEX, + index_keys, + ) + .await + } + + /// Fetches events from the retry queue that are ready to be retried (next_retry_at <= now) + /// # Arguments + /// * `now` - Current time in milliseconds since epoch + /// * `limit` - Maximum number of events to fetch per batch + /// # Returns + /// A vector of (index_key, score) pairs; empty when no events are ready. + #[tracing::instrument(name = "retry.index.fetch_ready", skip_all)] + pub async fn fetch_ready( + now: i64, + limit: Option, + ) -> Result, EventProcessorError> { + Self::try_from_index_sorted_set( + &RETRY_MANAGER_EVENTS_INDEX, + Some(now as f64), // max_score (start → max in get_range) + None, // min_score (end → min in get_range) + Some(0), // skip + limit, + SortOrder::Ascending, + Some(RETRY_MANAGER_PREFIX), + ) + .await + .map(Option::unwrap_or_default) + .map_err(|e| EventProcessorError::generic(format!("Failed to fetch retry events: {}", e))) + } } diff --git a/nexus-watcher/src/events/retry/mod.rs b/nexus-watcher/src/events/retry/mod.rs index 53f112654..b54071abb 100644 --- a/nexus-watcher/src/events/retry/mod.rs +++ b/nexus-watcher/src/events/retry/mod.rs @@ -1 +1,9 @@ pub mod event; +pub mod processor; +pub mod scheduler; +pub mod store; + +pub use event::RetryEvent; +pub use processor::RetryProcessor; +pub use scheduler::{InitialBackoff, RetryScheduler}; +pub use store::{InMemoryRetryStore, RedisRetryStore, RetryStore}; diff --git a/nexus-watcher/src/events/retry/processor.rs b/nexus-watcher/src/events/retry/processor.rs new file mode 100644 index 000000000..717e19a19 --- /dev/null +++ b/nexus-watcher/src/events/retry/processor.rs @@ -0,0 +1,221 @@ +use std::cmp::min; +use std::path::PathBuf; +use std::sync::Arc; + +use chrono::{DateTime, Utc}; +use nexus_common::config::EventRetryConfig; +use nexus_common::models::event::{Event, EventProcessorError, ParseResult}; +use nexus_common::WatcherConfig; +use tokio::sync::watch::Receiver; +use tracing::{debug, info, warn}; + +use super::store::{RedisRetryStore, RetryStore}; +use super::RetryEvent; +use super::RetryScheduler; +use crate::events::{DefaultEventHandler, EventHandler, Moderation}; +use crate::service::indexer::TEventProcessor; + +/// Maximum number of retry events to fetch per batch to avoid memory spikes +const RETRY_BATCH_SIZE: usize = 100; + +/// Processor for retrying events that failed due to missing dependencies +pub struct RetryProcessor { + pub files_path: PathBuf, + pub event_handler: Arc, + pub shutdown_rx: Receiver, + pub config: EventRetryConfig, + /// Persistence backend for retry events. Production wiring uses + /// [`RedisRetryStore`]; tests swap in an in-memory store for isolation. + pub store: Arc, +} + +#[async_trait::async_trait] +impl TEventProcessor for RetryProcessor { + fn files_path(&self) -> &PathBuf { + &self.files_path + } + + fn event_handler(&self) -> &Arc { + &self.event_handler + } + + fn instance_name(&self) -> String { + "RetryProcessor".to_string() + } + + fn retry_scheduler(&self) -> Option<&Arc> { + None + } + + async fn run_internal(self: Arc) -> Result<(), EventProcessorError> { + let now = Utc::now().timestamp_millis(); + + loop { + let events = self.fetch_ready_events(now).await?; + + if events.is_empty() { + debug!("No more events ready for retry"); + return Ok(()); + } + + info!("Processing batch of {} retry events", events.len()); + + for (index_key, retry_event) in events { + if *self.shutdown_rx.borrow() { + debug!("Shutdown detected; exiting retry processing loop"); + return Ok(()); + } + + self.process_retry_event(&index_key, retry_event).await?; + } + } + } +} + +impl RetryProcessor { + pub fn new(config: &WatcherConfig, shutdown_rx: Receiver) -> Self { + let moderation = Moderation::from_config(config); + let store: Arc = Arc::new(RedisRetryStore::new()); + Self { + files_path: config.stack.files_path.clone(), + event_handler: Arc::new(DefaultEventHandler::new(moderation)), + shutdown_rx, + config: config.retry.clone(), + store, + } + } + + /// Fetch events from the retry queue that are ready to be retried. + /// Resolved `(index_key, RetryEvent)` pairs are returned directly by the + /// store; stale-entry cleanup is the store's responsibility. + async fn fetch_ready_events( + &self, + now: i64, + ) -> Result, EventProcessorError> { + self.store.fetch_ready(now, Some(RETRY_BATCH_SIZE)).await + } + + /// Process a single retry event + async fn process_retry_event( + &self, + index_key: &str, + retry_event: RetryEvent, + ) -> Result<(), EventProcessorError> { + // Reconstruct the event line and parse the event + // Event format is "METHOD URI" (e.g., "PUT pubky://...") + let event_line = format!("{} {}", retry_event.event_type, retry_event.event_uri); + + // Parse the event from the line - if corrupted, remove and continue + let event = match Event::parse_event(&event_line, self.files_path().clone()) { + Ok(ParseResult::Parsed(event)) => event, + Ok(ParseResult::Skipped) | Err(_) => { + warn!("Corrupted retry entry for key {index_key}, removing: '{event_line}'"); + self.store.remove(index_key).await?; + return Ok(()); + } + Ok(ParseResult::UnrecognizedUri { reason, .. }) => { + warn!("Unrecognized URI in retry entry for key {index_key}, removing: {reason}"); + self.store.remove(index_key).await?; + return Ok(()); + } + }; + + let ev_uri = &retry_event.event_uri; + let ev_retry_count = retry_event.retry_count; + + // Call event_handler directly to get the actual error (bypassing handle_event/handle_error) + let event_handle_res = self.event_handler().handle(&event).await.inspect_err(|e| { + // In case of error, log it before the error itself is classified and handled + // Error handling could itself throw an error. We log it here to pre-empt this possibility. + warn!("Retry event handling failed: {e}"); + }); + + match event_handle_res { + Ok(()) => { + // Success - event was processed, remove from retry queue + debug!("Retry successful for event: {ev_uri}"); + self.store.remove(index_key).await?; + } + Err(e) if !e.is_retryable() => { + // Non-retryable error (ParseFailed, etc.) - dead-letter immediately + warn!("Event {ev_uri} failed with non-retryable error, dead-lettering: {e}"); + self.store.remove(index_key).await?; + } + Err(e) if e.is_infrastructure() => { + // Infrastructure errors (Neo4j/Redis failures) must NOT count against the + // application-level max_retries limit. Reschedule with backoff but do NOT + // increment retry_count, then propagate to stop the current batch. + self.reschedule(&retry_event, index_key, &e, false).await?; + return Err(e); + } + Err(e) if ev_retry_count >= self.get_max_retries_for_err(&e) => { + warn!("Event {ev_uri} exceeded max retries ({ev_retry_count}), dead-lettering"); + self.store.remove(index_key).await?; + } + Err(e) => { + // Schedule retry with backoff (increments retry_count) + self.reschedule(&retry_event, index_key, &e, true).await?; + } + } + + Ok(()) + } + + /// Reschedule an event for retry with exponential backoff. + /// + /// When `increment_count` is `true` the retry budget is consumed (application-level + /// errors). When `false` the counter stays unchanged — used for infrastructure + /// errors that should not count against the retry limit. + async fn reschedule( + &self, + retry_event: &RetryEvent, + index_key: &str, + error: &EventProcessorError, + increment_count: bool, + ) -> Result<(), EventProcessorError> { + let new_retry_count = match increment_count { + true => retry_event.retry_count + 1, + false => retry_event.retry_count, + }; + + // Calculate backoff based on error type + let (initial, max) = self.config.get_backoff_params(error); + // Use retry_count (not new_retry_count) so first retry uses 2^0 * initial = initial + let backoff_secs = self.calculate_backoff(retry_event.retry_count, initial, max); + + let now = Utc::now().timestamp_millis(); + let next_retry_at = now + (backoff_secs as i64 * 1000); + + let mut updated_event = retry_event.clone(); + updated_event.retry_count = new_retry_count; + updated_event.next_retry_at = next_retry_at; + + self.store.put(index_key, &updated_event).await?; + + let retry_time = + DateTime::::from_timestamp_millis(next_retry_at).unwrap_or_else(Utc::now); + info!( + "Rescheduling {} for {:?} (backoff: {}s, retry_count: {})", + retry_event.event_uri, retry_time, backoff_secs, new_retry_count + ); + + Ok(()) + } + + /// Calculate exponential backoff + fn calculate_backoff(&self, retry_count: u32, initial: u64, max: u64) -> u64 { + let exponential = 2u64 + .checked_pow(retry_count) + .and_then(|p| initial.checked_mul(p)) + .unwrap_or(max); + min(exponential, max) + } + + fn get_max_retries_for_err(&self, e: &EventProcessorError) -> u32 { + if e.is_missing_dependency() { + self.config.max_dependency_retries + } else { + self.config.max_retries + } + } +} diff --git a/nexus-watcher/src/events/retry/scheduler.rs b/nexus-watcher/src/events/retry/scheduler.rs new file mode 100644 index 000000000..41ceb3687 --- /dev/null +++ b/nexus-watcher/src/events/retry/scheduler.rs @@ -0,0 +1,74 @@ +use std::sync::Arc; + +use chrono::Utc; +use tracing::warn; + +use nexus_common::models::event::{Event, EventProcessorError}; +use nexus_common::WatcherConfig; + +use super::{RedisRetryStore, RetryEvent, RetryStore}; + +/// Initial backoff durations applied when an event first lands on the retry queue. +/// Subsequent reschedules use exponential backoff inside [`super::RetryProcessor`]. +#[derive(Debug, Clone, Copy)] +pub struct InitialBackoff { + pub missing_dep_ms: i64, + pub transient_ms: i64, +} + +impl InitialBackoff { + pub fn from_config(config: &WatcherConfig) -> Self { + Self { + missing_dep_ms: config.retry.initial_missing_dep_backoff_secs as i64 * 1000, + transient_ms: config.retry.initial_backoff_secs as i64 * 1000, + } + } +} + +/// Enqueues failed events onto the retry queue. Created once per watcher and +/// shared (`Arc`) with every event processor so that processors don't need to +/// carry backoff state themselves. +pub struct RetryScheduler { + store: Arc, + initial: InitialBackoff, +} + +impl RetryScheduler { + pub fn new(store: Arc, initial: InitialBackoff) -> Self { + Self { store, initial } + } + + pub fn from_config(config: &WatcherConfig) -> Self { + Self::new( + Arc::new(RedisRetryStore::new()), + InitialBackoff::from_config(config), + ) + } + + pub async fn queue_missing_dep(&self, event: &Event) -> Result<(), EventProcessorError> { + self.enqueue(event, self.initial.missing_dep_ms, "missing dependency") + .await + } + + pub async fn queue_transient(&self, event: &Event) -> Result<(), EventProcessorError> { + self.enqueue(event, self.initial.transient_ms, "client error") + .await + } + + async fn enqueue( + &self, + event: &Event, + initial_backoff_ms: i64, + reason: &str, + ) -> Result<(), EventProcessorError> { + let next_retry_at = Utc::now().timestamp_millis() + initial_backoff_ms; + let retry_event = + RetryEvent::new(event.event_type.clone(), event.uri.clone(), next_retry_at); + + // New EventRetries for the same URI will reset the retry_count + // The HS state changed since the earlier event, so we disregard previous retry attempts + self.store.put(&event.uri, &retry_event).await?; + warn!("Queued event for retry ({}): {}", reason, event.uri); + Ok(()) + } +} diff --git a/nexus-watcher/src/events/retry/store.rs b/nexus-watcher/src/events/retry/store.rs new file mode 100644 index 000000000..662e570b7 --- /dev/null +++ b/nexus-watcher/src/events/retry/store.rs @@ -0,0 +1,171 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use tokio::sync::Mutex; +use tracing::debug; + +use nexus_common::models::event::EventProcessorError; + +use super::RetryEvent; + +/// Storage backend for [`RetryEvent`]s. +/// +/// Abstracts persistence so the processor can run against Redis in production and +/// against a per-test in-memory store under `cargo test`, keeping parallel tests +/// from stomping on each other's queue state. +#[async_trait] +pub trait RetryStore: Send + Sync { + /// Insert or replace the event stored under `index_key`. + async fn put(&self, index_key: &str, event: &RetryEvent) -> Result<(), EventProcessorError>; + + /// Retrieve the event for `index_key`, if any. + async fn get(&self, index_key: &str) -> Result, EventProcessorError>; + + /// Remove `index_key` from the store. No-op if absent. + async fn remove(&self, index_key: &str) -> Result<(), EventProcessorError>; + + /// Return all events with `next_retry_at <= now`, ordered ascending by + /// `next_retry_at`, capped at `limit` if provided. + /// + /// Implementations are responsible for cleaning up any internal inconsistencies + /// (e.g. Redis sorted set entries that point at missing JSON state), so the + /// caller always receives fully-resolved `(key, event)` pairs. + async fn fetch_ready( + &self, + now: i64, + limit: Option, + ) -> Result, EventProcessorError>; +} + +/// Redis-backed [`RetryStore`], delegating to the `RetryEvent::*` helpers that +/// wrap the Redis sorted-set + JSON-state layout. +pub struct RedisRetryStore; + +impl RedisRetryStore { + pub fn new() -> Self { + Self + } +} + +impl Default for RedisRetryStore { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl RetryStore for RedisRetryStore { + async fn put(&self, index_key: &str, event: &RetryEvent) -> Result<(), EventProcessorError> { + event.put_to_index(index_key).await?; + Ok(()) + } + + async fn get(&self, index_key: &str) -> Result, EventProcessorError> { + Ok(RetryEvent::get_from_index(index_key).await?) + } + + async fn remove(&self, index_key: &str) -> Result<(), EventProcessorError> { + RetryEvent::remove_from_index(index_key).await?; + Ok(()) + } + + async fn fetch_ready( + &self, + now: i64, + limit: Option, + ) -> Result, EventProcessorError> { + let key_score_pairs = RetryEvent::fetch_ready(now, limit).await?; + + // Batch-fetch JSON state for every candidate in a single JSON.MGET. + let keys: Vec<&str> = key_score_pairs.iter().map(|(k, _)| k.as_str()).collect(); + let maybe_events = RetryEvent::get_multiple_from_index(&keys).await?; + + let mut events = Vec::with_capacity(key_score_pairs.len()); + let mut stale: Vec = Vec::new(); + for ((index_key, _score), maybe_event) in key_score_pairs.into_iter().zip(maybe_events) { + match maybe_event { + Some(event) => events.push((index_key, event)), + None => { + // Sorted-set entry with no JSON state — tombstone, clean up and skip. + debug!("Stale retry entry detected for key {index_key}, cleaning up"); + stale.push(index_key); + } + } + } + + if !stale.is_empty() { + let refs: Vec<&str> = stale.iter().map(String::as_str).collect(); + RetryEvent::remove_stale_index_entries(&refs).await?; + } + + Ok(events) + } +} + +/// In-memory [`RetryStore`] intended for unit/integration tests that want +/// per-test isolation without spinning up Redis state. +/// +/// Each instance is independent, so parallel tests that each own their own +/// `InMemoryRetryStore` cannot observe or mutate each other's events. +pub struct InMemoryRetryStore { + inner: Mutex>, +} + +impl InMemoryRetryStore { + pub fn new() -> Self { + Self { + inner: Mutex::new(HashMap::new()), + } + } +} + +impl Default for InMemoryRetryStore { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl RetryStore for InMemoryRetryStore { + async fn put(&self, index_key: &str, event: &RetryEvent) -> Result<(), EventProcessorError> { + self.inner + .lock() + .await + .insert(index_key.to_owned(), event.clone()); + Ok(()) + } + + async fn get(&self, index_key: &str) -> Result, EventProcessorError> { + Ok(self.inner.lock().await.get(index_key).cloned()) + } + + async fn remove(&self, index_key: &str) -> Result<(), EventProcessorError> { + self.inner.lock().await.remove(index_key); + Ok(()) + } + + async fn fetch_ready( + &self, + now: i64, + limit: Option, + ) -> Result, EventProcessorError> { + let guard = self.inner.lock().await; + let mut ready: Vec<(String, RetryEvent)> = guard + .iter() + .filter(|(_, event)| event.next_retry_at <= now) + .map(|(key, event)| (key.clone(), event.clone())) + .collect(); + // Ascending by (score, key) to match Redis sorted-set semantics + // (same-score members are ordered lexicographically). + ready.sort_by(|(key_a, event_a), (key_b, event_b)| { + event_a + .next_retry_at + .cmp(&event_b.next_retry_at) + .then_with(|| key_a.cmp(key_b)) + }); + if let Some(limit) = limit { + ready.truncate(limit); + } + Ok(ready) + } +} diff --git a/nexus-watcher/src/service/backoff.rs b/nexus-watcher/src/service/backoff.rs index 4bea299b6..819192ad3 100644 --- a/nexus-watcher/src/service/backoff.rs +++ b/nexus-watcher/src/service/backoff.rs @@ -3,12 +3,14 @@ use std::time::{Duration, Instant}; use tracing::info; +#[derive(Clone)] struct BackoffState { next_backoff_secs: u64, backoff_until: Instant, } /// Tracks per-homeserver failure counts and exponential backoff windows. +#[derive(Clone)] pub struct HomeserverBackoff { initial_backoff_secs: u64, max_backoff_secs: u64, diff --git a/nexus-watcher/src/service/indexer/homeserver.rs b/nexus-watcher/src/service/indexer/homeserver.rs new file mode 100644 index 000000000..05e7e700e --- /dev/null +++ b/nexus-watcher/src/service/indexer/homeserver.rs @@ -0,0 +1,136 @@ +use super::TEventProcessor; +use crate::events::retry::RetryScheduler; +use crate::events::EventHandler; +use nexus_common::db::PubkyConnector; +use nexus_common::models::event::EventProcessorError; +use nexus_common::models::homeserver::Homeserver; +use pubky::Method; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::watch::Receiver; +use tracing::{debug, error, info, warn}; + +/// Event processor for the default homeserver +pub struct HsEventProcessor { + /// The default HS endpoint this processor fetches events from + pub homeserver: Homeserver, + + /// See [WatcherConfig::events_limit] + pub limit: u16, + pub files_path: PathBuf, + pub event_handler: Arc, + pub shutdown_rx: Receiver, + /// Scheduler used to enqueue failed events onto the retry queue + pub retry_scheduler: Arc, +} + +#[async_trait::async_trait] +impl TEventProcessor for HsEventProcessor { + fn files_path(&self) -> &PathBuf { + &self.files_path + } + + fn event_handler(&self) -> &Arc { + &self.event_handler + } + + fn instance_name(&self) -> String { + format!("HsEventProcessor with HS ID: {}", self.homeserver.id) + } + + fn retry_scheduler(&self) -> Option<&Arc> { + Some(&self.retry_scheduler) + } + + async fn run_internal(self: Arc) -> Result<(), EventProcessorError> { + let maybe_event_lines = self + .poll_events() + .await + .inspect_err(|e| error!("Error polling events: {e:?}"))?; + + match maybe_event_lines { + None => debug!("No new events"), + Some(event_lines) => { + info!("Processing {} event lines", event_lines.len()); + self.process_event_lines(event_lines).await?; + } + } + + Ok(()) + } +} + +impl HsEventProcessor { + /// Polls new events from the homeserver. + /// + /// It sends a GET request to the homeserver's events endpoint + /// using the current cursor and a specified limit. It retrieves new event + /// URIs in a newline-separated format, processes it into a vector of strings, + /// and returns the result. + #[tracing::instrument(name = "events.poll", skip_all, fields(homeserver = %self.homeserver.id))] + async fn poll_events(&self) -> Result>, EventProcessorError> { + debug!("Polling new events from homeserver"); + + let response_text = { + let pubky = PubkyConnector::get()?; + let url = format!( + "https://{}/events/?cursor={}&limit={}", + self.homeserver.id, self.homeserver.cursor, self.limit + ); + + let response = pubky + .client() + .request(Method::GET, &url) + .send() + .await + .map_err(|e| EventProcessorError::client_error(e.to_string()))?; + + response + .text() + .await + .map_err(|e| EventProcessorError::client_error(e.to_string()))? + }; + + let lines: Vec = response_text.trim().lines().map(String::from).collect(); + debug!("Homeserver response lines {:?}", lines); + + if lines.is_empty() || (lines.len() == 1 && lines[0].is_empty()) { + return Ok(None); + } + + Ok(Some(lines)) + } + + /// Processes a batch of event lines retrieved from the homeserver. + /// + /// This function implements the retry logic: + /// - On infrastructure error: stops the batch, cursor is not saved, next tick replays from same position + /// - On MissingDependency: stores event in retry queue, continues processing + /// - On 404 (blob not found): skips indexing, continues processing + /// - On InvalidEventLine/SkipIndexing: logs and continues + /// + /// # Parameters + /// - `lines`: A vector of strings representing event lines retrieved from the homeserver. + #[tracing::instrument(name = "event_batch.process", skip_all, fields(batch.size = lines.len()))] + pub async fn process_event_lines(&self, lines: Vec) -> Result<(), EventProcessorError> { + for line in &lines { + if *self.shutdown_rx.borrow() { + debug!(hs_id = %self.homeserver.id, "Shutdown detected; exiting event processing loop"); + return Ok(()); + } + + if let Some(cursor) = line.strip_prefix("cursor: ") { + info!("Received cursor for the next request: {cursor}"); + match Homeserver::try_from_cursor(self.homeserver.id.clone(), cursor) { + Ok(hs) => hs.put_to_index().await?, + Err(e) => warn!("{e}"), + } + continue; + } + + self.process_event_line(line).await?; + } + + Ok(()) + } +} diff --git a/nexus-watcher/src/service/indexer/key_based.rs b/nexus-watcher/src/service/indexer/key_based.rs new file mode 100644 index 000000000..30514c8b3 --- /dev/null +++ b/nexus-watcher/src/service/indexer/key_based.rs @@ -0,0 +1,307 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use futures::StreamExt; +use nexus_common::db::{PubkyConnector, RedisOps}; +use nexus_common::models::event::{Event, EventProcessorError}; +use nexus_common::models::homeserver::Homeserver; +use nexus_common::models::user::{user_hs_cursor_key, UserDetails}; +use pubky::{Event as StreamEvent, EventCursor, PublicKey}; +use tokio::sync::watch::Receiver; +use tracing::{debug, error, info}; + +use super::TEventProcessor; +use crate::events::retry::RetryScheduler; +use crate::events::EventHandler; +use crate::service::user_hs_resolver; + +#[async_trait::async_trait] +pub trait KeyBasedEventSource: Send + Sync + 'static { + async fn fetch_events( + &self, + hs_pk: &PublicKey, + user_pk: &PublicKey, + cursor: EventCursor, + limit: u16, + ) -> Result, EventProcessorError>; +} + +pub struct PubkyKeyBasedEventSource; + +#[async_trait::async_trait] +impl KeyBasedEventSource for PubkyKeyBasedEventSource { + async fn fetch_events( + &self, + hs_pk: &PublicKey, + user_pk: &PublicKey, + cursor: EventCursor, + limit: u16, + ) -> Result, EventProcessorError> { + let pubky = PubkyConnector::get()?; + + // We are building the stream without the live flag, so it performs an HTTP GET and closes. + // See rustdoc of EventStreamBuilder::live() + let mut stream = pubky + .event_stream_for(hs_pk) + .add_users(vec![(user_pk, Some(cursor))])? + .limit(limit) + .path("/pub/") + .subscribe() + .await + .inspect_err(|e| error!("Failed to subscribe to event stream: {e:?}"))?; + + let mut events = Vec::new(); + while let Some(result) = stream.next().await { + events.push(result?); + } + + Ok(events) + } +} + +/// Event processor for non-default HSs, where the user-specific `/events-stream` endpoint is used +pub struct KeyBasedEventProcessor { + /// The HS endpoint this processor fetches events from + pub homeserver: Homeserver, + + /// Max events the homeserver will send before closing the stream. + /// Bounds execution time per user, preventing timeout and starvation. + pub limit: u16, + pub files_path: PathBuf, + pub event_handler: Arc, + pub event_source: Arc, + /// Scheduler used to enqueue failed events onto the retry queue + pub retry_scheduler: Arc, + pub shutdown_rx: Receiver, +} + +#[async_trait::async_trait] +impl TEventProcessor for KeyBasedEventProcessor { + fn files_path(&self) -> &PathBuf { + &self.files_path + } + + fn event_handler(&self) -> &Arc { + &self.event_handler + } + + fn instance_name(&self) -> String { + format!("KeyBasedEventProcessor with HS ID: {}", self.homeserver.id) + } + + fn retry_scheduler(&self) -> Option<&Arc> { + Some(&self.retry_scheduler) + } + + async fn run_internal(self: Arc) -> Result<(), EventProcessorError> { + let hs_id = self.homeserver.id.to_string(); + + let hs_pk: PublicKey = hs_id.parse().map_err(|_| { + EventProcessorError::client_error(format!("Invalid homeserver public key: {hs_id}")) + })?; + + let users = self + .resolve_users_with_cursors(&hs_id) + .await + .inspect_err(|e| error!("Failed to resolve users for HS {hs_id}: {e:?}"))?; + + if users.is_empty() { + debug!("No users on HS {hs_id}, skipping"); + return Ok(()); + } + + info!("Found {} users on HS {hs_id}", users.len()); + + // TODO: Process users concurrently (bounded semaphore) to reduce per-HS latency + // when many users share a non-default homeserver. + for (user_pk, cursor) in &users { + if *self.shutdown_rx.borrow() { + debug!("Shutdown detected; stopping user iteration for HS {hs_id}"); + break; + } + + if let Err(err) = self.process_user(&hs_pk, &hs_id, user_pk, *cursor).await { + let user_id = user_pk.z32(); + if err.is_infrastructure() { + error!( + hs_id = %hs_id, + user = %user_id, + action = "abort_hs", + error = ?err, + "Infrastructure error while processing user; aborting homeserver run", + ); + return Err(err); + } + + error!( + hs_id = %hs_id, + user = %user_id, + action = "skip_user", + error = ?err, + "Non-infrastructure user error; continuing with next user", + ); + } + } + + Ok(()) + } +} + +impl KeyBasedEventProcessor { + /// Resolves monitored users on this homeserver and reads their cursors from Redis. + #[tracing::instrument(name = "dx.users.resolve", skip_all, fields(homeserver = %hs_id))] + async fn resolve_users_with_cursors( + &self, + hs_id: &str, + ) -> Result, EventProcessorError> { + let user_ids = user_hs_resolver::get_user_ids_by_homeserver(hs_id).await?; + debug!("Resolved {} user(s) on HS {hs_id}", user_ids.len()); + + let mut users = Vec::with_capacity(user_ids.len()); + for user_id in &user_ids { + let Ok(user_pk) = user_id.parse::() else { + error!("Invalid user public key '{user_id}' on HS {hs_id}, skipping"); + continue; + }; + // TODO Batch fetch cursors from Redis, when many users share a non-default homeserver. + let cursor = Self::read_user_cursor(user_id, hs_id).await?; + users.push((user_pk, cursor)); + } + + Ok(users) + } + + /// Subscribes to the event stream for a single user and processes incoming events. + /// + /// Each user gets their own `limit` budget, ensuring fair progress regardless + /// of how many events other users have produced. + #[tracing::instrument(name = "dx.user_events.process", skip_all, fields( + homeserver = %hs_id, + user = %user_pk.z32(), + ))] + async fn process_user( + &self, + hs_pk: &PublicKey, + hs_id: &str, + user_pk: &PublicKey, + cursor: EventCursor, + ) -> Result<(), EventProcessorError> { + let stream_events = self + .event_source + .fetch_events(hs_pk, user_pk, cursor, self.limit) + .await?; + + let user_id = user_pk.z32(); + let (latest_cursor, result) = self + .process_user_events(hs_id, &user_id, stream_events) + .await; + + if let Some(cursor_val) = latest_cursor { + if let Err(write_err) = Self::write_user_cursor(&user_id, hs_id, cursor_val).await { + // TODO: Queue failed cursor writes in the retry manager so they + // can be recovered without re-processing events. + error!( + hs_id = %hs_id, + user = %user_id, + cursor = cursor_val, + cursor_write_error = ?write_err, + "Best-effort cursor persist failed; events may be re-processed on next run", + ); + } + } + + result + } + + /// Processes already-fetched events for a single user stream. + /// + /// Returns the latest cursor that is safe to persist, plus the processing + /// result. Cursor advancement is intentionally skipped for `UserIdMismatch` + /// and handler errors so those events are fetched again on the next run. + async fn process_user_events( + &self, + hs_id: &str, + user_id: &str, + stream_events: Vec, + ) -> (Option, Result<(), EventProcessorError>) { + let mut latest_cursor: Option = None; + + for stream_event in stream_events { + if *self.shutdown_rx.borrow() { + debug!(hs_id = %hs_id, user = %user_id, "Shutdown detected; exiting event loop"); + break; + } + + let cursor_id = stream_event.cursor.id(); + + match Event::from_stream_event(&stream_event, self.files_path.clone()) { + Ok(Some(event)) => { + // External homeservers must not index another user's URI. + if let Err(err) = Self::validate_user_id(hs_id, &event, user_id) { + return (latest_cursor, Err(err)); + } + + if let Err(err) = self.handle_event(&event).await { + return (latest_cursor, Err(err)); + } + } + Ok(None) => { /* resource not handled by Nexus, skip */ } + Err(e) => { + error!(%hs_id, %user_id, %cursor_id, "Skipping unparseable stream event: {e}"); + } + } + + // Advance after successful handling, unsupported resources, or + // logged parse errors. UserIdMismatch and handler errors return + // before this point, so their cursor is not persisted. + latest_cursor = Some(cursor_id); + } + + (latest_cursor, Ok(())) + } + + fn validate_user_id( + hs_id: &str, + event: &Event, + expected_user_id: &str, + ) -> Result<(), EventProcessorError> { + let event_user_id = event.parsed_uri.user_id().as_str(); + if event_user_id != expected_user_id { + return Err(EventProcessorError::UserIdMismatch { + hs_id: hs_id.into(), + expected_user_id: expected_user_id.into(), + event_user_id: event_user_id.into(), + }); + } + + Ok(()) + } + + /// Reads the per-user event cursor from the `USER_HS_CURSOR` sorted set in Redis. + /// + /// Returns `EventCursor(0)` when the user has no cursor entry (newly ingested). + /// Propagates Redis errors instead of silently rewinding to 0. + /// + /// The cursor is stored as the score (f64) of the homeserver member. + /// f64 is exact for integer values up to 2^53 (~9 quadrillion), which is + /// practically unreachable for monotonically incrementing event IDs. + async fn read_user_cursor( + user_id: &str, + hs_id: &str, + ) -> Result { + let key = user_hs_cursor_key(user_id); + let score = UserDetails::check_sorted_set_member(None, &key, &[hs_id]).await?; + Ok(EventCursor::new(score.unwrap_or(0) as u64)) + } + + /// Persists the per-user event cursor back to the `USER_HS_CURSOR` sorted set. + async fn write_user_cursor( + user_id: &str, + hs_id: &str, + cursor: u64, + ) -> Result<(), EventProcessorError> { + let key = user_hs_cursor_key(user_id); + UserDetails::put_index_sorted_set(&key, &[(cursor as f64, hs_id)], None, None).await?; + Ok(()) + } +} diff --git a/nexus-watcher/src/service/indexer/mod.rs b/nexus-watcher/src/service/indexer/mod.rs new file mode 100644 index 000000000..2c2558681 --- /dev/null +++ b/nexus-watcher/src/service/indexer/mod.rs @@ -0,0 +1,204 @@ +mod homeserver; +mod key_based; + +pub use homeserver::HsEventProcessor; +pub use key_based::{KeyBasedEventProcessor, KeyBasedEventSource, PubkyKeyBasedEventSource}; +use nexus_common::models::event::ParseResult; +use std::{fmt::Display, path::PathBuf, sync::Arc, time::Duration}; + +use tracing::Instrument; + +use nexus_common::models::event::{Event, EventProcessorError}; +use tracing::{debug, error, warn}; + +use crate::events::retry::RetryScheduler; +use crate::events::EventHandler; +use crate::service::PROCESSING_TIMEOUT_SECS; + +/// Possible error types of an event processor run +#[derive(Debug)] +pub enum RunError { + Internal(EventProcessorError), + Panicked, + TimedOut, +} + +impl RunError { + pub fn is_panic(&self) -> bool { + matches!(self, RunError::Panicked) + } + + pub fn is_timeout(&self) -> bool { + matches!(self, RunError::TimedOut) + } +} + +impl std::error::Error for RunError {} + +impl Display for RunError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RunError::Internal(err) => write!(f, "Internal error: {err}"), + RunError::Panicked => write!(f, "Execution panicked"), + RunError::TimedOut => write!(f, "Execution timed out"), + } + } +} + +/// Asynchronous event processor interface for the Watcher service. +/// +/// This trait represents a component that can process events asynchronously and can be +/// gracefully shut down through a watch channel. +/// +/// # Implementation Notes +/// - Implementors should regularly check the `shutdown_rx` channel for shutdown signals +/// and terminate gracefully when received +/// - The method returns an `EventProcessorError` to allow for typed error handling across +/// different processor implementations +#[async_trait::async_trait] +pub trait TEventProcessor: Send + Sync + 'static { + fn files_path(&self) -> &PathBuf; + + /// Returns the event handler used to process events. + /// + /// This allows for flexible event handling implementations, including mocked versions for testing. + fn event_handler(&self) -> &Arc; + + /// Returns the instance name of the event processor, used in the monitoring and tracing spans. + /// + /// For instances mapped to a specific HS, this should include the HS ID. + fn instance_name(&self) -> String; + + /// Returns the retry scheduler used by [`Self::handle_error`] to enqueue failed + /// events for later retry. Returns `None` when the processor bypasses + /// [`Self::handle_error`] and manages retries on its own (e.g. [`RetryProcessor`](crate::events::retry::RetryProcessor)). + fn retry_scheduler(&self) -> Option<&Arc> { + None + } + + async fn run(self: Arc) -> Result<(), RunError> { + let timeout = self + .custom_timeout() + .unwrap_or(Duration::from_secs(PROCESSING_TIMEOUT_SECS)); + + let instance_name = self.instance_name(); + let span = tracing::info_span!("event_processor.run", service = %instance_name); + let handle = tokio::spawn(self.run_internal().instrument(span)); + + let join_result = tokio::time::timeout(timeout, handle) + .await + .inspect_err(|_| error!("Event processor timed out for {instance_name}")) + .map_err(|_| RunError::TimedOut)?; + + // The JoinError can be: + // - join_error.is_panic() => panic by the inner future + // - join_error.is_cancelled() => inner future was abruptly interrupted, for example + // - JoinHandle::abort() is called on the handle + // - the Tokio runtime is shut down + // In our model, we don't trigger such interruptions. Instead we use the shutdown signal + // to gracefully stop the event processing loop. Therefore we consider all JoinErrors as panics. + let run_internal_result = join_result + .inspect_err(|je| error!("JoinError by event processor for {instance_name}: {je:?}")) + .map_err(|_| RunError::Panicked)?; + + run_internal_result + .inspect_err(|e| error!("Event processor failed for {instance_name}: {e:?}")) + .map_err(RunError::Internal) + } + + /// Runs the event processor asynchronously. + /// + /// Returns `Ok(())` on a clean exit, or `Err(EventProcessorError)` on failure. + async fn run_internal(self: Arc) -> Result<(), EventProcessorError>; + + /// Optional custom timeout for this event processor. + /// + /// If not set, the [`PROCESSING_TIMEOUT_SECS`] is applied. + fn custom_timeout(&self) -> Option { + None + } + + /// Parses a single event line and dispatches to [`Self::handle_event`]. + /// Unknown resource events are handled via `HomeserverParsedUri::UnknownResource` → + /// `DefaultEventHandler` → `tag::sync_put_resource` (main flow). + async fn process_event_line(&self, line: &str) -> Result<(), EventProcessorError> { + match Event::parse_event(line, self.files_path().clone()) { + Err(e) => error!("{e}"), + Ok(ParseResult::Skipped) => {} + Ok(ParseResult::UnrecognizedUri { reason, .. }) => { + // Should not normally occur — UnknownResource parsing happens in HomeserverParsedUri + warn!("Unrecognized event URI: {reason}"); + } + Ok(ParseResult::Parsed(event)) => { + debug!("Processing event: {:?}", event); + self.handle_event(&event).await?; + } + } + + Ok(()) + } + + /// Handles an error of event processing from event processing (e.g. logging, scheduling retries). + /// + /// Called in the event processing loop. + /// + /// Returns: + /// - `Ok(())` - Continue processing the batch (non-retryable errors are dropped, retryable + /// ones are queued for retry) + /// - `Err(e)` - Stop processing and return error (for infrastructure errors) + async fn handle_error( + &self, + event: &Event, + error: EventProcessorError, + ) -> Result<(), EventProcessorError> { + if error.is_infrastructure() { + warn!("Infrastructure error, stopping batch: {error}"); + return Err(error); + } + + if !error.is_retryable() { + debug!("Non-retryable error, skipping event {}: {error}", event.uri); + return Ok(()); + } + + let Some(scheduler) = self.retry_scheduler() else { + return Ok(()); + }; + + if error.is_missing_dependency() { + scheduler.queue_missing_dep(event).await + } else { + warn!("Retryable error, queuing event for retry: {error}"); + scheduler.queue_transient(event).await + } + } + + /// Processes an event and delegates to [`Self::handle_error`] on failure. + #[tracing::instrument( + name = "event.process", + skip_all, + fields( + event.resource = %event.parsed_uri.resource(), + event.uri = %event.uri, + event.r#type = %event.event_type, + event.user_id = %event.parsed_uri.user_id(), + event.resource_id = event.parsed_uri.resource().id().unwrap_or_default(), + instance = %self.instance_name(), + otel.status_code = tracing::field::Empty, + otel.status_message = tracing::field::Empty, + ) + )] + async fn handle_event(&self, event: &Event) -> Result<(), EventProcessorError> { + let span = tracing::Span::current(); + if let Err(e) = self.event_handler().handle(event).await { + span.record("otel.status_code", "ERROR"); + span.record("otel.status_message", tracing::field::display(&e)); + + self.handle_error(event, e).await?; + } else { + span.record("otel.status_code", "OK"); + } + + Ok(()) + } +} diff --git a/nexus-watcher/src/service/mod.rs b/nexus-watcher/src/service/mod.rs index 6b43ca5d8..b2c72e9c0 100644 --- a/nexus-watcher/src/service/mod.rs +++ b/nexus-watcher/src/service/mod.rs @@ -1,27 +1,32 @@ pub mod backoff; mod constants; -mod processor; -mod processor_runner; -mod stats; -mod traits; +pub mod indexer; +pub mod runner; +pub mod stats; +mod task_runner; +pub mod user_hs_resolver; /// Module exports pub use constants::{PROCESSING_TIMEOUT_SECS, WATCHER_CONFIG_FILE_NAME}; -use nexus_common::types::DynError; -pub use processor::EventProcessor; -pub use processor_runner::EventProcessorRunner; -pub use traits::{TEventProcessor, TEventProcessorRunner}; +pub use indexer::{HsEventProcessor, KeyBasedEventProcessor, RunError, TEventProcessor}; +pub use runner::{HsEventProcessorRunner, KeyBasedEventProcessorRunner, TEventProcessorRunner}; +pub(crate) use task_runner::{run_periodic_tasks, PeriodicTask}; + +/// Sleep interval for the retry processor (10 seconds) +const RETRY_PROCESSOR_SLEEP: u64 = 10_000; +use crate::events::retry::RetryProcessor; +use crate::service::task_runner::task_results_into_result; use crate::NexusWatcherBuilder; use nexus_common::file::ConfigLoader; use nexus_common::models::homeserver::Homeserver; +use nexus_common::types::DynError; use nexus_common::utils::create_shutdown_rx; use nexus_common::{DaemonConfig, WatcherConfig}; -use pubky_app_specs::PubkyId; use std::path::PathBuf; +use std::sync::Arc; use tokio::sync::watch::Receiver; -use tokio::time::Duration; -use tracing::{debug, error, info}; +use tracing::{debug, info}; pub struct NexusWatcher {} @@ -68,38 +73,60 @@ impl NexusWatcher { NexusWatcherBuilder(watcher_config).start(shutdown_rx).await } - pub async fn start( - mut shutdown_rx: Receiver, - config: WatcherConfig, - ) -> Result<(), DynError> { + /// Starts the Nexus Watcher with parallel periodic task loops. + /// + /// Currently runs three tasks: + /// 1. **Default homeserver**: Processes events from the default homeserver defined in [`WatcherConfig`]. + /// 2. **External homeservers**: Processes events from all external monitored homeservers, excluding the default. + /// 3. **User HS resolver**: Resolves each user's homeserver and persists `HOSTED_BY` relationships. + /// + /// The event-processing tasks share the same tick interval ([`WatcherConfig::watcher_sleep`]), + /// while the HS resolver uses its own interval ([`WatcherConfig::hs_resolver_sleep`]). + /// All tasks listen for the shutdown signal to exit gracefully. If any task panics, + /// an internal cancellation signal is sent so that sibling tasks can finish their + /// current iteration and exit. + pub async fn start(shutdown_rx: Receiver, config: WatcherConfig) -> Result<(), DynError> { debug!(?config, "Running NexusWatcher with "); - let config_hs = PubkyId::try_from(config.homeserver.as_str())?; - Homeserver::persist_if_unknown(config_hs).await?; - - let mut interval = tokio::time::interval(Duration::from_millis(config.watcher_sleep)); - let ev_processor_runner = EventProcessorRunner::from_config(&config, shutdown_rx.clone()); - let mut backoff = crate::service::backoff::HomeserverBackoff::new( - config.initial_backoff_secs, - config.max_backoff_secs, - ); - - loop { - tokio::select! { - _ = shutdown_rx.changed() => { - info!("SIGINT received, exiting Nexus Watcher loop"); - break; - } - _ = interval.tick() => { - debug!("Indexing homeservers…"); - _ = ev_processor_runner - .run_all(&mut backoff) - .await - .inspect_err(|e| error!("Failed to start event processors run: {e}")); - } - } - } + Homeserver::persist_if_unknown(config.homeserver.clone()).await?; + + let watcher_sleep = config.watcher_sleep; + let hs_resolver_sleep = config.hs_resolver_sleep; + let hs_resolver_ttl = config.hs_resolver_ttl; + + let hs_runner = Arc::new(HsEventProcessorRunner::from_config( + &config, + shutdown_rx.clone(), + )); + let key_based_runner = Arc::new(KeyBasedEventProcessorRunner::from_config( + &config, + shutdown_rx.clone(), + )); + + // Create retry processor + let retry_processor = Arc::new(RetryProcessor::new(&config, shutdown_rx.clone())); + + let tasks = vec![ + PeriodicTask::new("default-homeserver", watcher_sleep, move || { + let runner = hs_runner.clone(); + async move { runner.run().await.map(|_| ()) } + }), + PeriodicTask::new("external-homeservers", watcher_sleep, move || { + let runner = key_based_runner.clone(); + async move { runner.run().await.map(|_| ()) } + }), + PeriodicTask::new("user-hs-resolver", hs_resolver_sleep, move || async move { + user_hs_resolver::run(hs_resolver_ttl).await + }), + PeriodicTask::new("retry-processor", RETRY_PROCESSOR_SLEEP, move || { + let processor = retry_processor.clone(); + async move { processor.run().await.map_err(DynError::from) } + }), + ]; + + let task_results = run_periodic_tasks(tasks, shutdown_rx).await; + info!("Nexus Watcher shut down gracefully"); - Ok(()) + task_results_into_result(task_results) } } diff --git a/nexus-watcher/src/service/processor.rs b/nexus-watcher/src/service/processor.rs deleted file mode 100644 index ab0d0c555..000000000 --- a/nexus-watcher/src/service/processor.rs +++ /dev/null @@ -1,246 +0,0 @@ -use nexus_common::models::event::{Event, EventProcessorError, EventType, ParseResult}; - -use crate::events::handle; -use crate::events::retry::event::RetryEvent; -use crate::events::Moderation; -use crate::service::traits::TEventProcessor; -use nexus_common::db::PubkyConnector; -use nexus_common::models::homeserver::Homeserver; -use opentelemetry::trace::{FutureExt, Span, TraceContextExt, Tracer}; -use opentelemetry::{global, Context, KeyValue}; -use pubky::Method; -use pubky_app_specs::PubkyId; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::sync::watch::Receiver; -use tracing::{debug, error, info, warn}; - -pub struct EventProcessor { - pub homeserver: Homeserver, - /// See [WatcherConfig::events_limit] - pub limit: u32, - pub files_path: PathBuf, - pub tracer_name: String, - pub moderation: Arc, - pub shutdown_rx: Receiver, -} - -#[async_trait::async_trait] -impl TEventProcessor for EventProcessor { - fn get_homeserver_id(&self) -> PubkyId { - self.homeserver.id.clone() - } - - async fn run_internal(self: Arc) -> Result<(), EventProcessorError> { - let maybe_event_lines = self - .poll_events() - .await - .inspect_err(|e| error!("Error polling events: {e:?}"))?; - - match maybe_event_lines { - None => debug!("No new events"), - Some(event_lines) => { - info!("Processing {} event lines", event_lines.len()); - self.process_event_lines(event_lines).await?; - } - } - - Ok(()) - } -} - -impl EventProcessor { - /// Polls new events from the homeserver. - /// - /// It sends a GET request to the homeserver's events endpoint - /// using the current cursor and a specified limit. It retrieves new event - /// URIs in a newline-separated format, processes it into a vector of strings, - /// and returns the result. - #[tracing::instrument(name = "events.poll", skip_all, fields(homeserver = %self.homeserver.id))] - async fn poll_events(&self) -> Result>, EventProcessorError> { - debug!("Polling new events from homeserver"); - - let response_text = { - let pubky = PubkyConnector::get()?; - let url = format!( - "https://{}/events/?cursor={}&limit={}", - self.homeserver.id, self.homeserver.cursor, self.limit - ); - - let response = pubky - .client() - .request(Method::GET, &url) - .send() - .await - .map_err(|e| EventProcessorError::client_error(e.to_string()))?; - - response - .text() - .await - .map_err(|e| EventProcessorError::client_error(e.to_string()))? - }; - - let lines: Vec = response_text.trim().lines().map(String::from).collect(); - debug!("Homeserver response lines {:?}", lines); - - if lines.is_empty() || (lines.len() == 1 && lines[0].is_empty()) { - return Ok(None); - } - - Ok(Some(lines)) - } - - /// Processes a batch of event lines retrieved from the homeserver. - /// - /// This function iterates over a vector of event URIs, handling each line based on its content: - /// - Lines starting with `cursor:` update the cursor for the homeserver and save it to the index. - /// - Other lines are parsed into events and processed accordingly. If parsing fails, an error is logged. - /// - /// # Parameters - /// - `lines`: A vector of strings representing event lines retrieved from the homeserver. - #[tracing::instrument(name = "event_batch.process", skip_all, fields(batch.size = lines.len()))] - pub async fn process_event_lines(&self, lines: Vec) -> Result<(), EventProcessorError> { - for line in &lines { - let id = self.homeserver.id.clone(); - - if *self.shutdown_rx.borrow() { - debug!("Shutdown detected while processing HS {id}, exiting event processing loop"); - return Ok(()); - } - - if let Some(cursor) = line.strip_prefix("cursor: ") { - info!("Received cursor for the next request: {cursor}"); - match Homeserver::try_from_cursor(id, cursor) { - Ok(hs) => hs.put_to_index().await?, - Err(e) => warn!("{e}"), - } - } else { - match Event::parse_event(line, self.files_path.clone()) { - Err(e) => error!("{e}"), - Ok(ParseResult::Skipped) => {} - Ok(ParseResult::UnrecognizedUri { - event_type, - uri, - reason, - }) => { - if !self.try_handle_universal_tag(&event_type, &uri).await { - error!("Cannot parse event URI: {reason}"); - } - } - Ok(ParseResult::Parsed(event)) => { - let tracer = global::tracer(self.tracer_name.clone()); - let mut span = tracer.start(event.parsed_uri.resource.to_string()); - span.set_attribute(KeyValue::new("event.uri", event.uri.clone())); - span.set_attribute(KeyValue::new( - "event.type", - event.event_type.to_string(), - )); - span.set_attribute(KeyValue::new( - "event.user_id", - event.parsed_uri.user_id.to_string(), - )); - span.set_attribute(KeyValue::new( - "event.resource_id", - event.parsed_uri.resource.id().unwrap_or("".to_string()), - )); - let cx = Context::new().with_span(span); - debug!("Processing event: {:?}", event); - self.handle_event(&event).with_context(cx).await?; - } - } - } - } - - Ok(()) - } - - /// Attempts to handle an unrecognized URI as a universal tag at an app-specific path. - /// Returns `true` if the event was claimed (regardless of success/failure). - async fn try_handle_universal_tag(&self, event_type: &EventType, uri: &str) -> bool { - let result = crate::events::handlers::universal_tag::try_handle(event_type, uri).await; - - let Some(result) = result else { - return false; - }; - - if let Err(e) = result { - match e { - EventProcessorError::InvalidEventLine(ref msg) => { - error!("Universal tag non-retryable: {msg}"); - } - _ => { - let index_key = format!("{event_type}:{uri}"); - let retry_event = RetryEvent::new(e); - error!("{}, {}", retry_event.error_type, index_key); - if let Err(err) = retry_event.put_to_index(index_key).await { - error!("Failed to enqueue universal tag retry: {err}"); - } - } - } - } - - true - } - - /// Processes an event and track the fail event it if necessary - /// # Parameters: - /// - `event`: The event to be processed - #[tracing::instrument( - name = "event.process", - skip_all, - fields( - event.resource = %event.parsed_uri.resource, - event.uri = %event.uri, - event.r#type = %event.event_type, - event.user_id = %event.parsed_uri.user_id, - event.resource_id = event.parsed_uri.resource.id().unwrap_or_default(), - homeserver = %self.homeserver.id, - otel.status_code = tracing::field::Empty, - otel.status_message = tracing::field::Empty, - ) - )] - async fn handle_event(&self, event: &Event) -> Result<(), EventProcessorError> { - let span = tracing::Span::current(); - if let Err(e) = handle(event, self.moderation.clone()).await { - span.record("otel.status_code", "ERROR"); - span.record("otel.status_message", tracing::field::display(&e)); - - if let Some((index_key, retry_event)) = extract_retry_event_info(event, e) { - error!("{}, {}", retry_event.error_type, index_key); - if let Err(err) = retry_event.put_to_index(index_key).await { - error!("Failed to put event to retry index: {}", err); - } - } - } else { - span.record("otel.status_code", "OK"); - } - Ok(()) - } -} - -/// Extracts retry-related information from an event and its associated error -/// -/// # Parameters -/// - `event`: Reference to the event for which retry information is being extracted -/// - `error`: Determines whether the event is eligible for a retry or should be discarded -fn extract_retry_event_info( - event: &Event, - error: EventProcessorError, -) -> Option<(String, RetryEvent)> { - let retry_event = match error { - EventProcessorError::InvalidEventLine(ref message) => { - error!("{}", message); - return None; - } - _ => RetryEvent::new(error), - }; - - // Generate a compress index to save in the cache - let index = match RetryEvent::generate_index_key(&event.uri) { - Some(retry_index) => retry_index, - None => { - return None; - } - }; - Some((format!("{}:{}", event.event_type, index), retry_event)) -} diff --git a/nexus-watcher/src/service/processor_runner.rs b/nexus-watcher/src/service/processor_runner.rs deleted file mode 100644 index cc096a9ff..000000000 --- a/nexus-watcher/src/service/processor_runner.rs +++ /dev/null @@ -1,89 +0,0 @@ -use crate::events::Moderation; -use crate::service::processor::EventProcessor; -use crate::service::traits::{TEventProcessor, TEventProcessorRunner}; -use nexus_common::models::homeserver::Homeserver; -use nexus_common::types::DynError; -use nexus_common::WatcherConfig; -use pubky_app_specs::PubkyId; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::sync::watch::Receiver; - -pub struct EventProcessorRunner { - /// See [WatcherConfig::events_limit] - pub limit: u32, - /// See [WatcherConfig::monitored_homeservers_limit] - pub monitored_homeservers_limit: usize, - pub files_path: PathBuf, - pub tracer_name: String, - pub moderation: Arc, - pub shutdown_rx: Receiver, - /// See [WatcherConfig::homeserver] - pub default_homeserver: PubkyId, -} - -impl EventProcessorRunner { - /// Creates a new instance from the provided configuration - pub fn from_config(config: &WatcherConfig, shutdown_rx: Receiver) -> Self { - Self { - limit: config.events_limit, - monitored_homeservers_limit: config.monitored_homeservers_limit, - files_path: config.stack.files_path.clone(), - tracer_name: config.stack.otlp.name.clone(), - moderation: Arc::new(Moderation { - id: config.moderation_id.clone(), - tags: config.moderated_tags.clone(), - }), - shutdown_rx, - default_homeserver: config.homeserver.clone(), - } - } -} - -#[async_trait::async_trait] -impl TEventProcessorRunner for EventProcessorRunner { - fn shutdown_rx(&self) -> Receiver { - self.shutdown_rx.clone() - } - - fn default_homeserver(&self) -> &str { - &self.default_homeserver - } - - fn monitored_homeservers_limit(&self) -> usize { - self.monitored_homeservers_limit - } - - async fn homeservers_by_priority(&self) -> Result, DynError> { - let mut hs_ids = Homeserver::get_all_from_graph().await?; - - // Move default homeserver to index 0 if it exists in the array to prioritize its processing - if let Some(default_pos) = hs_ids - .iter() - .position(|hs_id| hs_id == self.default_homeserver()) - { - let default_hs = hs_ids.remove(default_pos); - hs_ids.insert(0, default_hs); - } - - Ok(hs_ids) - } - - /// Creates and returns a new event processor instance for the specified homeserver - async fn build(&self, homeserver_id: String) -> Result, DynError> { - let homeserver_id = PubkyId::try_from(&homeserver_id)?; - let homeserver = Homeserver::get_by_id(homeserver_id) - .await? - .ok_or("Homeserver not found")?; - - // Create a new event processor instance with the specified homeserver - Ok(Arc::new(EventProcessor { - homeserver, - limit: self.limit, - files_path: self.files_path.clone(), - tracer_name: self.tracer_name.clone(), - moderation: self.moderation.clone(), - shutdown_rx: self.shutdown_rx.clone(), - })) - } -} diff --git a/nexus-watcher/src/service/runner/homeserver.rs b/nexus-watcher/src/service/runner/homeserver.rs new file mode 100644 index 000000000..8f187e9c7 --- /dev/null +++ b/nexus-watcher/src/service/runner/homeserver.rs @@ -0,0 +1,69 @@ +use super::TEventProcessorRunner; +use crate::events::retry::RetryScheduler; +use crate::events::{DefaultEventHandler, EventHandler, Moderation}; +use crate::service::indexer::{HsEventProcessor, TEventProcessor}; +use nexus_common::models::homeserver::Homeserver; +use nexus_common::types::DynError; +use nexus_common::WatcherConfig; +use pubky_app_specs::PubkyId; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::watch::Receiver; + +pub struct HsEventProcessorRunner { + /// See [WatcherConfig::events_limit] + pub limit: u16, + pub files_path: PathBuf, + pub event_handler: Arc, + pub shutdown_rx: Receiver, + /// See [WatcherConfig::homeserver] + pub default_homeserver: PubkyId, + /// Scheduler shared with every processor this runner builds + pub retry_scheduler: Arc, +} + +impl HsEventProcessorRunner { + /// Creates a new instance from the provided configuration + pub fn from_config(config: &WatcherConfig, shutdown_rx: Receiver) -> Self { + Self { + limit: config.events_limit, + files_path: config.stack.files_path.clone(), + event_handler: Arc::new(DefaultEventHandler::new(Moderation::from_config(config))), + shutdown_rx, + default_homeserver: config.homeserver.clone(), + retry_scheduler: Arc::new(RetryScheduler::from_config(config)), + } + } + + pub fn default_homeserver(&self) -> &str { + &self.default_homeserver + } +} + +#[async_trait::async_trait] +impl TEventProcessorRunner for HsEventProcessorRunner { + fn shutdown_rx(&self) -> Receiver { + self.shutdown_rx.clone() + } + + /// Creates and returns a new event processor instance for the specified homeserver + async fn build(&self, homeserver_id: String) -> Result, DynError> { + let homeserver_id = PubkyId::try_from(&homeserver_id)?; + let homeserver = Homeserver::get_by_id(homeserver_id) + .await? + .ok_or("Homeserver not found")?; + + Ok(Arc::new(HsEventProcessor { + homeserver, + limit: self.limit, + files_path: self.files_path.clone(), + event_handler: self.event_handler.clone(), + shutdown_rx: self.shutdown_rx.clone(), + retry_scheduler: self.retry_scheduler.clone(), + })) + } + + async fn pre_run(&self) -> Result, DynError> { + Ok(vec![self.default_homeserver.to_string()]) + } +} diff --git a/nexus-watcher/src/service/runner/key_based.rs b/nexus-watcher/src/service/runner/key_based.rs new file mode 100644 index 000000000..e415b86e3 --- /dev/null +++ b/nexus-watcher/src/service/runner/key_based.rs @@ -0,0 +1,156 @@ +use super::TEventProcessorRunner; +use crate::events::retry::RetryScheduler; +use crate::events::{DefaultEventHandler, EventHandler, Moderation}; +use crate::service::backoff::HomeserverBackoff; +use crate::service::indexer::{ + KeyBasedEventProcessor, KeyBasedEventSource, PubkyKeyBasedEventSource, TEventProcessor, +}; +use crate::service::stats::{ProcessedStats, ProcessorRunStatus, RunAllProcessorsStats}; +use nexus_common::models::homeserver::Homeserver; +use nexus_common::types::DynError; +use nexus_common::WatcherConfig; +use pubky_app_specs::PubkyId; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::{watch::Receiver, Mutex}; +use tracing::{debug, info, warn}; + +/// Runner for [KeyBasedEventProcessor] +pub struct KeyBasedEventProcessorRunner { + /// See [WatcherConfig::key_based_events_limit] + pub limit: u16, + + /// See [WatcherConfig::monitored_homeservers_limit] + pub monitored_hs_limit: usize, + + pub files_path: PathBuf, + pub event_handler: Arc, + pub event_source: Arc, + pub shutdown_rx: Receiver, + + /// Default homeserver ID, excluded from the external targets list + pub default_homeserver: PubkyId, + + /// Per-target exponential backoff state + pub backoff: Mutex, + + /// Scheduler shared with every processor this runner builds + pub retry_scheduler: Arc, +} + +impl KeyBasedEventProcessorRunner { + /// Creates a new instance from the provided configuration + pub fn from_config(config: &WatcherConfig, shutdown_rx: Receiver) -> Self { + Self { + limit: config.key_based_events_limit, + monitored_hs_limit: config.monitored_homeservers_limit, + files_path: config.stack.files_path.clone(), + event_handler: Arc::new(DefaultEventHandler::new(Moderation::from_config(config))), + event_source: Arc::new(PubkyKeyBasedEventSource), + shutdown_rx, + default_homeserver: config.homeserver.clone(), + backoff: Mutex::new(HomeserverBackoff::new( + config.initial_backoff_secs, + config.max_backoff_secs, + )), + retry_scheduler: Arc::new(RetryScheduler::from_config(config)), + } + } + + /// Returns the homeserver IDs relevant for this run, ordered by their priority. + /// The default homeserver is excluded from this list. + async fn hs_by_priority(&self) -> Result, DynError> { + let hs_ids = Homeserver::get_all_active_from_graph().await?; + let default_hs = self.default_homeserver.as_str(); + + // Exclude the default homeserver from the list, as it is processed separately + // The default HS is not expected to be active, but we still filter as an extra precaution + let hs_ids: Vec = hs_ids + .into_iter() + .filter(|hs_id| hs_id != default_hs) + .collect(); + + Ok(hs_ids) + } +} + +#[async_trait::async_trait] +impl TEventProcessorRunner for KeyBasedEventProcessorRunner { + fn shutdown_rx(&self) -> Receiver { + self.shutdown_rx.clone() + } + + async fn build(&self, hs_id: String) -> Result, DynError> { + let homeserver_id = PubkyId::try_from(&hs_id)?; + let homeserver = Homeserver::get_by_id(homeserver_id) + .await? + .ok_or("Homeserver not found")?; + + Ok(Arc::new(KeyBasedEventProcessor { + homeserver, + limit: self.limit, + files_path: self.files_path.clone(), + event_handler: self.event_handler.clone(), + event_source: self.event_source.clone(), + retry_scheduler: self.retry_scheduler.clone(), + shutdown_rx: self.shutdown_rx.clone(), + })) + } + + async fn pre_run(&self) -> Result, DynError> { + let hs_ids = self.hs_by_priority().await?; + let max_index = std::cmp::min(self.monitored_hs_limit, hs_ids.len()); + Ok(hs_ids[..max_index].to_vec()) + } + + async fn backoff_should_skip(&self, hs_id: &str) -> Option { + let backoff = self.backoff.lock().await; + if backoff.should_skip(hs_id) { + debug!(hs_id = %hs_id, "Skipping homeserver in backoff"); + Some(ProcessorRunStatus::Skipped) + } else { + None + } + } + + async fn backoff_on_result(&self, hs_id: &str, status: &ProcessorRunStatus) { + let mut backoff = self.backoff.lock().await; + if *status == ProcessorRunStatus::Ok { + backoff.record_success(hs_id); + } else { + backoff.record_failure(hs_id); + } + } + + async fn post_run(&self, stats: RunAllProcessorsStats) -> ProcessedStats { + for individual_run_stat in &stats.stats { + let hs_id = &individual_run_stat.hs_id; + let duration = individual_run_stat.duration; + let status = &individual_run_stat.status; + debug!( + hs_id = %hs_id, + duration = ?duration, + status = ?status, + "Event processor run completed" + ); + } + + let count_ok = stats.count_ok(); + let count_error = stats.count_error(); + let count_panic = stats.count_panic(); + let count_timeout = stats.count_timeout(); + let count_failed_to_build = stats.count_failed_to_build(); + let count_skipped = stats.count_skipped(); + let had_issues = count_error + count_panic + count_timeout + count_failed_to_build > 0; + + if had_issues { + warn!("Run result: {count_ok} ok, {count_skipped} skipped (backoff), {count_failed_to_build} failed to build, {count_error} error, {count_panic} panic, {count_timeout} timeout"); + } else if count_skipped > 0 { + info!("Run result: {count_ok} ok, {count_skipped} skipped (backoff)"); + } else { + debug!("Run result: {count_ok} ok"); + } + + ProcessedStats(stats) + } +} diff --git a/nexus-watcher/src/service/runner/mod.rs b/nexus-watcher/src/service/runner/mod.rs new file mode 100644 index 000000000..7683359b0 --- /dev/null +++ b/nexus-watcher/src/service/runner/mod.rs @@ -0,0 +1,112 @@ +mod homeserver; +mod key_based; + +pub use homeserver::HsEventProcessorRunner; +pub use key_based::KeyBasedEventProcessorRunner; + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use nexus_common::types::DynError; +use tokio::sync::watch::Receiver; +use tracing::{error, info}; + +use crate::service::{ + indexer::{RunError, TEventProcessor}, + stats::{ProcessedStats, ProcessorRunStatus, RunAllProcessorsStats}, +}; + +pub fn status_from_run_result(result: Result<(), RunError>) -> ProcessorRunStatus { + match result { + Ok(_) => ProcessorRunStatus::Ok, + Err(RunError::Internal(_)) => ProcessorRunStatus::Error, + Err(RunError::Panicked) => ProcessorRunStatus::Panic, + Err(RunError::TimedOut) => ProcessorRunStatus::Timeout, + } +} + +/// The orchestrator that helps build and run event processors in the Watcher service. +/// +/// # Implementation Notes +/// - The `build` method should create and return a fully configured event processor ready for immediate use +/// - Implementors should ensure that created processors are properly isolated and don't share mutable state unless explicitly intended +#[async_trait::async_trait] +pub trait TEventProcessorRunner: Send + Sync { + /// Returns the shutdown signal receiver + fn shutdown_rx(&self) -> Receiver; + + /// Creates and returns a new event processor instance for the specified homeserver. + /// + /// # Parameters + /// * `hs_id` - The homeserver PubkyId. Represents the homeserver this event processor will + /// fetch and process events from. + /// + /// # Returns + /// A reference to the event processor instance, ready to be executed with its `run` method. + /// + /// # Errors + /// Returns an error if the event processor couldn't be built + async fn build(&self, hs_id: String) -> Result, DynError>; + + /// Pre-processing step before the main run loop. + /// + /// Determines the list of target HS IDs to process in this run cycle. + async fn pre_run(&self) -> Result, DynError>; + + /// Post-processing of the run results. + /// + /// No-op default implementation. Callers that perform post-processing should overwrite this. + async fn post_run(&self, stats: RunAllProcessorsStats) -> ProcessedStats { + ProcessedStats(stats) + } + + /// Main run loop: builds and runs event processors for the relevant targets. + /// + /// # Returns + /// Statistics about the event processor run results, summarized as [`ProcessedStats`] + async fn run(&self) -> Result { + let hs_ids = self.pre_run().await?; + let mut run_stats = RunAllProcessorsStats::default(); + + for hs_id in hs_ids { + if *self.shutdown_rx().borrow() { + info!(hs_id = %hs_id, "Shutdown detected; exiting run loop"); + break; + } + + if let Some(skip_status) = self.backoff_should_skip(&hs_id).await { + run_stats.add_run_result(hs_id, Duration::ZERO, skip_status); + continue; + } + + let t0 = Instant::now(); + let status = match self.build(hs_id.clone()).await { + Ok(event_processor) => status_from_run_result(event_processor.run().await), + Err(e) => { + error!(hs_id = %hs_id, error = %e, "Failed to build event processor"); + ProcessorRunStatus::FailedToBuild + } + }; + let duration = t0.elapsed(); + + self.backoff_on_result(&hs_id, &status).await; + run_stats.add_run_result(hs_id, duration, status); + } + + let processed_stats = self.post_run(run_stats).await; + Ok(processed_stats) + } + + /// Called before processing a homeserver, to check if backoff mechanism indicates it + /// should be skipped. Return `Some(status)` to skip it. + /// + /// No-op default implementation. Runners that use backoff should overwrite as needed. + async fn backoff_should_skip(&self, _hs_id: &str) -> Option { + None + } + + /// Called after a homeserver is processed (build + run), to update its backoff status. + /// + /// No-op default implementation. Runners that use backoff should overwrite as needed. + async fn backoff_on_result(&self, _hs_id: &str, _status: &ProcessorRunStatus) {} +} diff --git a/nexus-watcher/src/service/stats.rs b/nexus-watcher/src/service/stats.rs index 1a8d89e4c..94f1d3b6b 100644 --- a/nexus-watcher/src/service/stats.rs +++ b/nexus-watcher/src/service/stats.rs @@ -22,7 +22,7 @@ pub struct RunAllProcessorsStats { } impl RunAllProcessorsStats { - pub(crate) fn add_run_result( + pub fn add_run_result( &mut self, hs_id: String, duration: Duration, diff --git a/nexus-watcher/src/service/task_runner.rs b/nexus-watcher/src/service/task_runner.rs new file mode 100644 index 000000000..c9ddc5d27 --- /dev/null +++ b/nexus-watcher/src/service/task_runner.rs @@ -0,0 +1,413 @@ +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use tokio::sync::watch::{self, Receiver}; +use tokio::task::JoinSet; +use tokio::time::{Duration, MissedTickBehavior}; +use tracing::{debug, error, info}; + +use nexus_common::types::DynError; + +/// A boxed async function that can be called repeatedly to produce a future. +/// +/// Each invocation represents one "tick" of a periodic task. +pub type TaskFn = + Arc Pin> + Send>> + Send + Sync>; + +/// A periodic task to be run by [`run_periodic_tasks`]. +/// +/// Each task has a name (for logging), an interval in milliseconds, and an +/// async function that will be called on every tick of its interval timer. +pub(crate) struct PeriodicTask { + name: String, + interval_ms: u64, + task_fn: TaskFn, +} + +impl PeriodicTask { + /// Create a [`PeriodicTask`] from a name, interval, and any `Fn` that returns a future. + /// + /// This handles the `Arc` + `Box::pin` wrapping so callers don't have to. + /// + /// ```ignore + /// PeriodicTask::new("my-task", 5000, move || { + /// let runner = runner.clone(); + /// async move { runner.do_work().await } + /// }) + /// ``` + pub fn new(name: impl Into, interval_ms: u64, f: F) -> Self + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + Self { + name: name.into(), + interval_ms, + task_fn: Arc::new(move || Box::pin(f())), + } + } +} + +/// Runs a set of [`PeriodicTask`]s concurrently, each on its own interval tick. +/// +/// Each task runs in a loop that: +/// 1. Waits for the next interval tick (using the task's own `interval_ms`) +/// 2. Calls the task's async function +/// 3. Checks for shutdown or cancellation signals +/// +/// If any task **panics**, an internal cancellation signal is sent so that all +/// other tasks stop after completing their current iteration. The function +/// then waits for all tasks to finish before returning. +/// +/// The `shutdown_rx` channel provides an external shutdown signal (e.g. Ctrl-C). +/// +/// # Returns +/// +/// A `Vec` with one entry per task, indicating how it finished. +pub(crate) async fn run_periodic_tasks( + tasks: Vec, + shutdown_rx: Receiver, +) -> Vec { + let (cancel_tx, cancel_rx) = watch::channel(false); + let mut join_set = JoinSet::new(); + let mut name_map: HashMap = HashMap::new(); + + for task in tasks { + let mut shutdown = shutdown_rx.clone(); + let mut cancel = cancel_rx.clone(); + let name = task.name.clone(); + let task_fn = task.task_fn; + let interval_ms = task.interval_ms; + + let handle = join_set.spawn(async move { + let mut interval = tokio::time::interval(Duration::from_millis(interval_ms)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { + // Enforce polling of branches from top to bottom + // This ensures shutdown / cancel signals are always checked first on every iteration + biased; + + _ = shutdown.changed() => { + info!("Shutdown received, exiting '{name}' loop"); + break; + } + _ = cancel.changed() => { + info!("Cancellation received, exiting '{name}' loop"); + break; + } + _ = interval.tick() => { + debug!("Running task: {name}"); + if let Err(e) = (task_fn)().await { + error!("Task '{name}' returned error: {e}"); + } + } + } + } + }); + name_map.insert(handle.id(), task.name); + } + + // Drain the JoinSet. Each completed task is checked for panics; + // a panic sends the cancellation signal so surviving siblings exit promptly. + let mut results = Vec::with_capacity(join_set.len()); + while let Some(join_result) = join_set.join_next_with_id().await { + let mut get_task_name_fn = |id| { + name_map + .remove(&id) + .unwrap_or_else(|| format!(" { + let name = get_task_name_fn(id); + results.push(TaskResult::completed(&name)); + } + Err(join_error) => { + let id = join_error.id(); + let name = get_task_name_fn(id); + error!("Task panicked: {name}"); + let _ = cancel_tx.send(true); + results.push(TaskResult::panicked(&name)); + } + } + } + + results +} + +/// The outcome of a single periodic task. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum TaskOutcome { + /// The task exited its loop normally (shutdown or cancellation signal). + Completed, + /// The task panicked, which triggered cancellation of siblings. + Panicked, +} + +/// Result of running a single periodic task. +#[derive(Debug, Clone)] +pub(crate) struct TaskResult { + name: String, + outcome: TaskOutcome, +} + +impl TaskResult { + fn completed(name: &str) -> Self { + TaskResult { + name: name.into(), + outcome: TaskOutcome::Completed, + } + } + + fn panicked(name: &str) -> Self { + TaskResult { + name: name.into(), + outcome: TaskOutcome::Panicked, + } + } +} + +/// Converts a list of task results into a `Result`, returning `Err` if any task panicked. +pub(crate) fn task_results_into_result(results: Vec) -> Result<(), DynError> { + let panicked: Vec = results + .into_iter() + .filter(|r| r.outcome == TaskOutcome::Panicked) + .map(|r| r.name) + .collect(); + + match panicked.is_empty() { + true => Ok(()), + false => Err(format!("Task(s) panicked: {}", panicked.join(", ")).into()), + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use std::sync::atomic::{AtomicU32, Ordering}; + + #[tokio::test] + async fn test_shutdown_stops_all_tasks() { + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + + let counter_a = Arc::new(AtomicU32::new(0)); + let counter_b = Arc::new(AtomicU32::new(0)); + + let ca = counter_a.clone(); + let cb = counter_b.clone(); + + let tasks = vec![ + PeriodicTask::new("task-a", 50, move || { + ca.fetch_add(1, Ordering::SeqCst); + async { Ok(()) } + }), + PeriodicTask::new("task-b", 50, move || { + cb.fetch_add(1, Ordering::SeqCst); + async { Ok(()) } + }), + ]; + + // Send shutdown after a short delay + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(150)).await; + let _ = shutdown_tx.send(true); + }); + + let results = run_periodic_tasks(tasks, shutdown_rx).await; + + assert_eq!(results.len(), 2); + for r in &results { + assert_eq!(r.outcome, TaskOutcome::Completed); + } + // Both tasks should have ticked at least once + assert!(counter_a.load(Ordering::SeqCst) >= 1); + assert!(counter_b.load(Ordering::SeqCst) >= 1); + } + + #[tokio::test] + async fn test_panic_cancels_sibling_tasks() { + let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + + let healthy_counter = Arc::new(AtomicU32::new(0)); + let hc = healthy_counter.clone(); + + let panicking_task_name = "panicking-task"; + let completed_task_name = "healthy-task"; + let tasks = vec![ + PeriodicTask::new(panicking_task_name, 50, || async { + panic!("intentional test panic"); + }), + PeriodicTask::new(completed_task_name, 50, move || { + hc.fetch_add(1, Ordering::SeqCst); + async { Ok(()) } + }), + ]; + + let results = run_periodic_tasks(tasks, shutdown_rx).await; + + let panicked: Vec<_> = results + .iter() + .filter(|r| r.outcome == TaskOutcome::Panicked) + .collect(); + let completed: Vec<_> = results + .iter() + .filter(|r| r.outcome == TaskOutcome::Completed) + .collect(); + + assert_eq!(panicked.len(), 1); + assert_eq!(completed.len(), 1); + + assert_eq!(panicked[0].name, panicking_task_name); + assert_eq!(completed[0].name, completed_task_name); + } + + #[tokio::test] + async fn test_task_errors_do_not_cancel_siblings() { + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + + let error_counter = Arc::new(AtomicU32::new(0)); + let healthy_counter = Arc::new(AtomicU32::new(0)); + let ec = error_counter.clone(); + let hc = healthy_counter.clone(); + + let tasks = vec![ + PeriodicTask::new("erroring-task", 50, move || { + ec.fetch_add(1, Ordering::SeqCst); + async { Err("simulated error".into()) } + }), + PeriodicTask::new("healthy-task", 50, move || { + hc.fetch_add(1, Ordering::SeqCst); + async { Ok(()) } + }), + ]; + + // Let them run for a bit then shut down + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(200)).await; + let _ = shutdown_tx.send(true); + }); + + let results = run_periodic_tasks(tasks, shutdown_rx).await; + + // Both tasks should have completed normally + for r in &results { + assert_eq!(r.outcome, TaskOutcome::Completed); + } + + // Both should have ticked multiple times + assert!(error_counter.load(Ordering::SeqCst) >= 2); + assert!(healthy_counter.load(Ordering::SeqCst) >= 2); + } + + /// A slow task (one whose execution time exceeds the tick interval) should + /// NOT queue up several back-to-back invocations. With `MissedTickBehavior::Skip` + /// the missed ticks are simply dropped, so the task runs roughly once per + /// `max(interval, task_duration)` rather than bursting. + #[tokio::test] + async fn test_slow_task_skips_missed_ticks() { + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + + let counter = Arc::new(AtomicU32::new(0)); + let c = counter.clone(); + + // Interval is 50 ms but the task sleeps for 150 ms, + // so without Skip the counter would burst to catch up. + let tasks = vec![PeriodicTask::new("slow-task", 50, move || { + let c = c.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(150)).await; + Ok(()) + } + })]; + + // Let the task run for ~500 ms – enough for ~3 slow iterations. + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let _ = shutdown_tx.send(true); + }); + + let results = run_periodic_tasks(tasks, shutdown_rx).await; + + assert_eq!(results.len(), 1); + assert_eq!(results[0].outcome, TaskOutcome::Completed); + + let ticks = counter.load(Ordering::SeqCst); + // With Skip behaviour and a 150 ms task on a 50 ms interval the task + // should execute roughly every 150 ms. Over 500 ms that is about 3–4 + // invocations. Without Skip (Burst, the default) the counter would + // race ahead to ~10. We assert a reasonable upper bound. + assert!( + ticks <= 5, + "expected at most 5 ticks (skip behaviour), but got {ticks}" + ); + assert!(ticks >= 1, "task should have ticked at least once"); + } + + #[test] + fn test_task_results_into_result_no_panics() { + let results = vec![ + TaskResult::completed("task-a"), + TaskResult::completed("task-b"), + ]; + assert!(task_results_into_result(results).is_ok()); + } + + #[test] + fn test_task_results_into_result_with_panic() { + let results = vec![ + TaskResult::panicked("task-a"), + TaskResult::completed("task-b"), + ]; + let err = task_results_into_result(results).unwrap_err(); + assert!(err.to_string().contains("task-a")); + } + + #[test] + fn test_task_results_into_result_all_panicked() { + let results = vec![ + TaskResult::panicked("task-a"), + TaskResult::panicked("task-b"), + ]; + let err = task_results_into_result(results).unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("task-a") && msg.contains("task-b")); + } + + /// Verify that a fast task whose execution time is well below the + /// interval still ticks at the expected cadence (Skip doesn't + /// interfere with normal operation). + #[tokio::test] + async fn test_fast_task_ticks_normally_with_skip() { + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + + let counter = Arc::new(AtomicU32::new(0)); + let c = counter.clone(); + + let tasks = vec![PeriodicTask::new("fast-task", 50, move || { + c.fetch_add(1, Ordering::SeqCst); + async { Ok(()) } + })]; + + // Run for ~250 ms ⇒ expect ~5 ticks (including the immediate first tick). + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(250)).await; + let _ = shutdown_tx.send(true); + }); + + let results = run_periodic_tasks(tasks, shutdown_rx).await; + + assert_eq!(results.len(), 1); + assert_eq!(results[0].outcome, TaskOutcome::Completed); + + let ticks = counter.load(Ordering::SeqCst); + assert!( + ticks >= 3, + "expected at least 3 ticks for a fast 50 ms task over 250 ms, got {ticks}" + ); + } +} diff --git a/nexus-watcher/src/service/traits/mod.rs b/nexus-watcher/src/service/traits/mod.rs deleted file mode 100644 index 4412e1580..000000000 --- a/nexus-watcher/src/service/traits/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod tevent_processor; -mod tevent_processor_runner; - -pub use tevent_processor::TEventProcessor; -pub use tevent_processor_runner::TEventProcessorRunner; diff --git a/nexus-watcher/src/service/traits/tevent_processor.rs b/nexus-watcher/src/service/traits/tevent_processor.rs deleted file mode 100644 index 2c79d0a9a..000000000 --- a/nexus-watcher/src/service/traits/tevent_processor.rs +++ /dev/null @@ -1,92 +0,0 @@ -use std::{fmt::Display, sync::Arc, time::Duration}; - -use nexus_common::models::event::EventProcessorError; -use pubky_app_specs::PubkyId; -use tracing::{error, Instrument}; - -use crate::service::PROCESSING_TIMEOUT_SECS; - -/// Possible error types of an event processor run -#[derive(Debug)] -pub enum RunError { - Internal(EventProcessorError), - Panicked, - TimedOut, -} - -impl RunError { - pub fn is_panic(&self) -> bool { - matches!(self, RunError::Panicked) - } - - pub fn is_timeout(&self) -> bool { - matches!(self, RunError::TimedOut) - } -} - -impl Display for RunError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - RunError::Internal(err) => write!(f, "Internal error: {err}"), - RunError::Panicked => write!(f, "Execution panicked"), - RunError::TimedOut => write!(f, "Execution timed out"), - } - } -} - -/// Asynchronous event processor interface for the Watcher service. -/// -/// This trait represents a component that can process events asynchronously and can be -/// gracefully shut down through a watch channel. -/// -/// # Implementation Notes -/// - Implementors should regularly check the `shutdown_rx` channel for shutdown signals -/// and terminate gracefully when received -/// - The method returns an `EventProcessorError` to allow for typed error handling across -/// different processor implementations -#[async_trait::async_trait] -pub trait TEventProcessor: Send + Sync + 'static { - fn get_homeserver_id(&self) -> PubkyId; - - async fn run(self: Arc) -> Result<(), RunError> { - let hs_id = self.get_homeserver_id().to_string(); - let timeout = self - .custom_timeout() - .unwrap_or(Duration::from_secs(PROCESSING_TIMEOUT_SECS)); - - let span = tracing::info_span!("event_processor.run", homeserver = %hs_id); - let handle = tokio::spawn(self.run_internal().instrument(span)); - - let join_result = tokio::time::timeout(timeout, handle) - .await - .inspect_err(|_| error!("Event processor timed out for {hs_id}")) - .map_err(|_| RunError::TimedOut)?; - - // The JoinError can be: - // - join_error.is_panic() => panic by the inner future - // - join_error.is_cancelled() => inner future was abruptly interrupted, for example - // - JoinHandle::abort() is called on the handle - // - the Tokio runtime is shut down - // In our model, we don't trigger such interruptions. Instead we use the shutdown signal - // to gracefully stop the event processing loop. Therefore we consider all JoinErrors as panics. - let run_internal_result = join_result - .inspect_err(|je| error!("JoinError while running event processor for {hs_id}: {je:?}")) - .map_err(|_| RunError::Panicked)?; - - run_internal_result - .inspect_err(|e| error!("Event processor failed for {hs_id}: {e:?}")) - .map_err(RunError::Internal) - } - - /// Runs the event processor asynchronously. - /// - /// Returns `Ok(())` on a clean exit, or `Err(EventProcessorError)` on failure. - async fn run_internal(self: Arc) -> Result<(), EventProcessorError>; - - /// Optional custom timeout for this event processor. - /// - /// If not set, the [`PROCESSING_TIMEOUT_SECS`] is applied. - fn custom_timeout(&self) -> Option { - None - } -} diff --git a/nexus-watcher/src/service/traits/tevent_processor_runner.rs b/nexus-watcher/src/service/traits/tevent_processor_runner.rs deleted file mode 100644 index 7f3a5bffb..000000000 --- a/nexus-watcher/src/service/traits/tevent_processor_runner.rs +++ /dev/null @@ -1,145 +0,0 @@ -use std::{sync::Arc, time::Instant}; - -use nexus_common::types::DynError; -use tokio::sync::watch::Receiver; -use tracing::{debug, error, info, warn}; - -use crate::service::{ - backoff::HomeserverBackoff, - stats::{ProcessedStats, ProcessorRunStatus, RunAllProcessorsStats}, - traits::{tevent_processor::RunError, TEventProcessor}, -}; - -/// Asynchronous wrapper that helps build and run event processors in the Watcher service. -/// -/// # Implementation Notes -/// - The `build` method should create and return a fully configured event processor -/// ready for immediate use -/// - Implementors should ensure that created processors are properly isolated and -/// don't share mutable state unless explicitly intended -#[async_trait::async_trait] -pub trait TEventProcessorRunner { - /// Returns the shutdown signal receiver - fn shutdown_rx(&self) -> Receiver; - - /// Returns the default homeserver ID for this runner. - /// This is used to prioritize the default homeserver when processing multiple homeservers. - fn default_homeserver(&self) -> &str; - - fn monitored_homeservers_limit(&self) -> usize; - - /// Returns the homeserver IDs relevant for this run, ordered by their priority. - /// - /// Contains all homeserver IDs from the graph, with the default homeserver prioritized at index 0. - async fn homeservers_by_priority(&self) -> Result, DynError>; - - /// Creates and returns a new event processor instance for the specified homeserver. - /// - /// # Parameters - /// * `homeserver_id` - The homeserver PubkyId. Represents the homeserver this event processor will - /// fetch and process events from. - /// - /// # Returns - /// A reference to the event processor instance, ready to be executed with its `run` method. - /// - /// # Errors - /// Throws a [`WatcherError`] if the event processor couldn't be built - async fn build(&self, homeserver_id: String) -> Result, DynError>; - - /// Decides the amount and order of homeservers from which events will be fetched and processed in `run_all`. - /// - /// # Returns - /// Considers the values of [TEventProcessorRunner::homeservers_by_priority]. - /// Depending on [TEventProcessorRunner::monitored_homeservers_limit], only a subset of this list may be returned. - async fn pre_run_all(&self) -> Result, DynError> { - let hs_ids = self.homeservers_by_priority().await?; - let max_index = std::cmp::min(self.monitored_homeservers_limit(), hs_ids.len()); - Ok(hs_ids[..max_index].to_vec()) - } - - /// Post-processing of the run results - async fn post_run_all(&self, stats: RunAllProcessorsStats) -> ProcessedStats { - for individual_run_stat in &stats.stats { - let hs_id = &individual_run_stat.hs_id; - let duration = individual_run_stat.duration; - let status = &individual_run_stat.status; - debug!("Event processor run for HS {hs_id}: duration {duration:?}, status {status:?}"); - } - - let count_ok = stats.count_ok(); - let count_error = stats.count_error(); - let count_panic = stats.count_panic(); - let count_timeout = stats.count_timeout(); - let count_failed_to_build = stats.count_failed_to_build(); - let count_skipped = stats.count_skipped(); - let had_issues = count_error + count_panic + count_timeout + count_failed_to_build > 0; - - if had_issues { - warn!( "Run result: {count_ok} ok, {count_skipped} skipped (backoff), {count_failed_to_build} failed to build, {count_error} error, {count_panic} panic, {count_timeout} timeout"); - } else if count_skipped > 0 { - info!("Run result: {count_ok} ok, {count_skipped} skipped (backoff)"); - } else { - debug!("Run result: {count_ok} ok"); - } - - ProcessedStats(stats) - } - - /// Runs event processors for all homeservers relevant for this run, with timeout protection. - /// - /// # Parameters - /// * `backoff` - Tracks per-homeserver exponential backoff; homeservers in an active backoff - /// window are skipped and their state is updated after each run. - /// - /// # Returns - /// Statistics about the event processor run results, summarized as [`RunAllProcessorsStats`] - async fn run_all(&self, backoff: &mut HomeserverBackoff) -> Result { - let hs_ids = self.pre_run_all().await?; - - let mut run_stats = RunAllProcessorsStats::default(); - - for hs_id in hs_ids { - if *self.shutdown_rx().borrow() { - info!("Shutdown detected in homeserver {hs_id}, exiting run_all loop"); - break; // Exit loop - } - - // Skip homeservers that are in a backoff window - if backoff.should_skip(&hs_id) { - debug!("Skipping homeserver {hs_id} (in backoff)"); - run_stats.add_run_result( - hs_id, - std::time::Duration::ZERO, - ProcessorRunStatus::Skipped, - ); - continue; - } - - let t0 = Instant::now(); - let status = match self.build(hs_id.clone()).await { - Ok(event_processor) => match event_processor.run().await { - Ok(_) => ProcessorRunStatus::Ok, - Err(RunError::Internal(_)) => ProcessorRunStatus::Error, - Err(RunError::Panicked) => ProcessorRunStatus::Panic, - Err(RunError::TimedOut) => ProcessorRunStatus::Timeout, - }, - Err(e) => { - error!("Failed to build event processor for homeserver: {hs_id}: {e}"); - ProcessorRunStatus::FailedToBuild - } - }; - let duration = t0.elapsed(); - - if status == ProcessorRunStatus::Ok { - backoff.record_success(&hs_id); - } else { - backoff.record_failure(&hs_id); - } - - run_stats.add_run_result(hs_id, duration, status); - } - - let processed_stats = self.post_run_all(run_stats).await; - Ok(processed_stats) - } -} diff --git a/nexus-watcher/src/service/user_hs_resolver.rs b/nexus-watcher/src/service/user_hs_resolver.rs new file mode 100644 index 000000000..1b85f772d --- /dev/null +++ b/nexus-watcher/src/service/user_hs_resolver.rs @@ -0,0 +1,414 @@ +//! # User Homeserver Resolver +//! +//! Periodic task that resolves each user's homeserver and persists +//! the `(:User)-[:HOSTED_BY]->(:Homeserver)` relationship in Neo4j. + +use nexus_common::db::{ + exec_single_row, fetch_key_from_graph, queries, GraphResult, PubkyConnector, +}; +use nexus_common::types::DynError; +use opentelemetry::global; +use opentelemetry::metrics::Histogram; +use pubky::PublicKey; +use pubky_app_specs::PubkyId; +use std::sync::LazyLock; +use tracing::{debug, error, warn}; + +static HS_RESOLVER_METRICS: LazyLock = LazyLock::new(HsResolverMetrics::new); + +/// Main entry point for one cycle of the periodic task. +/// +/// `ttl_ms` controls the minimum time before a user's mapping is re-resolved. +/// Users whose `HOSTED_BY.resolved_at` is newer than `ttl_ms` are skipped. +pub async fn run(ttl_ms: u64) -> Result<(), DynError> { + let user_ids = get_users_needing_resolution(ttl_ms).await?; + if user_ids.is_empty() { + debug!("No users need homeserver resolution"); + HS_RESOLVER_METRICS.run_total.record(0, &[]); + HS_RESOLVER_METRICS.run_failed.record(0, &[]); + return Ok(()); + } + + // Convert user_ids to user_pks + let user_pks: Vec = user_ids + .iter() + .filter_map(|user_id| { + // For the user_ids that fail to convert, we log an error message and skip them + user_id + .parse::() + .map_err(|e| error!("Failed to parse user_id {user_id}: {e}")) + .ok() + }) + .collect(); + + let attempted = user_pks.len() as u64; + debug!("Resolving homeservers for {} users", attempted); + + let mut failed = 0u64; + + // As of pubky 0.7.0 parallel resolution is possible but unreliable. This was tried: + // - with the singleton Pubky client (up to 10% unresolved nodes with 10 req. in parallel) + // - with a relay-only Pubky client (up to 95% unresolved nodes with 10 req. in parallel) + // "unresolved nodes" = no HS was found using `get_homeserver_of(&user_pk)` for users with a known HS. + // + // The most reliable method remains sequential querying. + // + // To minimize the chance that User PKs are too close to each other and therefore might hit + // the same DHT node, which can cause that node to interpret this as spammy requests and therefore + // fail / refuse to resolve some of the queries, we order the User PKs such that every new query lands + // as far as possible from all previous queries in the PK keyspace. + // + // To achieve this, we use bisection ordering. + for user_pk in bisection_order_user_pks(user_pks) { + if let Err(e) = resolve_user(&user_pk).await { + failed += 1; + warn!("Failed to resolve HS for user {}: {e}", user_pk.z32()); + } + } + + HS_RESOLVER_METRICS.run_total.record(attempted, &[]); + HS_RESOLVER_METRICS.run_failed.record(failed, &[]); + + Ok(()) +} + +// Bisection ordering sorts the User PKs such that every new PK is as far as possible from all +// previously queried PKs in the keyspace. +// +// The algorithm: +// 1. Sort all PKs lexicographically. +// 2. Reorder via BFS over the implicit binary-search-tree layout of the sorted array: +// emit the midpoint of each interval, then recurse into left and right halves. +// +// For a sorted array [K0..=K7] this produces [K4, K2, K6, K1, K3, K5, K7, K0], ensuring +// each successive query lands as far as possible from all previous ones in the keyspace. +fn bisection_order_user_pks(unsorted_pks: Vec) -> Vec { + let mut sorted_pks = unsorted_pks; + sorted_pks.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes())); + + let n = sorted_pks.len(); + let mut bisection_result = Vec::with_capacity(n); + // Each entry is a half-open interval [lo, hi) of the sorted slice to process. + let mut queue = std::collections::VecDeque::new(); + if n > 0 { + queue.push_back((0usize, n)); + } + while let Some((lo, hi)) = queue.pop_front() { + if lo >= hi { + continue; + } + let mid = lo + (hi - lo) / 2; + bisection_result.push(sorted_pks[mid].clone()); + queue.push_back((lo, mid)); + queue.push_back((mid + 1, hi)); + } + bisection_result +} + +/// Fetches user IDs whose homeserver mapping is stale or missing. +/// +/// A mapping is considered stale when its `resolved_at` timestamp is older +/// than `ttl_ms` milliseconds ago. +async fn get_users_needing_resolution(ttl_ms: u64) -> GraphResult> { + let query = queries::get::get_users_needing_hs_resolution(ttl_ms); + let maybe_user_ids = fetch_key_from_graph(query, "user_ids").await?; + Ok(maybe_user_ids.unwrap_or_default()) +} + +/// Resolves a single user's homeserver and persists the HOSTED_BY relationship. +async fn resolve_user(user_pk: &PublicKey) -> Result<(), DynError> { + let pubky = PubkyConnector::get()?; + + let user_id = user_pk.z32(); + let Some(hs_pk) = pubky.get_homeserver_of(user_pk).await else { + // No PKDNS record: remove stale HOSTED_BY edge + let query = queries::del::remove_user_homeserver(&user_id); + exec_single_row(query).await?; + + debug!("User {user_id} has no published homeserver, removed HOSTED_BY"); + return Ok(()); + }; + + let hs_id = PubkyId::try_from(&hs_pk.into_inner().to_z32())?; + + let query = queries::put::set_user_homeserver(&user_id, &hs_id); + exec_single_row(query).await?; + + debug!("User {user_id} -> HS {hs_id}"); + Ok(()) +} + +/// Returns all user IDs hosted on a given homeserver. +pub async fn get_user_ids_by_homeserver(hs_id: &str) -> GraphResult> { + let query = queries::get::get_users_by_homeserver(hs_id); + let maybe_user_ids = fetch_key_from_graph(query, "user_ids").await?; + Ok(maybe_user_ids.unwrap_or_default()) +} + +struct HsResolverMetrics { + run_total: Histogram, + run_failed: Histogram, +} + +impl HsResolverMetrics { + fn new() -> Self { + let meter = global::meter("hs-resolver-meter"); + + Self { + run_total: meter + .u64_histogram("nexus.task.hs-resolver.total") + .with_description("Number of attempted HS resolutions in each resolver run") + .build(), + run_failed: meter + .u64_histogram("nexus.task.hs-resolver.failed") + .with_description("Number of failed HS resolutions in each resolver run") + .build(), + } + } +} + +// TODO Move tests to separate module? (switch to WatcherTest::setup()) +#[cfg(test)] +mod tests { + use super::*; + use nexus_common::db::exec_single_row; + use nexus_common::db::graph::Query; + use nexus_common::types::DynError; + use nexus_common::{StackConfig, StackManager}; + use pubky::Keypair; + + async fn setup() -> Result<(), DynError> { + StackManager::setup(&StackConfig::default()).await + } + + /// Helper: create a User node in the graph + async fn create_test_user(user_id: &str) -> GraphResult<()> { + let query = Query::new( + "create_test_user", + "MERGE (u:User {id: $id}) + SET u.name = 'test', u.indexed_at = 0 + RETURN u;", + ) + .param("id", user_id); + exec_single_row(query).await + } + + /// Helper: clean up test data + async fn cleanup_test_user(user_id: &str) -> GraphResult<()> { + let query = queries::del::delete_user(user_id); + exec_single_row(query).await + } + + #[tokio_shared_rt::test(shared)] + async fn test_set_user_homeserver_graph_query() -> Result<(), DynError> { + setup().await?; + + let user_id = "hs_resolver_test_user_001"; + let hs_id_a = "hs_resolver_test_hs_aaa"; + let hs_id_b = "hs_resolver_test_hs_bbb"; + + create_test_user(user_id).await?; + + // Set initial homeserver + let query = queries::put::set_user_homeserver(user_id, hs_id_a); + exec_single_row(query).await?; + + // Switch to a different homeserver + let query = queries::put::set_user_homeserver(user_id, hs_id_b); + exec_single_row(query).await?; + + // Cleanup + cleanup_test_user(user_id).await?; + + Ok(()) + } + + #[tokio_shared_rt::test(shared)] + async fn test_set_user_homeserver_idempotent() -> Result<(), DynError> { + setup().await?; + + let user_id = "hs_resolver_test_user_noop"; + let hs_id = "hs_resolver_test_hs_noop"; + + create_test_user(user_id).await?; + + // Set homeserver for the first time + let query = queries::put::set_user_homeserver(user_id, hs_id); + exec_single_row(query).await?; + + // Set same homeserver again (should reuse HS, e.g. not create any orphan HS) + let query = queries::put::set_user_homeserver(user_id, hs_id); + exec_single_row(query).await?; + + // Cleanup + cleanup_test_user(user_id).await?; + + Ok(()) + } + + #[tokio_shared_rt::test(shared)] + async fn test_get_users_needing_resolution_ttl() -> Result<(), DynError> { + setup().await?; + + let user_fresh = "ttl_test_user_fresh"; + let user_stale = "ttl_test_user_stale"; + let user_no_hs = "ttl_test_user_no_hs"; + let hs_id = "ttl_test_hs"; + + create_test_user(user_fresh).await?; + create_test_user(user_stale).await?; + create_test_user(user_no_hs).await?; + + // Give user_fresh a recently resolved mapping + exec_single_row(queries::put::set_user_homeserver(user_fresh, hs_id)).await?; + + // Give user_stale a mapping with an old resolved_at (1 hour ago) + let stale_query = Query::new( + "set_stale_hs", + "MATCH (u:User {id: $user_id}) + MERGE (hs:Homeserver {id: $hs_id}) + MERGE (u)-[r:HOSTED_BY]->(hs) + SET r.resolved_at = timestamp() - 7200000", + ) + .param("user_id", user_stale) + .param("hs_id", hs_id); + exec_single_row(stale_query).await?; + + // user_no_hs has no HOSTED_BY at all + + // With a 1-hour TTL: user_fresh should be skipped, user_stale and user_no_hs returned + let mut needing = get_users_needing_resolution(3_600_000).await?; + needing.sort(); + + assert!( + !needing.contains(&user_fresh.to_string()), + "Recently resolved user should be skipped" + ); + assert!( + needing.contains(&user_stale.to_string()), + "Stale user should need resolution" + ); + assert!( + needing.contains(&user_no_hs.to_string()), + "User without HOSTED_BY should need resolution" + ); + + // Cleanup + cleanup_test_user(user_fresh).await?; + cleanup_test_user(user_stale).await?; + cleanup_test_user(user_no_hs).await?; + + Ok(()) + } + + #[tokio_shared_rt::test(shared)] + async fn test_get_user_ids_by_homeserver() -> Result<(), DynError> { + setup().await?; + + let user_a = "hs_users_test_user_aaa"; + let user_b = "hs_users_test_user_bbb"; + let user_c = "hs_users_test_user_ccc"; + let hs_one = "hs_users_test_hs_one"; + let hs_two = "hs_users_test_hs_two"; + + create_test_user(user_a).await?; + create_test_user(user_b).await?; + create_test_user(user_c).await?; + + // Host user_a and user_b on hs_one, user_c on hs_two + exec_single_row(queries::put::set_user_homeserver(user_a, hs_one)).await?; + exec_single_row(queries::put::set_user_homeserver(user_b, hs_one)).await?; + exec_single_row(queries::put::set_user_homeserver(user_c, hs_two)).await?; + + // Query users on hs_one + let mut users = get_user_ids_by_homeserver(hs_one).await?; + users.sort(); + assert_eq!(users, vec![user_a, user_b]); + + // Query users on hs_two + let users = get_user_ids_by_homeserver(hs_two).await?; + assert_eq!(users, vec![user_c]); + + // Query unknown HS returns empty + let users = get_user_ids_by_homeserver("nonexistent_hs").await?; + assert!(users.is_empty()); + + // Cleanup + cleanup_test_user(user_a).await?; + cleanup_test_user(user_b).await?; + cleanup_test_user(user_c).await?; + + Ok(()) + } + + #[test] + fn test_bisection_order() { + // Empty and single-element edge cases. + assert!(bisection_order_user_pks(vec![]).is_empty()); + let lone = Keypair::random().public_key(); + let result = bisection_order_user_pks(vec![lone.clone()]); + assert_eq!(result[0].as_bytes(), lone.as_bytes()); + + // For 8 keys, verify the full BFS-bisection permutation. + // Sort first to establish the ground-truth lexicographic order. + let mut sorted: Vec = (0..8).map(|_| Keypair::random().public_key()).collect(); + sorted.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes())); + + // BFS over a sorted array of 8 elements (half-open intervals) emits: + // (0,8)→4, (0,4)→2, (5,8)→6, (0,2)→1, (3,4)→3, (5,6)→5, (7,8)→7, (0,1)→0 + let expected_indices: [usize; 8] = [4, 2, 6, 1, 3, 5, 7, 0]; + + let result = bisection_order_user_pks(sorted.clone()); + assert_eq!(result.len(), 8); + for (pos, &idx) in expected_indices.iter().enumerate() { + assert_eq!( + result[pos].as_bytes(), + sorted[idx].as_bytes(), + "position {pos}: expected sorted[{idx}]" + ); + } + } + + #[tokio_shared_rt::test(shared)] + async fn test_remove_user_homeserver() -> Result<(), DynError> { + setup().await?; + + let user_id = "remove_hosted_by_test_user"; + let hs_id = "remove_hosted_by_test_hs"; + + create_test_user(user_id).await?; + + // Assign a homeserver + exec_single_row(queries::put::set_user_homeserver(user_id, hs_id)).await?; + let users = get_user_ids_by_homeserver(hs_id).await?; + assert!( + users.contains(&user_id.to_string()), + "user should be hosted" + ); + + // Remove the HOSTED_BY edge + exec_single_row(queries::del::remove_user_homeserver(user_id)).await?; + + // User is no longer listed on that homeserver + let users = get_user_ids_by_homeserver(hs_id).await?; + assert!( + !users.contains(&user_id.to_string()), + "user should no longer be hosted after removal" + ); + + // User now needs resolution again (no HOSTED_BY edge) + let needing = get_users_needing_resolution(3_600_000).await?; + assert!( + needing.contains(&user_id.to_string()), + "user without HOSTED_BY should need resolution" + ); + + // Removing again is a no-op (no error) + exec_single_row(queries::del::remove_user_homeserver(user_id)).await?; + + // Cleanup + cleanup_test_user(user_id).await?; + + Ok(()) + } +} diff --git a/nexus-watcher/tests/event_processor/bookmarks/fail_index.rs b/nexus-watcher/tests/event_processor/bookmarks/fail_index.rs index 78246acea..b85211384 100644 --- a/nexus-watcher/tests/event_processor/bookmarks/fail_index.rs +++ b/nexus-watcher/tests/event_processor/bookmarks/fail_index.rs @@ -63,8 +63,8 @@ async fn test_homeserver_bookmark_without_user() -> Result<()> { // Simulate the event processor to handle the event. // If the event processor were activated, the test would not catch the missing dependency // error, and it would pass successfully - let moderation_ref = test.event_processor_runner.moderation.clone(); - let sync_fail = retrieve_and_handle_event_line(&bookmark_event, moderation_ref) + let event_handler = test.event_processor_runner.event_handler.clone(); + let sync_fail = retrieve_and_handle_event_line(&bookmark_event, event_handler) .await .map_err(|e| error!("SYNC ERROR: {:?}", e)) .is_err(); diff --git a/nexus-watcher/tests/event_processor/bookmarks/retry_bookmark.rs b/nexus-watcher/tests/event_processor/bookmarks/retry_bookmark.rs index a7671d72a..a4167cbc9 100644 --- a/nexus-watcher/tests/event_processor/bookmarks/retry_bookmark.rs +++ b/nexus-watcher/tests/event_processor/bookmarks/retry_bookmark.rs @@ -2,13 +2,11 @@ use crate::event_processor::utils::watcher::{ assert_eventually_exists, HomeserverHashIdPath, WatcherTest, }; use anyhow::Result; -use nexus_common::models::event::{EventProcessorError, EventType}; -use nexus_watcher::events::retry::event::RetryEvent; +use nexus_watcher::events::retry::RetryEvent; use pubky::Keypair; use pubky_app_specs::{ bookmark_uri_builder, post_uri_builder, traits::HashId, PubkyAppBookmark, PubkyAppUser, }; -use tokio::time; /// The user profile is stored in the homeserver. Missing the post to connect the bookmark #[tokio_shared_rt::test(shared)] @@ -44,48 +42,21 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { // PUT bookmark test.put(&user_kp, &bookmark_path, bookmark).await?; - let put_index_key = format!( - "{}:{}", - EventType::Put, - RetryEvent::generate_index_key(&bookmark_absolute_url).unwrap() - ); + let index_key = bookmark_absolute_url.clone(); - assert_eventually_exists(&put_index_key).await; + assert_eventually_exists(&index_key).await; - let timestamp = RetryEvent::check_uri(&put_index_key).await.unwrap(); - assert!(timestamp.is_some()); + assert!(RetryEvent::check_uri(&index_key).await.unwrap()); - let event_retry = RetryEvent::get_from_index(&put_index_key).await.unwrap(); + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); let event_state = event_retry.unwrap(); assert_eq!(event_state.retry_count, 0); + assert_eq!(event_state.event_uri, bookmark_absolute_url); - let dependency_uri = format!("{fake_user_id}:posts:{fake_post_id}"); - match event_state.error_type { - EventProcessorError::MissingDependency { dependency } => { - assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], dependency_uri); - } - _ => panic!("The error type has to be MissingDependency type"), - }; - - // DEL bookmark — bookmark was never indexed, so DEL returns Ok (no-op) + // DEL bookmark — bookmark was never indexed, so DEL returns Ok (no-op, no retry created) test.del(&user_kp, &bookmark_path).await?; - let del_index_key = format!( - "{}:{}", - EventType::Del, - RetryEvent::generate_index_key(&bookmark_absolute_url).unwrap() - ); - - // DEL should succeed silently — no retry event created - time::sleep(std::time::Duration::from_millis(500)).await; - let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); - assert!( - event_retry.is_none(), - "DEL of non-existent bookmark should not create a retry event" - ); - Ok(()) } diff --git a/nexus-watcher/tests/event_processor/follows/fail_index.rs b/nexus-watcher/tests/event_processor/follows/fail_index.rs index 248083114..08a504ec4 100644 --- a/nexus-watcher/tests/event_processor/follows/fail_index.rs +++ b/nexus-watcher/tests/event_processor/follows/fail_index.rs @@ -1,10 +1,10 @@ use crate::event_processor::utils::watcher::{retrieve_and_handle_event_line, WatcherTest}; use anyhow::Result; use pubky::Keypair; -use pubky_app_specs::PubkyAppUser; +use pubky_app_specs::{follow_uri_builder, PubkyAppUser}; use tracing::error; -/// The follower user is stored in the homeserver but it is not in sync with the graph +/// Verifies that a follow fails with MissingDependency when either party is not yet indexed. #[tokio_shared_rt::test(shared)] async fn test_homeserver_follow_cannot_complete() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -19,10 +19,9 @@ async fn test_homeserver_follow_cannot_complete() -> Result<()> { }; let follower_id = test.create_user(&follower_kp, &follower).await?; - // Switch OFF the event processor to simulate the pending events to index + // Switch OFF event processing — followee signs up but is not indexed test = test.remove_event_processing().await; - // Create a key but it would not be synchronised in the graph let followee_kp = Keypair::random(); let followee = PubkyAppUser { bio: Some("test_homeserver_follow_cannot_complete".to_string()), @@ -33,45 +32,44 @@ async fn test_homeserver_follow_cannot_complete() -> Result<()> { }; let shadow_followee_id = test.create_user(&followee_kp, &followee).await?; - let follow_url = test + let _follow_path = test .create_follow(&follower_kp, &shadow_followee_id) .await?; - // Create raw event line to retrieve the content from the homeserver - let follow_event = format!("PUT {follow_url}"); + // Full URI required by Event::parse_event + let follow_event = format!( + "PUT {}", + follow_uri_builder(follower_id.clone(), shadow_followee_id.clone()) + ); - // Simulate the event processor to handle the event. - // If the event processor were activated, the test would not catch the missing dependency - // error, and it would pass successfully - let moderation_ref = test.event_processor_runner.moderation.clone(); - let sync_fail = retrieve_and_handle_event_line(&follow_event, moderation_ref) + let event_handler = test.event_processor_runner.event_handler.clone(); + let sync_fail = retrieve_and_handle_event_line(&follow_event, event_handler) .await .map_err(|e| error!("SYNC ERROR: {:?}", e)) .is_err(); assert!( sync_fail, - "It seems that relationship exists, which should not be possible. Event processor should be disconnected" + "Follow indexing should fail: followee is not yet indexed" ); - // Create a follow in opposite direction - let opposite_follow = test.create_follow(&followee_kp, &follower_id).await?; + // Opposite direction: followee follows follower (follower IS indexed, followee is NOT) + let _opposite_follow_path = test.create_follow(&followee_kp, &follower_id).await?; - // Create raw event line to retrieve the content from the homeserver - let opposite_follow_event = format!("PUT {opposite_follow}"); + let opposite_follow_event = format!( + "PUT {}", + follow_uri_builder(shadow_followee_id, follower_id) + ); - // Simulate the event processor to handle the event. - // If the event processor were activated, the test would not catch the missing dependency - // error, and it would pass successfully - let moderation_ref = test.event_processor_runner.moderation.clone(); - let sync_fail = retrieve_and_handle_event_line(&opposite_follow_event, moderation_ref) + let event_handler = test.event_processor_runner.event_handler.clone(); + let sync_fail = retrieve_and_handle_event_line(&opposite_follow_event, event_handler) .await .map_err(|e| error!("SYNC ERROR: {:?}", e)) .is_err(); assert!( sync_fail, - "It seems that relationship exists, which should not be possible. Event processor should be disconnected" + "Follow indexing should fail: follower (shadow) is not yet indexed" ); Ok(()) diff --git a/nexus-watcher/tests/event_processor/follows/retry_follow.rs b/nexus-watcher/tests/event_processor/follows/retry_follow.rs index e728ae40e..c86836ca3 100644 --- a/nexus-watcher/tests/event_processor/follows/retry_follow.rs +++ b/nexus-watcher/tests/event_processor/follows/retry_follow.rs @@ -1,9 +1,8 @@ use crate::event_processor::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; -use nexus_common::models::event::{EventProcessorError, EventType}; -use nexus_watcher::events::retry::event::RetryEvent; +use nexus_watcher::events::retry::RetryEvent; use pubky::Keypair; -use pubky_app_specs::{follow_uri_builder, PubkyAppUser, PubkyId}; +use pubky_app_specs::{follow_uri_builder, PubkyAppUser}; /// The user profile is stored in the homeserver. Missing the followee to connect with follower #[tokio_shared_rt::test(shared)] @@ -12,7 +11,6 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { let followee_keypair = Keypair::random(); let followee_id = followee_keypair.public_key().to_z32(); - let followee_pubky_id = PubkyId::try_from(&followee_id).unwrap(); // In that case, that user will act as a NotSyncUser or user not registered in pubky.app // It will not have a profile.json test.register_user(&followee_keypair).await?; @@ -27,18 +25,13 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { }; let follower_id = test.create_user(&follower_kp, &follower_user).await?; - let follow_path = test.create_follow(&follower_kp, &followee_id).await?; + let _follow_path = test.create_follow(&follower_kp, &followee_id).await?; let follow_absolute_url = follow_uri_builder(follower_id, followee_id.clone()); - let index_key = format!( - "{}:{}", - EventType::Put, - RetryEvent::generate_index_key(&follow_absolute_url).unwrap() - ); + let index_key = follow_absolute_url.clone(); assert_eventually_exists(&index_key).await; - let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); - assert!(timestamp.is_some()); + assert!(RetryEvent::check_uri(&index_key).await.unwrap()); let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -47,17 +40,5 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { assert_eq!(event_state.retry_count, 0); - let dependency_key = RetryEvent::generate_index_key_from_uri(&followee_pubky_id.to_uri()); - - match event_state.error_type { - EventProcessorError::MissingDependency { dependency } => { - assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], dependency_key) - } - _ => panic!("The error type has to be MissingDependency type"), - }; - - test.del(&follower_kp, &follow_path).await?; - Ok(()) } diff --git a/nexus-watcher/tests/event_processor/homeserver/mod.rs b/nexus-watcher/tests/event_processor/homeserver/mod.rs index 0e715adca..9422e016f 100644 --- a/nexus-watcher/tests/event_processor/homeserver/mod.rs +++ b/nexus-watcher/tests/event_processor/homeserver/mod.rs @@ -1,3 +1,4 @@ +mod active_homeservers; mod ingest_homeservers_from_follow_events; mod ingest_homeservers_from_post_events; mod ingest_homeservers_from_tag_events; diff --git a/nexus-watcher/tests/event_processor/homeserver/utils.rs b/nexus-watcher/tests/event_processor/homeserver/utils.rs deleted file mode 100644 index 337e3291f..000000000 --- a/nexus-watcher/tests/event_processor/homeserver/utils.rs +++ /dev/null @@ -1,8 +0,0 @@ -use crate::event_processor::utils::watcher::WatcherTest; -use anyhow::Result; -use pubky::PublicKey; - -pub async fn create_external_test_homeserver(test: &mut WatcherTest) -> Result { - let homeserver_id = test.testnet.create_random_homeserver().await?.public_key(); - Ok(homeserver_id) -} diff --git a/nexus-watcher/tests/event_processor/mod.rs b/nexus-watcher/tests/event_processor/mod.rs index c1870ed84..634ef8069 100644 --- a/nexus-watcher/tests/event_processor/mod.rs +++ b/nexus-watcher/tests/event_processor/mod.rs @@ -1,7 +1,6 @@ mod bookmarks; mod files; mod follows; -mod homeserver; mod mentions; mod network; mod posts; diff --git a/nexus-watcher/tests/event_processor/posts/fail_reply.rs b/nexus-watcher/tests/event_processor/posts/fail_reply.rs index cd7f796eb..745bd3c52 100644 --- a/nexus-watcher/tests/event_processor/posts/fail_reply.rs +++ b/nexus-watcher/tests/event_processor/posts/fail_reply.rs @@ -52,8 +52,8 @@ async fn test_homeserver_post_reply_without_post_parent() -> Result<()> { // Simulate the event processor to handle the event. // If the event processor were activated, the test would not catch the missing dependency // error, and it would pass successfully - let moderation_ref = test.event_processor_runner.moderation.clone(); - let sync_fail = retrieve_and_handle_event_line(&post_event, moderation_ref) + let event_handler = test.event_processor_runner.event_handler.clone(); + let sync_fail = retrieve_and_handle_event_line(&post_event, event_handler) .await .map_err(|e| error!("SYNC ERROR: {:?}", e)) .is_err(); diff --git a/nexus-watcher/tests/event_processor/posts/fail_repost.rs b/nexus-watcher/tests/event_processor/posts/fail_repost.rs index dabf90e12..14c9d06b2 100644 --- a/nexus-watcher/tests/event_processor/posts/fail_repost.rs +++ b/nexus-watcher/tests/event_processor/posts/fail_repost.rs @@ -66,8 +66,8 @@ async fn test_homeserver_post_repost_without_post_parent() -> Result<()> { // Simulate the event processor to handle the event. // If the event processor were activated, the test would not catch the missing dependency // error, and it would pass successfully - let moderation_ref = test.event_processor_runner.moderation.clone(); - let sync_fail = retrieve_and_handle_event_line(&post_homeserver_uri, moderation_ref) + let event_handler = test.event_processor_runner.event_handler.clone(); + let sync_fail = retrieve_and_handle_event_line(&post_homeserver_uri, event_handler) .await .map_err(|e| error!("SYNC ERROR: {:?}", e)) .is_err(); diff --git a/nexus-watcher/tests/event_processor/posts/retry_all.rs b/nexus-watcher/tests/event_processor/posts/retry_all.rs index c02f2059f..e7076d995 100644 --- a/nexus-watcher/tests/event_processor/posts/retry_all.rs +++ b/nexus-watcher/tests/event_processor/posts/retry_all.rs @@ -1,7 +1,6 @@ use crate::event_processor::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; -use nexus_common::models::event::{EventProcessorError, EventType}; -use nexus_watcher::events::retry::event::RetryEvent; +use nexus_watcher::events::retry::RetryEvent; use pubky::Keypair; use pubky_app_specs::{ post_uri_builder, PubkyAppPost, PubkyAppPostEmbed, PubkyAppPostKind, PubkyAppUser, @@ -41,20 +40,15 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { attachments: None, }; - let (repost_reply_id, repost_reply_path) = test.create_post(&user_kp, &repost_reply).await?; + let (repost_reply_id, _repost_reply_path) = test.create_post(&user_kp, &repost_reply).await?; let repost_reply_absolute_url = post_uri_builder(user_id, repost_reply_id); - let index_key = format!( - "{}:{}", - EventType::Put, - RetryEvent::generate_index_key(&repost_reply_absolute_url).unwrap() - ); + let index_key = repost_reply_absolute_url.clone(); assert_eventually_exists(&index_key).await; - let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); - assert!(timestamp.is_some()); + assert!(RetryEvent::check_uri(&index_key).await.unwrap()); let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -62,44 +56,5 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { let event_state = event_retry.unwrap(); assert_eq!(event_state.retry_count, 0); - match event_state.error_type { - EventProcessorError::MissingDependency { dependency } => { - assert_eq!(dependency.len(), 2); - assert_eq!( - dependency[0], - RetryEvent::generate_index_key(&reply_absolute_uri).unwrap() - ); - assert_eq!( - dependency[1], - RetryEvent::generate_index_key(&repost_absolute_uri).unwrap() - ); - } - _ => panic!("The error type has to be MissingDependency type"), - }; - - test.del(&user_kp, &repost_reply_path).await?; - - let del_index_key = format!( - "{}:{}", - EventType::Del, - RetryEvent::generate_index_key(&repost_reply_absolute_url).unwrap() - ); - - assert_eventually_exists(&del_index_key).await; - - let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); - assert!(timestamp.is_some()); - - let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); - assert!(event_retry.is_some()); - - let event_state = event_retry.unwrap(); - assert_eq!(event_state.retry_count, 0); - - match event_state.error_type { - EventProcessorError::SkipIndexing => (), - _ => panic!("The error type has to be SkipIndexing type"), - }; - Ok(()) } diff --git a/nexus-watcher/tests/event_processor/posts/retry_post.rs b/nexus-watcher/tests/event_processor/posts/retry_post.rs index d54d3563c..2f1824d20 100644 --- a/nexus-watcher/tests/event_processor/posts/retry_post.rs +++ b/nexus-watcher/tests/event_processor/posts/retry_post.rs @@ -1,7 +1,6 @@ use crate::event_processor::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; -use nexus_common::models::event::{EventProcessorError, EventType}; -use nexus_watcher::events::retry::event::RetryEvent; +use nexus_watcher::events::retry::RetryEvent; use pubky::Keypair; use pubky_app_specs::{post_uri_builder, PubkyAppPost, PubkyAppPostKind}; @@ -25,20 +24,15 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { attachments: None, }; - let (post_id, post_path) = test.create_post(&user_kp, &post).await?; + let (post_id, _post_path) = test.create_post(&user_kp, &post).await?; let post_absolute_url = post_uri_builder(user_id.clone(), post_id); - let index_key = format!( - "{}:{}", - EventType::Put, - RetryEvent::generate_index_key(&post_absolute_url).unwrap() - ); + let index_key = post_absolute_url.clone(); assert_eventually_exists(&index_key).await; - let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); - assert!(timestamp.is_some()); + assert!(RetryEvent::check_uri(&index_key).await.unwrap()); let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -46,42 +40,5 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { let event_state = event_retry.unwrap(); assert_eq!(event_state.retry_count, 0); - let dependency_absolute_uri = format!("pubky://{user_id}/pub/pubky.app/profile.json"); - - match event_state.error_type { - EventProcessorError::MissingDependency { dependency } => { - assert_eq!(dependency.len(), 1); - assert_eq!( - dependency[0], - RetryEvent::generate_index_key(&dependency_absolute_uri).unwrap() - ); - } - _ => panic!("The error type has to be MissingDependency type"), - }; - - test.del(&user_kp, &post_path).await?; - - let del_index_key = format!( - "{}:{}", - EventType::Del, - RetryEvent::generate_index_key(&post_absolute_url).unwrap() - ); - - assert_eventually_exists(&del_index_key).await; - - let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); - assert!(timestamp.is_some()); - - let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); - assert!(event_retry.is_some()); - - let event_state = event_retry.unwrap(); - assert_eq!(event_state.retry_count, 0); - - match event_state.error_type { - EventProcessorError::SkipIndexing => (), - _ => panic!("The error type has to be SkipIndexing type"), - }; - Ok(()) } diff --git a/nexus-watcher/tests/event_processor/posts/retry_reply.rs b/nexus-watcher/tests/event_processor/posts/retry_reply.rs index 47fad9d0e..445f73b45 100644 --- a/nexus-watcher/tests/event_processor/posts/retry_reply.rs +++ b/nexus-watcher/tests/event_processor/posts/retry_reply.rs @@ -1,7 +1,6 @@ use crate::event_processor::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; -use nexus_common::models::event::{EventProcessorError, EventType}; -use nexus_watcher::events::retry::event::RetryEvent; +use nexus_watcher::events::retry::RetryEvent; use pubky::Keypair; use pubky_app_specs::{post_uri_builder, PubkyAppPost, PubkyAppPostKind, PubkyAppUser}; @@ -35,20 +34,15 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { attachments: None, }; - let (reply_id, reply_path) = test.create_post(&user_kp, &reply_post).await?; + let (reply_id, _reply_path) = test.create_post(&user_kp, &reply_post).await?; let reply_absolute_url = post_uri_builder(user_id, reply_id); - let index_key = format!( - "{}:{}", - EventType::Put, - RetryEvent::generate_index_key(&reply_absolute_url).unwrap() - ); + let index_key = reply_absolute_url.clone(); assert_eventually_exists(&index_key).await; - let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); - assert!(timestamp.is_some()); + assert!(RetryEvent::check_uri(&index_key).await.unwrap()); let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -56,40 +50,5 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { let event_state = event_retry.unwrap(); assert_eq!(event_state.retry_count, 0); - match event_state.error_type { - EventProcessorError::MissingDependency { dependency } => { - assert_eq!(dependency.len(), 1); - assert_eq!( - dependency[0], - RetryEvent::generate_index_key(&dependency_absolute_uri).unwrap() - ); - } - _ => panic!("The error type has to be MissingDependency type"), - }; - - test.del(&user_kp, &reply_path).await?; - - let del_index_key = format!( - "{}:{}", - EventType::Del, - RetryEvent::generate_index_key(&reply_absolute_url).unwrap() - ); - - assert_eventually_exists(&del_index_key).await; - - let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); - assert!(timestamp.is_some()); - - let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); - assert!(event_retry.is_some()); - - let event_state = event_retry.unwrap(); - assert_eq!(event_state.retry_count, 0); - - match event_state.error_type { - EventProcessorError::SkipIndexing => (), - _ => panic!("The error type has to be SkipIndexing type"), - }; - Ok(()) } diff --git a/nexus-watcher/tests/event_processor/posts/retry_repost.rs b/nexus-watcher/tests/event_processor/posts/retry_repost.rs index 88bd5ae35..c723a2aa5 100644 --- a/nexus-watcher/tests/event_processor/posts/retry_repost.rs +++ b/nexus-watcher/tests/event_processor/posts/retry_repost.rs @@ -1,7 +1,6 @@ use crate::event_processor::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; -use nexus_common::models::event::{EventProcessorError, EventType}; -use nexus_watcher::events::retry::event::RetryEvent; +use nexus_watcher::events::retry::RetryEvent; use pubky::Keypair; use pubky_app_specs::{ post_uri_builder, PubkyAppPost, PubkyAppPostEmbed, PubkyAppPostKind, PubkyAppUser, @@ -40,20 +39,15 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { attachments: None, }; - let (repost_id, repost_path) = test.create_post(&user_kp, &repost_post).await?; + let (repost_id, _repost_path) = test.create_post(&user_kp, &repost_post).await?; let repost_absolute_url = post_uri_builder(user_id, repost_id); - let index_key = format!( - "{}:{}", - EventType::Put, - RetryEvent::generate_index_key(&repost_absolute_url).unwrap() - ); + let index_key = repost_absolute_url.clone(); assert_eventually_exists(&index_key).await; - let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); - assert!(timestamp.is_some()); + assert!(RetryEvent::check_uri(&index_key).await.unwrap()); let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -61,40 +55,5 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { let event_state = event_retry.unwrap(); assert_eq!(event_state.retry_count, 0); - match event_state.error_type { - EventProcessorError::MissingDependency { dependency } => { - assert_eq!(dependency.len(), 1); - assert_eq!( - dependency[0], - RetryEvent::generate_index_key(&dependency_absolute_uri).unwrap() - ); - } - _ => panic!("The error type has to be MissingDependency type"), - }; - - test.del(&user_kp, &repost_path).await?; - - let del_index_key = format!( - "{}:{}", - EventType::Del, - RetryEvent::generate_index_key(&repost_absolute_url).unwrap() - ); - - assert_eventually_exists(&del_index_key).await; - - let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); - assert!(timestamp.is_some()); - - let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); - assert!(event_retry.is_some()); - - let event_state = event_retry.unwrap(); - assert_eq!(event_state.retry_count, 0); - - match event_state.error_type { - EventProcessorError::SkipIndexing => (), - _ => panic!("The error type has to be SkipIndexing type"), - }; - Ok(()) } diff --git a/nexus-watcher/tests/event_processor/tags/fail_index.rs b/nexus-watcher/tests/event_processor/tags/fail_index.rs index 683423f0b..8946c30cf 100644 --- a/nexus-watcher/tests/event_processor/tags/fail_index.rs +++ b/nexus-watcher/tests/event_processor/tags/fail_index.rs @@ -5,13 +5,11 @@ use anyhow::{anyhow, Result}; use chrono::Utc; use nexus_watcher::service::TEventProcessorRunner; use pubky::Keypair; -use pubky_app_specs::{ - post_uri_builder, - traits::{HasIdPath, HashId}, - PubkyAppPost, PubkyAppTag, PubkyAppUser, -}; +use pubky_app_specs::{post_uri_builder, traits::HashId, PubkyAppPost, PubkyAppTag, PubkyAppUser}; use tracing::error; +/// Verifies that tagging fails with MissingDependency when the tagger or tagged resource +/// is not yet indexed in the graph. #[tokio_shared_rt::test(shared)] async fn test_homeserver_tag_cannot_add_while_index() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -26,11 +24,9 @@ async fn test_homeserver_tag_cannot_add_while_index() -> Result<()> { }; let tagged_user_id = test.create_user(&tagged_keypair, &tagged_user).await?; - // Switch OFF the event processor to simulate the pending events to index - // In that case, shadow user + // Switch OFF event processing — shadow_user signs up on the homeserver but is not indexed test = test.remove_event_processing().await; - // Create a key but it would not be synchronised in nexus let shadow_user_kp = Keypair::random(); let shadow_user = PubkyAppUser { bio: Some("test_homeserver_tag_user_not_found".to_string()), @@ -39,43 +35,34 @@ async fn test_homeserver_tag_cannot_add_while_index() -> Result<()> { name: "Watcher:CannotTag:Tagger:Sync".to_string(), status: None, }; - let _shadow_user_id = test.create_user(&shadow_user_kp, &shadow_user).await?; - - // => Create user tag - let label = "friendly"; + let shadow_user_id = test.create_user(&shadow_user_kp, &shadow_user).await?; + // => User tag: shadow_user tags tagged_user's profile let tag = PubkyAppTag { uri: format!("pubky://{tagged_user_id}/pub/pubky.app/profile.json"), - label: label.to_string(), + label: "friendly".to_string(), created_at: Utc::now().timestamp_millis(), }; - + let tag_id = tag.create_id(); let tag_path = tag.hs_path(); - let tag_blob = serde_json::to_vec(&tag)?; - // PUT user tag - test.put(&shadow_user_kp, &tag_path, tag_blob).await?; + test.put(&shadow_user_kp, &tag_path, &tag).await?; - // Create raw event line to retrieve the content from the homeserver - let tag_event = format!("PUT {tag_path}"); + // Full URI required by Event::parse_event + let tag_event = format!("PUT pubky://{shadow_user_id}/pub/pubky.app/tags/{tag_id}"); - // Simulate the event processor to handle the event. - // If the event processor were activated, the test would not catch the missing dependency - // error, and it would pass successfully - let moderation_ref = test.event_processor_runner.moderation.clone(); - let sync_fail = retrieve_and_handle_event_line(&tag_event, moderation_ref) + let event_handler = test.event_processor_runner.event_handler.clone(); + let sync_fail = retrieve_and_handle_event_line(&tag_event, event_handler) .await .map_err(|e| error!("SYNC ERROR: {:?}", e)) .is_err(); assert!( sync_fail, - "It seems that tagged node exists, which should not be possible. Event processor should be disconnected" + "Tag indexing should fail: tagger (shadow_user) is not yet indexed" ); - // Build the event processor and run it to sync all the previous events with the event processor - // We do this because earlier, the runner's event processing has been turned off temporarily - // but at this point we are ready to run the event processing + // Sync all pending events so shadow_user is now in the graph test.event_processor_runner .build(test.homeserver_id.to_string()) .await @@ -84,7 +71,7 @@ async fn test_homeserver_tag_cannot_add_while_index() -> Result<()> { .await .map_err(|e| anyhow!(e))?; - // => Create post tag + // => Post tag: shadow_user tags a post that hasn't been indexed yet let post = PubkyAppPost { content: "Watcher:CannotTag:Post:unSync".to_string(), kind: PubkyAppPost::default().kind, @@ -94,33 +81,27 @@ async fn test_homeserver_tag_cannot_add_while_index() -> Result<()> { }; let (post_id, _post_path) = test.create_post(&tagged_keypair, &post).await?; - let label = "merkle_tree"; - - let tag = PubkyAppTag { + let post_tag = PubkyAppTag { uri: post_uri_builder(tagged_user_id, post_id), - label: label.to_string(), + label: "merkle_tree".to_string(), created_at: Utc::now().timestamp_millis(), }; - let tag_blob = serde_json::to_vec(&tag)?; - let tag_relative_url = PubkyAppTag::create_path(&tag.create_id()); - // PUT post tag - test.put(&shadow_user_kp, &tag_path, tag_blob).await?; - - // Create raw event line to retrieve the content from the homeserver - let tag_event = format!("PUT {tag_relative_url}"); - - // Simulate the event processor to handle the event. - // If the event processor were activated, the test would not catch the missing dependency - // error, and it would pass successfully - let moderation_ref = test.event_processor_runner.moderation.clone(); - let sync_fail = retrieve_and_handle_event_line(&tag_event, moderation_ref) + let post_tag_id = post_tag.create_id(); + let post_tag_path = post_tag.hs_path(); + + test.put(&shadow_user_kp, &post_tag_path, &post_tag).await?; + + let tag_event = format!("PUT pubky://{shadow_user_id}/pub/pubky.app/tags/{post_tag_id}"); + + let event_handler = test.event_processor_runner.event_handler.clone(); + let sync_fail = retrieve_and_handle_event_line(&tag_event, event_handler) .await .map_err(|e| error!("SYNC ERROR: {:?}", e)) .is_err(); assert!( sync_fail, - "It seems that tagged node exists, which should not be possible. Event processor should be disconnected" + "Tag indexing should fail: tagged post is not yet indexed" ); Ok(()) diff --git a/nexus-watcher/tests/event_processor/tags/resource_utils.rs b/nexus-watcher/tests/event_processor/tags/resource_utils.rs index d15927ad5..8b911ec8e 100644 --- a/nexus-watcher/tests/event_processor/tags/resource_utils.rs +++ b/nexus-watcher/tests/event_processor/tags/resource_utils.rs @@ -65,8 +65,8 @@ pub async fn check_resource_in_sorted_set( /// Compute the resource_id for a given URI (for test assertions) pub fn compute_resource_id(uri: &str) -> String { - let (normalized, _) = nexus_common::models::resource::normalize_uri(uri).unwrap(); - nexus_common::models::resource::resource_id(&normalized) + let (normalized, _) = nexus_common::universal_tag::normalize::normalize_uri(uri).unwrap(); + nexus_common::universal_tag::normalize::resource_id(&normalized) } fn resource_tag_query(resource_id: &str, label: &str) -> Query { diff --git a/nexus-watcher/tests/event_processor/tags/retry_post_tag.rs b/nexus-watcher/tests/event_processor/tags/retry_post_tag.rs index 38987a0d8..133c6b1ce 100644 --- a/nexus-watcher/tests/event_processor/tags/retry_post_tag.rs +++ b/nexus-watcher/tests/event_processor/tags/retry_post_tag.rs @@ -3,8 +3,7 @@ use crate::event_processor::utils::watcher::{ }; use anyhow::Result; use chrono::Utc; -use nexus_common::models::event::{EventProcessorError, EventType}; -use nexus_watcher::events::retry::event::RetryEvent; +use nexus_watcher::events::retry::RetryEvent; use pubky::Keypair; use pubky_app_specs::{post_uri_builder, tag_uri_builder}; use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; @@ -53,16 +52,11 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { // to let write the indexes test.put(&tagger_kp, &tag_path, tag).await?; - let index_key = format!( - "{}:{}", - EventType::Put, - RetryEvent::generate_index_key(&tag_absolute_url).unwrap() - ); + let index_key = tag_absolute_url.clone(); assert_eventually_exists(&index_key).await; - let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); - assert!(timestamp.is_some()); + assert!(RetryEvent::check_uri(&index_key).await.unwrap()); let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -70,18 +64,5 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { let event_state = event_retry.unwrap(); assert_eq!(event_state.retry_count, 0); - match event_state.error_type { - EventProcessorError::MissingDependency { dependency } => { - assert_eq!(dependency.len(), 1); - assert_eq!( - dependency[0], - RetryEvent::generate_index_key(&dependency_absolute_uri).unwrap() - ); - } - _ => panic!("The error type has to be MissingDependency type"), - }; - - test.del(&tagger_kp, &tag_path).await?; - Ok(()) } diff --git a/nexus-watcher/tests/event_processor/tags/retry_user_tag.rs b/nexus-watcher/tests/event_processor/tags/retry_user_tag.rs index fc5d28611..1563ef5b9 100644 --- a/nexus-watcher/tests/event_processor/tags/retry_user_tag.rs +++ b/nexus-watcher/tests/event_processor/tags/retry_user_tag.rs @@ -3,8 +3,7 @@ use crate::event_processor::utils::watcher::{ }; use anyhow::Result; use chrono::Utc; -use nexus_common::models::event::{EventProcessorError, EventType}; -use nexus_watcher::events::retry::event::RetryEvent; +use nexus_watcher::events::retry::RetryEvent; use pubky::Keypair; use pubky_app_specs::{tag_uri_builder, user_uri_builder}; use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; @@ -46,16 +45,11 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { // to let write the indexes test.put(&tagger_kp, &tag_path, tag).await?; - let index_key = format!( - "{}:{}", - EventType::Put, - RetryEvent::generate_index_key(&tag_absolute_url).unwrap() - ); + let index_key = tag_absolute_url.clone(); assert_eventually_exists(&index_key).await; - let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); - assert!(timestamp.is_some()); + assert!(RetryEvent::check_uri(&index_key).await.unwrap()); let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -63,18 +57,5 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { let event_state = event_retry.unwrap(); assert_eq!(event_state.retry_count, 0); - match event_state.error_type { - EventProcessorError::MissingDependency { dependency } => { - assert_eq!(dependency.len(), 1); - assert_eq!( - dependency[0], - RetryEvent::generate_index_key(&dependency_absolute_uri).unwrap() - ); - } - _ => panic!("The error type has to be MissingDependency type"), - }; - - test.del(&tagger_kp, &tag_path).await?; - Ok(()) } diff --git a/nexus-watcher/tests/event_processor/utils/mod.rs b/nexus-watcher/tests/event_processor/utils/mod.rs index 5e00f0e06..042e4d6a4 100644 --- a/nexus-watcher/tests/event_processor/utils/mod.rs +++ b/nexus-watcher/tests/event_processor/utils/mod.rs @@ -1,12 +1,4 @@ -use nexus_watcher::events::Moderation; -use pubky_app_specs::PubkyId; - pub mod watcher; -/// Default Moderation settings for tests -pub fn default_moderation_tests() -> Moderation { - let id = PubkyId::try_from("uo7jgkykft4885n8cruizwy6khw71mnu5pq3ay9i8pw1ymcn85ko") - .expect("Hardcoded test moderation key should be valid"); - let tags = Vec::from(["label_to_moderate".to_string()]); - Moderation { id, tags } -} +// Re-export shared test utilities +pub use crate::utils::default_moderation_tests; diff --git a/nexus-watcher/tests/event_processor/utils/watcher.rs b/nexus-watcher/tests/event_processor/utils/watcher.rs index 895e78910..51b365d62 100644 --- a/nexus-watcher/tests/event_processor/utils/watcher.rs +++ b/nexus-watcher/tests/event_processor/utils/watcher.rs @@ -1,17 +1,20 @@ +use crate::event_processor::utils::default_moderation_tests; use anyhow::{anyhow, Error, Result}; use base32::{encode, Alphabet}; use chrono::Utc; use nexus_common::db::PubkyConnector; use nexus_common::get_files_dir_pathbuf; use nexus_common::get_files_dir_test_pathbuf; -use nexus_common::models::event::{Event, EventProcessorError, ParseResult}; +use nexus_common::models::event::ParseResult; +use nexus_common::models::event::{Event, EventProcessorError}; use nexus_common::models::file::FileDetails; use nexus_common::models::homeserver::Homeserver; use nexus_common::models::traits::Collection; use nexus_common::{StackConfig, StackManager}; use nexus_watcher::events::retry::event::RetryEvent; -use nexus_watcher::events::{handle, Moderation}; -use nexus_watcher::service::EventProcessorRunner; +use nexus_watcher::events::retry::{InitialBackoff, RedisRetryStore, RetryScheduler, RetryStore}; +use nexus_watcher::events::{DefaultEventHandler, EventHandler}; +use nexus_watcher::service::HsEventProcessorRunner; use nexus_watcher::service::TEventProcessorRunner; use pubky::Keypair; use pubky::PublicKey; @@ -28,8 +31,6 @@ use std::sync::Arc; use std::time::Duration; use tracing::debug; -use crate::event_processor::utils::default_moderation_tests; - static COUNTER: AtomicU64 = AtomicU64::new(0); /// Generate a unique post ID for tests. @@ -52,7 +53,7 @@ pub struct WatcherTest { /// The homeserver ID pub homeserver_id: PubkyId, /// The event processor runner - pub event_processor_runner: EventProcessorRunner, + pub event_processor_runner: HsEventProcessorRunner, /// Whether to ensure event processing is complete pub ensure_event_processing: bool, } @@ -74,20 +75,29 @@ impl WatcherTest { /// that are designed specifically for test scenarios and should not be used in production. /// /// # Returns - /// Returns a fully configured `EventProcessorRunner` ready for use in tests. - fn create_test_event_processor_runner(default_homeserver: PubkyId) -> EventProcessorRunner { - let moderation = Arc::new(default_moderation_tests()); + /// Returns a fully configured `HsEventProcessorRunner` ready for use in tests. + fn create_test_event_processor_runner(default_homeserver: PubkyId) -> HsEventProcessorRunner { + let event_handler: Arc = + Arc::new(DefaultEventHandler::new(default_moderation_tests())); let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); - EventProcessorRunner { + let store: Arc = Arc::new(RedisRetryStore::new()); + let retry_scheduler = Arc::new(RetryScheduler::new( + store, + InitialBackoff { + missing_dep_ms: 60_000, + transient_ms: 10_000, + }, + )); + + HsEventProcessorRunner { limit: 1000, - monitored_homeservers_limit: 100, files_path: get_files_dir_test_pathbuf(), - tracer_name: "test".to_string(), - moderation, + event_handler, shutdown_rx, default_homeserver, + retry_scheduler, } } @@ -111,8 +121,7 @@ impl WatcherTest { // WARNING: testnet initialization is time expensive, we only init one per process // TODO: Maybe we should create a single testnet network (singleton and push there more homeservers) - // This can be further sped up by using Testnet::new_unseeded() with pubky-testnet 0.7.x - let mut testnet = Testnet::new().await?; + let mut testnet = Testnet::new_unseeded().await?; testnet.create_http_relay().await?; // Create a random homeserver with a random public key @@ -345,16 +354,11 @@ impl WatcherTest { /// Throws an error if event parsing fails pub async fn retrieve_and_handle_event_line( event_line: &str, - moderation: Arc, + event_handler: Arc, ) -> Result<(), EventProcessorError> { match Event::parse_event(event_line, get_files_dir_pathbuf())? { - ParseResult::Parsed(event) => handle(&event, moderation).await, - ParseResult::Skipped => Ok(()), - - // Propagate UnrecognizedUri as error, because this test helper is only meant for standard event handling - ParseResult::UnrecognizedUri { reason, .. } => Err(EventProcessorError::InvalidEventLine( - format!("Cannot parse event URI: {reason}"), - )), + ParseResult::Parsed(event) => event_handler.handle(&event).await, + ParseResult::Skipped | ParseResult::UnrecognizedUri { .. } => Ok(()), } } @@ -377,11 +381,8 @@ pub async fn assert_eventually_exists(event_index: &str) { SLEEP_MS * attempt as u64 ); match RetryEvent::check_uri(event_index).await { - Ok(timeframe) => { - if timeframe.is_some() { - return; - } - } + Ok(true) => return, + Ok(false) => {} Err(e) => panic!("Error while getting index: {e:?}"), }; // Nap time diff --git a/nexus-watcher/tests/homeserver/active_homeservers.rs b/nexus-watcher/tests/homeserver/active_homeservers.rs new file mode 100644 index 000000000..7082c4652 --- /dev/null +++ b/nexus-watcher/tests/homeserver/active_homeservers.rs @@ -0,0 +1,115 @@ +use crate::event_processor::utils::watcher::WatcherTest; +use nexus_common::db::exec_single_row; +use nexus_common::db::queries; +use nexus_common::models::homeserver::Homeserver; +use nexus_common::types::DynError; +use pubky::Keypair; +use pubky_app_specs::{PubkyAppUser, PubkyId}; + +/// Helper: create a PubkyAppUser with a given name. +fn make_test_user(name: &str) -> PubkyAppUser { + PubkyAppUser { + bio: Some(format!("bio-{name}")), + image: None, + links: None, + name: name.to_string(), + status: None, + } +} + +/// Helper: create an orphan homeserver (no users) in the graph, return its PubkyId. +async fn create_orphan_hs() -> Result { + let keys = Keypair::random(); + let id = PubkyId::try_from(&keys.public_key().to_z32())?; + let hs = Homeserver::new(id.clone()); + hs.put_to_graph().await?; + Ok(id) +} + +/// Active homeserver listing: orphan exclusion, sort order, and stability after user reassignment. +#[tokio_shared_rt::test(shared)] +async fn test_get_all_active_homeservers() -> Result<(), DynError> { + let mut test = WatcherTest::setup().await?; + + // -- Orphan HSs (no users) must be excluded -- + let orphan1 = create_orphan_hs().await?; + let orphan2 = create_orphan_hs().await?; + + // -- HS-A: 1 user -- + let hs_a = create_orphan_hs().await?; + let kp_a1 = Keypair::random(); + let id_a1 = test + .create_user(&kp_a1, &make_test_user("Watcher:ActiveHS:A1")) + .await?; + exec_single_row(queries::put::set_user_homeserver(&id_a1, &hs_a)).await?; + + // -- HS-B: 2 users -- + let hs_b = create_orphan_hs().await?; + let kp_b1 = Keypair::random(); + let id_b1 = test + .create_user(&kp_b1, &make_test_user("Watcher:ActiveHS:B1")) + .await?; + exec_single_row(queries::put::set_user_homeserver(&id_b1, &hs_b)).await?; + + let kp_b2 = Keypair::random(); + let id_b2 = test + .create_user(&kp_b2, &make_test_user("Watcher:ActiveHS:B2")) + .await?; + exec_single_row(queries::put::set_user_homeserver(&id_b2, &hs_b)).await?; + + let hs_ids = Homeserver::get_all_active_from_graph().await?; + + // Orphans excluded + assert!( + !hs_ids.contains(&orphan1.to_string()), + "orphan1 should be excluded" + ); + assert!( + !hs_ids.contains(&orphan2.to_string()), + "orphan2 should be excluded" + ); + + // Both active HSs included, HS-B (2 users) before HS-A (1 user) + let pos_a = hs_ids + .iter() + .position(|id| id == &hs_a.to_string()) + .expect("HS-A missing"); + let pos_b = hs_ids + .iter() + .position(|id| id == &hs_b.to_string()) + .expect("HS-B missing"); + assert!(pos_b < pos_a, "HS-B (2 users) should precede HS-A (1 user)"); + + // -- Reassign one user from HS-B to HS-A; both HSs must stay active -- + exec_single_row(queries::put::set_user_homeserver(&id_b2, &hs_a)).await?; + + let hs_ids = Homeserver::get_all_active_from_graph().await?; + assert!( + hs_ids.contains(&hs_a.to_string()), + "HS-A should still be active" + ); + assert!( + hs_ids.contains(&hs_b.to_string()), + "HS-B should still be active after partial removal" + ); + + // Now HS-A has 2 users, HS-B has 1 — order should flip + let pos_a = hs_ids + .iter() + .position(|id| id == &hs_a.to_string()) + .unwrap(); + let pos_b = hs_ids + .iter() + .position(|id| id == &hs_b.to_string()) + .unwrap(); + assert!( + pos_a < pos_b, + "HS-A (2 users) should now precede HS-B (1 user)" + ); + + // Cleanup + test.cleanup_user(&kp_a1).await?; + test.cleanup_user(&kp_b1).await?; + test.cleanup_user(&kp_b2).await?; + Ok(()) +} diff --git a/nexus-watcher/tests/homeserver/mod.rs b/nexus-watcher/tests/homeserver/mod.rs new file mode 100644 index 000000000..9765a2d94 --- /dev/null +++ b/nexus-watcher/tests/homeserver/mod.rs @@ -0,0 +1 @@ +mod active_homeservers; diff --git a/nexus-watcher/tests/mod.rs b/nexus-watcher/tests/mod.rs index 068173fe2..aade7252a 100644 --- a/nexus-watcher/tests/mod.rs +++ b/nexus-watcher/tests/mod.rs @@ -1,2 +1,5 @@ mod event_processor; +mod homeserver; mod service; +mod user_ingestion; +pub mod utils; diff --git a/nexus-watcher/tests/service/event_processing_multiple_homeservers.rs b/nexus-watcher/tests/service/event_processing_multiple_homeservers.rs index eca5a628a..340ab636d 100644 --- a/nexus-watcher/tests/service/event_processing_multiple_homeservers.rs +++ b/nexus-watcher/tests/service/event_processing_multiple_homeservers.rs @@ -3,7 +3,6 @@ use crate::service::utils::{ MockEventProcessorRunner, }; use anyhow::Result; -use nexus_watcher::service::backoff::HomeserverBackoff; use nexus_watcher::service::TEventProcessorRunner; use std::time::Duration; @@ -22,6 +21,7 @@ async fn test_multiple_homeserver_event_processing() -> Result<()> { processor_status, None, shutdown_rx.clone(), + Some(1), ) .await; } @@ -34,17 +34,15 @@ async fn test_multiple_homeserver_event_processing() -> Result<()> { processor_status, None, shutdown_rx.clone(), + Some(1), ) .await; let runner = MockEventProcessorRunner::new(event_processor_list, 4, shutdown_rx); - let stats = runner - .run_all(&mut HomeserverBackoff::default()) - .await - .unwrap() - .0; - assert_eq!(stats.count_ok(), 3); + // run excludes the default homeserver (the first one), so only 3 are processed + let stats = runner.run().await.unwrap().0; + assert_eq!(stats.count_ok(), 2); assert_eq!(stats.count_error(), 1); assert_eq!(stats.count_panic(), 0); assert_eq!(stats.count_timeout(), 0); @@ -67,20 +65,18 @@ async fn test_multi_hs_event_processing_with_homeserver_limit() -> Result<()> { processor_status, None, shutdown_rx.clone(), + Some(1), ) .await; } assert_eq!(event_processor_list.len(), 5); // Ensure 5 HSs are available - let hs_limit = 3; // Configure a monitored_homeservers_limit of 3 + let hs_limit = 3; let runner = MockEventProcessorRunner::new(event_processor_list, hs_limit, shutdown_rx); - let stats = runner - .run_all(&mut HomeserverBackoff::default()) - .await - .unwrap() - .0; + // run excludes the default HS, so 4 non-default HSs available, limited to 3 + let stats = runner.run().await.unwrap().0; - assert_eq!(stats.count_ok(), 3); // 3 successful ones, due to the limit (5 HSs were available) + assert_eq!(stats.count_ok(), 3); // 3 successful ones, due to the limit assert_eq!(stats.count_timeout(), 0); assert_eq!(stats.count_error(), 0); assert_eq!(stats.count_panic(), 0); @@ -103,24 +99,25 @@ async fn test_multi_hs_event_processing_with_homeserver_limit_one() -> Result<() processor_status, None, shutdown_rx.clone(), + Some(1), ) .await; } assert_eq!(event_processor_list.len(), 5); // Ensure 5 HSs are available - // Check that, when the limit is 1, only the default (first) homeserver is considered + // Check that, when the limit is 1, only one non-default homeserver is considered let runner_one = MockEventProcessorRunner::new(event_processor_list, 1, shutdown_rx); - let hs_list = runner_one.pre_run_all().await.unwrap(); + let hs_list = runner_one.pre_run().await.unwrap(); assert_eq!(hs_list.len(), 1); - assert_eq!(hs_list.first().unwrap(), &runner_one.default_homeserver()); - - let stats_one = runner_one - .run_all(&mut HomeserverBackoff::default()) - .await - .unwrap() - .0; - assert_eq!(stats_one.count_ok(), 1); // 1 successful, due to the limit (5 HSs were available) + assert_ne!( + hs_list.first().unwrap(), + &runner_one.default_homeserver(), + "Default homeserver should be excluded from pre_run" + ); + + let stats_one = runner_one.run().await.unwrap().0; + assert_eq!(stats_one.count_ok(), 1); // 1 successful, due to the limit assert_eq!(stats_one.count_timeout(), 0); assert_eq!(stats_one.count_error(), 0); assert_eq!(stats_one.count_panic(), 0); @@ -136,6 +133,9 @@ async fn test_multi_hs_event_processing_with_timeout() -> Result<()> { let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); // Create 3 random homeservers with timeout limit + // Index 0: 0s sleep (default, excluded from run) + // Index 1: 2s sleep + // Index 2: 4s sleep for index in 0..3 { let processor_status = MockEventProcessorResult::Success; create_random_homeservers_and_persist( @@ -144,18 +144,17 @@ async fn test_multi_hs_event_processing_with_timeout() -> Result<()> { processor_status, EVENT_PROCESSOR_TIMEOUT, shutdown_rx.clone(), + Some(1), ) .await; } let runner = MockEventProcessorRunner::new(event_processor_list, 3, shutdown_rx); - let stats = runner - .run_all(&mut HomeserverBackoff::default()) - .await - .unwrap() - .0; - assert_eq!(stats.count_ok(), 1); // 1 success + // run excludes the default HS (0s sleep), so only index 1 and 2 are processed. + // Both have sleep durations exceeding the 1s timeout. + let stats = runner.run().await.unwrap().0; + assert_eq!(stats.count_ok(), 0); // no successes assert_eq!(stats.count_timeout(), 2); // 2 failures due to timeout assert_eq!(stats.count_error(), 0); assert_eq!(stats.count_panic(), 0); @@ -178,6 +177,7 @@ async fn test_multi_hs_event_processing_with_panic() -> Result<()> { processor_status, None, shutdown_rx.clone(), + Some(1), ) .await; } @@ -191,18 +191,16 @@ async fn test_multi_hs_event_processing_with_panic() -> Result<()> { processor_status, None, shutdown_rx.clone(), + Some(1), ) .await; } let runner = MockEventProcessorRunner::new(event_processor_list, 5, shutdown_rx); - let stats = runner - .run_all(&mut HomeserverBackoff::default()) - .await - .unwrap() - .0; - assert_eq!(stats.count_ok(), 3); // 3 expected to succeed + // run excludes the default HS (first success), so 2 success + 2 panic are processed + let stats = runner.run().await.unwrap().0; + assert_eq!(stats.count_ok(), 2); // 2 expected to succeed (3 - 1 default) assert_eq!(stats.count_timeout(), 0); assert_eq!(stats.count_error(), 0); assert_eq!(stats.count_panic(), 2); // 2 expected to panic diff --git a/nexus-watcher/tests/service/event_processor_prioritization.rs b/nexus-watcher/tests/service/event_processor_prioritization.rs index 8d0438c74..aa9c34e64 100644 --- a/nexus-watcher/tests/service/event_processor_prioritization.rs +++ b/nexus-watcher/tests/service/event_processor_prioritization.rs @@ -4,25 +4,41 @@ use crate::service::utils::{create_mock_event_processors, setup, MockEventProces use anyhow::Result; use nexus_common::models::homeserver::Homeserver; use nexus_common::types::DynError; -use nexus_watcher::service::EventProcessorRunner; -use nexus_watcher::service::TEventProcessorRunner; +use nexus_watcher::events::retry::{InitialBackoff, RedisRetryStore, RetryScheduler, RetryStore}; +use nexus_watcher::events::{DefaultEventHandler, EventHandler}; +use nexus_watcher::service::backoff::HomeserverBackoff; +use nexus_watcher::service::indexer::PubkyKeyBasedEventSource; +use nexus_watcher::service::{KeyBasedEventProcessorRunner, TEventProcessorRunner}; use pubky_app_specs::PubkyId; use std::path::PathBuf; use std::sync::Arc; +use tokio::sync::Mutex; #[tokio_shared_rt::test(shared)] -async fn test_event_processor_runner_default_homeserver_prioritization() -> Result<(), DynError> { +async fn test_event_processor_runner_default_homeserver_excluded() -> Result<(), DynError> { // Initialize the test setup().await?; - let runner = EventProcessorRunner { - default_homeserver: PubkyId::try_from(HS_IDS[3]).unwrap(), - shutdown_rx: tokio::sync::watch::channel(false).1, + let event_handler: Arc = + Arc::new(DefaultEventHandler::new(default_moderation_tests())); + let store: Arc = Arc::new(RedisRetryStore::new()); + let retry_scheduler = Arc::new(RetryScheduler::new( + store, + InitialBackoff { + missing_dep_ms: 60_000, + transient_ms: 10_000, + }, + )); + let runner = KeyBasedEventProcessorRunner { limit: 1000, - monitored_homeservers_limit: HS_IDS.len(), + monitored_hs_limit: HS_IDS.len(), files_path: PathBuf::from("/tmp/nexus-watcher-test"), - tracer_name: "test".to_string(), - moderation: Arc::new(default_moderation_tests()), + event_handler, + event_source: Arc::new(PubkyKeyBasedEventSource), + shutdown_rx: tokio::sync::watch::channel(false).1, + default_homeserver: PubkyId::try_from(HS_IDS[3]).unwrap(), + backoff: Mutex::new(HomeserverBackoff::default()), + retry_scheduler, }; // Persist the homeservers @@ -31,16 +47,18 @@ async fn test_event_processor_runner_default_homeserver_prioritization() -> Resu hs.put_to_graph().await.unwrap(); } - // Prioritize the default homeserver - let hs_ids = runner.homeservers_by_priority().await?; - assert_eq!(hs_ids[0], HS_IDS[3]); + // The default homeserver should be excluded from the list + let hs_ids = runner.pre_run().await?; + assert!( + !hs_ids.contains(&HS_IDS[3].to_string()), + "Default homeserver should be excluded from pre_run" + ); Ok(()) } #[tokio_shared_rt::test(shared)] -async fn test_mock_event_processor_runner_default_homeserver_prioritization() -> Result<(), DynError> -{ +async fn test_mock_event_processor_runner_default_homeserver_excluded() -> Result<(), DynError> { // Initialize the test setup().await?; @@ -51,7 +69,7 @@ async fn test_mock_event_processor_runner_default_homeserver_prioritization() -> let runner = MockEventProcessorRunner { event_processors, - monitored_homeservers_limit: 100, + monitored_hs_limit: 100, shutdown_rx: tokio::sync::watch::channel(false).1, }; @@ -61,9 +79,12 @@ async fn test_mock_event_processor_runner_default_homeserver_prioritization() -> hs.put_to_graph().await.unwrap(); } - // Prioritize the default homeserver - let hs_ids = runner.homeservers_by_priority().await?; - assert_eq!(hs_ids[0], HS_IDS[0]); + // The default homeserver (HS_IDS[0]) should be excluded from the list + let hs_ids = runner.hs_by_priority().await?; + assert!( + !hs_ids.contains(&HS_IDS[0].to_string()), + "Default homeserver should be excluded from hs_by_priority" + ); Ok(()) } diff --git a/nexus-watcher/tests/service/hs_event_processor.rs b/nexus-watcher/tests/service/hs_event_processor.rs new file mode 100644 index 000000000..e2b50a7bd --- /dev/null +++ b/nexus-watcher/tests/service/hs_event_processor.rs @@ -0,0 +1,157 @@ +use crate::service::utils::common::create_mock_handler; +use crate::service::utils::{new_in_memory_store, setup, TEST_USER_ID}; +use anyhow::Result; +use nexus_common::models::event::EventProcessorError; +use nexus_common::models::homeserver::Homeserver; +use nexus_watcher::events::retry::{InitialBackoff, RetryScheduler, RetryStore}; +use nexus_watcher::events::EventHandler; +use nexus_watcher::service::HsEventProcessor; +use pubky_app_specs::{post_uri_builder, PubkyId}; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::watch; + +/// Assemble an [`HsEventProcessor`] for tests. Tests bypass `poll_events` by +/// calling `process_event_lines` directly with constructed event lines. +fn build_processor( + store: Arc, + event_handler: Arc, + shutdown_rx: watch::Receiver, +) -> Arc { + let retry_scheduler = Arc::new(RetryScheduler::new( + store, + InitialBackoff { + missing_dep_ms: 60_000, + transient_ms: 10_000, + }, + )); + let hs_id = PubkyId::try_from(TEST_USER_ID).expect("Valid test Pubky ID"); + + Arc::new(HsEventProcessor { + homeserver: Homeserver::new(hs_id), + limit: 100, + files_path: PathBuf::from("/tmp/test"), + event_handler, + shutdown_rx, + retry_scheduler, + }) +} + +// ============================================================================ +// Batch continues after a single event fails +// A retryable application error on one event must not halt the batch — later +// events still need to be handed to the event handler. +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_batch_continues_after_single_failure() -> Result<()> { + setup().await?; + + let first_post_id = "failone"; + let second_post_id = "failtwo"; + let first_uri = post_uri_builder(TEST_USER_ID.to_string(), first_post_id.to_string()); + let second_uri = post_uri_builder(TEST_USER_ID.to_string(), second_post_id.to_string()); + + let lines = vec![format!("PUT {first_uri}"), format!("PUT {second_uri}")]; + + let store = new_in_memory_store(); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + + // Handler returns a retryable error for every event — both events should + // therefore be enqueued by the RetryScheduler. If the batch halted on the + // first failure, only the first URI would be present in the store. + let handler = create_mock_handler( + Err(EventProcessorError::Generic("handler fails".to_string())), + None, + ); + let processor = build_processor(store.clone(), handler.clone(), shutdown_rx); + + let result = processor.process_event_lines(lines).await; + assert!( + result.is_ok(), + "Retryable application error must not stop the batch" + ); + + // Both events were processed (handler called twice), proving the batch + // continued past the first failure. + assert_eq!( + handler.get_handle_count(), + 2, + "Handler must be called for both events — batch continued past failure" + ); + + assert!( + store.get(&first_uri).await?.is_some(), + "First event must be queued for retry" + ); + assert!( + store.get(&second_uri).await?.is_some(), + "Second event must be queued for retry — proves the batch continued past the first failure" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Infrastructure error stops the batch +// Infrastructure errors propagate out of `handle_error`, short-circuiting the +// loop so the cursor is not advanced past unprocessed events. +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_batch_stops_on_infrastructure_error() -> Result<()> { + setup().await?; + + let first_post_id = "infraone"; + let second_post_id = "infratwo"; + let first_uri = post_uri_builder(TEST_USER_ID.to_string(), first_post_id.to_string()); + let second_uri = post_uri_builder(TEST_USER_ID.to_string(), second_post_id.to_string()); + + let lines = vec![format!("PUT {first_uri}"), format!("PUT {second_uri}")]; + + let store = new_in_memory_store(); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + + // Scope the infrastructure error to the first event only. The handler + // returns Ok(()) for non-matching events, so if the batch continued past + // the first failure, the second event would succeed. The invocation + // counter provides the definitive proof: handle_count == 1 proves the + // handler was called exactly once (first event), and the second event + // was never reached. + let handler = create_mock_handler( + Err(EventProcessorError::IndexOperationFailed( + true, + "simulated infra failure".to_string(), + )), + Some(first_post_id), + ); + let processor = build_processor(store.clone(), handler.clone(), shutdown_rx); + + let result = processor.process_event_lines(lines).await; + assert!( + result.is_err(), + "Infrastructure error must propagate and stop the batch" + ); + + // Definitive proof: handler was called exactly once, so the batch stopped + // after the first event and never reached the second. + assert_eq!( + handler.get_handle_count(), + 1, + "Handler must be called exactly once — batch stopped on infrastructure error" + ); + + // Infrastructure errors bypass the retry scheduler entirely. + assert!( + store.get(&first_uri).await?.is_none(), + "Infrastructure errors must not be queued for retry" + ); + assert!( + store.get(&second_uri).await?.is_none(), + "Second event must not be queued — batch should have stopped at the first failure" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} diff --git a/nexus-watcher/tests/service/key_based_event_processor.rs b/nexus-watcher/tests/service/key_based_event_processor.rs new file mode 100644 index 000000000..4b56376a0 --- /dev/null +++ b/nexus-watcher/tests/service/key_based_event_processor.rs @@ -0,0 +1,644 @@ +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Result; +use chrono::Utc; +use nexus_common::db::{exec_single_row, graph::Query, queries, RedisOps}; +use nexus_common::models::event::{Event, EventProcessorError}; +use nexus_common::models::homeserver::Homeserver; +use nexus_common::models::user::{user_hs_cursor_key, UserDetails}; +use nexus_common::types::DynError; +use nexus_watcher::events::retry::{InitialBackoff, RetryScheduler}; +use nexus_watcher::events::EventHandler; +use nexus_watcher::service::indexer::{KeyBasedEventProcessor, RunError, TEventProcessor}; +use pubky::{Event as StreamEvent, EventCursor, EventType, Keypair, PubkyResource, PublicKey}; +use pubky_app_specs::PubkyId; +use tokio::sync::watch; + +use crate::service::utils::{ + create_mock_handler, create_random_homeservers_and_persist, new_in_memory_store, setup, + MockEventProcessorResult, MockKeyBasedEventSource, +}; + +/// Verifies `TEventProcessor::run` maps elapsed execution to a timeout error. +#[tokio_shared_rt::test(shared)] +async fn processor_run_returns_timeout_error() -> Result<(), DynError> { + setup().await?; + + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + let mut processors = Vec::new(); + create_random_homeservers_and_persist( + &mut processors, + Some(Duration::from_millis(50)), + MockEventProcessorResult::Success, + Some(Duration::from_millis(1)), + shutdown_rx, + None, + ) + .await; + + let err = Arc::new(processors.pop().expect("processor should be created")) + .run() + .await + .unwrap_err(); + + assert!(err.is_timeout(), "expected timeout, got {err:?}"); + Ok(()) +} + +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_skips_unrecognized_events() -> Result<(), DynError> { + setup().await?; + + // Create a homeserver with one hosted user to resolve during the run. + let (_hs_keypair, homeserver) = create_homeserver().await?; + let user_id = create_user_on_homeserver(&homeserver).await?; + + // Return one unrecognized event followed by one valid pubky.app event for the same user. + let source = Arc::new( + MockKeyBasedEventSource::default() + .with_events(vec![vec![ + stream_event(1, &user_id, "/pub/other.app/profile.json")?, + stream_event(2, &user_id, "/pub/pubky.app/profile.json")?, + ]]) + .await, + ); + + let handler = create_mock_handler(Ok(()), None); + let processor = processor(homeserver, handler.clone(), source.clone()); + + processor.run().await?; + + // The unrecognized event is skipped, while the valid event is handled. + assert_eq!(handler.get_handle_count(), 1); + + // The processor fetched events only for the hosted user. + assert_eq!(source.calls().await, vec![user_id]); + + Ok(()) +} + +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_stops_mismatched_user_stream_but_continues_other_users( +) -> Result<(), DynError> { + setup().await?; + + // Create a homeserver with two hosted users to resolve during the run. + let (_hs_keypair, homeserver) = create_homeserver().await?; + let user_a_id = create_user_on_homeserver(&homeserver).await?; + let user_b_id = create_user_on_homeserver(&homeserver).await?; + + // This ID is not hosted on the homeserver; it simulates a malicious or broken event source. + let user_c_id = Keypair::random().public_key().to_z32(); + + // For the first hosted user, return an event whose URI belongs to a different user. + // The following valid event for the same hosted user must not be processed after that mismatch. + let source = Arc::new( + MockKeyBasedEventSource::default() + .with_user_events(vec![ + ( + user_a_id.clone(), + vec![ + stream_event(1, &user_c_id, "/pub/pubky.app/profile.json")?, + stream_event(2, &user_a_id, "/pub/pubky.app/profile.json")?, + ], + ), + // For the second hosted user, return a valid event to prove processing continues. + ( + user_b_id.clone(), + vec![stream_event(3, &user_b_id, "/pub/pubky.app/profile.json")?], + ), + ]) + .await, + ); + + // Wire the processor to the user-keyed mock source and handler. + let handler = create_mock_handler(Ok(()), None); + let hs_id = homeserver.id.to_string(); + let processor = processor(homeserver, handler.clone(), source.clone()); + + // Run one processing pass. User-level mismatches should be logged and skipped, not fail the run. + let result = processor.run().await; + + assert!(result.is_ok()); + + // Both hosted users were fetched from the same homeserver despite the first user's mismatch. + let calls = source.calls().await; + assert_eq!(calls.len(), 2); + assert!(calls.contains(&user_a_id)); + assert!(calls.contains(&user_b_id)); + + // Only the other user's event was handled; the valid event after the mismatch was skipped. + let handled_uris = handler.get_handled_uris(); + assert_eq!(handled_uris.len(), 1); + assert!(handled_uris.iter().all(|uri| !uri.contains(&user_a_id))); + assert!(handled_uris.iter().any(|uri| uri.contains(&user_b_id))); + + // The mismatched user's cursor must not be persisted: the bad event is the first in the + // batch, so `latest_cursor` is never set and no write to the USER_HS_CURSOR set should occur. + let cursor_a = + UserDetails::check_sorted_set_member(None, &user_hs_cursor_key(&user_a_id), &[&hs_id]) + .await?; + assert!( + cursor_a.is_none(), + "user_a cursor must not be advanced past the mismatched event, got {cursor_a:?}", + ); + + Ok(()) +} + +/// Verifies an empty hosted-user set exits successfully without fetching events. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_returns_ok_without_users() -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + let source = Arc::new(MockKeyBasedEventSource::default()); + let handler = create_mock_handler(Ok(()), None); + let processor = processor(homeserver, handler.clone(), source.clone()); + + processor.run().await?; + + assert!(source.calls().await.is_empty()); + assert_eq!(handler.get_handle_count(), 0); + + Ok(()) +} + +/// Verifies invalid resolved user IDs are skipped while valid users still run. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_skips_invalid_resolved_user_id() -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + let valid_user_id = create_user_on_homeserver(&homeserver).await?; + let invalid_user_id = "not-a-pubky-user"; + create_invalid_user_on_homeserver(&homeserver, invalid_user_id).await?; + + let source = Arc::new( + MockKeyBasedEventSource::default() + .with_user_events(vec![( + valid_user_id.clone(), + vec![stream_event( + 1, + &valid_user_id, + "/pub/pubky.app/profile.json", + )?], + )]) + .await, + ); + let handler = create_mock_handler(Ok(()), None); + let processor = processor(homeserver, handler.clone(), source.clone()); + + processor.run().await?; + + assert_eq!(source.calls().await, vec![valid_user_id]); + assert_eq!(handler.get_handle_count(), 1); + + Ok(()) +} + +/// Verifies Redis cursor read failures abort before fetching user events. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_propagates_cursor_read_errors() -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + let user_id = create_user_on_homeserver(&homeserver).await?; + let cursor_key = user_hs_cursor_key(&user_id); + test_user_details(&user_id)? + .put_index_json(&cursor_key, Some("Sorted".into()), None) + .await?; + + let source = Arc::new(MockKeyBasedEventSource::default()); + let handler = create_mock_handler(Ok(()), None); + let processor = processor(homeserver, handler, source.clone()); + + let err = processor.run().await.unwrap_err(); + + assert_internal_index_operation_failed(err); + assert!(source.calls().await.is_empty()); + + Ok(()) +} + +/// Verifies stored per-user cursors and configured limits are passed to the source. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_passes_stored_cursor_and_limit_to_source() -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + let hs_id = homeserver.id.to_string(); + let user_id = create_user_on_homeserver(&homeserver).await?; + let cursor_key = user_hs_cursor_key(&user_id); + UserDetails::put_index_sorted_set(&cursor_key, &[(42.0, hs_id.as_str())], None, None).await?; + + let source = Arc::new(MockKeyBasedEventSource::default()); + let handler = create_mock_handler(Ok(()), None); + let processor = processor_with_limit(homeserver, handler, source.clone(), 17); + + processor.run().await?; + + assert_eq!(source.call_details().await, vec![(user_id, 42, 17)]); + + Ok(()) +} + +/// Verifies successful event processing persists the last stream cursor. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_persists_latest_cursor_after_success() -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + let hs_id = homeserver.id.to_string(); + let user_id = create_user_on_homeserver(&homeserver).await?; + let source = Arc::new( + MockKeyBasedEventSource::default() + .with_events(vec![vec![ + stream_event(1, &user_id, "/pub/pubky.app/profile.json")?, + stream_event(4, &user_id, "/pub/pubky.app/profile.json")?, + ]]) + .await, + ); + let handler = create_mock_handler(Ok(()), None); + let processor = processor(homeserver, handler.clone(), source); + + processor.run().await?; + + assert_eq!(handler.get_handle_count(), 2); + assert_eq!(user_cursor(&user_id, &hs_id).await?, Some(4)); + + Ok(()) +} + +/// Verifies cursor persistence stops at the last safe event before a mismatch. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_persists_last_safe_cursor_before_mismatch() -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + let hs_id = homeserver.id.to_string(); + let user_id = create_user_on_homeserver(&homeserver).await?; + let mismatched_user_id = Keypair::random().public_key().to_z32(); + let source = Arc::new( + MockKeyBasedEventSource::default() + .with_events(vec![vec![ + stream_event(5, &user_id, "/pub/pubky.app/profile.json")?, + stream_event(6, &mismatched_user_id, "/pub/pubky.app/profile.json")?, + ]]) + .await, + ); + let handler = create_mock_handler(Ok(()), None); + let processor = processor(homeserver, handler.clone(), source); + + processor.run().await?; + + assert_eq!(handler.get_handle_count(), 1); + assert_eq!(user_cursor(&user_id, &hs_id).await?, Some(5)); + + Ok(()) +} + +/// Verifies infrastructure fetch errors abort the homeserver run immediately. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_aborts_on_infrastructure_fetch_error() -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + create_user_on_homeserver(&homeserver).await?; + create_user_on_homeserver(&homeserver).await?; + let source = Arc::new( + MockKeyBasedEventSource::default() + .with_results(vec![Err(EventProcessorError::IndexOperationFailed( + true, + "redis unavailable".into(), + ))]) + .await, + ); + let handler = create_mock_handler(Ok(()), None); + let processor = processor(homeserver, handler.clone(), source.clone()); + + let err = processor.run().await.unwrap_err(); + + assert_internal_infrastructure_index_operation_failed(err); + assert_eq!(source.calls().await.len(), 1); + assert_eq!(handler.get_handle_count(), 0); + + Ok(()) +} + +/// Verifies non-infrastructure fetch errors skip only the affected user. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_continues_after_non_infrastructure_fetch_error() -> Result<(), DynError> +{ + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + let user_a_id = create_user_on_homeserver(&homeserver).await?; + let user_b_id = create_user_on_homeserver(&homeserver).await?; + let source = Arc::new( + MockKeyBasedEventSource::default() + .with_user_results(vec![ + ( + user_a_id.clone(), + Err(EventProcessorError::Generic("bad user stream".into())), + ), + ( + user_b_id.clone(), + Ok(vec![stream_event( + 9, + &user_b_id, + "/pub/pubky.app/profile.json", + )?]), + ), + ]) + .await, + ); + let handler = create_mock_handler(Ok(()), None); + let processor = processor(homeserver, handler.clone(), source.clone()); + + processor.run().await?; + + let calls = source.calls().await; + assert_eq!(calls.len(), 2); + assert!(calls.contains(&user_a_id)); + assert!(calls.contains(&user_b_id)); + assert_eq!(handler.get_handle_count(), 1); + + Ok(()) +} + +/// Verifies infrastructure handler failures abort without advancing the cursor. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_aborts_and_keeps_cursor_on_infrastructure_handler_error( +) -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + let hs_id = homeserver.id.to_string(); + let user_id = create_user_on_homeserver(&homeserver).await?; + let source = Arc::new( + MockKeyBasedEventSource::default() + .with_events(vec![vec![stream_event( + 9, + &user_id, + "/pub/pubky.app/profile.json", + )?]]) + .await, + ); + let handler = create_mock_handler( + Err(EventProcessorError::IndexOperationFailed( + true, + "redis unavailable".into(), + )), + None, + ); + let processor = processor(homeserver, handler.clone(), source); + + let err = processor.run().await.unwrap_err(); + + assert_internal_infrastructure_index_operation_failed(err); + assert_eq!(handler.get_handle_count(), 1); + assert_eq!(user_cursor(&user_id, &hs_id).await?, None); + + Ok(()) +} + +/// Verifies an already-signaled shutdown exits before fetching any user events. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_does_not_fetch_when_shutdown_is_already_set() -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + create_user_on_homeserver(&homeserver).await?; + let source = Arc::new(MockKeyBasedEventSource::default()); + let handler = create_mock_handler(Ok(()), None); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + shutdown_tx + .send(true) + .expect("shutdown receiver should exist"); + let processor = + processor_with_shutdown(homeserver, handler.clone(), source.clone(), shutdown_rx); + + processor.run().await?; + + assert!(source.calls().await.is_empty()); + assert_eq!(handler.get_handle_count(), 0); + + Ok(()) +} + +/// Verifies shutdown during one user stops that stream and prevents later users. +#[tokio_shared_rt::test(shared)] +async fn key_based_processor_stops_current_and_next_users_after_shutdown() -> Result<(), DynError> { + setup().await?; + + let (_hs_keypair, homeserver) = create_homeserver().await?; + let hs_id = homeserver.id.to_string(); + let user_a_id = create_user_on_homeserver(&homeserver).await?; + let user_b_id = create_user_on_homeserver(&homeserver).await?; + let source = Arc::new( + MockKeyBasedEventSource::default() + .with_user_events(vec![ + ( + user_a_id.clone(), + vec![ + stream_event(1, &user_a_id, "/pub/pubky.app/profile.json")?, + stream_event(2, &user_a_id, "/pub/pubky.app/profile.json")?, + ], + ), + ( + user_b_id.clone(), + vec![ + stream_event(1, &user_b_id, "/pub/pubky.app/profile.json")?, + stream_event(2, &user_b_id, "/pub/pubky.app/profile.json")?, + ], + ), + ]) + .await, + ); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let handler = Arc::new(ShutdownOnFirstHandle::new(shutdown_tx)); + let processor = + processor_with_shutdown(homeserver, handler.clone(), source.clone(), shutdown_rx); + + processor.run().await?; + + let calls = source.calls().await; + assert_eq!(calls.len(), 1); + assert_eq!(handler.handle_count(), 1); + assert_eq!(user_cursor(&calls[0], &hs_id).await?, Some(1)); + + Ok(()) +} + +async fn create_homeserver() -> Result<(Keypair, Homeserver), DynError> { + let keypair = Keypair::random(); + let homeserver_id = PubkyId::try_from(keypair.public_key().to_z32().as_str())?; + let homeserver = Homeserver::new(homeserver_id); + homeserver.put_to_graph().await?; + Ok((keypair, homeserver)) +} + +async fn create_user_on_homeserver(homeserver: &Homeserver) -> Result { + let user_id = PubkyId::try_from(Keypair::random().public_key().to_z32().as_str())?; + let user = UserDetails { + id: user_id.clone(), + name: "key-based-processor-test-user".into(), + bio: None, + status: None, + links: None, + image: None, + indexed_at: Utc::now().timestamp_millis(), + }; + + exec_single_row(queries::put::create_user(&user)?).await?; + exec_single_row(queries::put::set_user_homeserver(&user_id, &homeserver.id)).await?; + + Ok(user_id.to_string()) +} + +async fn create_invalid_user_on_homeserver( + homeserver: &Homeserver, + user_id: &str, +) -> Result<(), DynError> { + exec_single_row( + Query::new( + "create_invalid_key_based_user", + "MERGE (u:User {id: $id}) SET u.name = $name", + ) + .param("id", user_id.to_string()) + .param("name", "invalid-key-based-processor-test-user".to_string()), + ) + .await?; + exec_single_row(queries::put::set_user_homeserver(user_id, &homeserver.id)).await?; + + Ok(()) +} + +fn test_user_details(user_id: &str) -> Result { + Ok(UserDetails { + id: PubkyId::try_from(user_id)?, + name: "key-based-processor-test-user".into(), + bio: None, + status: None, + links: None, + image: None, + indexed_at: Utc::now().timestamp_millis(), + }) +} + +async fn user_cursor(user_id: &str, hs_id: &str) -> Result, DynError> { + Ok(UserDetails::check_sorted_set_member(None, &user_hs_cursor_key(user_id), &[hs_id]).await?) +} + +fn stream_event(cursor: u64, user_id: &str, path: &str) -> Result { + let user_pk: PublicKey = user_id.parse()?; + + Ok(StreamEvent { + event_type: EventType::Delete, + resource: PubkyResource::new(user_pk, path)?, + cursor: EventCursor::new(cursor), + }) +} + +fn processor( + homeserver: Homeserver, + handler: Arc, + source: Arc, +) -> Arc { + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + processor_with_options(homeserver, handler, source, 100, shutdown_rx) +} + +fn processor_with_limit( + homeserver: Homeserver, + handler: Arc, + source: Arc, + limit: u16, +) -> Arc { + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + processor_with_options(homeserver, handler, source, limit, shutdown_rx) +} + +fn processor_with_shutdown( + homeserver: Homeserver, + handler: Arc, + source: Arc, + shutdown_rx: watch::Receiver, +) -> Arc { + processor_with_options(homeserver, handler, source, 100, shutdown_rx) +} + +fn processor_with_options( + homeserver: Homeserver, + handler: Arc, + source: Arc, + limit: u16, + shutdown_rx: watch::Receiver, +) -> Arc { + Arc::new(KeyBasedEventProcessor { + homeserver, + limit, + files_path: PathBuf::from("/tmp/nexus-watcher-test"), + event_handler: handler, + event_source: source, + retry_scheduler: Arc::new(RetryScheduler::new( + new_in_memory_store(), + InitialBackoff { + missing_dep_ms: 60_000, + transient_ms: 10_000, + }, + )), + shutdown_rx, + }) +} + +fn assert_internal_index_operation_failed(err: RunError) { + match err { + RunError::Internal(EventProcessorError::IndexOperationFailed(_, _)) => {} + other => panic!("expected internal index operation failure, got {other:?}"), + } +} + +fn assert_internal_infrastructure_index_operation_failed(err: RunError) { + match err { + RunError::Internal(EventProcessorError::IndexOperationFailed(true, _)) => {} + other => panic!("expected internal infrastructure index operation failure, got {other:?}"), + } +} + +/// Test handler that signals shutdown after handling its first event. +/// +/// This lets shutdown-path tests verify that the processor persists the first +/// safe cursor, stops the current user stream, and does not fetch later users. +struct ShutdownOnFirstHandle { + shutdown_tx: watch::Sender, + handle_count: AtomicUsize, +} + +impl ShutdownOnFirstHandle { + fn new(shutdown_tx: watch::Sender) -> Self { + Self { + shutdown_tx, + handle_count: AtomicUsize::new(0), + } + } + + fn handle_count(&self) -> usize { + self.handle_count.load(Ordering::SeqCst) + } +} + +#[async_trait::async_trait] +impl EventHandler for ShutdownOnFirstHandle { + async fn handle(&self, _event: &Event) -> Result<(), EventProcessorError> { + if self.handle_count.fetch_add(1, Ordering::SeqCst) == 0 { + let _ = self.shutdown_tx.send(true); + } + + Ok(()) + } +} diff --git a/nexus-watcher/tests/service/mod.rs b/nexus-watcher/tests/service/mod.rs index 0b77f7bc8..e7b796f64 100644 --- a/nexus-watcher/tests/service/mod.rs +++ b/nexus-watcher/tests/service/mod.rs @@ -1,5 +1,8 @@ pub mod event_processing_multiple_homeservers; pub mod event_processor_prioritization; +pub mod hs_event_processor; +pub mod key_based_event_processor; pub mod mock_event_processor; +pub mod retry_processor; pub mod signal; pub mod utils; diff --git a/nexus-watcher/tests/service/retry_processor.rs b/nexus-watcher/tests/service/retry_processor.rs new file mode 100644 index 000000000..75c334a12 --- /dev/null +++ b/nexus-watcher/tests/service/retry_processor.rs @@ -0,0 +1,1176 @@ +use crate::service::utils::common::create_mock_handler; +use crate::service::utils::{new_in_memory_store, setup, TEST_USER_ID}; +use anyhow::Result; +use chrono::Utc; +use nexus_common::config::EventRetryConfig; +use nexus_common::db::kv::RedisOps; +use nexus_common::models::event::{EventProcessorError, EventType}; +use nexus_watcher::events::retry::{RedisRetryStore, RetryEvent, RetryProcessor, RetryStore}; +use nexus_watcher::events::EventHandler; +use nexus_watcher::service::TEventProcessor; +use pubky_app_specs::post_uri_builder; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::watch; + +/// Test helper to create an EventRetryConfig with custom values +fn create_test_config( + max_retries: u32, + max_dependency_retries: u32, + initial_backoff_secs: u64, + max_backoff_secs: u64, + initial_missing_dep_backoff_secs: u64, + max_missing_dep_backoff_secs: u64, +) -> EventRetryConfig { + EventRetryConfig { + max_retries, + max_dependency_retries, + initial_backoff_secs, + max_backoff_secs, + initial_missing_dep_backoff_secs, + max_missing_dep_backoff_secs, + } +} + +/// Test helper to create a test RetryEvent with a valid URI +fn create_test_retry_event( + post_id: &str, + event_type: EventType, + retry_count: u32, + next_retry_at: i64, +) -> RetryEvent { + let event_uri = post_uri_builder(TEST_USER_ID.to_string(), post_id.to_string()); + RetryEvent { + retry_count, + event_type, + event_uri, + next_retry_at, + } +} + +/// Test helper to create a resource key for a test event, matching the format the scheduler uses. +fn create_resource_key(post_id: &str) -> String { + post_uri_builder(TEST_USER_ID.to_string(), post_id.to_string()) +} + +/// Assemble a [`RetryProcessor`] for tests with the given store, config, and handler. +fn build_processor( + store: Arc, + config: EventRetryConfig, + event_handler: Arc, + shutdown_rx: watch::Receiver, +) -> Arc { + Arc::new(RetryProcessor { + files_path: PathBuf::from("/tmp/test"), + event_handler, + shutdown_rx, + config, + store, + }) +} + +// ============================================================================ +// Backoff - first retry uses initial value +// calculate_backoff(0, 60, 3600) returns 60 (2^0 * initial) +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_backoff_first_retry_uses_initial_value() -> Result<()> { + setup().await?; + + let post_id = "backoff1st"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create and store a retry event with retry_count = 0 + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 0, // First retry attempt + now - 1000, + ); + store.put(&resource_key, &retry_event).await?; + + // Create processor with initial_backoff_secs = 60 + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + create_mock_handler( + Err(EventProcessorError::Generic("retry error".to_string())), + Some(post_id), + ), + shutdown_rx, + ); + + // Process through the public API + let _ = processor.run_internal().await; + + // Verify event was re-queued with backoff + let updated_event = store + .get(&resource_key) + .await? + .expect("Event should be re-queued"); + + // First retry (retry_count = 0) should use initial backoff (60 seconds = 60000 ms) + let expected_next_retry = now + 60_000; + assert!( + updated_event.next_retry_at >= expected_next_retry - 1000, + "First retry should use initial backoff value (2^0 * 60 = 60s)" + ); + assert!( + updated_event.next_retry_at <= expected_next_retry + 1000, + "First retry should use initial backoff value (2^0 * 60 = 60s)" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Backoff - exponential growth +// calculate_backoff(3, 10, 3600) returns 80 (2^3 * initial) +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_backoff_exponential_growth() -> Result<()> { + setup().await?; + + let post_id = "backoffexp"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create and store a retry event with retry_count = 3 + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 3, // Third retry attempt + now - 1000, + ); + store.put(&resource_key, &retry_event).await?; + + // Create processor with initial_backoff_secs = 10 + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 10, 3600, 60, 3600), + create_mock_handler( + Err(EventProcessorError::Generic("retry error".to_string())), + Some(post_id), + ), + shutdown_rx, + ); + + // Process through the public API + let _ = processor.run_internal().await; + + // Verify event was re-queued with exponential backoff + let updated_event = store + .get(&resource_key) + .await? + .expect("Event should be re-queued"); + + // Retry 3 should have backoff of 2^3 * 10 = 80 seconds = 80000 ms + let expected_next_retry = now + 80_000; + assert!( + updated_event.next_retry_at >= expected_next_retry - 1000, + "Retry 3 should have backoff of 2^3 * 10 = 80s" + ); + assert!( + updated_event.next_retry_at <= expected_next_retry + 1000, + "Retry 3 should have backoff of 2^3 * 10 = 80s" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Infrastructure error at max_retries does NOT dead-letter +// This is the key regression test for the P2 Infrastructure bug. +// Even when retry_count >= max_retries, an infrastructure error must NOT be +// dead-lettered — it must be re-queued with retry_count unchanged so the +// event can be retried indefinitely until the infrastructure recovers. +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_infrastructure_error_at_max_retries_does_not_dead_letter() -> Result<()> { + setup().await?; + + let post_id = "inframax"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create a retry event already at max_retries (10) + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 10, // At max_retries — would be dead-lettered by application errors + now - 1000, + ); + store.put(&resource_key, &retry_event).await?; + + // Create processor with max_retries = 10 + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + create_mock_handler( + Err(EventProcessorError::GraphQueryFailed( + true, // is_infrastructure = true + "Database connection failed".to_string(), + )), + Some(post_id), + ), + shutdown_rx, + ); + + // Process through the public API — infrastructure error must NOT dead-letter + let result = processor.run_internal().await; + assert!( + result.is_err(), + "Infrastructure error should propagate, not dead-letter" + ); + + // Verify event was NOT removed — it should still be in the queue for retry + let updated_event = store.get(&resource_key).await?.expect( + "Event must NOT be dead-lettered; infrastructure errors don't count against max_retries", + ); + + assert_eq!( + updated_event.retry_count, 10, + "retry_count must remain 10 (unchanged) — infrastructure errors do not increment retry_count" + ); + + // next_retry_at should have been advanced with backoff + assert!( + updated_event.next_retry_at > now, + "next_retry_at should be in the future after infrastructure error backoff" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Backoff - capped at max +// Large retry count returns max, never exceeds ceiling +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_backoff_capped_at_max() -> Result<()> { + setup().await?; + + let post_id = "backoffcap"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create and store a retry event with retry_count = 6 + // 2^6 * 60 = 3840, which exceeds max_backoff_secs (3600), so it should be capped + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 6, // Large retry count + now - 1000, + ); + store.put(&resource_key, &retry_event).await?; + + // Create processor with initial_backoff_secs = 60, max_backoff_secs = 3600 + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + create_mock_handler( + Err(EventProcessorError::Generic("retry error".to_string())), + Some(post_id), + ), + shutdown_rx, + ); + + // Process through the public API + let _ = processor.run_internal().await; + + // Verify event was re-queued with capped backoff + let updated_event = store + .get(&resource_key) + .await? + .expect("Event should be re-queued"); + + // Backoff should be capped at max (3600 seconds = 3600000 ms) + let expected_next_retry = now + 3_600_000; + assert!( + updated_event.next_retry_at >= expected_next_retry - 1000, + "Backoff should be capped at max value (3600s)" + ); + assert!( + updated_event.next_retry_at <= expected_next_retry + 1000, + "Backoff should be capped at max value (3600s)" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Retry success removes from queue +// Handler returns Ok(()), event is removed from retry index +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_retry_success_removes_from_queue() -> Result<()> { + setup().await?; + + let post_id = "successrmv"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create and store a retry event + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 0, + now - 1000, // Ready for retry (in the past) + ); + store.put(&resource_key, &retry_event).await?; + + // Verify event exists in index + assert!( + store.get(&resource_key).await?.is_some(), + "Event should exist in index before processing" + ); + + // Create processor with handler that returns success + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + create_mock_handler(Ok(()), Some(post_id)), + shutdown_rx, + ); + + // Process through the public API + let _ = processor.run_internal().await; + + // Verify event was removed from index after processing + assert!( + store.get(&resource_key).await?.is_none(), + "Event should be removed from index after successful retry" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Retry 404 removes from queue +// Handler returns PubkyClientError with 404 message, event is removed (content gone, no point retrying) +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_retry_404_removes_from_queue() -> Result<()> { + setup().await?; + + let post_id = "r404remove"; + let resource_key = create_resource_key(post_id); + let event_uri = post_uri_builder(TEST_USER_ID.to_string(), post_id.to_string()); + let store = new_in_memory_store(); + + // Create and store a retry event + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event(post_id, EventType::Put, 0, now - 1000); + store.put(&resource_key, &retry_event).await?; + + // Verify event exists in index + assert!( + store.get(&resource_key).await?.is_some(), + "Event should exist in index before processing" + ); + + // Create processor with handler that returns 404 + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + create_mock_handler( + Err(EventProcessorError::PubkyClientError( + nexus_common::db::PubkyClientError::NotFound404 { message: event_uri }, + )), + Some(post_id), + ), + shutdown_rx, + ); + + // Process through the public API + let _ = processor.run_internal().await; + + // Verify event was removed from index (404 means content is gone) + assert!( + store.get(&resource_key).await?.is_none(), + "Event should be removed from index after 404 error" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Infrastructure error schedules retry without incrementing retry_count +// Handler returns infrastructure error, event is re-queued WITHOUT incrementing +// retry_count — infrastructure failures must not consume the application-level +// retry budget. next_retry_at is still advanced via exponential backoff. +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_transient_error_schedules_retry() -> Result<()> { + setup().await?; + + let post_id = "transientr"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create and store a retry event with retry_count = 0 + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 0, // First retry attempt + now - 1000, + ); + store.put(&resource_key, &retry_event).await?; + + // Create processor with handler that returns infrastructure error + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + create_mock_handler( + Err(EventProcessorError::GraphQueryFailed( + true, // is_infrastructure = true + "Database connection failed".to_string(), + )), + Some(post_id), + ), + shutdown_rx, + ); + + // Process through the public API - this will propagate the infrastructure error + let result = processor.run_internal().await; + assert!(result.is_err(), "Infrastructure error should propagate"); + + // Verify event was re-queued with retry_count UNCHANGED (infrastructure errors + // do not consume the application-level retry budget). + let updated_event = store + .get(&resource_key) + .await? + .expect("Event should be re-queued after transient error"); + + assert_eq!( + updated_event.retry_count, 0, + "Retry count should remain 0 for infrastructure errors" + ); + + // Verify next_retry_at is set with transient backoff (60 seconds = 60000 ms) + let expected_next_retry = now + 60_000; + assert!( + updated_event.next_retry_at >= expected_next_retry - 1000, + "Next retry should be scheduled with transient backoff (60s)" + ); + assert!( + updated_event.next_retry_at <= expected_next_retry + 1000, + "Next retry should be scheduled with transient backoff (60s)" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// MissingDependency schedules retry +// Handler returns MissingDependency, event is re-queued with dependency backoff params +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_missing_dependency_schedules_retry() -> Result<()> { + setup().await?; + + let post_id = "missingdep"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create and store a retry event with retry_count = 0 + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 0, // First retry attempt + now - 1000, + ); + store.put(&resource_key, &retry_event).await?; + + // Create processor with handler that returns MissingDependency + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 300, 18000), // 300s initial for deps + create_mock_handler( + Err(EventProcessorError::MissingDependency { + dependency: vec!["some_dependency".to_string()], + }), + Some(post_id), + ), + shutdown_rx, + ); + + // Process through the public API + let _ = processor.run_internal().await; + + // Verify event was re-queued with incremented retry_count + let updated_event = store + .get(&resource_key) + .await? + .expect("Event should be re-queued after missing dependency error"); + + assert_eq!( + updated_event.retry_count, 1, + "Retry count should be incremented to 1" + ); + + // Verify next_retry_at is set with dependency backoff (300 seconds = 300000 ms) + let expected_next_retry = now + 300_000; + assert!( + updated_event.next_retry_at >= expected_next_retry - 1000, + "Next retry should be scheduled with dependency backoff (300s)" + ); + assert!( + updated_event.next_retry_at <= expected_next_retry + 1000, + "Next retry should be scheduled with dependency backoff (300s)" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Dead-letter after max transient retries +// Event with retry_count >= max_retries for an APPLICATION error is removed +// without retrying. Infrastructure errors NO LONGER count against max_retries. +// Uses a Generic error (application-level transient) to test the dead-letter path. +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_dead_letter_after_max_transient_retries() -> Result<()> { + setup().await?; + + let post_id = "dltransmax"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create and store a retry event that has exceeded max_retries (10) + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 10, // At max_retries + now - 1000, + ); + store.put(&resource_key, &retry_event).await?; + + // Verify event exists in index + assert!( + store.get(&resource_key).await?.is_some(), + "Event should exist in index before processing" + ); + + // Create processor with max_retries = 10 + // Uses Generic error (application-level transient, NOT infrastructure) + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + create_mock_handler( + Err(EventProcessorError::Generic( + "transient application failure".to_string(), + )), + Some(post_id), + ), + shutdown_rx, + ); + + // Process through the public API — event should be dead-lettered + let _ = processor.run_internal().await; + + // Verify event was removed from index (dead-lettered) + assert!( + store.get(&resource_key).await?.is_none(), + "Event should be dead-lettered (removed) after max transient retries" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Dead-letter after max dependency retries +// retry_count >= max_dependency_retries is removed without retrying +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_dead_letter_after_max_dependency_retries() -> Result<()> { + setup().await?; + + let post_id = "dldepndmax"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create and store a retry event that has exceeded max_dependency_retries (50) + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 50, // At max_dependency_retries + now - 1000, + ); + store.put(&resource_key, &retry_event).await?; + + // Verify event exists in index + assert!( + store.get(&resource_key).await?.is_some(), + "Event should exist in index before processing" + ); + + // Create processor with max_dependency_retries = 50 + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + create_mock_handler( + Err(EventProcessorError::MissingDependency { + dependency: vec!["some_dependency".to_string()], + }), + Some(post_id), + ), + shutdown_rx, + ); + + // Process through the public API + let _ = processor.run_internal().await; + + // Verify event was removed from index (dead-lettered) + assert!( + store.get(&resource_key).await?.is_none(), + "Event should be dead-lettered (removed) after max dependency retries" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Stale sorted set entry cleaned up +// Redis-specific: a sorted-set entry without a matching JSON state should be +// detected and removed by RedisRetryStore::fetch_ready. This test bypasses +// InMemoryRetryStore because the inconsistency doesn't exist in that backend — +// it's Redis layout detail. We exercise RedisRetryStore directly. +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_stale_sorted_set_entry_cleaned_up() -> Result<()> { + setup().await?; + + let post_id = "staleclnup"; + let resource_key = create_resource_key(post_id); + + // Manually add a stale entry to the sorted set only (no JSON state). + let now = Utc::now().timestamp_millis(); + RetryEvent::put_index_sorted_set( + &["events"], + &[(now as f64, &resource_key)], + Some("RetryManager"), + None, + ) + .await?; + + // Sanity: the stale entry is visible in the raw sorted set. + let raw_before = RetryEvent::fetch_ready(now, None).await?; + assert!( + raw_before.iter().any(|(key, _)| key == &resource_key), + "Stale entry should be present in sorted set before cleanup" + ); + + // RedisRetryStore::fetch_ready should silently drop-and-clean stale entries: + // they're sorted-set members with no corresponding JSON state. + let store = RedisRetryStore::new(); + let ready = store.fetch_ready(now, None).await?; + assert!( + !ready.iter().any(|(key, _)| key == &resource_key), + "Stale entry {resource_key} should be filtered out by RedisRetryStore::fetch_ready" + ); + + // And it should actually be removed from the sorted set (not just filtered). + let raw_after = RetryEvent::fetch_ready(now, None).await?; + assert!( + !raw_after.iter().any(|(key, _)| key == &resource_key), + "Stale entry {resource_key} should be removed from sorted set after cleanup" + ); + + Ok(()) +} + +// ============================================================================ +// Shutdown interrupts batch +// Shutdown signal set mid-batch stops processing remaining events and returns Ok(()) +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_shutdown_interrupts_batch() -> Result<()> { + setup().await?; + + // Create multiple retry events + let num_events = 5; + let now = Utc::now().timestamp_millis(); + let store = new_in_memory_store(); + + for i in 0..num_events { + let post_id = format!("shutdown{}", i); + let event_uri = post_uri_builder(TEST_USER_ID.to_string(), post_id); + let resource_key = event_uri.clone(); + + let retry_event = RetryEvent { + retry_count: 0, + event_type: EventType::Put, + event_uri, + next_retry_at: now - 1000, + }; + store.put(&resource_key, &retry_event).await?; + } + + // Create processor; shutdown is set before run_internal so nothing is actually + // processed. + let handler = create_mock_handler(Ok(()), Some("shutdown")); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + handler.clone(), + shutdown_rx, + ); + + // Trigger shutdown before processing + shutdown_tx.send(true)?; + + // Run the processor - should return Ok(()) immediately due to shutdown + let result: Result<(), EventProcessorError> = processor.run_internal().await; + + assert!( + result.is_ok(), + "Processor should return Ok(()) when shutdown is triggered" + ); + + // Handler must not be called — shutdown short-circuits before any processing. + assert_eq!( + handler.get_handle_count(), + 0, + "Handler must not be called when shutdown is triggered before processing" + ); + + // Verify events are still in the queue (not processed due to shutdown) + for i in 0..num_events { + let resource_key = post_uri_builder(TEST_USER_ID.to_string(), format!("shutdown{}", i)); + assert!( + store.get(&resource_key).await?.is_some(), + "Event {} should still be in queue (not processed due to shutdown)", + i + ); + } + + Ok(()) +} + +// ============================================================================ +// Infrastructure error stops batch +// Infrastructure error from processing propagates up, halting the batch +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_infrastructure_error_stops_batch() -> Result<()> { + setup().await?; + + // Create multiple retry events + let num_events = 3; + let now = Utc::now().timestamp_millis(); + let store = new_in_memory_store(); + + for i in 0..num_events { + let post_id = format!("infrastop{}", i); + let resource_key = post_uri_builder(TEST_USER_ID.to_string(), post_id.clone()); + let event_uri = post_uri_builder(TEST_USER_ID.to_string(), post_id); + + let retry_event = RetryEvent { + retry_count: 0, + event_type: EventType::Put, + event_uri, + next_retry_at: now - 1000, + }; + store.put(&resource_key, &retry_event).await?; + } + + // Create processor with handler that returns infrastructure error for our events only + let handler = create_mock_handler( + Err(EventProcessorError::GraphQueryFailed( + true, // is_infrastructure = true + "Critical database failure".to_string(), + )), + Some("infrastop"), + ); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + handler.clone(), + shutdown_rx, + ); + + // Run the processor - should propagate infrastructure error + let result: Result<(), EventProcessorError> = processor.run_internal().await; + + // Verify the error propagated up + assert!( + result.is_err(), + "Processor should propagate infrastructure error" + ); + + // Handler called exactly once — infrastructure error halted the batch + // after the first event, so remaining events were never reached. + assert_eq!( + handler.get_handle_count(), + 1, + "Handler must be called exactly once — batch stopped on infrastructure error" + ); + + // Verify the error is an infrastructure error + let err = result.unwrap_err(); + assert!( + err.is_infrastructure(), + "Error should be an infrastructure error" + ); + + // InMemoryRetryStore sorts same-score events lexicographically by key, + // matching Redis sorted-set semantics. So event 0 is processed first. + // Infrastructure errors do NOT increment retry_count — they preserve the + // application-level retry budget. + let first_key = post_uri_builder(TEST_USER_ID.to_string(), "infrastop0".to_string()); + let first_event = store + .get(&first_key) + .await? + .expect("First event should still be in queue (re-queued after error)"); + assert_eq!( + first_event.retry_count, 0, + "First event should have retry_count unchanged (infrastructure errors do not increment retry_count)" + ); + + // Remaining events should be untouched (retry_count still 0) + for i in 1..num_events { + let resource_key = post_uri_builder(TEST_USER_ID.to_string(), format!("infrastop{}", i)); + let event = store + .get(&resource_key) + .await? + .expect("Event should still be in queue"); + assert_eq!( + event.retry_count, 0, + "Event {} should be untouched (retry_count = 0), batch halted before reaching it", + i + ); + } + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Empty batch returns Ok +// No events in queue - processor returns Ok(()) +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_empty_batch_returns_ok() -> Result<()> { + setup().await?; + + // Fresh in-memory store is empty by construction. + let store = new_in_memory_store(); + let handler = create_mock_handler(Ok(()), Some("empty")); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store, + create_test_config(10, 50, 60, 3600, 60, 3600), + handler.clone(), + shutdown_rx, + ); + + // No events in queue - should return Ok(()) + let result: Result<(), EventProcessorError> = processor.run_internal().await; + assert!(result.is_ok(), "Empty batch should return Ok(())"); + + // No events, handler must never be called. + assert_eq!( + handler.get_handle_count(), + 0, + "Handler must not be called when no events are in queue" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// DEL event retry success +// DEL events reconstruct correctly and are removed from queue on success +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_del_event_retry_success() -> Result<()> { + setup().await?; + + let post_id = "delretrys"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create a DEL retry event + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event(post_id, EventType::Del, 0, now - 1000); + store.put(&resource_key, &retry_event).await?; + + // Create processor with handler that returns success + let handler = create_mock_handler(Ok(()), Some(post_id)); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + handler.clone(), + shutdown_rx, + ); + + let _ = processor.run_internal().await; + + // Handler called once for the DEL event. + assert_eq!( + handler.get_handle_count(), + 1, + "Handler must be called exactly once for the DEL event" + ); + + // Verify DEL event was removed from queue after successful processing + assert!( + store.get(&resource_key).await?.is_none(), + "DEL event should be removed from queue after successful retry" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Non-retryable error removes event immediately +// Handler returns a non-retryable error (e.g. InvalidEventLine), event is +// dead-lettered without incrementing retry_count +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_non_retryable_error_removes_event() -> Result<()> { + setup().await?; + + let post_id = "nonretrybl"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event(post_id, EventType::Put, 0, now - 1000); + store.put(&resource_key, &retry_event).await?; + + // Create processor with handler that returns a non-retryable error + let handler = create_mock_handler( + Err(EventProcessorError::InvalidEventLine( + "malformed data".to_string(), + )), + Some(post_id), + ); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + handler.clone(), + shutdown_rx, + ); + + let result = processor.run_internal().await; + assert!(result.is_ok(), "Non-retryable error should not propagate"); + + // Handler called once for the event before dead-lettering. + assert_eq!( + handler.get_handle_count(), + 1, + "Handler must be called exactly once before non-retryable error removes event" + ); + + // Event should be removed (dead-lettered immediately, not re-queued) + assert!( + store.get(&resource_key).await?.is_none(), + "Non-retryable error should cause immediate removal from queue" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Batch continues after a single event fails +// A retryable application error on one event must not halt the batch — later +// events still need to be processed. +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_batch_continues_after_single_failure() -> Result<()> { + setup().await?; + + // Both events share the same next_retry_at so they are fetched in the same + // batch. The test is order-independent: regardless of which event is + // processed first, the failing one is re-queued and the succeeding one is + // removed — proving the batch continued past the failure. + let failing_post_id = "failbatch1"; + let succeeding_post_id = "okbatch2"; + let failing_key = create_resource_key(failing_post_id); + let succeeding_key = create_resource_key(succeeding_post_id); + + let store = new_in_memory_store(); + let now = Utc::now().timestamp_millis(); + store + .put( + &failing_key, + &create_test_retry_event(failing_post_id, EventType::Put, 0, now - 1000), + ) + .await?; + store + .put( + &succeeding_key, + &create_test_retry_event(succeeding_post_id, EventType::Put, 0, now - 1000), + ) + .await?; + + // MockEventHandler's `target_uri_substring` scopes the error to the failing + // post_id; the succeeding event's URI doesn't match and so falls through to + // Ok(()). + let handler = create_mock_handler( + Err(EventProcessorError::Generic( + "first handler fails".to_string(), + )), + Some(failing_post_id), + ); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + handler.clone(), + shutdown_rx, + ); + + let result = processor.run_internal().await; + assert!( + result.is_ok(), + "Retryable application error must not stop the batch" + ); + + // Handler called twice — once for each event — proving the batch + // continued past the first failure. + assert_eq!( + handler.get_handle_count(), + 2, + "Handler must be called for both events — batch continued past failure" + ); + + // First event failed with a retryable Generic error — re-queued with + // retry_count incremented. + let requeued = store + .get(&failing_key) + .await? + .expect("Failing event should remain in queue for retry"); + assert_eq!( + requeued.retry_count, 1, + "Failing event should have retry_count incremented after retryable failure" + ); + + // Second event was reached despite the first failing, and its handler + // returned Ok(()), so the entry must have been removed. + assert!( + store.get(&succeeding_key).await?.is_none(), + "Processor must continue past a failed event and process the next one" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} + +// ============================================================================ +// Future next_retry_at events are not picked up +// Events with next_retry_at in the future should not be fetched or processed +// ============================================================================ + +#[tokio_shared_rt::test(shared)] +async fn test_future_events_not_picked_up() -> Result<()> { + setup().await?; + + let post_id = "futureevnt"; + let resource_key = create_resource_key(post_id); + let store = new_in_memory_store(); + + // Create a retry event scheduled far in the future + let now = Utc::now().timestamp_millis(); + let retry_event = create_test_retry_event( + post_id, + EventType::Put, + 0, + now + 600_000, // 10 minutes in the future + ); + store.put(&resource_key, &retry_event).await?; + + // Create processor with handler that would fail if called + let handler = create_mock_handler( + Err(EventProcessorError::Generic( + "should not be called".to_string(), + )), + Some(post_id), + ); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let processor = build_processor( + store.clone(), + create_test_config(10, 50, 60, 3600, 60, 3600), + handler.clone(), + shutdown_rx, + ); + + let result = processor.run_internal().await; + assert!(result.is_ok(), "Should return Ok when no ready events"); + + // Handler must never be called — no ready events in the batch. + assert_eq!( + handler.get_handle_count(), + 0, + "Handler must not be called when no events are ready" + ); + + // Event should still be in the queue, untouched + let event = store + .get(&resource_key) + .await? + .expect("Future event should remain in queue"); + assert_eq!( + event.retry_count, 0, + "Future event should not have been processed (retry_count unchanged)" + ); + + let _ = shutdown_tx.send(true); + Ok(()) +} diff --git a/nexus-watcher/tests/service/signal.rs b/nexus-watcher/tests/service/signal.rs index 5ad17217d..8cd8182d6 100644 --- a/nexus-watcher/tests/service/signal.rs +++ b/nexus-watcher/tests/service/signal.rs @@ -3,7 +3,6 @@ use crate::service::utils::{ MockEventProcessorRunner, }; use anyhow::Result; -use nexus_watcher::service::backoff::HomeserverBackoff; use nexus_watcher::service::TEventProcessorRunner; use std::time::Duration; use tokio::time::sleep; @@ -15,6 +14,9 @@ async fn test_shutdown_signal() -> Result<()> { let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); // Create 3 random homeservers with timeout limit + // Index 0: 0s sleep (default, excluded from run) + // Index 1: 2s sleep + // Index 2: 4s sleep for index in 0..3 { let processor_status = MockEventProcessorResult::Success; create_random_homeservers_and_persist( @@ -23,6 +25,7 @@ async fn test_shutdown_signal() -> Result<()> { processor_status, None, shutdown_rx.clone(), + Some(1), ) .await; } @@ -38,16 +41,14 @@ async fn test_shutdown_signal() -> Result<()> { } }); - let stats = runner - .run_all(&mut HomeserverBackoff::default()) - .await - .unwrap() - .0; + let stats = runner.run().await.unwrap().0; - // We created 3 HSs, each with different execution durations (0s, 2s, 4s) - // We triggered the shutdown signal 1s after start - assert_eq!(stats.count_ok(), 2); // 2 processors run without errors (of the 3, the 3rd one didn't even start) - assert_eq!(stats.count_error(), 0); // no processors fail, because no erratic or unexpected behavior was triggered + // run excludes the default HS (0s sleep). + // Of the remaining 2 (2s, 4s sleep), the shutdown signal fires after 1s. + // The 2s HS starts running, detects shutdown and exits early with Ok. + // The 4s HS doesn't start because shutdown is detected before it begins. + assert_eq!(stats.count_ok(), 1); // 1 processor exited gracefully + assert_eq!(stats.count_error(), 0); assert_eq!(stats.count_panic(), 0); assert_eq!(stats.count_timeout(), 0); diff --git a/nexus-watcher/tests/service/utils/common.rs b/nexus-watcher/tests/service/utils/common.rs new file mode 100644 index 000000000..a9d3e910a --- /dev/null +++ b/nexus-watcher/tests/service/utils/common.rs @@ -0,0 +1,27 @@ +use crate::utils::MockEventHandler; +use nexus_common::models::event::EventProcessorError; +use nexus_watcher::events::retry::{InMemoryRetryStore, RetryStore}; +use std::sync::{Arc, Mutex}; + +pub const TEST_USER_ID: &str = "uo7jgkykft4885n8cruizwy6khw71mnu5pq3ay9i8pw1ymcn85ko"; + +pub fn new_in_memory_store() -> Arc { + Arc::new(InMemoryRetryStore::new()) +} + +/// Create a mock event handler with an invocation counter. +/// +/// The returned handler tracks how many times `handle()` was called via +/// its `handle_count` field. Wrap it in `Arc` before passing to processors. +pub fn create_mock_handler( + result: Result<(), EventProcessorError>, + target_substring: Option<&str>, +) -> Arc { + MockEventHandler { + result, + target_uri_substring: target_substring.map(str::to_string), + handle_count: Arc::new(Mutex::new(0)), + handled_uris: Arc::new(Mutex::new(Vec::new())), + } + .into() +} diff --git a/nexus-watcher/tests/service/utils/key_based_event_source.rs b/nexus-watcher/tests/service/utils/key_based_event_source.rs new file mode 100644 index 000000000..cb5d62642 --- /dev/null +++ b/nexus-watcher/tests/service/utils/key_based_event_source.rs @@ -0,0 +1,89 @@ +use std::collections::{HashMap, VecDeque}; + +use tokio::sync::Mutex; + +use nexus_common::models::event::EventProcessorError; +use nexus_watcher::service::indexer::KeyBasedEventSource; +use pubky::{Event as StreamEvent, EventCursor, PublicKey}; + +type FetchEventsResult = Result, EventProcessorError>; + +#[derive(Default)] +pub struct MockKeyBasedEventSource { + /// Event batches returned in fetch order. + /// Useful when user ordering is not important and tests only care about processor flow. + events: Mutex>, + + /// Event batches returned by requested user ID. + /// Useful when graph user ordering is intentionally not part of the assertion. + user_events: Mutex>, + + /// User IDs, cursors, and limits requested from the mock, in fetch order. + /// Useful for asserting the processor continued to, or stopped before, specific users. + calls: Mutex>, +} + +impl MockKeyBasedEventSource { + pub async fn with_events(self, events: Vec>) -> Self { + *self.events.lock().await = events.into_iter().map(Ok).collect(); + self + } + + pub async fn with_results(self, results: Vec) -> Self { + *self.events.lock().await = results.into(); + self + } + + pub async fn with_user_events(self, events: Vec<(String, Vec)>) -> Self { + *self.user_events.lock().await = events + .into_iter() + .map(|(user_id, events)| (user_id, Ok(events))) + .collect(); + self + } + + pub async fn with_user_results(self, results: Vec<(String, FetchEventsResult)>) -> Self { + *self.user_events.lock().await = results.into_iter().collect(); + self + } + + pub async fn calls(&self) -> Vec { + self.calls + .lock() + .await + .iter() + .map(|(user_id, _, _)| user_id.clone()) + .collect() + } + + pub async fn call_details(&self) -> Vec<(String, u64, u16)> { + self.calls.lock().await.clone() + } +} + +#[async_trait::async_trait] +impl KeyBasedEventSource for MockKeyBasedEventSource { + async fn fetch_events( + &self, + _hs_pk: &PublicKey, + user_pk: &PublicKey, + cursor: EventCursor, + limit: u16, + ) -> Result, EventProcessorError> { + let user_id = user_pk.z32(); + self.calls + .lock() + .await + .push((user_id.clone(), cursor.id(), limit)); + + if let Some(events) = self.user_events.lock().await.remove(&user_id) { + return events; + } + + self.events + .lock() + .await + .pop_front() + .unwrap_or_else(|| Ok(Vec::new())) + } +} diff --git a/nexus-watcher/tests/service/utils/mod.rs b/nexus-watcher/tests/service/utils/mod.rs index d7d140e28..93e3a84ac 100644 --- a/nexus-watcher/tests/service/utils/mod.rs +++ b/nexus-watcher/tests/service/utils/mod.rs @@ -1,8 +1,12 @@ +pub mod common; +mod key_based_event_source; mod processor; mod processor_runner; mod result; mod setup; +pub use common::{create_mock_handler, new_in_memory_store, TEST_USER_ID}; +pub use key_based_event_source::MockKeyBasedEventSource; pub use processor::{ create_mock_event_processors, create_random_homeservers_and_persist, MockEventProcessor, }; diff --git a/nexus-watcher/tests/service/utils/processor.rs b/nexus-watcher/tests/service/utils/processor.rs index 50a4f72b1..ba1664757 100644 --- a/nexus-watcher/tests/service/utils/processor.rs +++ b/nexus-watcher/tests/service/utils/processor.rs @@ -1,8 +1,16 @@ +use std::path::PathBuf; use std::sync::Arc; +use crate::service::utils::common::create_mock_handler; use crate::service::utils::{MockEventProcessorResult, HS_IDS}; +use chrono::Utc; +use nexus_common::db::exec_single_row; +use nexus_common::db::queries; use nexus_common::models::event::EventProcessorError; use nexus_common::models::homeserver::Homeserver; +use nexus_common::models::user::UserDetails; +use nexus_watcher::events::retry::RetryScheduler; +use nexus_watcher::events::EventHandler; use nexus_watcher::service::TEventProcessor; use pubky::Keypair; use pubky_app_specs::PubkyId; @@ -17,18 +25,32 @@ pub struct MockEventProcessor { sleep_duration: Option, custom_timeout: Option, shutdown_rx: Receiver, + files_path: PathBuf, + event_handler: Arc, } #[async_trait::async_trait] impl TEventProcessor for MockEventProcessor { - fn get_homeserver_id(&self) -> PubkyId { - self.homeserver_id.clone() + fn files_path(&self) -> &PathBuf { + &self.files_path + } + + fn event_handler(&self) -> &Arc { + &self.event_handler } fn custom_timeout(&self) -> Option { self.custom_timeout } + fn instance_name(&self) -> String { + format!("MockEventProcessor for HS ID: {}", self.homeserver_id) + } + + fn retry_scheduler(&self) -> Option<&Arc> { + None + } + async fn run_internal(self: Arc) -> Result<(), EventProcessorError> { // Simulate a long-running task if needed, but be responsive to shutdown // This simulates the processing of event lines, which can take a while but can be interrupted by the shutdown signal @@ -50,13 +72,17 @@ impl TEventProcessor for MockEventProcessor { } } -/// Create a random homeserver and add it to the event processor list +/// Create a random homeserver and add it to the event processor list. +/// +/// If `create_active_users` is `Some(n)`, `n` test users will be created in the +/// graph and linked to this homeserver via `HOSTED_BY`. pub async fn create_random_homeservers_and_persist( event_processor_list: &mut Vec, sleep_duration: Option, processor_status: MockEventProcessorResult, custom_timeout: Option, shutdown_rx: Receiver, + create_active_users: Option, ) { let homeserver_keypair = Keypair::random(); let homeserver_public_key = homeserver_keypair.public_key().to_z32(); @@ -66,12 +92,35 @@ pub async fn create_random_homeservers_and_persist( .await .unwrap(); + // Create test users linked to this homeserver via HOSTED_BY + if let Some(count) = create_active_users { + for _ in 0..count { + let user_keypair = Keypair::random(); + let user_id = PubkyId::try_from(user_keypair.public_key().to_z32().as_str()).unwrap(); + let user = UserDetails { + id: user_id.clone(), + name: "test-user".to_string(), + bio: None, + status: None, + links: None, + image: None, + indexed_at: Utc::now().timestamp_millis(), + }; + let create_query = queries::put::create_user(&user).unwrap(); + exec_single_row(create_query).await.unwrap(); + let link_query = queries::put::set_user_homeserver(&user_id, &homeserver_id); + exec_single_row(link_query).await.unwrap(); + } + } + let event_processor = MockEventProcessor { homeserver_id, sleep_duration, processor_status, custom_timeout, shutdown_rx, + files_path: PathBuf::from("/tmp/mock"), + event_handler: create_mock_handler(Ok(()), None), }; event_processor_list.push(event_processor); } @@ -82,6 +131,7 @@ pub fn create_mock_event_processors( shutdown_rx: Receiver, ) -> Vec { use MockEventProcessorResult::*; + let event_handler = create_mock_handler(Ok(()), None); [ (HS_IDS[0], None, Success), (HS_IDS[1], None, Error("Event processor error!".into())), @@ -97,6 +147,8 @@ pub fn create_mock_event_processors( processor_status: status, custom_timeout, shutdown_rx: shutdown_rx.clone(), + files_path: PathBuf::from("/tmp/mock"), + event_handler: event_handler.clone(), }, ) .collect() diff --git a/nexus-watcher/tests/service/utils/processor_runner.rs b/nexus-watcher/tests/service/utils/processor_runner.rs index 0fa8f0526..eb8667f57 100644 --- a/nexus-watcher/tests/service/utils/processor_runner.rs +++ b/nexus-watcher/tests/service/utils/processor_runner.rs @@ -10,7 +10,7 @@ use tokio::sync::watch::Receiver; pub struct MockEventProcessorRunner { /// The event processors to be used by the runner pub event_processors: Vec>, - pub monitored_homeservers_limit: usize, + pub monitored_hs_limit: usize, pub shutdown_rx: Receiver, } @@ -18,7 +18,7 @@ impl MockEventProcessorRunner { /// Creates a new instance from the provided event processors pub fn new( event_processors: Vec, - monitored_homeservers_limit: usize, + monitored_hs_limit: usize, shutdown_rx: Receiver, ) -> Self { let arcs: Vec> = @@ -26,39 +26,18 @@ impl MockEventProcessorRunner { Self { event_processors: arcs, - monitored_homeservers_limit, + monitored_hs_limit, shutdown_rx, } } -} - -#[async_trait::async_trait] -impl TEventProcessorRunner for MockEventProcessorRunner { - fn shutdown_rx(&self) -> Receiver { - self.shutdown_rx.clone() - } - - fn default_homeserver(&self) -> &str { - // Use first mock homeserver ID if available, otherwise fallback to mock constant - self.event_processors - .first() - .map(|s| s.homeserver_id.as_str()) - .unwrap_or("8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo") - } - - fn monitored_homeservers_limit(&self) -> usize { - self.monitored_homeservers_limit - } - - async fn homeservers_by_priority(&self) -> Result, DynError> { - let persistedhs_ids = Homeserver::get_all_from_graph().await?; + pub async fn hs_by_priority(&self) -> Result, DynError> { + let persisted_hs_ids = Homeserver::get_all_active_from_graph().await?; let mut hs_ids = vec![]; - // Skip the homeserver IDs that are not part of the runner's event processors for mock_event_processor in self.event_processors.iter() { let hs_id = mock_event_processor.homeserver_id.to_string(); - if persistedhs_ids.contains(&hs_id) { + if persisted_hs_ids.contains(&hs_id) && hs_id != self.default_homeserver() { hs_ids.push(hs_id); } } @@ -66,17 +45,38 @@ impl TEventProcessorRunner for MockEventProcessorRunner { Ok(hs_ids) } + pub fn default_homeserver(&self) -> &str { + // Use first mock homeserver ID if available, otherwise fallback to mock constant + self.event_processors + .first() + .map(|s| s.homeserver_id.as_str()) + .unwrap_or("8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo") + } +} + +#[async_trait::async_trait] +impl TEventProcessorRunner for MockEventProcessorRunner { + fn shutdown_rx(&self) -> Receiver { + self.shutdown_rx.clone() + } + /// Returns the event processor for the specified homeserver. /// /// The mock event processor was pre-built and given to the mock runner on initialization, so this returns a reference to it. - async fn build(&self, homeserver_id: String) -> Result, DynError> { + async fn build(&self, hs_id: String) -> Result, DynError> { let mock_event_processor = self .event_processors .iter() - .find(|p| p.homeserver_id.to_string() == homeserver_id) + .find(|p| p.homeserver_id.to_string() == hs_id) .cloned() - .ok_or(format!("No MockEventProcessor for HS ID: {homeserver_id}"))?; + .ok_or(format!("No MockEventProcessor for HS ID: {hs_id}"))?; Ok(mock_event_processor) } + + async fn pre_run(&self) -> Result, DynError> { + let hs_ids = self.hs_by_priority().await?; + let max_index = std::cmp::min(self.monitored_hs_limit, hs_ids.len()); + Ok(hs_ids[..max_index].to_vec()) + } } diff --git a/nexus-watcher/tests/event_processor/homeserver/ingest_homeservers_from_follow_events.rs b/nexus-watcher/tests/user_ingestion/ingest_user_from_follow_events.rs similarity index 52% rename from nexus-watcher/tests/event_processor/homeserver/ingest_homeservers_from_follow_events.rs rename to nexus-watcher/tests/user_ingestion/ingest_user_from_follow_events.rs index c40805c3b..33958b00e 100644 --- a/nexus-watcher/tests/event_processor/homeserver/ingest_homeservers_from_follow_events.rs +++ b/nexus-watcher/tests/user_ingestion/ingest_user_from_follow_events.rs @@ -1,46 +1,34 @@ -use crate::event_processor::{ - homeserver::utils::create_external_test_homeserver, utils::watcher::WatcherTest, -}; +use super::utils::{assert_user_ingested, create_external_test_homeserver}; +use crate::event_processor::utils::watcher::WatcherTest; use anyhow::Result; -use nexus_common::models::homeserver::Homeserver; use pubky::Keypair; -use pubky_app_specs::{PubkyAppUser, PubkyId}; +use pubky_app_specs::PubkyAppUser; #[tokio_shared_rt::test(shared)] async fn test_follow_on_unknown_homeserver() -> Result<()> { let mut test = WatcherTest::setup().await?; - // Create a separate homeserver for the followee let followee_hs_pk = create_external_test_homeserver(&mut test).await?; - let followee_hs_id = PubkyId::try_from(&followee_hs_pk.to_z32()).unwrap(); - // Create followee let followee_kp = Keypair::random(); let followee_id = followee_kp.public_key().to_z32(); - // Register the followee PK in the new homeserver - // We only need the record mapping, not necessarily the profile.json being uploaded test.register_user_in_hs(&followee_kp, &followee_hs_pk) .await?; - // Create follower user let follower_kp = Keypair::random(); let follower_user = PubkyAppUser { bio: Some("test_follow_on_unknown_homeserver".to_string()), image: None, links: None, - name: "Watcher:Homeserver:Follow".to_string(), + name: "Watcher:UserIngestion:Follow".to_string(), status: None, }; let _follower_id = test.create_user(&follower_kp, &follower_user).await?; - // Follow the followee test.create_follow(&follower_kp, &followee_id).await?; - assert!(Homeserver::get_by_id(followee_hs_id) - .await - .unwrap() - .is_some()); + assert_user_ingested(&followee_id, &followee_hs_pk).await; Ok(()) } diff --git a/nexus-watcher/tests/event_processor/homeserver/ingest_homeservers_from_post_events.rs b/nexus-watcher/tests/user_ingestion/ingest_user_from_post_events.rs similarity index 62% rename from nexus-watcher/tests/event_processor/homeserver/ingest_homeservers_from_post_events.rs rename to nexus-watcher/tests/user_ingestion/ingest_user_from_post_events.rs index cde4f7d81..0311eb731 100644 --- a/nexus-watcher/tests/event_processor/homeserver/ingest_homeservers_from_post_events.rs +++ b/nexus-watcher/tests/user_ingestion/ingest_user_from_post_events.rs @@ -1,35 +1,27 @@ -use super::utils::create_external_test_homeserver; +use super::utils::{assert_user_ingested, create_external_test_homeserver}; use crate::event_processor::utils::watcher::WatcherTest; use anyhow::Result; -use nexus_common::models::homeserver::Homeserver; +use nexus_common::models::user::UserDetails; use pubky::Keypair; use pubky_app_specs::{ post_uri_builder, traits::TimestampId, PubkyAppPost, PubkyAppPostEmbed, PubkyAppPostKind, - PubkyAppUser, PubkyId, + PubkyAppUser, }; #[tokio_shared_rt::test(shared)] async fn test_reply_to_post_on_unknown_homeserver() -> Result<()> { let mut test = WatcherTest::setup().await?; - // Create a separate homeserver let parent_author_hs_pk = create_external_test_homeserver(&mut test).await?; - // Create parent post author let parent_author_kp = Keypair::random(); let parent_author_id = parent_author_kp.public_key().to_z32(); - let parent_author_hs_id = PubkyId::try_from(&parent_author_hs_pk.to_z32()).unwrap(); - - // Register the parent author PK in the new homeserver - // We only need the record mapping, not necessarily the profile.json being uploaded test.register_user_in_hs(&parent_author_kp, &parent_author_hs_pk) .await?; - // Create parent Post - // We only need its ID, not necessarily to upload it on the new HS let parent_post = PubkyAppPost { - content: "Watcher:ReplyHomeserverIngest:User:Post".to_string(), + content: "Watcher:ReplyUserIngest:User:Post".to_string(), kind: PubkyAppPostKind::Short, parent: None, embed: None, @@ -39,12 +31,11 @@ async fn test_reply_to_post_on_unknown_homeserver() -> Result<()> { let parent_post_absolute_uri = post_uri_builder(parent_author_id.clone(), parent_post_id.clone()); - // Create reply, written by a separate reply author, on the main test homeserver let reply_author = PubkyAppUser { bio: Some("test_reply_to_post_on_unknown_homeserver_reply".to_string()), image: None, links: None, - name: "Watcher:ReplyHomeserverIngest:Reply:User".to_string(), + name: "Watcher:ReplyUserIngest:Reply:User".to_string(), status: None, }; let reply_author_kp = Keypair::random(); @@ -59,11 +50,8 @@ async fn test_reply_to_post_on_unknown_homeserver() -> Result<()> { }; let (_reply_id, reply_path) = test.create_post(&reply_author_kp, &reply).await?; - // Check if new HS was ingested - let root_author_hs = Homeserver::get_by_id(parent_author_hs_id).await.unwrap(); - assert!(root_author_hs.is_some()); + assert_user_ingested(&parent_author_id, &parent_author_hs_pk).await; - // Cleanup test.cleanup_user(&reply_author_kp).await?; test.cleanup_post(&reply_author_kp, &reply_path).await?; @@ -74,23 +62,16 @@ async fn test_reply_to_post_on_unknown_homeserver() -> Result<()> { async fn test_repost_of_post_on_unknown_homeserver() -> Result<()> { let mut test = WatcherTest::setup().await?; - // Create a separate homeserver let original_author_hs_pk = create_external_test_homeserver(&mut test).await?; - let original_author_hs_id = PubkyId::try_from(&original_author_hs_pk.to_z32()).unwrap(); - // Create original post author let original_author_kp = Keypair::random(); let original_author_id = original_author_kp.public_key().to_z32(); - // Register the original author PK in the new homeserver - // We only need the record mapping, not necessarily the profile.json being uploaded test.register_user_in_hs(&original_author_kp, &original_author_hs_pk) .await?; - // Create original Post - // We only need its ID, not necessarily to upload it on the new HS let original_post = PubkyAppPost { - content: "Watcher:RepostHomeserverIngest:Original:Post".to_string(), + content: "Watcher:RepostUserIngest:Original:Post".to_string(), kind: PubkyAppPostKind::Short, parent: None, embed: None, @@ -99,12 +80,11 @@ async fn test_repost_of_post_on_unknown_homeserver() -> Result<()> { let original_post_id = original_post.create_id(); let original_post_uri = post_uri_builder(original_author_id.clone(), original_post_id.clone()); - // Create repost, written by a separate repost author, on the main test homeserver let repost_author = PubkyAppUser { bio: Some("test_repost_of_post_on_unknown_homeserver_repost".to_string()), image: None, links: None, - name: "Watcher:RepostHomeserverIngest:Repost:User".to_string(), + name: "Watcher:RepostUserIngest:Repost:User".to_string(), status: None, }; let repost_author_kp = Keypair::random(); @@ -123,11 +103,8 @@ async fn test_repost_of_post_on_unknown_homeserver() -> Result<()> { let (_repost_id, repost_path) = test.create_post(&repost_author_kp, &repost).await?; - // Check if new HS was ingested - let original_author_hs = Homeserver::get_by_id(original_author_hs_id).await.unwrap(); - assert!(original_author_hs.is_some()); + assert_user_ingested(&original_author_id, &original_author_hs_pk).await; - // Cleanup test.cleanup_user(&repost_author_kp).await?; test.cleanup_post(&repost_author_kp, &repost_path).await?; @@ -138,15 +115,10 @@ async fn test_repost_of_post_on_unknown_homeserver() -> Result<()> { async fn test_post_and_mention_users_on_unknown_homeserver() -> Result<()> { let mut test = WatcherTest::setup().await?; - // Create three separate homeservers for three mentioned users, each on one HS let user_1_hs_pk = create_external_test_homeserver(&mut test).await?; - let user_1_hs_id = PubkyId::try_from(&user_1_hs_pk.to_z32()).unwrap(); let user_2_hs_pk = create_external_test_homeserver(&mut test).await?; - let user_2_hs_id = PubkyId::try_from(&user_2_hs_pk.to_z32()).unwrap(); let user_3_hs_pk = create_external_test_homeserver(&mut test).await?; - let user_3_hs_id = PubkyId::try_from(&user_3_hs_pk.to_z32()).unwrap(); - // Create three users, which will be later mentioned in the test post let user_1_kp = Keypair::random(); let user_1_id = user_1_kp.public_key().to_z32(); let user_2_kp = Keypair::random(); @@ -154,25 +126,21 @@ async fn test_post_and_mention_users_on_unknown_homeserver() -> Result<()> { let user_3_kp = Keypair::random(); let user_3_id = user_3_kp.public_key().to_z32(); - // Register each new user in their respective homeserver - // We only need the record mapping, not necessarily the profile.json being uploaded test.register_user_in_hs(&user_1_kp, &user_1_hs_pk).await?; test.register_user_in_hs(&user_2_kp, &user_2_hs_pk).await?; test.register_user_in_hs(&user_3_kp, &user_3_hs_pk).await?; - // Create the test post on the main test homeserver, created by a known user (author) let post_author = PubkyAppUser { bio: Some("test_post_and_mention_users_on_unknown_homeserver".to_string()), image: None, links: None, - name: "Watcher:MentionHomeserverIngest:User".to_string(), + name: "Watcher:MentionUserIngest:User".to_string(), status: None, }; let post_author_kp = Keypair::random(); let _post_author_id = test.create_user(&post_author_kp, &post_author).await?; let post = PubkyAppPost { - // The post content references the PKs of the external users content: format!("Hey pubky{user_1_id}, pubky{user_2_id} and pubky{user_3_id}!"), kind: PubkyAppPostKind::Short, parent: None, @@ -181,13 +149,12 @@ async fn test_post_and_mention_users_on_unknown_homeserver() -> Result<()> { }; let (_post_id, post_path) = test.create_post(&post_author_kp, &post).await?; - // Check if the new homeserver of the first unknown mentioned user was ingested ... - assert!(Homeserver::get_by_id(user_1_hs_id).await.unwrap().is_some()); - // ... and the the HS of the other mentioned users are not ingested - assert!(Homeserver::get_by_id(user_2_hs_id).await.unwrap().is_none()); - assert!(Homeserver::get_by_id(user_3_hs_id).await.unwrap().is_none()); + // The first unknown mentioned user should have been ingested ... + assert_user_ingested(&user_1_id, &user_1_hs_pk).await; + // ... but the others are not ingested (only first mention triggers ingestion) + assert!(UserDetails::get_by_id(&user_2_id).await.unwrap().is_none()); + assert!(UserDetails::get_by_id(&user_3_id).await.unwrap().is_none()); - // Cleanup test.cleanup_user(&post_author_kp).await?; test.cleanup_post(&post_author_kp, &post_path).await?; diff --git a/nexus-watcher/tests/event_processor/homeserver/ingest_homeservers_from_tag_events.rs b/nexus-watcher/tests/user_ingestion/ingest_user_from_tag_events.rs similarity index 57% rename from nexus-watcher/tests/event_processor/homeserver/ingest_homeservers_from_tag_events.rs rename to nexus-watcher/tests/user_ingestion/ingest_user_from_tag_events.rs index c69258b3c..9aa591122 100644 --- a/nexus-watcher/tests/event_processor/homeserver/ingest_homeservers_from_tag_events.rs +++ b/nexus-watcher/tests/user_ingestion/ingest_user_from_tag_events.rs @@ -1,75 +1,56 @@ -use crate::event_processor::{ - homeserver::utils::create_external_test_homeserver, - utils::watcher::{HomeserverHashIdPath, WatcherTest}, -}; +use super::utils::{assert_user_ingested, create_external_test_homeserver}; +use crate::event_processor::utils::watcher::{HomeserverHashIdPath, WatcherTest}; use anyhow::Result; use chrono::Utc; -use nexus_common::models::homeserver::Homeserver; use pubky::Keypair; use pubky_app_specs::{ post_uri_builder, traits::TimestampId, user_uri_builder, PubkyAppPost, PubkyAppPostKind, - PubkyAppTag, PubkyAppUser, PubkyId, + PubkyAppTag, PubkyAppUser, }; #[tokio_shared_rt::test(shared)] async fn test_tag_post_on_unknown_homeserver() -> Result<()> { let mut test = WatcherTest::setup().await?; - // Create a separate homeserver for the tagged post let tagged_post_hs_pk = create_external_test_homeserver(&mut test).await?; - // Create tagged post author let tagged_post_author_kp = Keypair::random(); let tagged_post_author_id = tagged_post_author_kp.public_key().to_z32(); - // Register the tagged post author PK in the new homeserver - // We only need the record mapping, not necessarily the profile.json being uploaded test.register_user_in_hs(&tagged_post_author_kp, &tagged_post_hs_pk) .await?; - // Create tagged post let post = PubkyAppPost { - content: "Watcher:Homeserver:Tagged:Post".to_string(), + content: "Watcher:UserIngestion:Tagged:Post".to_string(), kind: PubkyAppPostKind::Short, parent: None, embed: None, attachments: None, }; - // We cannot PUT that event because the tagged user is not signed up (missing profile.json, missing graph node) - // That one will force in the post event handler to ingest the homeserver of the tagged user - // because it will throw a MissingDependency error let post_id = post.create_id(); let post_uri = post_uri_builder(tagged_post_author_id.clone(), post_id.clone()); - // Create tagger user let tagger_author_kp = Keypair::random(); let tagger_user = PubkyAppUser { bio: Some("test_tag_post_on_unknown_homeserver".to_string()), image: None, links: None, - name: "Watcher:Homeserver:Tagger:User".to_string(), + name: "Watcher:UserIngestion:Tagger:User".to_string(), status: None, }; test.create_user(&tagger_author_kp, &tagger_user).await?; - // Add a tag to the post let tag = PubkyAppTag { uri: post_uri.clone(), label: "test".to_string(), created_at: Utc::now().timestamp_millis(), }; - // PUT tag let tag_path = tag.hs_path(); test.put(&tagger_author_kp, &tag_path, tag).await?; - // Check if the new homeserver of the unknown tagged user was ingested - let tagged_post_hs_id = PubkyId::try_from(&tagged_post_hs_pk.to_z32()).unwrap(); - assert!(Homeserver::get_by_id(tagged_post_hs_id) - .await - .unwrap() - .is_some()); + assert_user_ingested(&tagged_post_author_id, &tagged_post_hs_pk).await; Ok(()) } @@ -78,48 +59,36 @@ async fn test_tag_post_on_unknown_homeserver() -> Result<()> { async fn test_tag_user_on_unknown_homeserver() -> Result<()> { let mut test = WatcherTest::setup().await?; - // Create a separate homeserver for the tagged post let tagged_user_hs_pk = create_external_test_homeserver(&mut test).await?; - // Create tagged post author let tagged_user_author_kp = Keypair::random(); let tagged_user_author_id = tagged_user_author_kp.public_key().to_z32(); - // Register the tagged post author PK in the new homeserver - // We only need the record mapping, not necessarily the profile.json being uploaded test.register_user_in_hs(&tagged_user_author_kp, &tagged_user_hs_pk) .await?; - // Create tagger user let tagger_author_kp = Keypair::random(); let tagger_user = PubkyAppUser { bio: Some("test_tag_user_on_unknown_homeserver".to_string()), image: None, links: None, - name: "Watcher:Homeserver:Tagger:User".to_string(), + name: "Watcher:UserIngestion:Tagger:User".to_string(), status: None, }; test.create_user(&tagger_author_kp, &tagger_user).await?; let tagged_user_uri = user_uri_builder(tagged_user_author_id.clone()); - // Add a tag to the user let tag = PubkyAppTag { uri: tagged_user_uri.clone(), label: "test".to_string(), created_at: Utc::now().timestamp_millis(), }; - // PUT tag let tag_path = tag.hs_path(); test.put(&tagger_author_kp, &tag_path, tag).await?; - // Check if the new homeserver of the unknown tagged user was ingested - let tagged_user_hs_id = PubkyId::try_from(&tagged_user_hs_pk.to_z32()).unwrap(); - assert!(Homeserver::get_by_id(tagged_user_hs_id) - .await - .unwrap() - .is_some()); + assert_user_ingested(&tagged_user_author_id, &tagged_user_hs_pk).await; Ok(()) } diff --git a/nexus-watcher/tests/user_ingestion/mod.rs b/nexus-watcher/tests/user_ingestion/mod.rs new file mode 100644 index 000000000..9a495eb8c --- /dev/null +++ b/nexus-watcher/tests/user_ingestion/mod.rs @@ -0,0 +1,4 @@ +mod ingest_user_from_follow_events; +mod ingest_user_from_post_events; +mod ingest_user_from_tag_events; +mod utils; diff --git a/nexus-watcher/tests/user_ingestion/utils.rs b/nexus-watcher/tests/user_ingestion/utils.rs new file mode 100644 index 000000000..9b1901e34 --- /dev/null +++ b/nexus-watcher/tests/user_ingestion/utils.rs @@ -0,0 +1,33 @@ +use crate::event_processor::utils::watcher::WatcherTest; +use anyhow::Result; +use nexus_common::db::kv::RedisOps; +use nexus_common::models::user::{user_hs_cursor_key, UserDetails}; +use pubky::PublicKey; + +pub async fn create_external_test_homeserver(test: &mut WatcherTest) -> Result { + let homeserver_id = test.testnet.create_random_homeserver().await?.public_key(); + Ok(homeserver_id) +} + +/// Asserts that a user was properly ingested: graph node exists and the +/// `USER_HS_CURSOR` sorted-set entry points to the expected homeserver. +pub async fn assert_user_ingested(user_id: &str, hs_pk: &PublicKey) { + let hs_id = hs_pk.to_z32(); + + let user = UserDetails::get_by_id(user_id) + .await + .expect("UserDetails::get_by_id failed"); + assert!( + user.is_some(), + "User {user_id} should be ingested in graph/cache" + ); + + let key = user_hs_cursor_key(user_id); + let cursor = UserDetails::check_sorted_set_member(None, &key, &[&hs_id]) + .await + .expect("check_sorted_set_member failed"); + assert!( + cursor.is_some(), + "USER_HS_CURSOR should map user {user_id} to homeserver {hs_id}" + ); +} diff --git a/nexus-watcher/tests/utils/mod.rs b/nexus-watcher/tests/utils/mod.rs new file mode 100644 index 000000000..430555c8e --- /dev/null +++ b/nexus-watcher/tests/utils/mod.rs @@ -0,0 +1,58 @@ +use nexus_common::models::event::{Event, EventProcessorError}; +use nexus_watcher::events::{EventHandler, Moderation}; +use pubky_app_specs::PubkyId; +use std::sync::Arc; +use std::sync::Mutex; + +/// Mock implementation of EventHandler for testing. +/// +/// If `target_uri_substring` is set, `result` only applies to events whose URI contains +/// the substring; all other events return `Ok(())`. +/// +/// In principle, some retry tests could be written as integration tests using [WatcherTest], +/// real local DHT homeservers, and real events. That would test more of the full pipeline. +/// However, [MockEventHandler] makes it possible to retry processor tests deterministically +/// force exact `handle()` outcomes, especially cases that are hard or flaky to create with real HSs. +pub struct MockEventHandler { + pub result: Result<(), EventProcessorError>, + pub target_uri_substring: Option, + /// Tracks how many times `handle()` was invoked. Shared via `Arc` so tests + /// can read the count after processing. + pub handle_count: Arc>, + pub handled_uris: Arc>>, +} + +impl MockEventHandler { + /// Returns the number of times `handle()` was called. + pub fn get_handle_count(&self) -> usize { + *self.handle_count.lock().unwrap() + } + + pub fn get_handled_uris(&self) -> Vec { + self.handled_uris.lock().unwrap().clone() + } +} + +#[async_trait::async_trait] +impl EventHandler for MockEventHandler { + async fn handle(&self, event: &Event) -> Result<(), EventProcessorError> { + // Increment invocation counter on every call + *self.handle_count.lock().unwrap() += 1; + self.handled_uris.lock().unwrap().push(event.uri.clone()); + + match &self.target_uri_substring { + Some(s) if !event.uri.contains(s) => Ok(()), + _ => self.result.clone(), + } + } +} + +/// Default Moderation settings for tests +/// Returns the real Moderation implementation configured with test moderator ID and tags +pub fn default_moderation_tests() -> Arc { + // Moderator ID from moderator_key.pkarr (52-char z32 encoded ID without pubky prefix) + let id = PubkyId::try_from("uo7jgkykft4885n8cruizwy6khw71mnu5pq3ay9i8pw1ymcn85ko") + .expect("Hardcoded test moderation key should be valid"); + let tags = Vec::from(["label_to_moderate".to_string()]); + Arc::new(Moderation { id, tags }) +} diff --git a/nexus-webapi/src/error.rs b/nexus-webapi/src/error.rs index 5beee2fff..c90af1726 100644 --- a/nexus-webapi/src/error.rs +++ b/nexus-webapi/src/error.rs @@ -35,9 +35,9 @@ pub enum Error { } impl Error { - pub fn invalid_input(message: &str) -> Self { + pub fn invalid_input>(message: T) -> Self { Error::InvalidInput { - message: message.to_string(), + message: message.into(), } } } diff --git a/nexus-webapi/src/routes/static/files.rs b/nexus-webapi/src/routes/static/files.rs index b71bbf30a..400f9885d 100644 --- a/nexus-webapi/src/routes/static/files.rs +++ b/nexus-webapi/src/routes/static/files.rs @@ -91,7 +91,7 @@ pub async fn static_files_handler( .ok_or(Error::FileNotFound {})?; if !VariantController::validate_variant_for_content_type(file.content_type.as_str(), &variant) { - return Err(Error::invalid_input(&format!( + return Err(Error::invalid_input(format!( "variant {} is not valid for content type {}", variant, file.content_type ))); diff --git a/nexus-webapi/src/routes/v0/bootstrap.rs b/nexus-webapi/src/routes/v0/bootstrap.rs index 1fa078216..39a4ce9b9 100644 --- a/nexus-webapi/src/routes/v0/bootstrap.rs +++ b/nexus-webapi/src/routes/v0/bootstrap.rs @@ -1,14 +1,13 @@ -use crate::routes::v0::endpoints::{self, PUT_HOMESERVER_ROUTE}; +use crate::routes::v0::endpoints::{self, BOOTSTRAP_ROUTE, INGEST_USER_ROUTE}; use crate::routes::AppState; -use crate::Result; -use crate::{routes::v0::endpoints::BOOTSTRAP_ROUTE, Error}; +use crate::{Error, Result}; use axum::extract::Path; use axum::routing::{get, put}; use axum::Json; use axum::Router; use nexus_common::models::bootstrap::{Bootstrap, ViewType}; -use nexus_common::models::homeserver::Homeserver; +use nexus_common::models::user::UserDetails; use pubky_app_specs::PubkyId; use tracing::debug; use utoipa::OpenApi; @@ -38,30 +37,30 @@ pub async fn bootstrap_handler( #[utoipa::path( put, - path = PUT_HOMESERVER_ROUTE, - description = "Ingest (start monitoring all events of) the Homeserver on which this User PK stores data at this time", + path = INGEST_USER_ROUTE, + description = "Ingest a user by resolving their homeserver and persisting a user node in the graph. If the user is already known, this is a no-op.", tag = "Bootstrap", params( ("user_id" = String, Path, description = "User Pubky ID") ), responses( - (status = 200, description = "Successfully added new homeserver"), + (status = 200, description = "User successfully ingested (or already known)"), (status = 500, description = "Internal server error") ) )] -pub async fn put_homeserver_handler(Path(user_id): Path) -> Result<()> { - debug!("PUT {PUT_HOMESERVER_ROUTE}, user_id:{user_id}"); +pub async fn ingest_user_handler(Path(user_id): Path) -> Result<()> { + debug!("PUT {INGEST_USER_ROUTE}, user_id:{user_id}"); let user_id = PubkyId::try_from(&user_id) - .map_err(|e| Error::invalid_input(&format!("Invalid user PK: {e}")))?; + .map_err(|e| Error::invalid_input(format!("Invalid user PK: {e}")))?; - Homeserver::maybe_ingest_for_user(&user_id).await?; + UserDetails::maybe_ingest_user(&user_id).await?; Ok(()) } #[derive(OpenApi)] #[openapi( - paths(bootstrap_handler, put_homeserver_handler), + paths(bootstrap_handler, ingest_user_handler), components(schemas(Bootstrap)) )] pub struct BootstrapApiDoc; @@ -69,5 +68,5 @@ pub struct BootstrapApiDoc; pub fn routes() -> Router { Router::new() .route(endpoints::BOOTSTRAP_ROUTE, get(bootstrap_handler)) - .route(endpoints::PUT_HOMESERVER_ROUTE, put(put_homeserver_handler)) + .route(endpoints::INGEST_USER_ROUTE, put(ingest_user_handler)) } diff --git a/nexus-webapi/src/routes/v0/endpoints.rs b/nexus-webapi/src/routes/v0/endpoints.rs index 17b8e23f7..5b0ae14b1 100644 --- a/nexus-webapi/src/routes/v0/endpoints.rs +++ b/nexus-webapi/src/routes/v0/endpoints.rs @@ -69,7 +69,7 @@ pub const NOTIFICATION_ROUTE: &str = concatcp!(USER_ROUTE, "/notifications"); // -- BOOTSTRAP endpoints - pub const BOOTSTRAP_ROUTE: &str = concatcp!(VERSION_ROUTE, "/bootstrap/{user_id}"); -pub const PUT_HOMESERVER_ROUTE: &str = concatcp!(VERSION_ROUTE, "/ingest/{user_id}"); +pub const INGEST_USER_ROUTE: &str = concatcp!(VERSION_ROUTE, "/ingest/{user_id}"); // -- RESOURCE endpoints -- const RESOURCE_PREFIX: &str = concatcp!(VERSION_ROUTE, "/resource"); diff --git a/nexus-webapi/src/routes/v0/resource/mod.rs b/nexus-webapi/src/routes/v0/resource/mod.rs index f8abfcfd7..6f70f9738 100644 --- a/nexus-webapi/src/routes/v0/resource/mod.rs +++ b/nexus-webapi/src/routes/v0/resource/mod.rs @@ -8,9 +8,10 @@ use axum::extract::{Path, Query}; use axum::routing::get; use axum::{Json, Router}; use nexus_common::models::resource::tag::TagResource; -use nexus_common::models::resource::{normalize_uri, resource_id, ResourceDetails}; +use nexus_common::models::resource::ResourceDetails; use nexus_common::models::tag::traits::{TagCollection, TaggersCollection}; use nexus_common::models::tag::TagDetails; +use nexus_common::universal_tag::normalize::{normalize_uri, resource_id}; use serde::{Deserialize, Serialize}; use tracing::debug; use utoipa::{OpenApi, ToSchema}; @@ -122,15 +123,14 @@ pub async fn resource_by_uri_handler( Query(query): Query, ) -> Result> { if query.uri.len() > MAX_URI_LENGTH { - return Err(Error::invalid_input(&format!( + return Err(Error::invalid_input(format!( "URI too long (max {MAX_URI_LENGTH} bytes)" ))); } debug!("GET {RESOURCE_BY_URI_ROUTE} uri:{}", query.uri); - let (normalized, _scheme) = - normalize_uri(&query.uri).map_err(|e| Error::InvalidInput { message: e })?; + let (normalized, _scheme) = normalize_uri(&query.uri).map_err(Error::invalid_input)?; let res_id = resource_id(&normalized); let tags = TagResource::get_by_id( diff --git a/nexus-webapi/src/routes/v0/search/tags.rs b/nexus-webapi/src/routes/v0/search/tags.rs index 9d8d5d312..6c4f9fd5f 100644 --- a/nexus-webapi/src/routes/v0/search/tags.rs +++ b/nexus-webapi/src/routes/v0/search/tags.rs @@ -62,7 +62,7 @@ fn sanitize_validate(tag_prefix: &str) -> Result { temp_tag .validate(None) - .map_err(|e| Error::invalid_input(&e.to_string()))?; + .map_err(|e| Error::invalid_input(e.to_string()))?; let sanitized = temp_tag.label; Ok(sanitized) diff --git a/nexus-webapi/src/routes/v0/search/users.rs b/nexus-webapi/src/routes/v0/search/users.rs index 926bb5ff9..027ca665d 100644 --- a/nexus-webapi/src/routes/v0/search/users.rs +++ b/nexus-webapi/src/routes/v0/search/users.rs @@ -73,7 +73,7 @@ pub async fn search_users_by_id_handler( ) -> Result> { let id_prefix = prefix; if id_prefix.trim().chars().count() < USER_ID_SEARCH_MIN_PREFIX_LEN { - return Err(Error::invalid_input(&format!( + return Err(Error::invalid_input(format!( "ID prefix must be at least {USER_ID_SEARCH_MIN_PREFIX_LEN} chars" ))); } diff --git a/nexus-webapi/src/routes/v0/stream/posts.rs b/nexus-webapi/src/routes/v0/stream/posts.rs index 77445c360..a19856c2a 100644 --- a/nexus-webapi/src/routes/v0/stream/posts.rs +++ b/nexus-webapi/src/routes/v0/stream/posts.rs @@ -51,7 +51,7 @@ impl PostStreamQuery { pub fn validate_tags(&self) -> AppResult<()> { if let Some(ref tags) = self.tags { if tags.len() > MAX_TAGS { - return Err(Error::invalid_input(&format!( + return Err(Error::invalid_input(format!( "Too many tags provided; maximum allowed is {MAX_TAGS}" ))); } diff --git a/nexus-webapi/src/routes/v0/stream/users.rs b/nexus-webapi/src/routes/v0/stream/users.rs index fb6f18c5d..fd5502465 100644 --- a/nexus-webapi/src/routes/v0/stream/users.rs +++ b/nexus-webapi/src/routes/v0/stream/users.rs @@ -257,7 +257,7 @@ fn build_user_stream_input( | UserStreamSource::Following | UserStreamSource::Friends | UserStreamSource::Recommended => { - return Err(Error::invalid_input(&format!( + return Err(Error::invalid_input(format!( "user_id query param must be provided for source '{}'", source_name(&source) ))); diff --git a/nexus-webapi/tests/utils/server.rs b/nexus-webapi/tests/utils/server.rs index da2c57041..75cb2138b 100644 --- a/nexus-webapi/tests/utils/server.rs +++ b/nexus-webapi/tests/utils/server.rs @@ -23,7 +23,7 @@ impl TestServiceServer { pub async fn get_test_server() -> &'static TestServiceServer { TEST_SERVER .get_or_init(|| async { - let testnet = pubky_testnet::Testnet::new().await.unwrap(); + let testnet = pubky_testnet::Testnet::new_unseeded().await.unwrap(); let nexus_api = Self::start_server(&testnet, false).await.unwrap(); TestServiceServer { nexus_api, testnet } }) @@ -37,7 +37,7 @@ impl TestServiceServer { pub async fn get_test_server_with_key_republisher() -> &'static TestServiceServer { TEST_SERVER_WITH_KEY_REPUBLISHER .get_or_init(|| async { - let testnet = pubky_testnet::Testnet::new().await.unwrap(); + let testnet = pubky_testnet::Testnet::new_unseeded().await.unwrap(); let nexus_api = Self::start_server(&testnet, true).await.unwrap(); TestServiceServer { nexus_api, testnet } })