diff --git a/design-docs/cluster-raft.md b/design-docs/cluster-raft.md index 495c41182e4..a14d7472f76 100644 --- a/design-docs/cluster-raft.md +++ b/design-docs/cluster-raft.md @@ -503,6 +503,42 @@ consensus (§4.3). Both require the new node to receive AE as a non-voting member before its NODE_JOIN is committed, adding complexity to the leader's replication logic. +#### Automatic learner promotion + +The leader runs a one-way reconciliation controller that promotes +caught-up learners to voters until the voting set reaches +`RAFT_TARGET_VOTERS` (5). It is intentionally minimal: + +- **Leader-only and one-way.** Only the leader promotes; it never + demotes. When `size >= RAFT_TARGET_VOTERS` it is a no-op. +- **Reuses ADD_VOTER.** There is no new node flag or entry type. Apply + goes through `clusterRaftApplyAddVoter`, the single membership-change + point. +- **Replacement-first demotion.** `CLUSTER DELVOTER` pre-validation + rejects a demote that would bring `size` below the target, forcing the + admin to add a replacement voter first. A manual `ADD_VOTER` may still + push `size` above the target (the controller does not demote). + +A candidate is eligible only when it is a learner, has finished joining +(no MEET flag), is not failed, has an active link, and its `match_index` +has caught up with the leader's last log index. + +To avoid overlapping membership transitions, the leader enforces **at +most one quorum-changing entry in flight**, where a transition's identity +is its entry type plus the target node id (the first `CLUSTER_NAMELEN` +bytes of the entry data). The check scans the committed-but-unapplied +entries `(last_applied, lastIndex]` and classifies the candidate as: + +- `NONE` — no quorum-changing entry in flight; proceed with full + pre-validation. +- `SAME` — the identical transition is already in flight; resend the + duplicate without re-running readiness checks (which would be fooled by + a `match_index` reset after a leader change). +- `CONFLICT` — a different transition is in flight; reject. + +The same gate fronts `clusterRaftPreValidate` (for manual and automatic +proposals) and the leader branch of `clusterRaftRetryProposals`. + ## Failure Detection The leader tracks `last_ack_time` per peer, updated on every AE_ACK @@ -881,7 +917,9 @@ targets. entries, especially don't trigger primary/replica failovers in a minority partition). - Log compaction / snapshotting for lagging followers. -- Automatic learner promotion (promote the first 5 or 7 nodes to voters). +- Configurable voter target and automatic failed-voter replacement (the + current controller hardcodes `RAFT_TARGET_VOTERS` and never replaces a + failed voter). - Chained learner replication. - Balancing voters over availability zones: the leader can use the per-node availability-zone info to add/remove voters so that voters diff --git a/src/cluster_raft.c b/src/cluster_raft.c index f81f8974802..a4b13b71cba 100644 --- a/src/cluster_raft.c +++ b/src/cluster_raft.c @@ -266,6 +266,7 @@ static bool nodeIsVoter(clusterNode *n) { #define REPL_OFFSETS_BROADCAST_PERIOD_MS 10000 #define RAFT_LOG_REWRITE_THRESHOLD 100 #define PROPOSAL_MAX_RETRIES 10 +#define RAFT_TARGET_VOTERS 5 /* Monotonic millisecond clock for timeouts and failure detection. * Unlike gettimeofday(), this is not affected by system clock adjustments. */ @@ -371,6 +372,15 @@ static clusterMsgSendBlock *clusterRaftBuildAllOffsetsMsg(void) { * Helpers * -------------------------------------------------------------------------- */ +/* Identity of a quorum-changing transition is (entry type, target node-id = + * the first CLUSTER_NAMELEN bytes of the entry data). Used to enforce + * "at most one membership transition in flight". */ +typedef enum { + RAFT_QC_NONE = 0, /* No quorum-changing entry in flight. */ + RAFT_QC_SAME, /* The same transition is already in flight. */ + RAFT_QC_CONFLICT, /* A different transition is in flight. */ +} raftQuorumChangeStatus; + static void clusterRaftPropose(sds entry, void *ctx, void (*callback)(void *ctx, const char *error)); static void clusterRaftDeferPendingProposals(void); static void clusterRaftCompletePendingProposal(int type, sds data, RaftProposalResult result); @@ -401,6 +411,10 @@ static void clusterRaftPersistNewLogEntries(uint64_t from); static uint64_t raftLogLastTerm(void); static int clusterRaftCanGrantVote(clusterRaftState *rs, uint64_t candidate_last_index, uint64_t candidate_last_term); static void clusterRaftStartElection(void); +static bool raftEntryAffectsQuorum(int type, sds data); +static raftQuorumChangeStatus clusterRaftQuorumChangeStatus(int type, sds data); +static clusterNode *clusterRaftSelectPromotionCandidate(void); +static void clusterRaftAutoPromoteLearner(void); static void clusterRaftRandomizeElectionTimeout(void) { mstime_t base = server.cluster_node_timeout; @@ -1112,6 +1126,12 @@ static bool parseSlotChangeEpochs(sds *argv, int argc, slotChangeEpochInfo *info * RAFT_RESULT_REJECTED describing why it failed. */ static RaftProposalResult clusterRaftPreValidate(int type, sds data) { RaftProposalResult result = RAFT_RESULT_OK; + /* Enforce at most one quorum-changing transition in flight. A different + * membership transition already in the log is rejected here so it never + * consumes log space. */ + if (clusterRaftQuorumChangeStatus(type, data) == RAFT_QC_CONFLICT) { + return RAFT_RESULT_REJECTED; + } switch (type) { case RAFT_ENTRY_FAILOVER: result = clusterRaftApplyFailover(data, 1); @@ -1324,6 +1344,58 @@ static uint64_t raftLogTermAt(uint64_t index) { return e ? e->term : 0; } +/* True if the entry changes the voting set (server.cluster->size). Safe on + * malformed/too-short data: never reads past the buffer. */ +static bool raftEntryAffectsQuorum(int type, sds data) { + if (sdslen(data) < CLUSTER_NAMELEN) return false; + + switch (type) { + case RAFT_ENTRY_ADD_VOTER: + case RAFT_ENTRY_DEL_VOTER: + return true; + case RAFT_ENTRY_NODE_JOIN: { + /* data: "
" */ + int argc; + sds *argv = sdssplitlen(data, sdslen(data), " ", 1, &argc); + bool changes = (argv && argc >= 3 && !strcasecmp(argv[2], "voter")); + if (argv) sdsfreesplitres(argv, argc); + return changes; + } + case RAFT_ENTRY_NODE_FORGET: { + clusterNode *node = clusterLookupNode(data, CLUSTER_NAMELEN); + return node != NULL && nodeIsVoter(node); + } + default: + return false; + } +} + +/* Classify the in-flight (committed but not yet applied) quorum-changing + * entries relative to a candidate transition. Scans (last_applied, + * raftLogLastIndex()] once. Returns: + * RAFT_QC_NONE — no quorum-changing entry in flight. + * RAFT_QC_SAME — the identical (type, target) transition is in flight. + * RAFT_QC_CONFLICT — a different transition is in flight. */ +static raftQuorumChangeStatus clusterRaftQuorumChangeStatus(int type, sds data) { + if (!raftEntryAffectsQuorum(type, data)) return RAFT_QC_NONE; + + clusterRaftState *rs = RAFT_STATE(); + int found_same = 0; + int found_conflict = 0; + for (uint64_t idx = rs->last_applied + 1; idx <= raftLogLastIndex(); idx++) { + raftLogEntry *e = raftLogGet(idx); + if (!e || !raftEntryAffectsQuorum(e->type, e->data)) continue; + if (e->type == type && memcmp(e->data, data, CLUSTER_NAMELEN) == 0) { + found_same = 1; + } else { + found_conflict = 1; + } + } + if (found_conflict) return RAFT_QC_CONFLICT; + if (found_same) return RAFT_QC_SAME; + return RAFT_QC_NONE; +} + /* Find a pending proposal matching type+data, fire its callback, and remove it. * Called both when an entry is applied (from raftLogApply) and when the leader * sends a REJECT for a forwarded proposal. */ @@ -2292,24 +2364,46 @@ static void clusterRaftRetryProposals(int include_pending) { while ((ln = listNext(&li)) != NULL) { raftPendingProposal *pp = listNodeValue(ln); - /* For deferred entries, update to latest shard epoch */ + /* For deferred entries, update to latest shard epoch. */ if (pp->deferred) { if (!raftRefreshEpochInData(pp->type, &pp->data)) continue; pp->deferred = 0; + } else if (!include_pending) { + continue; + } - if (rs->role == RAFT_ROLE_LEADER) { - RaftProposalResult pre_result = clusterRaftPreValidate(pp->type, pp->data); - if (pre_result != RAFT_RESULT_OK) { - if (pp->callback) pp->callback(pp->ctx, raftProposalResultMsg(pre_result)); - sdsfree(pp->data); - zfree(pp); - listDelNode(rs->pending_proposals, ln); + if (rs->role == RAFT_ROLE_LEADER) { + raftQuorumChangeStatus status = clusterRaftQuorumChangeStatus(pp->type, pp->data); + + /* A different membership transition is in flight: reject. */ + if (status == RAFT_QC_CONFLICT) { + if (pp->callback) pp->callback(pp->ctx, GENERIC_PROPOSAL_REJECTION_MSG); + sdsfree(pp->data); + zfree(pp); + listDelNode(rs->pending_proposals, ln); + continue; + } + + /* The identical transition is already in the log; it will + * complete the pending proposal when applied. */ + if (status == RAFT_QC_SAME) continue; + + /* No membership transition in flight: run full pre-validation. */ + RaftProposalResult pre_result = clusterRaftPreValidate(pp->type, pp->data); + if (pre_result != RAFT_RESULT_OK) { + if (pre_result == RAFT_RESULT_STALE_EPOCH && pp->retries > 0) { + pp->retries--; + pp->deferred = 1; + rs->todo_retry_deferred = 1; continue; } + if (pp->callback) pp->callback(pp->ctx, raftProposalResultMsg(pre_result)); + sdsfree(pp->data); + zfree(pp); + listDelNode(rs->pending_proposals, ln); + continue; } - } else { - if (!include_pending) continue; } clusterRaftSendProposal(pp, leader_link); @@ -2322,6 +2416,53 @@ static void clusterRaftAssertLearnerRole(void) { serverAssert((rs->role == RAFT_ROLE_LEARNER) == nodeIsLearner(myself)); } +/* Pick a learner eligible for automatic promotion to voter. A candidate must + * be a learner that finished joining (no MEET flag), is not marked failed, + * has an active link, and has fully caught up with the leader's log. */ +static clusterNode *clusterRaftSelectPromotionCandidate(void) { + uint64_t last = raftLogLastIndex(); + dictIterator *di = dictGetSafeIterator(server.cluster->nodes); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + clusterNode *n = dictGetVal(de); + if (!nodeIsLearner(n)) continue; + if (n->flags & CLUSTER_NODE_MEET) continue; + if (nodeFailed(n)) continue; + if (!n->link) continue; + if (RAFT_NODE(n)->match_index < last) continue; + dictReleaseIterator(di); + return n; + } + dictReleaseIterator(di); + return NULL; +} + +/* Leader-only, one-way reconciliation controller: promote caught-up learners + * to voters until the voting set reaches RAFT_TARGET_VOTERS. Reuses the + * ADD_VOTER path. Never demotes; when size >= target it is a no-op. Stops + * while any membership transition is in flight. */ +static void clusterRaftAutoPromoteLearner(void) { + clusterRaftState *rs = RAFT_STATE(); + if (rs->role != RAFT_ROLE_LEADER) return; + if (server.cluster->size >= RAFT_TARGET_VOTERS) return; + + clusterNode *candidate = clusterRaftSelectPromotionCandidate(); + if (!candidate) return; + + sds data = sdsnewlen(candidate->name, CLUSTER_NAMELEN); + /* Only one membership transition in flight at a time. */ + if (clusterRaftQuorumChangeStatus(RAFT_ENTRY_ADD_VOTER, data) != RAFT_QC_NONE) { + sdsfree(data); + return; + } + + sds entry = sdsnew("ADD_VOTER "); + entry = sdscatsds(entry, data); + clusterRaftPropose(entry, NULL, NULL); + sdsfree(data); + sdsfree(entry); +} + static void clusterRaftCron(void) { clusterRaftState *rs = RAFT_STATE(); mstime_t now = monotonicMs(); @@ -2434,6 +2575,7 @@ static void clusterRaftCron(void) { } myself->repl_offset = my_offset; clusterRaftDetectFailures(now); + clusterRaftAutoPromoteLearner(); } } @@ -3646,6 +3788,12 @@ static RaftProposalResult clusterRaftApplyDelVoter(sds data, int validate_only) } if (validate_only) { + /* Replacement-first: refuse to demote below the auto-promotion + * target. The caller must add a replacement voter first. */ + if (server.cluster->size - 1 < RAFT_TARGET_VOTERS) { + sdsfreesplitres(argv, argc); + return RAFT_RESULT_REJECTED; + } sdsfreesplitres(argv, argc); return RAFT_RESULT_OK; } @@ -3685,7 +3833,13 @@ static RaftProposalResult clusterRaftApplyNodeForget(sds data, int validate_only goto reject; } - if (validate_only) goto done; + if (validate_only) { + /* Replacement-first: refuse to forget a voter below the target. */ + if (nodeIsVoter(node) && server.cluster->size - 1 < RAFT_TARGET_VOTERS) { + goto reject; + } + goto done; + } /* Save shard_id before deleting the node. */ char shard_id[CLUSTER_NAMELEN]; diff --git a/tests/support/cluster_raft.tcl b/tests/support/cluster_raft.tcl index cc5d56d89da..c9aea5734e3 100644 --- a/tests/support/cluster_raft.tcl +++ b/tests/support/cluster_raft.tcl @@ -1,70 +1,129 @@ # Shared helpers for Raft cluster integration tests. -proc get_cluster_info_field {client field} { - set info [$client CLUSTER INFO] - foreach line [split $info "\n"] { - set line [string trim $line "\r"] - if {[string match "${field}:*" $line]} { - return [lindex [split $line ":"] 1] - } +proc raft_add_voter {leader node_id} { + set leader_client [Rn $leader] + # Wait until NODE_JOIN finished: learner, or already a voting member. + wait_for_condition 100 100 { + [expr {[llength [cluster_get_node_by_id $leader $node_id]] > 0 && + ([cluster_has_flag [cluster_get_node_by_id $leader $node_id] learner] || + ![cluster_has_flag [cluster_get_node_by_id $leader $node_id] handshake])}] + } else { + fail "Node $node_id did not finish joining on leader $leader" + } + set err "" + # The auto-promoter may have already promoted this learner, making a + # subsequent ADDVOTER fail with "already a voter". Treat either + # "promoted" or "ADDVOTER succeeded" as success (avoids a TOCTOU race). + wait_for_condition 200 100 { + [expr {![cluster_has_flag [cluster_get_node_by_id $leader $node_id] learner] || + ![catch {$leader_client CLUSTER ADDVOTER $node_id} err]}] + } else { + fail "Could not promote $node_id to voter: $err" } - return "" } -proc raft_cluster_nodes_line {client node_id} { - foreach line [split [$client CLUSTER NODES] "\n"] { - set line [string trim $line "\r"] - if {[string match "$node_id *" $line]} { - return $line - } - } - return "" +# --------------------------------------------------------------------------- +# Raft wire protocol helpers for simulating cluster nodes over the cluster bus. +# --------------------------------------------------------------------------- + +# Build a raft wire message: "RAFT" + 4-byte big-endian length + payload. +proc raft_msg {payload} { + set len [expr {8 + [string length $payload]}] + set hdr "RAFT" + append hdr [binary format I $len] + append hdr $payload + return $hdr +} + +# Connect to a node's cluster bus port and return the socket. +proc raft_connect {host port} { + set fd [socket $host $port] + fconfigure $fd -translation binary -buffering full + return $fd } -proc raft_node_visible {client node_id} { - return [expr {[raft_cluster_nodes_line $client $node_id] ne ""}] +# Send a raft message on a cluster bus connection. +proc raft_send {fd payload} { + puts -nonewline $fd [raft_msg $payload] + flush $fd } -proc raft_node_has_flag {client node_id flag} { - set line [raft_cluster_nodes_line $client $node_id] - if {$line eq ""} { - return 0 +# Read a raft message from a cluster bus connection. Returns the payload. +proc raft_recv {fd {timeout 5000}} { + fconfigure $fd -blocking 0 + set deadline [expr {[clock milliseconds] + $timeout}] + set hdr "" + while {[string length $hdr] < 8} { + append hdr [read $fd [expr {8 - [string length $hdr]}]] + if {[string length $hdr] < 8} { + if {[clock milliseconds] > $deadline} { + error "timeout reading raft header" + } + after 10 + } + } + if {[string range $hdr 0 3] ne "RAFT"} { + error "bad raft header: [string range $hdr 0 3]" + } + binary scan [string range $hdr 4 7] I totlen + set paylen [expr {$totlen - 8}] + set payload "" + while {[string length $payload] < $paylen} { + append payload [read $fd [expr {$paylen - [string length $payload]}]] + if {[string length $payload] < $paylen} { + if {[clock milliseconds] > $deadline} { + error "timeout reading raft payload (got [string length $payload]/$paylen)" + } + after 10 + } } - return [string match "*${flag}*" [lindex [split $line] 2]] + return $payload } -proc raft_add_voter {leader node_id} { - # Wait until NODE_JOIN finished: learner, or already a voting member. - wait_for_condition 100 100 { - [expr {[raft_node_visible $leader $node_id] && ( - [raft_node_has_flag $leader $node_id learner] || - ![raft_node_has_flag $leader $node_id handshake] - )}] - } else { - fail "Node $node_id did not finish joining on $leader" +# Listen on a random port, run optional setup code, accept one connection, then +# close the listener. Returns the accepted client fd and sets the listen port in +# the upvar port_var. +proc raft_listen_and_accept {port_var {timeout 5000} {before_accept {}}} { + upvar $port_var listen_port + set ::_raft_accepted "" + proc _raft_on_accept {fd addr port} { + fconfigure $fd -translation binary -buffering full + set ::_raft_accepted $fd } - if {![raft_node_has_flag $leader $node_id learner]} { - return + set listen_fd [socket -server _raft_on_accept -myaddr 127.0.0.1 0] + set listen_port [lindex [fconfigure $listen_fd -sockname] 2] + if {$before_accept ne {}} { + uplevel 1 $before_accept } - set err "" - wait_for_condition 200 100 { - [expr {![catch {$leader CLUSTER ADDVOTER $node_id} err]}] - } else { - fail "Could not promote $node_id to voter: $err" + set accept_after [after $timeout {set ::_raft_accepted timeout}] + vwait ::_raft_accepted + after cancel $accept_after + close $listen_fd + if {$::_raft_accepted eq "timeout"} { + error "timeout waiting for connection" } + return $::_raft_accepted +} + +# Connect a fake node to a cluster bus port and complete the HELLO/HI handshake. +proc raft_connect_fake_node {host cport fake_id fake_addr} { + set fd [raft_connect $host $cport] + raft_send $fd "HELLO $fake_id $fake_addr" + set reply [raft_recv $fd] + assert_match "HI *" $reply + return $fd } -# TEST HARNESS ONLY — not production behavior. -# -# On the wire, CLUSTER MEET still proposes NODE_JOIN ... learner. -# start_cluster historically assumed every joined node could vote -# (pre-learner). Generic cluster tests still rely on that topology, -# so after MEET we explicitly ADDVOTER each peer. -# -# Do NOT copy this into product code. Tests that need learners must -# use start_multiple_servers (or equivalent) and promote themselves. -proc raft_promote_start_cluster_voters {node_count} { - for {set i 1} {$i < $node_count} {incr i} { - raft_add_voter [srv 0 client] [R $i CLUSTER MYID] +# Reply AE_ACK to an AE message. last_index defaults to the AE's prev-log-idx +# plus its entry count; pass it explicitly to simulate a lagging follower. +proc raft_reply_ae_ack {fd ae_msg repl_offset {last_index ""}} { + set lines [split $ae_msg "\n"] + set fields [split [lindex $lines 0] " "] + # AE + set term [lindex $fields 2] + if {$last_index eq ""} { + set last_index [expr {[lindex $fields 3] + [lindex $fields 6]}] } + raft_send $fd "AE_ACK $term 1 $last_index $repl_offset" + return $last_index } diff --git a/tests/support/cluster_util.tcl b/tests/support/cluster_util.tcl index bb89de16b6d..95b8c2afcc8 100644 --- a/tests/support/cluster_util.tcl +++ b/tests/support/cluster_util.tcl @@ -2,7 +2,6 @@ source tests/support/cli.tcl source tests/support/cluster.tcl -source tests/support/cluster_raft.tcl proc config_set_all_nodes {keyword value} { for {set j 0} {$j < [llength $::servers]} {incr j} { @@ -244,11 +243,8 @@ proc cluster_setup {masters replicas node_count slot_allocator replica_allocator } } - # See raft_promote_start_cluster_voters: test-only multi-voter restore. - if {$::cluster_raft} { - raft_promote_start_cluster_voters $node_count - } - + # Learners are promoted to voters automatically by the leader's + # auto-promotion controller (RAFT_TARGET_VOTERS). $slot_allocator $masters $replicas wait_for_cluster_propagation diff --git a/tests/unit/cluster/cluster-raft-autopromote.tcl b/tests/unit/cluster/cluster-raft-autopromote.tcl new file mode 100644 index 00000000000..fbf3a38315a --- /dev/null +++ b/tests/unit/cluster/cluster-raft-autopromote.tcl @@ -0,0 +1,386 @@ +# Test automatic learner promotion for the Raft cluster protocol: the leader +# promotes caught-up learners to voters up to RAFT_TARGET_VOTERS (5), reusing +# the ADD_VOTER path, with replacement-first DELVOTER pre-validation and an +# "at most one membership transition in flight" guard. + +source tests/support/cluster_raft.tcl + +# Have node 0 meet peers 1..n-1 (star formation) and return their node ids. +proc raft_meet_peers {n} { + set r0 [srv 0 client] + for {set i 1} {$i < $n} {incr i} { + $r0 CLUSTER MEET [srv -$i host] [srv -$i port] + } + set peer_ids [list] + for {set i 1} {$i < $n} {incr i} { + lappend peer_ids [R $i CLUSTER MYID] + } + return $peer_ids +} + +# Return the first learner id among the given peer ids, or "" if none. +proc raft_find_learner {leader ids} { + foreach id $ids { + if {[cluster_has_flag [cluster_get_node_by_id $leader $id] learner]} { + return $id + } + } + return "" +} + +# Return the first voting (non-learner) id among the given peer ids, skipping +# an optional excluded id, or "" if none. +proc raft_find_voter {leader ids {exclude ""}} { + foreach id $ids { + if {$id eq $exclude} continue + if {![cluster_has_flag [cluster_get_node_by_id $leader $id] learner]} { + return $id + } + } + return "" +} + +# Return the unique ADD_VOTER targets among the unapplied (in-flight) log +# entries persisted in nodes.conf. A full rewrite (SAVECONFIG) writes only +# entries with index > last_applied as "log" lines. +proc raft_inflight_add_voter_targets {client} { + $client CLUSTER SAVECONFIG + set dir [lindex [$client CONFIG GET dir] 1] + set path "$dir/nodes.conf" + if {![file exists $path]} { + return [list] + } + set fp [open $path r] + set text [read $fp] + close $fp + set targets [list] + foreach line [split $text "\n"] { + set fields [split [string trim $line "\r"] " "] + # Format: log + if {[lindex $fields 0] eq "log" && [lindex $fields 4] eq "ADD_VOTER"} { + lappend targets [lindex $fields 5] + } + } + return [lsort -unique $targets] +} + +tags {external:skip cluster singledb} { + +start_multiple_servers 5 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 1000}} { + test "Raft autopromote: 1 voter + 4 learners converge to 5 voters" { + raft_meet_peers 5 + # No manual ADDVOTER — the controller must promote all four learners. + wait_for_condition 200 100 { + [CI 0 cluster_size] == 5 && + [CI 1 cluster_size] == 5 && + [CI 4 cluster_size] == 5 + } else { + fail "Sizes: [CI 0 cluster_size] [CI 1 cluster_size] [CI 4 cluster_size]" + } + } +} + +start_multiple_servers 3 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 1000}} { + test "Raft autopromote: fewer than target nodes converge without hanging" { + raft_meet_peers 3 + wait_for_condition 100 100 { + [CI 0 cluster_size] == 3 && + [CI 1 cluster_size] == 3 && + [CI 2 cluster_size] == 3 + } else { + fail "Sizes: [CI 0 cluster_size] [CI 1 cluster_size] [CI 2 cluster_size]" + } + } +} + +start_multiple_servers 5 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 1000}} { + test "Raft autopromote: stops at exactly the target number of voters" { + set r0 [srv 0 client] + raft_meet_peers 5 + wait_for_condition 200 100 { + [CI 0 cluster_size] == 5 + } else { + fail "size=[CI 0 cluster_size]" + } + # Give the controller a chance to overshoot; it must not. + after 2000 + assert_equal 5 [CI 0 cluster_size] + assert_equal 0 [llength [raft_inflight_add_voter_targets $r0]] + } +} + +start_multiple_servers 6 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 1000}} { + test "Raft autopromote: manual ADDVOTER above target is not demoted" { + set peer_ids [raft_meet_peers 6] + wait_for_condition 200 100 { + [CI 0 cluster_size] == 5 + } else { + fail "size=[CI 0 cluster_size]" + } + # Promote the remaining learner by hand, pushing size to 6. + set remaining [raft_find_learner 0 $peer_ids] + assert {$remaining ne ""} + raft_add_voter 0 $remaining + wait_for_condition 100 100 { + [CI 0 cluster_size] == 6 + } else { + fail "size=[CI 0 cluster_size]" + } + # The one-way controller must not demote it back to 5. + after 2000 + assert_equal 6 [CI 0 cluster_size] + } +} + +start_multiple_servers 5 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 1000}} { + test "Raft autopromote: DELVOTER below target is rejected (replacement-first)" { + set r0 [srv 0 client] + set peer_ids [raft_meet_peers 5] + wait_for_condition 200 100 { + [CI 0 cluster_size] == 5 + } else { + fail "size=[CI 0 cluster_size]" + } + # Demoting a voter would leave 4 < target. + set target [raft_find_voter 0 $peer_ids] + assert {$target ne ""} + catch {$r0 CLUSTER DELVOTER $target} err + assert_match "*rejected*" $err + assert_equal 5 [CI 0 cluster_size] + } +} + +start_multiple_servers 6 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 1000}} { + test "Raft autopromote: DELVOTER at/above target is allowed" { + set r0 [srv 0 client] + set peer_ids [raft_meet_peers 6] + wait_for_condition 200 100 { + [CI 0 cluster_size] == 5 + } else { + fail "size=[CI 0 cluster_size]" + } + set remaining [raft_find_learner 0 $peer_ids] + assert {$remaining ne ""} + raft_add_voter 0 $remaining + wait_for_condition 100 100 { + [CI 0 cluster_size] == 6 + } else { + fail "size=[CI 0 cluster_size]" + } + # Now demoting a non-leader voter (6 -> 5) is allowed. + set target [raft_find_voter 0 $peer_ids $remaining] + assert {$target ne ""} + $r0 CLUSTER DELVOTER $target + wait_for_condition 100 100 { + [CI 0 cluster_size] == 5 + } else { + fail "size=[CI 0 cluster_size]" + } + } +} + +start_multiple_servers 6 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 1000}} { + test "Raft autopromote: paused voter does not cause promotion beyond target" { + set peer_ids [raft_meet_peers 6] + wait_for_condition 200 100 { + [CI 0 cluster_size] == 5 + } else { + fail "size=[CI 0 cluster_size]" + } + set remaining [raft_find_learner 0 $peer_ids] + assert {$remaining ne ""} + # Pause a non-leader voter. The controller must not promote the 6th + # learner to replace it (target is 5, already reached). + set pause_i 0 + for {set i 0} {$i < [llength $peer_ids]} {incr i} { + if {![cluster_has_flag [cluster_get_node_by_id 0 [lindex $peer_ids $i]] learner]} { + set pause_i [expr {$i + 1}] + break + } + } + assert {$pause_i > 0} + pause_process [srv -$pause_i pid] + after 2000 + assert_equal 5 [CI 0 cluster_size] + assert {[cluster_has_flag [cluster_get_node_by_id 0 $remaining] learner]} + resume_process [srv -$pause_i pid] + } +} + +start_server {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 5000}} { + test "Raft autopromote: learner that has not caught up is not promoted" { + set fake_id [string repeat "9" 40] + set fake_shard [string repeat "8" 40] + + # Listen for the leader's outbound connection so AE flows to us. MEET + # triggers NODE_JOIN(learner) which commits under quorum=1 (we don't + # count as a voter). + set fake_cport 0 + set fd [raft_listen_and_accept fake_cport 5000 { + set fake_port [expr {$fake_cport - 10000}] + set fake_addr "127.0.0.1:${fake_port}@${fake_cport},,tls-port=0,shard-id=$fake_shard" + set meet_client [valkey_deferring_client] + $meet_client CLUSTER MEET 127.0.0.1 $fake_port + }] + + # Leader connects outbound: HELLO + MEET(singleton). + set reply [raft_recv $fd 5000] + assert_match "HELLO *" $reply + set reply [raft_recv $fd 5000] + assert_match "MEET *" $reply + raft_send $fd "HI $fake_id $fake_addr" + raft_send $fd "ADD_ME" + + # Leader commits NODE_JOIN(learner) and sends AE carrying it. + set reply [raft_recv $fd 5000] + assert_match "AE *" $reply + + # Reply with a lagging last-log-index so the leader never sees us as + # caught up (match_index stays 0). + raft_reply_ae_ack $fd $reply 0 0 + + wait_for_condition 50 100 { + [cluster_has_flag [cluster_get_node_by_id 0 $fake_id] learner] + } else { + fail "learner did not join" + } + + # The catch-up gate must keep us a learner. + after 2000 + assert {[cluster_has_flag [cluster_get_node_by_id 0 $fake_id] learner]} + + close $fd + $meet_client close + } +} + +start_multiple_servers 3 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 15000}} { + test "Raft autopromote: at most one distinct membership transition in flight" { + raft_meet_peers 3 + wait_for_condition 100 100 { + [CI 0 cluster_size] == 3 + } else { + fail "size=[CI 0 cluster_size]" + } + + # Pause both follower voters so the leader can append but not commit a + # membership transition (quorum = 2 of 3). node-timeout is large enough + # that the leader won't step down within the observation window. + pause_process [srv -1 pid] + pause_process [srv -2 pid] + + set cport [expr {[srv 0 port] + 10000}] + set f1 [string repeat "a" 40] + set f2 [string repeat "b" 40] + set a1 "127.0.0.1:9001@19001,,tls-port=0,shard-id=[string repeat "c" 40]" + set a2 "127.0.0.1:9002@19002,,tls-port=0,shard-id=[string repeat "d" 40]" + + # First membership transition (NODE_JOIN voter) is accepted and stays + # in flight because quorum is unavailable. + set fd1 [raft_connect_fake_node 127.0.0.1 $cport $f1 $a1] + raft_send $fd1 "PROPOSE NODE_JOIN $f1 $a1 voter" + after 500 + + # A second, different membership transition must be rejected by the + # "at most one in flight" gate. + set fd2 [raft_connect_fake_node 127.0.0.1 $cport $f2 $a2] + raft_send $fd2 "PROPOSE NODE_JOIN $f2 $a2 voter" + set reply [raft_recv $fd2 2000] + assert_match "REJECT *" $reply + + close $fd1 + close $fd2 + resume_process [srv -1 pid] + resume_process [srv -2 pid] + } +} + +start_multiple_servers 6 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 500}} { + test "Raft autopromote: leader change preserves the voter set and ADDVOTER works" { + set peer_ids [raft_meet_peers 6] + wait_for_condition 200 100 { + [CI 0 cluster_size] == 5 + } else { + fail "size=[CI 0 cluster_size]" + } + set remaining [raft_find_learner 0 $peer_ids] + assert {$remaining ne ""} + + set old_term [CI 0 cluster_raft_current_term] + pause_process [srv 0 pid] + wait_for_condition 200 100 { + [CI 1 cluster_raft_current_term] > $old_term && + ([CI 1 cluster_raft_role] eq "leader" || + [CI 2 cluster_raft_role] eq "leader" || + [CI 3 cluster_raft_role] eq "leader" || + [CI 4 cluster_raft_role] eq "leader" || + [CI 5 cluster_raft_role] eq "leader") + } else { + resume_process [srv 0 pid] + fail "no new leader elected after pause" + } + + set new_leader_idx 0 + for {set i 1} {$i < 6} {incr i} { + if {[CI $i cluster_raft_role] eq "leader"} { + set new_leader_idx $i + break + } + } + assert {$new_leader_idx > 0} + + # The new leader must still promote the remaining learner. + raft_add_voter $new_leader_idx $remaining + resume_process [srv 0 pid] + wait_for_condition 200 100 { + [CI 0 cluster_size] == 6 && [CI 1 cluster_size] == 6 + } else { + fail "Sizes after leader change: [CI 0 cluster_size] [CI 1 cluster_size]" + } + } +} + +start_multiple_servers 6 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 1000}} { + test "Raft autopromote: concurrent DELVOTER does not drop below target" { + set peer_ids [raft_meet_peers 6] + wait_for_condition 200 100 { + [CI 0 cluster_size] == 5 + } else { + fail "size=[CI 0 cluster_size]" + } + set remaining [raft_find_learner 0 $peer_ids] + raft_add_voter 0 $remaining + wait_for_condition 100 100 { + [CI 0 cluster_size] == 6 + } else { + fail "size=[CI 0 cluster_size]" + } + + set a [raft_find_voter 0 $peer_ids] + set b [raft_find_voter 0 $peer_ids $a] + assert {$a ne "" && $b ne ""} + + # Fire two DELVOTERs concurrently. Exactly one may win (6 -> 5); the + # other is rejected (either by the in-flight gate or because it would + # go 5 -> 4, below target). + set c1 [valkey_deferring_client] + set c2 [valkey_deferring_client] + $c1 CLUSTER DELVOTER $a + $c2 CLUSTER DELVOTER $b + set s1 [catch {$c1 read} r1] + set s2 [catch {$c2 read} r2] + $c1 close + $c2 close + + assert {($s1 == 0) != ($s2 == 0)} + wait_for_condition 100 100 { + [CI 0 cluster_size] == 5 + } else { + fail "size=[CI 0 cluster_size]" + } + after 1000 + assert_equal 5 [CI 0 cluster_size] + } +} + +} ;# tags diff --git a/tests/unit/cluster/cluster-raft-meet.tcl b/tests/unit/cluster/cluster-raft-meet.tcl index 7068e5c93d6..edfb031bade 100644 --- a/tests/unit/cluster/cluster-raft-meet.tcl +++ b/tests/unit/cluster/cluster-raft-meet.tcl @@ -1,22 +1,18 @@ # Test CLUSTER MEET scenarios for the Raft cluster protocol. -source tests/support/cluster_raft.tcl - tags {external:skip cluster singledb} { start_multiple_servers 2 {overrides {cluster-enabled yes cluster-protocol raft}} { test "Raft MEET: two singletons" { set r0 [srv 0 client] - set r1 [srv -1 client] $r0 CLUSTER MEET [srv -1 host] [srv -1 port] - raft_add_voter $r0 [$r1 CLUSTER MYID] wait_for_condition 50 100 { - [get_cluster_info_field $r0 cluster_size] == 2 && - [get_cluster_info_field $r1 cluster_size] == 2 + [CI 0 cluster_size] == 2 && + [CI 1 cluster_size] == 2 } else { - fail "Cluster size: [get_cluster_info_field $r0 cluster_size] [get_cluster_info_field $r1 cluster_size]" + fail "Cluster size: [CI 0 cluster_size] [CI 1 cluster_size]" } } } @@ -28,16 +24,13 @@ start_multiple_servers 5 {overrides {cluster-enabled yes cluster-protocol raft}} for {set i 1} {$i < 5} {incr i} { $r0 CLUSTER MEET [srv -$i host] [srv -$i port] } - for {set i 1} {$i < 5} {incr i} { - raft_add_voter $r0 [[srv -$i client] CLUSTER MYID] - } wait_for_condition 100 200 { - [get_cluster_info_field $r0 cluster_size] == 5 && - [get_cluster_info_field [srv -1 client] cluster_size] == 5 && - [get_cluster_info_field [srv -4 client] cluster_size] == 5 + [CI 0 cluster_size] == 5 && + [CI 1 cluster_size] == 5 && + [CI 4 cluster_size] == 5 } else { - fail "Sizes: [get_cluster_info_field $r0 cluster_size] [get_cluster_info_field [srv -1 client] cluster_size] [get_cluster_info_field [srv -4 client] cluster_size]" + fail "Sizes: [CI 0 cluster_size] [CI 1 cluster_size] [CI 4 cluster_size]" } } } @@ -47,18 +40,13 @@ start_multiple_servers 5 {overrides {cluster-enabled yes cluster-protocol raft}} for {set i 1} {$i < 5} {incr i} { [srv -$i client] CLUSTER MEET [srv 0 host] [srv 0 port] } - # The first initiator becomes leader; promote all joined learners there. - set leader [srv -1 client] - foreach idx {0 -1 -2 -3 -4} { - raft_add_voter $leader [[srv $idx client] CLUSTER MYID] - } wait_for_condition 100 200 { - [get_cluster_info_field [srv 0 client] cluster_size] == 5 && - [get_cluster_info_field [srv -1 client] cluster_size] == 5 && - [get_cluster_info_field [srv -4 client] cluster_size] == 5 + [CI 0 cluster_size] == 5 && + [CI 1 cluster_size] == 5 && + [CI 4 cluster_size] == 5 } else { - fail "Sizes: [get_cluster_info_field [srv 0 client] cluster_size] [get_cluster_info_field [srv -1 client] cluster_size] [get_cluster_info_field [srv -4 client] cluster_size]" + fail "Sizes: [CI 0 cluster_size] [CI 1 cluster_size] [CI 4 cluster_size]" } } } @@ -70,16 +58,13 @@ start_multiple_servers 5 {overrides {cluster-enabled yes cluster-protocol raft}} for {set i 0} {$i < 4} {incr i} { [srv -$i client] CLUSTER MEET [srv -[expr {$i+1}] host] [srv -[expr {$i+1}] port] } - for {set i 1} {$i < 5} {incr i} { - raft_add_voter $r0 [[srv -$i client] CLUSTER MYID] - } wait_for_condition 100 200 { - [get_cluster_info_field $r0 cluster_size] == 5 && - [get_cluster_info_field [srv -1 client] cluster_size] == 5 && - [get_cluster_info_field [srv -4 client] cluster_size] == 5 + [CI 0 cluster_size] == 5 && + [CI 1 cluster_size] == 5 && + [CI 4 cluster_size] == 5 } else { - fail "Sizes: [get_cluster_info_field $r0 cluster_size] [get_cluster_info_field [srv -1 client] cluster_size] [get_cluster_info_field [srv -4 client] cluster_size]" + fail "Sizes: [CI 0 cluster_size] [CI 1 cluster_size] [CI 4 cluster_size]" } } } @@ -87,12 +72,10 @@ start_multiple_servers 5 {overrides {cluster-enabled yes cluster-protocol raft}} start_multiple_servers 2 {overrides {cluster-enabled yes cluster-protocol raft}} { test "Raft MEET: addslots after meet" { set r0 [srv 0 client] - set r1 [srv -1 client] $r0 CLUSTER MEET [srv -1 host] [srv -1 port] - raft_add_voter $r0 [$r1 CLUSTER MYID] wait_for_condition 50 100 { - [get_cluster_info_field $r0 cluster_size] == 2 + [CI 0 cluster_size] == 2 } else { fail "Cluster did not form" } @@ -100,14 +83,14 @@ start_multiple_servers 2 {overrides {cluster-enabled yes cluster-protocol raft}} $r0 CLUSTER ADDSLOTSRANGE 0 16383 wait_for_condition 50 100 { - [get_cluster_info_field $r0 cluster_slots_assigned] == 16384 && - [get_cluster_info_field $r1 cluster_slots_assigned] == 16384 + [CI 0 cluster_slots_assigned] == 16384 && + [CI 1 cluster_slots_assigned] == 16384 } else { - fail "Slots: [get_cluster_info_field $r0 cluster_slots_assigned] [get_cluster_info_field $r1 cluster_slots_assigned]" + fail "Slots: [CI 0 cluster_slots_assigned] [CI 1 cluster_slots_assigned]" } - assert_equal ok [get_cluster_info_field $r0 cluster_state] - assert_equal ok [get_cluster_info_field $r1 cluster_state] + assert_equal ok [CI 0 cluster_state] + assert_equal ok [CI 1 cluster_state] } } @@ -115,13 +98,11 @@ start_multiple_servers 4 {overrides {cluster-enabled yes cluster-protocol raft}} test "Raft MEET: merging two clusters is rejected" { # Form two separate 2-node clusters. [srv 0 client] CLUSTER MEET [srv -1 host] [srv -1 port] - raft_add_voter [srv 0 client] [[srv -1 client] CLUSTER MYID] [srv -2 client] CLUSTER MEET [srv -3 host] [srv -3 port] - raft_add_voter [srv -2 client] [[srv -3 client] CLUSTER MYID] wait_for_condition 50 100 { - [get_cluster_info_field [srv 0 client] cluster_size] == 2 && - [get_cluster_info_field [srv -2 client] cluster_size] == 2 + [CI 0 cluster_size] == 2 && + [CI 2 cluster_size] == 2 } else { fail "Two clusters did not form" } diff --git a/tests/unit/cluster/cluster-raft-prevote.tcl b/tests/unit/cluster/cluster-raft-prevote.tcl index 470f22fb067..e4d3e22a2cf 100644 --- a/tests/unit/cluster/cluster-raft-prevote.tcl +++ b/tests/unit/cluster/cluster-raft-prevote.tcl @@ -1,15 +1,9 @@ # Test pre-vote behavior in real raft clusters. -source tests/support/cluster_raft.tcl - tags {external:skip cluster singledb} { start_cluster 3 0 {overrides {cluster-protocol raft cluster-node-timeout 500}} { test "Raft Cluster: leader failure, successful pre-vote and election" { - set r0 [srv 0 client] - raft_add_voter $r0 [[srv -1 client] CLUSTER MYID] - raft_add_voter $r0 [[srv -2 client] CLUSTER MYID] - wait_for_condition 50 100 { [CI 0 cluster_size] == 3 && [CI 1 cluster_size] == 3 && diff --git a/tests/unit/cluster/cluster-raft-proto.tcl b/tests/unit/cluster/cluster-raft-proto.tcl index ac077bed32b..246b2db2c09 100644 --- a/tests/unit/cluster/cluster-raft-proto.tcl +++ b/tests/unit/cluster/cluster-raft-proto.tcl @@ -2,114 +2,10 @@ # A single valkey-server is started and we connect to its cluster bus port, # speaking the raft wire protocol directly. -# Build a raft wire message: "RAFT" + 4-byte big-endian length + payload. -proc raft_msg {payload} { - set len [expr {8 + [string length $payload]}] - set hdr "RAFT" - append hdr [binary format I $len] - append hdr $payload - return $hdr -} - -# Connect to a node's cluster bus port and return the socket. -proc raft_connect {host port} { - set fd [socket $host $port] - fconfigure $fd -translation binary -buffering full - return $fd -} - -# Send a raft message on a cluster bus connection. -proc raft_send {fd payload} { - puts -nonewline $fd [raft_msg $payload] - flush $fd -} - -# Read a raft message from a cluster bus connection. Returns the payload. -proc raft_recv {fd {timeout 5000}} { - # Read 8-byte header - fconfigure $fd -blocking 0 - set deadline [expr {[clock milliseconds] + $timeout}] - set hdr "" - while {[string length $hdr] < 8} { - append hdr [read $fd [expr {8 - [string length $hdr]}]] - if {[string length $hdr] < 8} { - if {[clock milliseconds] > $deadline} { - error "timeout reading raft header" - } - after 10 - } - } - if {[string range $hdr 0 3] ne "RAFT"} { - error "bad raft header: [string range $hdr 0 3]" - } - binary scan [string range $hdr 4 7] I totlen - set paylen [expr {$totlen - 8}] - set payload "" - while {[string length $payload] < $paylen} { - append payload [read $fd [expr {$paylen - [string length $payload]}]] - if {[string length $payload] < $paylen} { - if {[clock milliseconds] > $deadline} { - error "timeout reading raft payload (got [string length $payload]/$paylen)" - } - after 10 - } - } - return $payload -} +source tests/support/cluster_raft.tcl tags {tls:skip external:skip cluster singledb} { -# Listen on a random port, run optional setup code, accept one connection, close -# the listener. Returns the accepted client fd. Sets the listen port in the -# upvar port_var before running the setup code. -proc raft_listen_and_accept {port_var {timeout 5000} {before_accept {}}} { - upvar $port_var listen_port - set ::_raft_accepted "" - proc _raft_on_accept {fd addr port} { - fconfigure $fd -translation binary -buffering full - set ::_raft_accepted $fd - } - set listen_fd [socket -server _raft_on_accept -myaddr 127.0.0.1 0] - set listen_port [lindex [fconfigure $listen_fd -sockname] 2] - if {$before_accept ne {}} { - uplevel 1 $before_accept - } - set accept_after [after $timeout {set ::_raft_accepted timeout}] - vwait ::_raft_accepted - after cancel $accept_after - close $listen_fd - if {$::_raft_accepted eq "timeout"} { - error "timeout waiting for connection" - } - return $::_raft_accepted -} - -# Connect a fake node to a cluster bus port and complete HELLO handshake. -# Returns the connected fd. -proc raft_connect_fake_node {host cport fake_id fake_addr} { - set fd [raft_connect $host $cport] - raft_send $fd "HELLO $fake_id $fake_addr" - set reply [raft_recv $fd] - assert_match "HI *" $reply - return $fd -} - -# Parse an AE message and reply with AE_ACK. -# Returns the last log index after applying the entries. -proc raft_reply_ae_ack {fd ae_msg repl_offset} { - set lines [split $ae_msg "\n"] - set fields [split [lindex $lines 0] " "] - # AE - set term [lindex $fields 2] - set prev_idx [lindex $fields 3] - set count [lindex $fields 6] - set last_index [expr {$prev_idx + $count}] - raft_send $fd "AE_ACK $term 1 $last_index $repl_offset" - return $last_index -} - -source tests/support/cluster_raft.tcl - test "Raft proto: connect to cluster bus and exchange HELLO" { start_server {overrides {cluster-enabled yes cluster-protocol raft}} { set port [srv 0 port] @@ -235,7 +131,7 @@ test "Raft proto: learner join, voter promotion, demotion, and forget" { set reply [raft_recv $fd] assert_match "AE_ACK 1 1 2 *" $reply assert_equal 1 [CI 0 cluster_size] - assert_match "*learner*" [raft_cluster_nodes_line [srv 0 client] $learner_id] + assert {[cluster_has_flag [cluster_get_node_by_id 0 $learner_id] learner]} R 0 CLUSTER SAVECONFIG set nodes_conf "[lindex [R 0 CONFIG GET dir] 1]/nodes.conf" set fp [open $nodes_conf r] @@ -249,7 +145,7 @@ test "Raft proto: learner join, voter promotion, demotion, and forget" { set reply [raft_recv $fd] assert_match "AE_ACK 1 1 3 *" $reply assert_equal 2 [CI 0 cluster_size] - assert {![string match "*learner*" [raft_cluster_nodes_line [srv 0 client] $learner_id]]} + assert {![cluster_has_flag [cluster_get_node_by_id 0 $learner_id] learner]} set ae "AE $leader_id 1 3 1 4 1\n" append ae "1 DEL_VOTER $learner_id" @@ -257,7 +153,7 @@ test "Raft proto: learner join, voter promotion, demotion, and forget" { set reply [raft_recv $fd] assert_match "AE_ACK 1 1 4 *" $reply assert_equal 1 [CI 0 cluster_size] - assert_match "*learner*" [raft_cluster_nodes_line [srv 0 client] $learner_id] + assert {[cluster_has_flag [cluster_get_node_by_id 0 $learner_id] learner]} set ae "AE $leader_id 1 4 1 5 1\n" append ae "1 NODE_FORGET $learner_id 0" @@ -265,7 +161,7 @@ test "Raft proto: learner join, voter promotion, demotion, and forget" { set reply [raft_recv $fd] assert_match "AE_ACK 1 1 5 *" $reply assert_equal 1 [CI 0 cluster_size] - assert_equal "" [raft_cluster_nodes_line [srv 0 client] $learner_id] + assert_equal {} [cluster_get_node_by_id 0 $learner_id] close $fd } @@ -596,7 +492,6 @@ test "Raft proto: leader sends REPL_OFFSETS after follower offset changes" { start_multiple_servers 2 {overrides {cluster-enabled yes cluster-protocol raft cluster-node-timeout 2000 loglevel debug}} { # Shared setup: form cluster and assign slots. R 0 CLUSTER MEET [srv -1 host] [srv -1 port] - raft_add_voter [srv 0 client] [R 1 CLUSTER MYID] wait_for_condition 50 100 { [CI 0 cluster_size] == 2 && diff --git a/tests/unit/cluster/cluster-raft.tcl b/tests/unit/cluster/cluster-raft.tcl index 567aa17fad2..062e3c4d514 100644 --- a/tests/unit/cluster/cluster-raft.tcl +++ b/tests/unit/cluster/cluster-raft.tcl @@ -1,8 +1,6 @@ # Test higher-level Raft cluster behavior that does not require direct # wire-protocol interaction from Tcl. -source tests/support/cluster_raft.tcl - tags {external:skip cluster singledb} { test "Raft: leader steps down after losing quorum freshness" { @@ -10,8 +8,6 @@ test "Raft: leader steps down after losing quorum freshness" { set r0 [srv 0 client] $r0 CLUSTER MEET [srv -1 host] [srv -1 port] $r0 CLUSTER MEET [srv -2 host] [srv -2 port] - raft_add_voter $r0 [[srv -1 client] CLUSTER MYID] - raft_add_voter $r0 [[srv -2 client] CLUSTER MYID] wait_for_condition 50 100 { [CI 0 cluster_size] == 3 && diff --git a/tests/unit/cluster/failover2.tcl b/tests/unit/cluster/failover2.tcl index 0fa1e6a4ec9..2aa9202d216 100644 --- a/tests/unit/cluster/failover2.tcl +++ b/tests/unit/cluster/failover2.tcl @@ -102,7 +102,7 @@ start_cluster 7 3 {tags {external:skip cluster cluster-raft:skip} overrides {clu } ;# start_cluster run_solo {cluster} { - start_cluster 32 15 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-node-timeout 15000}} { + start_cluster 32 15 {tags {external:skip cluster cluster-raft:skip} overrides {cluster-ping-interval 1000 cluster-node-timeout 15000}} { test "Multiple primary nodes are down, rank them based on the failed primary" { # Killing these primary nodes. for {set j 0} {$j < 15} {incr j} {