From d9232393c9f17d39a059053cb2460ae386b12eec Mon Sep 17 00:00:00 2001 From: Jim Brunner Date: Tue, 10 Mar 2026 10:31:59 +0000 Subject: [PATCH 01/18] New command flag to indicate modification of first key only Signed-off-by: Jim Brunner --- src/commands.h | 1 + src/module.c | 2 ++ src/server.c | 23 ++++++++++++++++ src/server.h | 4 +++ src/unit/test_cmdflags.cpp | 56 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+) create mode 100644 src/unit/test_cmdflags.cpp diff --git a/src/commands.h b/src/commands.h index eef77e6c560..dde1d4a2bf2 100644 --- a/src/commands.h +++ b/src/commands.h @@ -57,6 +57,7 @@ typedef enum { #define CMD_MODULE_GETCHANNELS (1ULL << 27) /* Use the modules getchannels interface. */ #define CMD_TOUCHES_ARBITRARY_KEYS (1ULL << 28) #define CMD_ALL_DBS (1ULL << 29) +#define CMD_WRITE_FIRSTKEY_ONLY (1ULL << 30) /* Command flags. Please don't forget to add command flag documentation in struct * serverCommand in server.h file. */ diff --git a/src/module.c b/src/module.c index 377a0c7f4d7..eecff84b112 100644 --- a/src/module.c +++ b/src/module.c @@ -2115,6 +2115,8 @@ int VM_SetCommandInfo(ValkeyModuleCommand *command, const ValkeyModuleCommandInf /* Update the legacy (first,last,step) spec and "movablekeys" flag used by the COMMAND command, * by trying to "glue" consecutive range key specs. */ populateCommandLegacyRangeSpec(cmd); + + detectWriteFirstkeyOnlyCommand(cmd); } if (info->args) { diff --git a/src/server.c b/src/server.c index 3f68250339e..b8a35af8e26 100644 --- a/src/server.c +++ b/src/server.c @@ -3384,6 +3384,27 @@ void commandAddSubcommand(struct serverCommand *parent, struct serverCommand *su serverAssert(hashtableAdd(parent->subcommands_ht, subcommand)); } +/* Automatically set CMD_WRITE_FIRSTKEY_ONLY for write commands where the first + * key is written, and other keys are read only. */ +void detectWriteFirstkeyOnlyCommand(struct serverCommand *c) { + c->flags &= ~CMD_WRITE_FIRSTKEY_ONLY; // Override if set elsewhere + if (!(c->flags & CMD_WRITE)) return; + if (c->key_specs_num < 2) return; + if (!(c->key_specs[0].flags & (CMD_KEY_OW | CMD_KEY_RW))) return; + if (c->key_specs[0].find_keys_type != KSPEC_FK_RANGE) return; + if (c->key_specs[0].fk.range.lastkey != 0) return; + + bool write_first_key_only = true; + for (int i = 1; i < c->key_specs_num; i++) { + if (!(c->key_specs[i].flags & CMD_KEY_RO) || (c->key_specs[i].flags & (CMD_KEY_RW | CMD_KEY_OW | CMD_KEY_RM))) { + write_first_key_only = false; + break; + } + } + + if (write_first_key_only) c->flags |= CMD_WRITE_FIRSTKEY_ONLY; +} + /* Recursively populate the command structure. * * On success, the function return C_OK. Otherwise, C_ERR is returned and we won't @@ -3407,6 +3428,8 @@ int populateCommandStructure(struct serverCommand *c) { /* Handle the legacy range spec and the "movablekeys" flag (must be done after populating all key specs). */ populateCommandLegacyRangeSpec(c); + detectWriteFirstkeyOnlyCommand(c); + /* Assign the ID used for ACL. */ c->id = ACLGetCommandID(c->fullname); diff --git a/src/server.h b/src/server.h index d34fed6851b..ece04824165 100644 --- a/src/server.h +++ b/src/server.h @@ -2675,6 +2675,9 @@ typedef int *commandDbIdArgs(robj **argv, int argc, int *count); * * CMD_ALL_DBS: The command works with all databases. * + * CMD_WRITE_FIRSTKEY_ONLY: The command must be CMD_WRITE. It only modifies the first key. + * Other keys are read-only. Example: SUNIONSTORE + * * The following additional flags are only used in order to put commands * in a specific ACL category. Commands can have multiple ACL categories. * See valkey.conf for the exact meaning of each. @@ -2871,6 +2874,7 @@ extern list *modules; /* Command metadata */ void populateCommandLegacyRangeSpec(struct serverCommand *c); +void detectWriteFirstkeyOnlyCommand(struct serverCommand *c); /* Utils */ mstime_t commandTimeSnapshot(void); diff --git a/src/unit/test_cmdflags.cpp b/src/unit/test_cmdflags.cpp new file mode 100644 index 00000000000..6c324c9e7ac --- /dev/null +++ b/src/unit/test_cmdflags.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "server.h" +} +extern hashtableType commandSetType; +extern hashtableType originalCommandSetType; + + +class CmdFlagsTest : public ::testing::Test { + protected: + void SetUp() override { + server.commands = hashtableCreate(&commandSetType); + server.orig_commands = hashtableCreate(&originalCommandSetType); + populateCommandTable(); + } +}; + + +TEST_F(CmdFlagsTest, TestWriteFirstkeyOnly) { + /* Each command with this flag is explicitly listed here to ensure: + * - new commands are not mistakenly detected as write firstkey only + * - commands which should be write firstkey only are detected. */ + const char *const writeFirstkeyCommands[] = { + "bitop", "geosearchstore", "pfmerge", "sdiffstore", "sinterstore", + "sunionstore", "zdiffstore", "zinterstore", "zrangestore", "zunionstore"}; + int expectedCount = sizeof(writeFirstkeyCommands) / sizeof(char *); + + int count = 0; + + hashtableIterator iter; + hashtableInitIterator(&iter, server.commands, 0); + struct serverCommand *c; + while (hashtableNext(&iter, (void **)&c)) { + if (c->flags & CMD_WRITE_FIRSTKEY_ONLY) { + count++; + bool found = false; + for (int i = 0; i < expectedCount; i++) { + if (strcmp(c->declared_name, writeFirstkeyCommands[i]) == 0) { + found = true; + break; + } + } + EXPECT_TRUE(found); + } + } + hashtableCleanupIterator(&iter); + + EXPECT_EQ(count, expectedCount); +} From bf9312edc6d09f45a24718fa2f24a0a7cfc38cf1 Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Tue, 21 Apr 2026 11:15:07 -0700 Subject: [PATCH 02/18] Client blocking mechanism for keys in use (forkless) A client blocking system (blockInUse) that prevents concurrent access to keys actively being modified by internal operations (e.g., bgIteration). The mechanism blocks clients attempting to access in-use keys and automatically unblocks them when keys become available. When a client is blocked by blockInUse, its read handler is removed so the event loop stops monitoring read events for that connection, preventing new commands from being buffered into c->querybuf while the client is waiting. The read handler is restored in processUnblockedClients() when the client is unblocked. To avoid leaking zombie file descriptors, clientsCronTcpIsClosing() is added to detect and free connections that were closed by the remote side while the read handler was removed. Signed-off-by: harrylin98 Signed-off-by: Jim Brunner --- src/blocked.c | 252 ++++++++++++++++++++++++-- src/connection.h | 12 +- src/module.c | 9 + src/networking.c | 7 +- src/rdma.c | 1 + src/server.c | 21 +++ src/server.h | 4 + src/socket.c | 24 +++ src/tls.c | 1 + src/unit/test_blocked.cpp | 369 ++++++++++++++++++++++++++++++++++++++ src/unit/wrappers.h | 2 + src/unix.c | 1 + 12 files changed, 688 insertions(+), 15 deletions(-) create mode 100644 src/unit/test_blocked.cpp diff --git a/src/blocked.c b/src/blocked.c index a8451a17f6c..9f6a0374823 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -77,6 +77,7 @@ static void handleClientsBlockedOnKey(readyList *rl); static void unblockClientOnKey(client *c, robj *key); static void moduleUnblockClientOnKey(client *c, robj *key); static void releaseBlockedEntry(client *c, dictEntry *de, int remove_key); +static void unlinkBlockInUseClient(client *c); void initClientBlockingState(client *c) { if (c->bstate) return; @@ -164,6 +165,7 @@ void processUnblockedClients(void) { serverAssert(ln != NULL); c = ln->value; listDelNode(server.unblocked_clients, ln); + serverAssert(c->flag.module || !c->flag.blocked); c->flag.unblocked = 0; if (c->flag.module) { @@ -173,16 +175,16 @@ void processUnblockedClients(void) { continue; } - /* Process remaining data in the input buffer, unless the client - * is blocked again. Actually processInputBuffer() checks that the - * client is not blocked before to proceed, but things may change and - * the code is conceptually more correct this way. */ - if (!c->flag.blocked) { - /* If we have a queued command, execute it now. */ - if (processPendingCommandAndInputBuffer(c) == C_ERR) { + if (c->conn && !connHasReadHandler(c->conn)) { + if (connSetReadHandler(c->conn, readQueryFromClient) == C_ERR) { + freeClient(c); continue; } } + /* If we have a queued command, execute it now. */ + if (processPendingCommandAndInputBuffer(c) == C_ERR) { + continue; + } beforeNextClient(c); } } @@ -215,20 +217,31 @@ void queueClientForReprocessing(client *c) { /* Unblock a client calling the right function depending on the kind * of operation the client is blocking for. */ void unblockClient(client *c, int queue_for_reprocessing) { - if (c->bstate->btype == BLOCKED_LIST || c->bstate->btype == BLOCKED_ZSET || c->bstate->btype == BLOCKED_STREAM) { + switch (c->bstate->btype) { + case BLOCKED_LIST: + case BLOCKED_ZSET: + case BLOCKED_STREAM: unblockClientWaitingData(c); - } else if (c->bstate->btype == BLOCKED_WAIT) { + break; + case BLOCKED_WAIT: unblockClientWaitingReplicas(c); - } else if (c->bstate->btype == BLOCKED_MODULE) { + break; + case BLOCKED_MODULE: if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c); unblockClientFromModule(c); - } else if (c->bstate->btype == BLOCKED_POSTPONE) { + break; + case BLOCKED_POSTPONE: serverAssert(c->bstate->postponed_list_node); listDelNode(server.postponed_clients, c->bstate->postponed_list_node); c->bstate->postponed_list_node = NULL; - } else if (c->bstate->btype == BLOCKED_SHUTDOWN) { + break; + case BLOCKED_SHUTDOWN: /* No special cleanup. */ - } else { + break; + case BLOCKED_INUSE: + unlinkBlockInUseClient(c); + break; + default: serverPanic("Unknown btype in unblockClient()."); } @@ -338,6 +351,10 @@ void disconnectOrRedirectAllBlockedClients(void) { * which the command is already in progress in a way. */ if (c->bstate->btype == BLOCKED_POSTPONE) continue; + /* BLOCKED_INUSE clients will reprocess their command when unblocked + * by the caller. Sending error replies here would be incorrect. */ + if (c->bstate->btype == BLOCKED_INUSE) continue; + if (server.cluster_enabled) { if (clusterRedirectBlockedClientIfNeeded(c)) unblockClientOnError(c, NULL); @@ -824,6 +841,197 @@ void unblockClientOnError(client *c, const char *err_str) { unblockClient(c, 1); } +/* ========================== BlockInUse ==================================== + * + * Client blocking mechanism for keys currently being processed by a + * background thread. + * + * Note: All blockInUse APIs must be called from the main thread only. + * + * This uses the BLOCKED_INUSE blocking type (via blockClient()) and tracks + * blocked keys per client in c->bstate->keys and a key→clients mapping in + * a static hashtable (inuse_key_to_clients). + * + * Workflow: + * 1. blockClientInUseOnKeys() blocks the client via + * blockClient(c, BLOCKED_INUSE) and records mappings in both + * c->bstate->keys and inuse_key_to_clients. + * 2. unblockClientsInUseOnKey() unblocks a single key. A client remains + * blocked until all its keys are unblocked. + * 3. A client is fully unblocked only when it has no remaining keys in its + * c->bstate->keys dict. + * 4. processUnblockedClients() restores the read handler and resumes the + * pending command. + */ + +/* Internal blockInUse data structures. + * + * Clients are blocked on key name, regardless of DB. This avoids complexity + * with DB swaps. A BLOCKED_INUSE client may get unblocked early due to + * unblocking a key with the same name on a different DB. In this case, the + * client will get reblocked when attempting to reprocess the command. */ +static hashtable *inuse_key_to_clients; /* Maps keys to keyToClientsEntry. */ + +/* ----------------------------- key_to_clients Hashtable Util ------------------------- */ + +typedef struct { + robj *key; + list *clients; +} keyToClientsEntry; + +// hashtable callback, returns an robj containing a string +static const void *keyToClientsGetKey(const void *entry) { + return ((keyToClientsEntry *)entry)->key; +} + +// hashtable callback +static void keyToClientsDestructor(void *entry) { + keyToClientsEntry *e = entry; + decrRefCount(e->key); + listRelease(e->clients); + zfree(e); +} + +static hashtableType keyToClientsHashtableType = { + .entryGetKey = keyToClientsGetKey, + .hashFunction = dictEncObjHash, + .keyCompare = dictEncObjKeyCompare, + .entryDestructor = keyToClientsDestructor, +}; + +// Return the list of clients blocked on key, or NULL if none exist. +static list *keyToClients_getBlockedClientsList(robj *key) { + keyToClientsEntry *entry; + if (hashtableFind(inuse_key_to_clients, key, (void **)&entry)) { + return entry->clients; + } + return NULL; +} + +/* Create a new keyToClientsEntry for key, add it to key_to_clients, + * and return its clients list. Precondition: the key must not already exist. */ +static list *keyToClients_addEntry(robj *key) { + keyToClientsEntry *entry = zcalloc(sizeof(keyToClientsEntry)); + entry->key = key; + incrRefCount(key); + entry->clients = listCreate(); + serverAssert(hashtableAdd(inuse_key_to_clients, entry)); + return entry->clients; +} + +/* ----------------------------- blockInUse API ----------------------------- */ + +static bool isClientBlockedInUse(client *c) { + return c->flag.blocked && c->bstate->btype == BLOCKED_INUSE; +} + +/* Block a client on a set of keys. Duplicate keys are deduplicated. + * + * Each key robj must contain an sds string value. Keys are simple names, + * independent of DB — a client may be unblocked early if the same key name + * in another DB is unblocked. + * + * The client remains blocked until ALL of its keys are unblocked via + * unblockClientsInUseOnKey(). + * + * The caller MUST set c->flag.pending_command = 1 before calling this function. + * This ensures the pending command is executed when the client is later + * unblocked via processPendingCommandAndInputBuffer(). + * The caller should then return without executing the command. */ +void blockClientInUseOnKeys(client *c, int num_keys, robj *keys[]) { + serverAssert(!c->flag.blocked && !c->flag.unblocked); + serverAssert(c->flag.pending_command == 1); + serverAssert(num_keys > 0); + serverAssert(!c->flag.replica); + + if (!inuse_key_to_clients) inuse_key_to_clients = hashtableCreate(&keyToClientsHashtableType); + + initClientBlockingState(c); + c->bstate->timeout = 0; + serverAssert(dictSize(c->bstate->keys) == 0); + + for (int i = 0; i < num_keys; ++i) { + robj *key = keys[i]; + serverAssert(key->type == OBJ_STRING); + + /* Deduplicate via bstate->keys dict */ + if (dictAdd(c->bstate->keys, key, NULL) != DICT_OK) continue; + incrRefCount(key); + + list *blockedClientsList = keyToClients_getBlockedClientsList(key); + if (!blockedClientsList) blockedClientsList = keyToClients_addEntry(key); + listAddNodeTail(blockedClientsList, c); + } + + serverAssert(dictSize(c->bstate->keys) > 0); + blockClient(c, BLOCKED_INUSE); + + /* Disable client's Read Handler to prevent reading commands while blocked */ + if (c->conn) { + connSetReadHandler(c->conn, NULL); + } +} + +/* Unblock clients blocked on the given key. + * + * A client is fully unblocked only when it has no remaining keys in its + * bstate->keys dict. Such clients are queued for reprocessing and resumed + * later during processUnblockedClients(). */ +void unblockClientsInUseOnKey(robj *key) { + list *blockedClientsList = keyToClients_getBlockedClientsList(key); + if (blockedClientsList == NULL) return; + + serverAssert(listLength(blockedClientsList) > 0); + + while (listLength(blockedClientsList) > 0) { + listNode *ln = listFirst(blockedClientsList); + client *c = listNodeValue(ln); + serverAssert(isClientBlockedInUse(c) && c->flag.unblocked == 0); + listDelNode(blockedClientsList, ln); + dictDelete(c->bstate->keys, key); + + if (dictSize(c->bstate->keys) == 0) { + unblockClient(c, 1); + } + } + + hashtableDelete(inuse_key_to_clients, key); +} + +/* Unblock all clients that are currently blocked by blockInUse, across all + * keys. Unblocked clients are queued for reprocessing and resumed during + * processUnblockedClients(). After this call, no clients remain blocked + * by blockInUse. */ +void unblockClientsInUseOnAllKeys(void) { + if (!inuse_key_to_clients) return; + hashtableIterator iter; + hashtableInitIterator(&iter, inuse_key_to_clients, HASHTABLE_ITER_SAFE); + keyToClientsEntry *e; + while (hashtableNext(&iter, (void **)&e)) { + unblockClientsInUseOnKey(e->key); + } + hashtableCleanupIterator(&iter); + serverAssert(server.blocked_clients_by_type[BLOCKED_INUSE] == 0); + serverAssert(hashtableSize(inuse_key_to_clients) == 0); +} + +/* Remove a client from all blockInUse key-to-clients mappings. + * Called from unblockClient() for BLOCKED_INUSE cleanup. */ +static void unlinkBlockInUseClient(client *c) { + if (!c->bstate->keys || dictSize(c->bstate->keys) == 0) return; + dictIterator *di = dictGetIterator(c->bstate->keys); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + robj *key = dictGetKey(de); + list *clientList = keyToClients_getBlockedClientsList(key); + serverAssert(clientList != NULL); + listDelNode(clientList, listSearchKey(clientList, c)); + if (listLength(clientList) == 0) hashtableDelete(inuse_key_to_clients, key); + } + dictReleaseIterator(di); + dictEmpty(c->bstate->keys, NULL); +} + void blockedBeforeSleep(void) { /* Handle precise timeouts of blocked clients. */ handleBlockedClientsTimeout(); @@ -847,3 +1055,21 @@ void blockedBeforeSleep(void) { /* Try to process pending commands for clients that were just unblocked. */ if (listLength(server.unblocked_clients)) processUnblockedClients(); } + +/* -------------------------------------------------------------------------- + * Test-only APIs for blockInUse + * -------------------------------------------------------------------------- */ + +/* Test-only: get the current number of blocked keys by blockInUse. */ +int getBlockInUseKeyCount(void) { + return inuse_key_to_clients ? hashtableSize(inuse_key_to_clients) : 0; +} + +/* Test-only: release the blockInUse hashtable. */ +void releaseBlockInUse(void) { + unblockClientsInUseOnAllKeys(); + if (inuse_key_to_clients) { + hashtableRelease(inuse_key_to_clients); + inuse_key_to_clients = NULL; + } +} diff --git a/src/connection.h b/src/connection.h index 5527ee2a769..60ed4572d30 100644 --- a/src/connection.h +++ b/src/connection.h @@ -160,7 +160,8 @@ typedef struct ConnectionType { struct user *(*get_peer_user)(connection *conn, sds *cert_username); /* Miscellaneous */ - int (*connIntegrityChecked)(void); // return 1 if connection type has built-in integrity checks + int (*connIntegrityChecked)(void); // return 1 if connection type has built-in integrity checks + int (*is_closing)(connection *conn); // return 1 if connection is closed } ConnectionType; struct connection { @@ -397,6 +398,15 @@ static inline int connHasReadHandler(connection *conn) { return conn->read_handler != NULL; } +/* Check if the remote side has closed the connection. */ +static inline int connIsClosing(connection *conn) { + if (!conn->type->is_closing) return 0; + return conn->type->is_closing(conn); +} + +/* Shared is_closing implementation for socket-based connections. */ +int connSocketIsClosing(connection *conn); + /* Associate a private data pointer with the connection */ static inline void connSetPrivateData(connection *conn, void *data) { conn->private_data = data; diff --git a/src/module.c b/src/module.c index eecff84b112..93a2fcefc6c 100644 --- a/src/module.c +++ b/src/module.c @@ -6972,6 +6972,15 @@ static void moduleCallCommandHelper(ValkeyModuleCtx *ctx, client *c, robj **argv server.replication_allowed = prev_replication_allowed; if (c->flag.blocked) { + if (c->flag.deny_blocking) { + /* The module did not pass ALLOW_BLOCK — it does not expect the + * command to block. Unblock the client and return an error. */ + c->flag.pending_command = 0; + unblockClient(c, 0); + addReplyError(c, "INUSE key is being processed."); + goto cleanup; + } + /* Blocking commands are not allowed when calling commands in scripting engines. */ serverAssert(!is_running_script); serverAssert(flags & VALKEYMODULE_CALL_ARGV_ALLOW_BLOCK); diff --git a/src/networking.c b/src/networking.c index abf29985a43..0d85bbc1500 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2084,6 +2084,10 @@ void unlinkClient(client *c) { /* Clear the tracking status. */ if (c->flag.tracking) disableTracking(c); + + /* Client must not be in blocked or unblocked state at this point. + * Guaranteed by freeClient ordering: unblockClient -> freeClientBlockingState -> unlinkClient. */ + serverAssert(!c->flag.blocked && !c->flag.unblocked); } /* Clear the client state to resemble a newly connected client. */ @@ -2214,7 +2218,7 @@ int freeClient(client *c) { /* Deallocate structures used to block on blocking ops. */ /* If there is any in-flight command, we don't record their duration. */ c->duration = 0; - if (c->flag.blocked) unblockClient(c, 1); + if (c->flag.blocked) unblockClient(c, 0); freeClientBlockingState(c); freeClientPubSubData(c); @@ -3998,6 +4002,7 @@ int processPendingCommandAndInputBuffer(client *c) { * But in case of a module blocked client (see RM_Call 'K' flag) we do not reach this code path. * So whenever we change the code here we need to consider if we need this change on module * blocked client as well */ + if (c->flag.close_asap) return C_ERR; if (c->flag.pending_command) { c->flag.pending_command = 0; if (processCommandAndResetClient(c) == C_ERR) { diff --git a/src/rdma.c b/src/rdma.c index 198721021a3..77ef6d6ffe3 100644 --- a/src/rdma.c +++ b/src/rdma.c @@ -1865,6 +1865,7 @@ static ConnectionType CT_RDMA = { /* Miscellaneous */ .connIntegrityChecked = NULL, + .is_closing = NULL, }; ConnectionType *connectionTypeRdma(void) { diff --git a/src/server.c b/src/server.c index b8a35af8e26..55a4ecfc0b3 100644 --- a/src/server.c +++ b/src/server.c @@ -1214,6 +1214,24 @@ void getExpensiveClientsInfo(size_t *in_usage, size_t *out_usage) { *out_usage = o; } +/* Detect and free zombie connections whose read handler was removed (e.g. + * BLOCKED_INUSE). Without a read handler the event loop won't notice the + * remote side closing, so these fds would leak until the fd limit is hit. */ +static bool clientsCronTcpIsClosing(client *c) { + if (!c->conn) return false; + + if (!connIsClosing(c->conn)) return false; + + if (server.verbosity <= LL_VERBOSE) { + sds client_info = catClientInfoString(sdsempty(), c, server.hide_user_data_from_log); + serverLog(LL_VERBOSE, "Client closed connection while blocked %s", client_info); + sdsfree(client_info); + } + + freeClientAsync(c); + return true; +} + /* This function is called by clientsTimeProc() and is used in order to perform * operations on clients that are important to perform constantly. For instance * we use this function in order to disconnect clients after a timeout, including @@ -1268,6 +1286,7 @@ static void clientsCron(int clients_this_cycle) { if (clientsCronResizeQueryBuffer(c)) continue; if (clientsCronResizeOutputBuffer(c, now)) continue; if (clientsCronTrackExpensiveClients(c, curr_peak_mem_usage_slot)) continue; + if (clientsCronTcpIsClosing(c)) continue; /* Iterating all the clients in getMemoryOverheadData() is too slow and * in turn would make the INFO command too slow. So we perform this @@ -4374,6 +4393,8 @@ void unprepareCommand(client *c) { * other operations can be performed by the caller. Otherwise * if C_ERR is returned the client was destroyed (i.e. after QUIT). */ int processCommand(client *c) { + serverAssert(!c->flag.blocked && !c->flag.unblocked); + if (!scriptIsTimedout()) { /* Both EXEC and scripts call call() directly so there should be * no way in_exec or scriptIsRunning() is 1. diff --git a/src/server.h b/src/server.h index ece04824165..22d1fbc1237 100644 --- a/src/server.h +++ b/src/server.h @@ -347,6 +347,7 @@ typedef enum blocking_type { BLOCKED_ZSET, /* BZPOP et al. */ BLOCKED_POSTPONE, /* Blocked by processCommand, re-try processing later. */ BLOCKED_SHUTDOWN, /* SHUTDOWN. */ + BLOCKED_INUSE, /* Key in use by background thread. */ BLOCKED_NUM, /* Number of blocked states. */ BLOCKED_END /* End of enumeration */ } blocking_type; @@ -3921,6 +3922,9 @@ void signalKeyAsReady(serverDb *db, robj *key, int type); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, int unblock_on_nokey); void blockClientShutdown(client *c); void blockPostponeClient(client *c); +void blockClientInUseOnKeys(client *c, int num_keys, robj *keys[]); +void unblockClientsInUseOnKey(robj *key); +void unblockClientsInUseOnAllKeys(void); void blockClientForReplicaAck(client *c, mstime_t timeout, long long offset, int numreplicas, int numlocal); void replicationRequestAckFromReplicas(void); void signalDeletedKeyAsReady(serverDb *db, robj *key, int type); diff --git a/src/socket.c b/src/socket.c index c9f9cae046e..55143e2d026 100644 --- a/src/socket.c +++ b/src/socket.c @@ -30,6 +30,10 @@ #include "server.h" #include "connhelpers.h" #include "io_threads.h" +#include +#ifdef __APPLE__ +#include +#endif /* The connections module provides a lean abstraction of network connections * to avoid direct socket and async event management across the server code base. @@ -418,6 +422,25 @@ static int connSocketGetType(void) { return CONN_TYPE_SOCKET; } +int connSocketIsClosing(connection *conn) { + if (aeGetFileEvents(server.el, conn->fd) != AE_NONE) return false; +#if defined(__linux__) + struct tcp_info info; + socklen_t infolen = sizeof(info); + if (getsockopt(conn->fd, IPPROTO_TCP, TCP_INFO, &info, &infolen) != 0 || infolen < sizeof(info)) return false; // Cannot retrieve TCP info + return (info.tcpi_state == TCP_CLOSE_WAIT || info.tcpi_state == TCP_CLOSE); +#elif defined(__APPLE__) + struct tcp_connection_info info; + socklen_t infolen = sizeof(info); + if (getsockopt(conn->fd, IPPROTO_TCP, TCP_CONNECTION_INFO, &info, &infolen) != 0 || infolen < sizeof(info)) return false; // Cannot retrieve TCP info + return (info.tcpi_state == TCPS_CLOSE_WAIT || info.tcpi_state == TCPS_CLOSED); +#else + /* Unsupported platform: zombie connection detection is not available. */ + UNUSED(conn); + return false; +#endif +} + static ConnectionType CT_Socket = { /* connection type */ .get_type = connSocketGetType, @@ -465,6 +488,7 @@ static ConnectionType CT_Socket = { /* Miscellaneous */ .connIntegrityChecked = NULL, + .is_closing = connSocketIsClosing, }; int connBlock(connection *conn) { diff --git a/src/tls.c b/src/tls.c index e443ce4d0a6..e4240250205 100644 --- a/src/tls.c +++ b/src/tls.c @@ -2022,6 +2022,7 @@ static ConnectionType CT_TLS = { /* Miscellaneous */ .connIntegrityChecked = connTLSIsIntegrityChecked, + .is_closing = connSocketIsClosing, }; diff --git a/src/unit/test_blocked.cpp b/src/unit/test_blocked.cpp new file mode 100644 index 00000000000..5fa78903492 --- /dev/null +++ b/src/unit/test_blocked.cpp @@ -0,0 +1,369 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" +extern "C" { +#include "server.h" +int getBlockInUseKeyCount(void); +void releaseBlockInUse(void); +} + +/* These unit tests were introduced at the time of inuse key blocking. Functions + * introduced earlier are tested only indirectly through integration tests. */ +class BlockedInuseTest : public ::testing::Test { + protected: + MockValkey mock; + RealValkey real; + static inline ConnectionType dummyConnType = {0}; + + static void SetUpTestSuite() { + memset(&server, 0, sizeof(valkeyServer)); + server.hz = CONFIG_DEFAULT_HZ; + dummyConnType.set_read_handler = dummySetReadHandler; + } + + static void TearDownTestSuite() { + releaseBlockInUse(); + } + + void SetUp() override { + server.unblocked_clients = listCreate(); + ASSERT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + ASSERT_EQ(getBlockInUseKeyCount(), 0); + } + + void TearDown() override { + ASSERT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + ASSERT_EQ(getBlockInUseKeyCount(), 0); + ASSERT_EQ(listLength(server.unblocked_clients), 0UL); + listRelease(server.unblocked_clients); + server.unblocked_clients = NULL; + } + + + static int dummySetReadHandler(connection *conn, ConnectionCallbackFunc func) { + conn->read_handler = func; + return C_OK; + } + + client *createFakeClient(int client_id) { + client *c = (client *)zcalloc(sizeof(client)); + c->id = client_id; + c->conn = (connection *)zcalloc(sizeof(connection)); + c->conn->type = &dummyConnType; + c->conn->read_handler = (ConnectionCallbackFunc)1; + c->flag.pending_command = 1; + return c; + } + + void freeFakeClient(client *c) { + freeClientBlockingState(c); + if (c->conn) zfree(c->conn); + zfree(c); + } + + void verifyClientBlockState(client *c, bool blocked, bool unblocked) { + EXPECT_EQ(c->flag.unblocked, unblocked); + EXPECT_EQ(c->flag.blocked && c->bstate->btype == BLOCKED_INUSE, blocked); + if (blocked || unblocked) { + EXPECT_EQ(c->conn->read_handler, nullptr); + } else { + EXPECT_NE(c->conn->read_handler, nullptr); + } + } +}; + +using BlockedInuseDeathTest = BlockedInuseTest; + + +TEST_F(BlockedInuseTest, blockInitialState) { + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + ASSERT_NE(server.unblocked_clients, nullptr); +} + +TEST_F(BlockedInuseTest, blockClientOnSingleKey) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + // Block + blockClientInUseOnKeys(c, 1, keys); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key->refcount, 3u); + + // Unblock + unblockClientsInUseOnKey(key); + verifyClientBlockState(c, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(key->refcount, 1u); + EXPECT_EQ(listLength(server.unblocked_clients), 1UL); + EXPECT_EQ(listFirst(server.unblocked_clients)->value, c); + + // Process unblocked client in event loop + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c, 0, 0); + EXPECT_EQ(key->refcount, 1u); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, blockClientClearsLeftoverTimeout) { + client *c = createFakeClient(1); + initClientBlockingState(c); + c->bstate->timeout = 1000; + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + blockClientInUseOnKeys(c, 1, keys); + EXPECT_EQ(c->bstate->timeout, 0); + + unblockClientsInUseOnKey(key); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + processUnblockedClients(); + + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, blockClientOnMultipleKeys) { + client *c = createFakeClient(1); + robj *key1 = createObject(OBJ_STRING, sdsnew("key1")); + robj *key2 = createObject(OBJ_STRING, sdsnew("key2")); + robj *keys[] = {key1, key2}; + + // Block + blockClientInUseOnKeys(c, 2, keys); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 2); + EXPECT_EQ(key1->refcount, 3u); + EXPECT_EQ(key2->refcount, 3u); + + // Unblock key1 + unblockClientsInUseOnKey(key1); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 3u); + + // Unblock key2, client gets unblocked + unblockClientsInUseOnKey(key2); + verifyClientBlockState(c, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 1u); + EXPECT_EQ(listLength(server.unblocked_clients), 1UL); + EXPECT_EQ(listFirst(server.unblocked_clients)->value, c); + + // Process unblocked client in event loop + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c, 0, 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 1u); + decrRefCount(key1); + decrRefCount(key2); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, blockMultipleClientsOnSameKey) { + client *c1 = createFakeClient(1); + client *c2 = createFakeClient(2); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + // Block + blockClientInUseOnKeys(c1, 1, keys); + blockClientInUseOnKeys(c2, 1, keys); + verifyClientBlockState(c1, 1, 0); + verifyClientBlockState(c2, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 2u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key->refcount, 4u); + + // Unblock + unblockClientsInUseOnKey(key); + verifyClientBlockState(c1, 0, 1); + verifyClientBlockState(c2, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(key->refcount, 1u); + EXPECT_EQ(listLength(server.unblocked_clients), 2UL); + + // Process client in event loop + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c1)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c1)).Times(1); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c2)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c2)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c1, 0, 0); + verifyClientBlockState(c2, 0, 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + + EXPECT_EQ(key->refcount, 1u); + decrRefCount(key); + freeFakeClient(c1); + freeFakeClient(c2); +} + +TEST_F(BlockedInuseTest, unblockBlockedClient) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + // Block + blockClientInUseOnKeys(c, 1, keys); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key->refcount, 3u); + + // Unblock client, simulate freeClient + unblockClient(c, 0); + EXPECT_FALSE(c->flag.blocked); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + EXPECT_EQ(key->refcount, 1u); + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, blockClientOnDuplicateKeys) { + client *c = createFakeClient(1); + robj *key1 = createObject(OBJ_STRING, sdsnew("foo")); + robj *key2 = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key1, key2}; + + // Block + blockClientInUseOnKeys(c, 2, keys); + verifyClientBlockState(c, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 1u); + EXPECT_EQ(getBlockInUseKeyCount(), 1); + EXPECT_EQ(key1->refcount, 3u); + EXPECT_EQ(key2->refcount, 1u); // Key is deduplicated, only blocked once + + // Unblock + unblockClientsInUseOnKey(key1); + verifyClientBlockState(c, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(listLength(server.unblocked_clients), 1UL); + EXPECT_EQ(listFirst(server.unblocked_clients)->value, c); + + // Process client in event loop + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c, 0, 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 1u); + decrRefCount(key1); + decrRefCount(key2); + freeFakeClient(c); +} + +TEST_F(BlockedInuseTest, unblockAllKeys) { + client *c1 = createFakeClient(1); + client *c2 = createFakeClient(2); + robj *key1 = createObject(OBJ_STRING, sdsnew("key1")); + robj *key2 = createObject(OBJ_STRING, sdsnew("key2")); + robj *keys1[] = {key1}; + robj *keys2[] = {key2}; + + // Block c1 on key1, c2 on key2 + blockClientInUseOnKeys(c1, 1, keys1); + blockClientInUseOnKeys(c2, 1, keys2); + verifyClientBlockState(c1, 1, 0); + verifyClientBlockState(c2, 1, 0); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 2u); + EXPECT_EQ(getBlockInUseKeyCount(), 2); + EXPECT_EQ(key1->refcount, 3u); + EXPECT_EQ(key2->refcount, 3u); + + // Unblock all + unblockClientsInUseOnAllKeys(); + verifyClientBlockState(c1, 0, 1); + verifyClientBlockState(c2, 0, 1); + EXPECT_EQ(server.blocked_clients_by_type[BLOCKED_INUSE], 0u); + EXPECT_EQ(getBlockInUseKeyCount(), 0); + EXPECT_EQ(listLength(server.unblocked_clients), 2UL); + + // Process clients + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c1)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c1)).Times(1); + EXPECT_CALL(mock, processPendingCommandAndInputBuffer(c2)).WillOnce(Return(C_OK)); + EXPECT_CALL(mock, beforeNextClient(c2)).Times(1); + processUnblockedClients(); + verifyClientBlockState(c1, 0, 0); + verifyClientBlockState(c2, 0, 0); + EXPECT_EQ(listLength(server.unblocked_clients), 0UL); + EXPECT_EQ(key1->refcount, 1u); + EXPECT_EQ(key2->refcount, 1u); + decrRefCount(key1); + decrRefCount(key2); + freeFakeClient(c1); + freeFakeClient(c2); +} + +TEST_F(BlockedInuseDeathTest, blockingOnKeysReplicaClient) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + c->flag.replica = 1; + EXPECT_DEATH(blockClientInUseOnKeys(c, 1, keys), ""); + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseDeathTest, blockingOnKeysNonStringType) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + keys[0]->type = OBJ_LIST; + EXPECT_DEATH(blockClientInUseOnKeys(c, 1, keys), ""); + keys[0]->type = OBJ_STRING; + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseDeathTest, blockingOnKeysZeroKeys) { + client *c = createFakeClient(1); + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + EXPECT_DEATH(blockClientInUseOnKeys(c, 0, keys), ""); + decrRefCount(key); + freeFakeClient(c); +} + +TEST_F(BlockedInuseDeathTest, blockingOnKeysWithoutPendingCommand) { + client *c = createFakeClient(1); + c->flag.pending_command = 0; + robj *key = createObject(OBJ_STRING, sdsnew("foo")); + robj *keys[] = {key}; + + EXPECT_DEATH(blockClientInUseOnKeys(c, 1, keys), ""); + decrRefCount(key); + freeFakeClient(c); +} diff --git a/src/unit/wrappers.h b/src/unit/wrappers.h index 89a2ee9fc4e..0d5be26effd 100644 --- a/src/unit/wrappers.h +++ b/src/unit/wrappers.h @@ -63,6 +63,8 @@ extern "C" { long long __wrap_aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, aeTimeProc *proc, void *clientData, aeEventFinalizerProc *finalizerProc); ssize_t __wrap_streamDecompressorFeed(streamDecompressor *decompressor, uint8_t *output, size_t output_capacity, const uint8_t *input, size_t input_len, size_t *input_consumed); void __wrap_zmadvise_dontneed(void *ptr, size_t size_hint); +int __wrap_processPendingCommandAndInputBuffer(client *c); +void __wrap_beforeNextClient(client *c); #undef protected #undef _Bool #undef typename diff --git a/src/unix.c b/src/unix.c index e5db7cbb9a1..e2b4ec656a4 100644 --- a/src/unix.c +++ b/src/unix.c @@ -214,6 +214,7 @@ static ConnectionType CT_Unix = { /* Miscellaneous */ .connIntegrityChecked = NULL, + .is_closing = NULL, }; int RedisRegisterConnectionTypeUnix(void) { From 1edaa0eafb2e5cd8502251428ca0d0b3667deeb8 Mon Sep 17 00:00:00 2001 From: Alina Liu Date: Thu, 12 Mar 2026 21:07:31 +0000 Subject: [PATCH 03/18] Allocate metadata for each key to support forkless save operations Signed-off-by: Alina Liu --- .config/typos.toml | 1 + src/config.c | 1 + src/object.c | 104 +++++++++++++- src/server.c | 8 ++ src/server.h | 35 +++-- src/unit/test_object.cpp | 253 ++++++++++++++++++++++++++--------- tests/unit/introspection.tcl | 1 + valkey.conf | 11 ++ 8 files changed, 331 insertions(+), 83 deletions(-) diff --git a/.config/typos.toml b/.config/typos.toml index d4bc2684c70..7a2fc0c1c73 100644 --- a/.config/typos.toml +++ b/.config/typos.toml @@ -59,6 +59,7 @@ seeked = "seeked" [type.c.extend-words] arange = "arange" +Threadsave = "Threadsave" fo = "fo" frst = "frst" limite = "limite" diff --git a/src/config.c b/src/config.c index ddf5715711f..80135ea266a 100644 --- a/src/config.c +++ b/src/config.c @@ -3381,6 +3381,7 @@ standardConfig static_configs[] = { createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_replica_ignore_maxmemory, 1, NULL, NULL), createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), createBoolConfig("activedefrag", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.active_defrag_enabled, CONFIG_ACTIVE_DEFRAG_DEFAULT, isValidActiveDefrag, NULL), + createBoolConfig("forkless-options-supported", NULL, IMMUTABLE_CONFIG, server.forkless_options_supported, 0, NULL, NULL), createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, server.syslog_enabled, 0, NULL, NULL), createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, server.cluster_enabled, 0, NULL, NULL), createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG | DENY_LOADING_CONFIG, server.aof_enabled, 0, NULL, updateAppendOnly), diff --git a/src/object.c b/src/object.c index eeb6e06e39e..fb5727e4a0e 100644 --- a/src/object.c +++ b/src/object.c @@ -80,6 +80,82 @@ void objectSetLRU(robj *o, unsigned int lru) { o->lru = lru; } +/* Get beginning of embedded data, which may contain expire, metadata, key, and/or value. + * Embedded data flags must be accurate when called. */ +static unsigned char *objectEmbeddedData(const robj *o) { + unsigned char *data = (void *)(o + 1); + if (o->hasembval) data -= sizeof(void *); + return data; +} + +/* ===================== Object Metadata Management ========================= */ + +/* Static variable to store metadata size. Set once at server initialization. */ +static size_t object_metadata_size = 0; + +/* Set the metadata size. + * Size should not be changed once set. */ +void objectSetMetadataSize(size_t size) { + /* Metadata size already set - only allow setting to the same value */ + if (object_metadata_size == size) return; + + /* When current size is 0 and the incoming size is not - setting for the first time */ + serverAssert(object_metadata_size == 0); + + /* Check that all databases are empty */ + if (server.db != NULL) { + for (int j = 0; j < server.dbnum; j++) { + if (server.db[j] != NULL) { + serverAssert(kvstoreSize(server.db[j]->keys) == 0); + } + } + } + + object_metadata_size = size; +} + +/* Calculate the size of metadata for an object. + * Returns the configured metadata size if the object has an embedded key, 0 otherwise. */ +size_t objectGetMetadataSize(const robj *o) { + if (o->hasembkey) return object_metadata_size; + return 0; +} + +/* Get a void pointer to the metadata for an object. + * Returns NULL if the object doesn't have metadata. + * The caller must cast this to the appropriate metadata structure type. + * + * Memory layout visualization for objects: + * + * ┌─────────────────────────────────────────────────────────────────┐ + * │ robj (struct serverObject) │ + * │ - type, encoding, lru, hasexpire, hasembkey, hasembval... │ + * ├─────────────────────────────────────────────────────────────────┤ + * │ expire field (optional, if hasexpire == 1) │ + * │ - long long (8 bytes) │ + * ├─────────────────────────────────────────────────────────────────┤ + * │ metadata (optional, if hasembkey == 1 && metadata_size > 0) │ + * │ - (object_metadata_size) │ ← objectGetMetadata returns pointer here + * ├─────────────────────────────────────────────────────────────────┤ + * │ embedded key (if hasembkey == 1) │ + * ├─────────────────────────────────────────────────────────────────┤ + * │ embedded value (if hasembval == 1) │ + * └─────────────────────────────────────────────────────────────────┘ + */ +void *objectGetMetadata(const robj *o) { + if (object_metadata_size == 0 || !o->hasembkey) return NULL; + + /* The memory after the struct where we embedded metadata. */ + unsigned char *data = objectEmbeddedData(o); + + /* If expire field exists, metadata is after it */ + if (o->hasexpire) { + data += sizeof(long long); + } + + return (void *)data; +} + /* ===================== Creation and parsing of objects ==================== */ /* Creates an object, optionally with embedded key and expire fields. The key @@ -93,10 +169,12 @@ static robj *createUnembeddedObjectWithKeyAndExpire(int type, void *val, const_s size_t key_sds_len = has_embkey ? sdslen(key) : 0; char key_sds_type = has_embkey ? sdsReqType(key_sds_len) : 0; size_t key_sds_size = has_embkey ? sdsReqSize(key_sds_len, key_sds_type) : 0; + size_t metadata_size = has_embkey ? object_metadata_size : 0; size_t min_size = sizeof(robj); if (has_expire) { min_size += sizeof(long long); } + min_size += metadata_size; if (has_embkey) { /* Size of embedded key, incl. 1 byte for prefixed sds hdr size. */ min_size += 1 + key_sds_size; @@ -129,6 +207,12 @@ static robj *createUnembeddedObjectWithKeyAndExpire(int type, void *val, const_s data += sizeof(long long); } + /* Initialize metadata to zero */ + if (metadata_size > 0) { + memset(data, 0, metadata_size); + data += metadata_size; + } + /* Copy embedded key. */ if (o->hasembkey) { *data++ = sdsHdrSize(key_sds_type); @@ -171,12 +255,6 @@ robj *createRawStringObject(const char *ptr, size_t len) { return createObject(OBJ_STRING, sdsnewlen(ptr, len)); } -/* Get beginning of embedded data, which may contain expire, key, and/or value. Embedded data flags must be accurate when called. */ -static unsigned char *objectEmbeddedData(const robj *o) { - unsigned char *data = (void *)(o + 1); - if (o->hasembval) data -= sizeof(void *); - return data; -} /* Creates a new embedded string object and copies the content of key, val_ptr * and expire to the new object. LRU is set to 0. */ @@ -190,6 +268,7 @@ static robj *createEmbeddedStringObjectWithKeyAndExpire(const char *val_ptr, char key_sds_type = has_embkey ? sdsReqType(key_sds_len) : 0; size_t key_sds_size = has_embkey ? sdsReqSize(key_sds_len, key_sds_type) : 0; size_t val_sds_size = sdsReqSize(val_len, SDS_TYPE_8); + size_t metadata_size = has_embkey ? object_metadata_size : 0; if (val_sds_size < sizeof(void *)) { val_sds_size = sizeof(void *); /* Ensure it's possible to "unembed" value later */ } @@ -199,6 +278,7 @@ static robj *createEmbeddedStringObjectWithKeyAndExpire(const char *val_ptr, if (expire != EXPIRY_NONE) { min_size += sizeof(long long); } + min_size += metadata_size; if (has_embkey) { /* Size of embedded key, incl. 1 byte for prefixed sds hdr size. */ min_size += 1 + key_sds_size; @@ -232,6 +312,12 @@ static robj *createEmbeddedStringObjectWithKeyAndExpire(const char *val_ptr, data += sizeof(long long); } + /* Initialize metadata to zero */ + if (metadata_size > 0) { + memset(data, 0, metadata_size); + data += metadata_size; + } + /* Copy embedded key. */ if (o->hasembkey) { *data++ = sdsHdrSize(key_sds_type); @@ -265,6 +351,7 @@ static bool shouldEmbedStringObject(size_t val_len, const_sds key, long long exp if (key) { size_t key_len = sdslen(key); size += sdsReqSize(key_len, sdsReqType(key_len)) + 1; /* 1 byte for prefixed sds hdr size */ + size += object_metadata_size; } size += (expire != EXPIRY_NONE) * sizeof(long long); size += sdsReqSize(val_len, SDS_TYPE_8); @@ -300,6 +387,8 @@ void *objectGetVal(const robj *o) { data += sizeof(long long); } if (o->hasembkey) { + /* Skip metadata */ + data += objectGetMetadataSize(o); /* Skip embedded key */ uint8_t hdr_size = *(uint8_t *)data; data += 1 + hdr_size; /* +1 for header size byte */ @@ -319,6 +408,9 @@ sds objectGetKey(const robj *o) { data += sizeof(long long); } if (o->hasembkey) { + /* Skip metadata */ + data += objectGetMetadataSize(o); + /* Skip header size byte */ uint8_t hdr_size = *(uint8_t *)data; data += 1 + hdr_size; return (sds)data; diff --git a/src/server.c b/src/server.c index 55a4ecfc0b3..4665cbeb77c 100644 --- a/src/server.c +++ b/src/server.c @@ -2373,6 +2373,7 @@ void initServerConfig(void) { for (j = 0; j < CONFIG_DEFAULT_BINDADDR_COUNT; j++) server.bindaddr[j] = zstrdup(default_bindaddr[j]); memset(server.listeners, 0x00, sizeof(server.listeners)); server.active_expire_enabled = 1; + server.forkless_options_supported = 0; server.lazy_expire_disabled = 0; server.skip_checksum_validation = 0; server.loading = 0; @@ -3066,6 +3067,13 @@ void initServer(void) { server.dbnum = server.cluster_enabled ? server.config_databases_cluster : server.config_databases; server.db = zcalloc(sizeof(serverDb *) * server.dbnum); + + /* Set object metadata size before creating any database key objects */ + if (server.forkless_options_supported) { + objectSetMetadataSize(sizeof(uint32_t)); /* This is a placeholder until Threadsave defines a metadata structure */ + /* 4 bytes for iterator_epoch for now*/ + } + createDatabaseIfNeeded(0); /* The default database should always exist */ evictionPoolAlloc(); /* Initialize the LRU keys pool. */ diff --git a/src/server.h b/src/server.h index 22d1fbc1237..e1253c15beb 100644 --- a/src/server.h +++ b/src/server.h @@ -800,25 +800,27 @@ typedef struct ValkeyModuleType moduleType; * The optional variable-sized embedded data has 2 possible layouts. If value is embedded (hasembval == 1) * the `val_ptr` pointer is not used - instead the val data is embedded: * - * +------+----------+-----+------------+----------+--------+-----------------+---------+------------+ - * | type | encoding | lru | has* flags | refcount | expire | key_header_size | key sds | value data | - * +------+----------+-----+------------+----------+--------+-----------------+---------+------------+ - * ^ ^ ^ ^ - * | | | | - * | | | +--- present because hasembval == 1 - * | | | - * | +-----------------+--- present if hasembkey == 1 + * +------+----------+-----+------------+----------+--------+----------+-----------------+---------+------------+ + * | type | encoding | lru | has* flags | refcount | expire | metadata | key_header_size | key sds | value data | + * +------+----------+-----+------------+----------+--------+----------+-----------------+---------+------------+ + * ^ ^ ^ ^ ^ + * | | | | | + * | | | | +--- present because hasembval == 1 + * | | | | + * | +----------+-----------------+--- present if hasembkey == 1 + * | * | * +--- present if hasexpire == 1 * * Otherwise value is not embedded and we use the `val_ptr` pointer: * - * +------+----------+-----+------------+----------+---------+--------+-----------------+---------+ - * | type | encoding | lru | has* flags | refcount | val_ptr | expire | key_header_size | key sds | - * +------+----------+-----+------------+----------+---------+--------+-----------------+---------+ - * ^ ^ ^ ^ - * | | | | - * | | +-----------------+--- present if hasembkey == 1 + * +------+----------+-----+------------+----------+---------+--------+----------+-----------------+---------+ + * | type | encoding | lru | has* flags | refcount | val_ptr | expire | metadata | key_header_size | key sds | + * +------+----------+-----+------------+----------+---------+--------+----------+-----------------+---------+ + * ^ ^ ^ ^ ^ + * | | | | | + * | | +----------+-----------------+--- present if hasembkey == 1 + * | | * | | * | +--- present if hasexpire == 1 * | @@ -2067,6 +2069,7 @@ struct valkeyServer { int rdb_checksum; /* Use RDB checksum? */ int rdb_del_sync_files; /* Remove RDB files used only for SYNC if the instance does not use persistence. */ + int forkless_options_supported; /* Enable forkless options support. */ time_t lastsave; /* Unix time of last successful save */ time_t lastbgsave_try; /* Unix time of last attempted bgsave */ time_t rdb_save_time_last; /* Time used by last RDB save run. */ @@ -3231,6 +3234,10 @@ void objectSetEncoding(robj *o, int encoding); unsigned int objectGetRefcount(const robj *o); unsigned int objectGetLRU(const robj *o); void objectSetLRU(robj *o, unsigned int lru); +/* Object metadata management */ +void objectSetMetadataSize(size_t size); +size_t objectGetMetadataSize(const robj *o); +void *objectGetMetadata(const robj *o); /* Synchronous I/O with timeout */ ssize_t syncWrite(int fd, char *ptr, ssize_t size, long long timeout); diff --git a/src/unit/test_object.cpp b/src/unit/test_object.cpp index 054802d0831..9e030bde002 100644 --- a/src/unit/test_object.cpp +++ b/src/unit/test_object.cpp @@ -15,7 +15,47 @@ extern "C" { #include "server.h" } +/* Metadata test helpers */ +typedef struct objMetadata { + uint32_t meta_int; +} objMetadata; + class ObjectTest : public ::testing::Test { + protected: + robj *createKeyValueObject(const char *k, const char *v) { + sds key = sdsnew(k); + robj *obj = createStringObject(v, strlen(v)); + robj *obj_with_key = objectSetKeyAndExpire(obj, key, -1); + sdsfree(key); + return obj_with_key; + } + + void objectSetMetaInt(robj *o, uint32_t metadata_int) { + objMetadata *meta = (objMetadata *)objectGetMetadata(o); + meta->meta_int = metadata_int; + } + + uint32_t objectGetMetaInt(const robj *o) { + objMetadata *meta = (objMetadata *)objectGetMetadata(o); + return meta->meta_int; + } + + /* Find the largest value length that still embeds with the given key and expire. */ + int findMaxEmbeddableValueLen(const char *key, long long expire) { + sds k = key ? sdsnew(key) : NULL; + + int len; + for (len = 1; len <= 256; len++) { + robj *obj = createStringObject(NULL, len); + if (k) obj = objectSetKeyAndExpire(obj, k, expire); + bool isEmbedded = (obj->encoding == OBJ_ENCODING_EMBSTR); + decrRefCount(obj); + if (!isEmbedded) break; + } + + sdsfree(k); + return len - 1; + } }; TEST_F(ObjectTest, object_with_key) { @@ -57,88 +97,71 @@ TEST_F(ObjectTest, object_with_key) { } TEST_F(ObjectTest, embedded_string_with_key) { - /* key of length 32 - type 8 */ - sds key = sdsnew("k:123456789012345678901234567890"); - ASSERT_EQ(sdslen(key), 32u); - - /* 32B key and 79B value should be embedded within 128B. Contents: - * - 8B robj (no ptr) + 1B key header size - * - 3B key header + 32B key + 1B null terminator - * - 3B val header + 79B val + 1B null terminator - * because no pointers are stored, there is no difference for 32 bit builds*/ - const char *short_value = "1234567890123456789012345678901234567890123456789012345678901234567890123456789"; - ASSERT_EQ(strlen(short_value), 79u); - robj *short_val_obj = createStringObject(short_value, strlen(short_value)); - robj *embstr_obj = objectSetKeyAndExpire(short_val_obj, key, -1); + const char *key = "k:123456789012345678901234567890"; + int max_len = findMaxEmbeddableValueLen(key, -1); + ASSERT_GT(max_len, 0); + + /* Value at max length should embed. */ + sds k1 = sdsnew(key); + robj *embstr_obj = createStringObject(NULL, max_len); + embstr_obj = objectSetKeyAndExpire(embstr_obj, k1, -1); ASSERT_EQ(embstr_obj->encoding, (unsigned)OBJ_ENCODING_EMBSTR); - ASSERT_EQ(sdslen(objectGetKey(embstr_obj)), 32u); - ASSERT_EQ(sdscmp(objectGetKey(embstr_obj), key), 0); - ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), 79u); - ASSERT_EQ(strcmp((const char *)objectGetVal(embstr_obj), short_value), 0); - - /* value of length 80 cannot be embedded with other contents within 128B */ - const char *longer_value = "12345678901234567890123456789012345678901234567890123456789012345678901234567890"; - ASSERT_EQ(strlen(longer_value), 80u); - robj *longer_val_obj = createStringObject(longer_value, strlen(longer_value)); - robj *raw_obj = objectSetKeyAndExpire(longer_val_obj, key, -1); + ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), (size_t)max_len); + + /* One byte more should not embed. */ + sds k2 = sdsnew(key); + robj *raw_obj = createStringObject(NULL, max_len + 1); + raw_obj = objectSetKeyAndExpire(raw_obj, k2, -1); ASSERT_EQ(raw_obj->encoding, (unsigned)OBJ_ENCODING_RAW); - ASSERT_EQ(sdslen(objectGetKey(raw_obj)), 32u); - ASSERT_EQ(sdscmp(objectGetKey(raw_obj), key), 0); - ASSERT_EQ(sdslen((sds)objectGetVal(raw_obj)), 80u); - ASSERT_EQ(strcmp((const char *)objectGetVal(raw_obj), longer_value), 0); + ASSERT_EQ(sdslen((sds)objectGetVal(raw_obj)), (size_t)(max_len + 1)); - sdsfree(key); + sdsfree(k1); + sdsfree(k2); decrRefCount(embstr_obj); decrRefCount(raw_obj); } TEST_F(ObjectTest, embedded_string_with_key_and_expire) { - /* key of length 32 - type 8 */ - sds key = sdsnew("k:123456789012345678901234567890"); - ASSERT_EQ(sdslen(key), 32u); - - /* 32B key and 71B value should be embedded within 128B. Contents: - * - 8B robj (no ptr) + 8B expire + 1B key header size - * - 3B key header + 32B key + 1B null terminator - * - 3B val header + 71B val + 1B null terminator - * because no pointers are stored, there is no difference for 32 bit builds*/ - const char *short_value = "12345678901234567890123456789012345678901234567890123456789012345678901"; - ASSERT_EQ(strlen(short_value), 71u); - robj *short_val_obj = createStringObject(short_value, strlen(short_value)); - robj *embstr_obj = objectSetKeyAndExpire(short_val_obj, key, 128); + const char *key = "k:123456789012345678901234567890"; + int max_len = findMaxEmbeddableValueLen(key, 128); + ASSERT_GT(max_len, 0); + + /* Adding an expire reduces the available space for the value. */ + int max_len_no_expire = findMaxEmbeddableValueLen(key, -1); + ASSERT_LT(max_len, max_len_no_expire); + + /* Value at max length should embed. */ + sds k1 = sdsnew(key); + robj *embstr_obj = createStringObject(NULL, max_len); + embstr_obj = objectSetKeyAndExpire(embstr_obj, k1, 128); ASSERT_EQ(embstr_obj->encoding, (unsigned)OBJ_ENCODING_EMBSTR); - ASSERT_EQ(sdslen(objectGetKey(embstr_obj)), 32u); - ASSERT_EQ(sdscmp(objectGetKey(embstr_obj), key), 0); - ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), 71u); - ASSERT_EQ(strcmp((const char *)objectGetVal(embstr_obj), short_value), 0); - - /* value of length 72 cannot be embedded with other contents within 128B */ - const char *longer_value = "123456789012345678901234567890123456789012345678901234567890123456789012"; - ASSERT_EQ(strlen(longer_value), 72u); - robj *longer_val_obj = createStringObject(longer_value, strlen(longer_value)); - robj *raw_obj = objectSetKeyAndExpire(longer_val_obj, key, 128); + + /* One byte more should not embed. */ + sds k2 = sdsnew(key); + robj *raw_obj = createStringObject(NULL, max_len + 1); + raw_obj = objectSetKeyAndExpire(raw_obj, k2, 128); ASSERT_EQ(raw_obj->encoding, (unsigned)OBJ_ENCODING_RAW); - ASSERT_EQ(sdslen(objectGetKey(raw_obj)), 32u); - ASSERT_EQ(sdscmp(objectGetKey(raw_obj), key), 0); - ASSERT_EQ(sdslen((sds)objectGetVal(raw_obj)), 72u); - ASSERT_EQ(strcmp((const char *)objectGetVal(raw_obj), longer_value), 0); - sdsfree(key); + sdsfree(k1); + sdsfree(k2); decrRefCount(embstr_obj); decrRefCount(raw_obj); } TEST_F(ObjectTest, embedded_value) { - /* with only value there is only 12B overhead, so we can embed up to 52B. - * 8B robj (no ptr) + 3B val header + 52B val + 1B null terminator */ - const char *val = "v:12345678901234567890123456789012345678901234567890"; - ASSERT_EQ(strlen(val), 52u); - robj *embstr_obj = createStringObject(val, strlen(val)); + /* Value-only object (no key): find the largest value that embeds. */ + int max_len = findMaxEmbeddableValueLen(NULL, -1); + ASSERT_GT(max_len, 0); + + robj *embstr_obj = createStringObject(NULL, max_len); ASSERT_EQ(embstr_obj->encoding, (unsigned)OBJ_ENCODING_EMBSTR); - ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), 52u); - ASSERT_EQ(strcmp((const char *)objectGetVal(embstr_obj), val), 0); + ASSERT_EQ(sdslen((sds)objectGetVal(embstr_obj)), (size_t)max_len); + + robj *raw_obj = createStringObject(NULL, max_len + 1); + ASSERT_EQ(raw_obj->encoding, (unsigned)OBJ_ENCODING_RAW); decrRefCount(embstr_obj); + decrRefCount(raw_obj); } TEST_F(ObjectTest, unembed_value) { @@ -166,3 +189,107 @@ TEST_F(ObjectTest, unembed_value) { sdsfree(key); decrRefCount(obj); } + + +TEST_F(ObjectTest, metadata_disabled) { + robj *obj_with_key = createKeyValueObject("testkey", "value"); + + ASSERT_EQ(objectGetMetadata(obj_with_key), nullptr); + ASSERT_EQ(objectGetMetadataSize(obj_with_key), 0u); + + decrRefCount(obj_with_key); +} + +TEST_F(ObjectTest, metadata_without_key) { + objectSetMetadataSize(sizeof(objMetadata)); + + robj *obj_no_key = createStringObject("value_without_key", 17); + + ASSERT_EQ(objectGetMetadata(obj_no_key), nullptr); + ASSERT_EQ(objectGetMetadataSize(obj_no_key), 0u); + + decrRefCount(obj_no_key); +} + +TEST_F(ObjectTest, metadata_with_key) { + objectSetMetadataSize(sizeof(objMetadata)); + + robj *obj_with_key = createKeyValueObject("testkey", "value"); + + ASSERT_EQ(objectGetMetadataSize(obj_with_key), sizeof(objMetadata)); + + objMetadata *meta = (objMetadata *)objectGetMetadata(obj_with_key); + ASSERT_NE(meta, nullptr); + EXPECT_EQ(meta->meta_int, 0u); + + decrRefCount(obj_with_key); +} + +TEST_F(ObjectTest, metadata_read_write) { + objectSetMetadataSize(sizeof(objMetadata)); + + robj *obj_with_key = createKeyValueObject("mykey", "myvalue"); + + ASSERT_EQ(objectGetMetadataSize(obj_with_key), sizeof(objMetadata)); + + objectSetMetaInt(obj_with_key, 12345); + EXPECT_EQ(objectGetMetaInt(obj_with_key), 12345u); + + objectSetMetaInt(obj_with_key, 67890); + EXPECT_EQ(objectGetMetaInt(obj_with_key), 67890u); + + decrRefCount(obj_with_key); +} + +TEST_F(ObjectTest, metadata_multiple_objects) { + objectSetMetadataSize(sizeof(objMetadata)); + + robj *obj_with_key1 = createKeyValueObject("key1", "val1"); + robj *obj_with_key2 = createKeyValueObject("key2", "val2"); + robj *obj_with_key3 = createKeyValueObject("key3", "val3"); + + ASSERT_EQ(objectGetMetadataSize(obj_with_key1), sizeof(objMetadata)); + ASSERT_EQ(objectGetMetadataSize(obj_with_key2), sizeof(objMetadata)); + ASSERT_EQ(objectGetMetadataSize(obj_with_key3), sizeof(objMetadata)); + + objectSetMetaInt(obj_with_key1, 100); + objectSetMetaInt(obj_with_key2, 200); + objectSetMetaInt(obj_with_key3, 300); + + EXPECT_EQ(objectGetMetaInt(obj_with_key1), 100u); + EXPECT_EQ(objectGetMetaInt(obj_with_key2), 200u); + EXPECT_EQ(objectGetMetaInt(obj_with_key3), 300u); + + objectSetMetaInt(obj_with_key2, 999); + EXPECT_EQ(objectGetMetaInt(obj_with_key1), 100u); + EXPECT_EQ(objectGetMetaInt(obj_with_key2), 999u); + EXPECT_EQ(objectGetMetaInt(obj_with_key3), 300u); + + decrRefCount(obj_with_key1); + decrRefCount(obj_with_key2); + decrRefCount(obj_with_key3); +} + +TEST_F(ObjectTest, metadata_changes_embed_threshold) { + /* Find the max embeddable value length without metadata, then verify + * that enabling metadata reduces it (some previously-embeddable objects + * become RAW). */ + const char *key = "k:123456789012345678901234567890"; + int max_without = findMaxEmbeddableValueLen(key, -1); + ASSERT_GT(max_without, 0); + + objectSetMetadataSize(sizeof(objMetadata)); + int max_with = findMaxEmbeddableValueLen(key, -1); + + /* Metadata takes space, so the threshold must shrink. */ + ASSERT_LT(max_with, max_without); + + /* An object that just fit before should now be RAW. */ + sds k = sdsnew(key); + robj *obj = createStringObject(NULL, max_without); + obj = objectSetKeyAndExpire(obj, k, -1); + ASSERT_EQ(obj->encoding, (unsigned)OBJ_ENCODING_RAW); + + sdsfree(k); + decrRefCount(obj); +} diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 324100f6eb1..25467a7d709 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -1393,6 +1393,7 @@ start_server {tags {"introspection"}} { rdma-rx-size rdma-bind rdma-port + forkless-options-supported } if {!$::tls} { diff --git a/valkey.conf b/valkey.conf index 10b6354bbb9..b1bf6a2225e 100644 --- a/valkey.conf +++ b/valkey.conf @@ -556,6 +556,17 @@ locale-collate "" # # hash-seed example-seed-val +# Enable support for forkless save operations by allocating metadata for each key. +# This is an immutable configuration that must be set at server startup and +# cannot be changed at runtime. +# +# When enabled, the server allocates 4 additional bytes per key. +# +# Note: This only enables the infrastructure support. The actual forkless save +# behavior is controlled separately by the 'forkless-enabled' runtime configuration. +# +# forkless-options-supported no + ################################ SNAPSHOTTING ################################ # Save the DB to disk. From 5abfbcdfc66ec91a49e7ff4d19040cff0cd5a7c4 Mon Sep 17 00:00:00 2001 From: harrylin98 Date: Fri, 24 Apr 2026 14:13:58 -0700 Subject: [PATCH 04/18] Fix crash when looking up blocked-inuse clients before initialization Signed-off-by: harrylin98 --- src/blocked.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/blocked.c b/src/blocked.c index 9f6a0374823..9e81795b0a3 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -901,6 +901,7 @@ static hashtableType keyToClientsHashtableType = { // Return the list of clients blocked on key, or NULL if none exist. static list *keyToClients_getBlockedClientsList(robj *key) { + if (!inuse_key_to_clients) return NULL; keyToClientsEntry *entry; if (hashtableFind(inuse_key_to_clients, key, (void **)&entry)) { return entry->clients; From 80297201e254634a6af9973961d593030f1cd515 Mon Sep 17 00:00:00 2001 From: Harry Lin <49881386+harrylin98@users.noreply.github.com> Date: Wed, 27 May 2026 08:58:53 -0700 Subject: [PATCH 05/18] Set pending_command flag consistently across all command execution paths (#3600) The `pending_command` flag indicates that a client has a fully parsed command ready for execution. This update ensures that the flag is set/cleared consistently across different execution paths. --------- Signed-off-by: harrylin98 Signed-off-by: Jim Brunner --- src/blocked.c | 17 ++++++----------- src/db.c | 2 ++ src/module.c | 7 +++++++ src/networking.c | 3 ++- src/replication.c | 10 ++++++++-- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 9e81795b0a3..affc733131c 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -495,10 +495,10 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo } } c->bstate->unblock_on_nokey = unblock_on_nokey; - /* Currently we assume key blocking will require reprocessing the command. - * However in case of modules, they have a different way to handle the reprocessing - * which does not require setting the pending command flag */ - if (btype != BLOCKED_MODULE) c->flag.pending_command = 1; + /* Key-blocked clients require pending_command for reprocessing on unblock. + * The caller must have set it (processInputBuffer for real clients, + * RM_Call for module fake clients). */ + serverAssert(c->flag.pending_command == 1); blockClient(c, btype); } @@ -730,8 +730,7 @@ void blockPostponeClient(client *c) { listAddNodeTail(server.postponed_clients, c); serverAssert(c->bstate->postponed_list_node == NULL); c->bstate->postponed_list_node = listLast(server.postponed_clients); - /* Mark this client to execute its command */ - c->flag.pending_command = 1; + serverAssert(c->flag.pending_command == 1); } /* Block client due to shutdown command */ @@ -762,7 +761,6 @@ static void unblockClientOnKey(client *c, robj *key) { /* In case this client was blocked on keys during command * we need to re process the command again */ if (c->flag.pending_command) { - c->flag.pending_command = 0; c->flag.reexecuting_command = 1; /* We want the command processing and the unblock handler (see RM_Call 'K' option) * to run atomically, this is why we must enter the execution unit here before @@ -935,10 +933,7 @@ static bool isClientBlockedInUse(client *c) { * The client remains blocked until ALL of its keys are unblocked via * unblockClientsInUseOnKey(). * - * The caller MUST set c->flag.pending_command = 1 before calling this function. - * This ensures the pending command is executed when the client is later - * unblocked via processPendingCommandAndInputBuffer(). - * The caller should then return without executing the command. */ + * The caller should return without executing the command after calling this. */ void blockClientInUseOnKeys(client *c, int num_keys, robj *keys[]) { serverAssert(!c->flag.blocked && !c->flag.unblocked); serverAssert(c->flag.pending_command == 1); diff --git a/src/db.c b/src/db.c index 2670d386dfd..7c0da08a303 100644 --- a/src/db.c +++ b/src/db.c @@ -1495,6 +1495,8 @@ void shutdownCommand(client *c) { return; } + /* Clear pending_command to avoid re-execution. */ + c->flag.pending_command = 0; blockClientShutdown(c); if (prepareForShutdown(c, flags) == C_OK) exit(0); /* If we're here, then shutdown is ongoing (the client is still blocked) or diff --git a/src/module.c b/src/module.c index 93a2fcefc6c..d7eb68db9df 100644 --- a/src/module.c +++ b/src/module.c @@ -6951,6 +6951,9 @@ static void moduleCallCommandHelper(ValkeyModuleCtx *ctx, client *c, robj **argv if (!(flags & VALKEYMODULE_CALL_ARGV_NO_AOF)) call_flags |= CMD_CALL_PROPAGATE_AOF; if (!(flags & VALKEYMODULE_CALL_ARGV_NO_REPLICAS)) call_flags |= CMD_CALL_PROPAGATE_REPL; } + /* Mirror processInputBuffer: set pending_command so that if the command + * blocks on keys, unblockClientOnKey will reprocess it on unblock. */ + c->flag.pending_command = 1; call(c, call_flags); /* Propagate database changes from the temporary client back to the context client @@ -8452,6 +8455,10 @@ ValkeyModuleBlockedClient *moduleBlockClient(ValkeyModuleCtx *ctx, c->bstate->timeout = timeout; blockClient(c, BLOCKED_MODULE); } + /* Module handles its own reply on unblock, so clear pending_command + * to prevent re-execution. Auth clients are the exception — they + * need re-execution after auth completes. */ + if (!auth_reply_callback) c->flag.pending_command = 0; /* Defer response until after being unblocked for a context originated from * keyspace notification events */ if (is_keyspace_notification) { diff --git a/src/networking.c b/src/networking.c index 0d85bbc1500..1c4cf32a636 100644 --- a/src/networking.c +++ b/src/networking.c @@ -3920,6 +3920,7 @@ void commandProcessed(client *c) { * since we have not applied the command. */ if (c->flag.blocked) return; + c->flag.pending_command = 0; reqresAppendResponse(c); clusterSlotStatsAddNetworkBytesInForUserClient(c); resetClient(c); @@ -4004,7 +4005,6 @@ int processPendingCommandAndInputBuffer(client *c) { * blocked client as well */ if (c->flag.close_asap) return C_ERR; if (c->flag.pending_command) { - c->flag.pending_command = 0; if (processCommandAndResetClient(c) == C_ERR) { return C_ERR; } @@ -4286,6 +4286,7 @@ int processInputBuffer(client *c) { } /* We are finally ready to execute the command. */ + c->flag.pending_command = 1; if (processCommandAndResetClient(c) == C_ERR) { /* If the client is no longer valid, we avoid exiting this * loop and trimming the client buffer later. So we return diff --git a/src/replication.c b/src/replication.c index 11bbb52897c..ddde359f0f2 100644 --- a/src/replication.c +++ b/src/replication.c @@ -5149,7 +5149,10 @@ void waitCommand(client *c) { } /* Otherwise, block the client and put it into our list of clients - * waiting for ack from replicas. */ + * waiting for ack from replicas. WAIT handles its own reply in + * processClientsWaitingReplicas, so clear pending_command to avoid + * being mistaken for a command that needs re-execution. */ + c->flag.pending_command = 0; blockClientForReplicaAck(c, timeout, offset, numreplicas, 0); /* Make sure that the server will send an ACK request to all the replicas @@ -5191,7 +5194,10 @@ void waitaofCommand(client *c) { } /* Otherwise, block the client and put it into our list of clients - * waiting for ack from replicas. */ + * waiting for ack from replicas. WAITAOF handles its own reply in + * pgit add rocessClientsWaitingReplicas, so clear pending_command to avoid + * being mistaken for a command that needs re-execution. */ + c->flag.pending_command = 0; blockClientForReplicaAck(c, timeout, offset, numreplicas, numlocal); /* Make sure that the server will send an ACK request to all the replicas From d18b12a37ed382c3b6c5d3094341fe0ce41f34ba Mon Sep 17 00:00:00 2001 From: Jim Brunner Date: Thu, 23 Jul 2026 10:21:18 -0700 Subject: [PATCH 06/18] Forkless, background iteration utility (forkless) (#3553) BgIteration - background iteration utility, the core of forkless operations. --------- Signed-off-by: Jim Brunner --- .config/typos.toml | 15 +- cmake/Modules/SourceFiles.cmake | 1 + src/Makefile | 1 + src/bgiteration.c | 2693 ++++++++++++++++++++++++++ src/bgiteration.h | 365 ++++ src/db.c | 11 +- src/defrag.c | 8 + src/expire.c | 8 +- src/hashtable.c | 24 +- src/hashtable.h | 2 + src/module.c | 10 +- src/module.h | 2 +- src/object.c | 5 +- src/server.c | 83 +- src/server.h | 21 +- src/unit/custom_matchers.hpp | 6 +- src/unit/test_bgiteration.cpp | 3172 +++++++++++++++++++++++++++++++ src/unit/wrappers.h | 9 + 18 files changed, 6407 insertions(+), 29 deletions(-) create mode 100644 src/bgiteration.c create mode 100644 src/bgiteration.h create mode 100644 src/unit/test_bgiteration.cpp diff --git a/.config/typos.toml b/.config/typos.toml index 7a2fc0c1c73..c2a9be035db 100644 --- a/.config/typos.toml +++ b/.config/typos.toml @@ -18,11 +18,16 @@ Collet = "Collet" # LZ4 author Yann Collet nd = "nd" Ba = "Ba" Addd = "Addd" +threadsave = "threadsave" + +[default.extend-identifiers] +dbe = "dbe" [default] extend-ignore-re = [ - "SELECTed", - "WATCHed", + "[A-Z]{2,}", # Acronyms (all caps) + "[A-Z]{2,}ed", # SELECTed, WATCHed, etc. + "[A-Z]{2,}s", # SELECTs, etc. ] [type.c] @@ -68,6 +73,9 @@ pn = "pn" seeked = "seeked" tre = "tre" +[type.cpp.extend-words] +fo = "fo" + [type.systemd.extend-words] # systemd = .conf ake = "ake" @@ -75,6 +83,3 @@ ake = "ake" [type.tcl.extend-words] fo = "fo" tre = "tre" - -[type.cpp.extend-words] -fo = "fo" diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake index 9aeebe03406..c735d9d32a9 100644 --- a/cmake/Modules/SourceFiles.cmake +++ b/cmake/Modules/SourceFiles.cmake @@ -38,6 +38,7 @@ set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/t_hash.c ${CMAKE_SOURCE_DIR}/src/config.c ${CMAKE_SOURCE_DIR}/src/aof.c + ${CMAKE_SOURCE_DIR}/src/bgiteration.c ${CMAKE_SOURCE_DIR}/src/pubsub.c ${CMAKE_SOURCE_DIR}/src/multi.c ${CMAKE_SOURCE_DIR}/src/debug.c diff --git a/src/Makefile b/src/Makefile index 3f8063584ac..23549fa49c2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -477,6 +477,7 @@ ENGINE_SERVER_OBJ = \ allocator_defrag.o \ anet.o \ aof.o \ + bgiteration.o \ bio.o \ bitops.o \ blocked.o \ diff --git a/src/bgiteration.c b/src/bgiteration.c new file mode 100644 index 00000000000..e0ef2a26c61 --- /dev/null +++ b/src/bgiteration.c @@ -0,0 +1,2693 @@ +/* + * Copyright Valkey Contributors. + * All rights reserved. + * SPDX-License-Identifier: BSD 3-Clause + */ + +#include "fmacros.h" +#include "bgiteration.h" +#include "dict.h" +#include "fifo.h" +#include "kvstore.h" +#include "monotonic.h" +#include "mutexqueue.h" +#include "server.h" + + +static bool receiveItemsBackFromOneIterator(bgIterator *it); + + +// Returns true if the cmd is a script command that may replicate. +static bool isScriptCallWriteCmd(struct serverCommand *cmd) { + return ((cmd->proc == fcallCommand) || (cmd->proc == evalCommand) || (cmd->proc == evalShaCommand)); +} + +/* The PFCOUNT command (which does NOT have the CMD_WRITE flag) modifies the underlying string and + * is replicated as a write. So it needs to be detected and handled specially. */ +static bool isWriteCmd(struct serverCommand *cmd) { + return ((cmd->flags & CMD_WRITE) || (cmd->proc == pfcountCommand) || (cmd->proc == execCommand) || (isScriptCallWriteCmd(cmd))); +} + +// Returns true if the command is a deletion based command (DEL or UNLINK) +static bool isDeleteCmd(struct serverCommand *cmd) { + return ((cmd->proc == delCommand) || (cmd->proc == unlinkCommand)); +} + + +/* This utility utilizes the main thread and background threads for processing. The API is split, + * with some of the functions intended for the main thread and others intended for the background + * clients. This sanity check ensures that we maintain thread safety, calling the API as intended. */ +static bool onValkeyMainThread(void) { + /* Modules interact with the main thread using a mutex. If a module owns the mutex, consider + * that equivalent to being on the main thread. */ + bool mightBeInModule = (atomic_load_explicit(&server.module_gil_acquired, memory_order_relaxed) == 0); + return (mightBeInModule || pthread_equal(server.main_thread_id, pthread_self()) != 0); +} + + +/* Parse a parameters robj, extracting a valid DBID. + * Returns FALSE if DBID isn't valid. */ +static bool getDbIdFromRobj(robj *obj, int *db_id) { + long long value; + if (getLongLongFromObject(obj, &value) != C_OK) return false; + if ((value < 0) || (value >= server.dbnum)) return false; + *db_id = (int)value; + return true; +} + +/* Parse the parameters of the COPY command, extracting the target DBID. + * Returns FALSE if the command would not run. */ +static bool getTargetDbIdForCopyCommand(int argc, robj **argv, int selected_dbid, int *target_dbid) { + const int COPY_COMMAND_OPTIONAL_ARG_START_INDEX = 3; + + *target_dbid = selected_dbid; + + for (int i = COPY_COMMAND_OPTIONAL_ARG_START_INDEX; i < argc; i++) { + if (!strcasecmp((char *)objectGetVal(argv[i]), "replace")) { + continue; + } else if (!strcasecmp((char *)objectGetVal(argv[i]), "db") && (i + 1 < argc)) { + /* Note the parsing here needs to perfectly match what we have in copyCommand. The + * following command is considered OK so we can't return here, but must continue to + * parse till the last db which is the one that's effectively used. + * COPY key1 key2 db 1 db 2 db 3 (This will use db 3) */ + if (!getDbIdFromRobj(argv[i + 1], target_dbid)) { + return false; // parse failure + } + i++; // Consume additional argument + } else { + return false; // parse failure + } + } + return true; +} + +/* Get parameters for the SWAPDB command. + * The optional permission_client allows for checking of a client's permission for swapdb. + * Returns true if command would be executed. */ +static bool getParamsForSwapdb(int argc, robj **argv, client *permission_client, int *id1_p, int *id2_p) { + static struct serverCommand *swapdb_cmd = NULL; + + // We don't need to check permissions in the replication phase + if (permission_client != NULL) { + if (swapdb_cmd == NULL) { + swapdb_cmd = lookupCommandByCString("swapdb"); + serverAssert(swapdb_cmd != NULL); + } + + int idxptr; + if (ACLCheckAllUserCommandPerm(permission_client->user, swapdb_cmd, argv, argc, + permission_client->db->id, &idxptr) != ACL_OK) return false; + } + + long long dbid1, dbid2; + if (argc != 3) return false; + if (server.cluster_enabled) return false; + if (getLongLongFromObject(argv[1], &dbid1) != C_OK) return false; + if (getLongLongFromObject(argv[2], &dbid2) != C_OK) return false; + if (dbid1 < 0 || dbid1 >= server.dbnum) return false; + if (dbid2 < 0 || dbid2 >= server.dbnum) return false; + if (dbid1 == dbid2) return false; // Valid, but doesn't do anything + + *id1_p = (int)dbid1; + *id2_p = (int)dbid2; + return true; +} + +/* Get parameters for the SELECT command. + * The optional permission_client allows for checking of a client's permission for select. + * Returns true if command would be executed. */ +static bool getParamsForSelect(int argc, robj **argv, client *permission_client, int *dbid_p) { + static struct serverCommand *select_cmd = NULL; + + // We don't need to check permissions in the replication phase + if (permission_client != NULL) { + if (select_cmd == NULL) { + select_cmd = lookupCommandByCString("select"); + serverAssert(select_cmd != NULL); + } + + int idxptr; + if (ACLCheckAllUserCommandPerm(permission_client->user, select_cmd, argv, argc, + permission_client->db->id, &idxptr) != ACL_OK) return false; + } + + long long dbid; + if (argc != 2) return false; + if (getLongLongFromObject(argv[1], &dbid) != C_OK) return false; + if (dbid < 0 || dbid >= server.dbnum) return false; + + *dbid_p = (int)dbid; + return true; +} + +static void pauseRehashForKvsHashtable(kvstore *kvs, int didx) { + hashtable *ht = kvstoreGetHashtable(kvs, didx); + if (ht != NULL) hashtablePauseRehashing(ht); +} + +static void resumeRehashForKvsHashtable(kvstore *kvs, int didx) { + hashtable *ht = kvstoreGetHashtable(kvs, didx); + if (ht != NULL) hashtableResumeRehashing(ht); +} + + +/* DictType for SDS->ptr. The SDS is referenced, no destructor. */ +static dictType sdsrefToPtrDictType = { + .entryGetKey = dictEntryGetKey, + .hashFunction = dictSdsHash, + .keyCompare = dictSdsKeyCompare, + .entryDestructor = zfree}; + + +/* Wrap decrRefCount() so that it can be used as a callback requiring void. */ +static void decrRefCountVoid(void *o) { + decrRefCount(o); +} + + +/* Concatenate argc/argv into a command string for debugging. */ +static sds createSdsFromClientArgv(int argc, robj **argv) { + sds cmd = sdsempty(); + for (int i = 0; i < argc; i++) { + robj *arg = getDecodedObject(argv[i]); // some objects are int encoded + cmd = sdscatprintf(cmd, "'%s' ", (char *)objectGetVal(arg)); + decrRefCount(arg); + } + return cmd; +} + + +// ########################################################################### + + +/* bgIteration internal (compile time) configuration values */ +enum { + BGITER_EARLY_ITERATE_DICT_INITIAL_SIZE = 16384, // Prevent initial rehashing + BGITER_MAX_CLONE_ITEM_BYTES = 512, // Max size item to clone + BGITER_MAX_CLONE_POOL_BYTES = (1 * 1024 * 1024), // Total limit for all cloned items + BGITER_QUEUE_INCREASE_INCR = 100, // Step size when increasing queue target + BGITER_QUEUE_MAX_LENGTH = 10000, // Max length for the dynamic queue + BGITER_CYCLE_DELAY_MS = 2, // Delay between calls on bgIteration timer + BGITER_CYCLE_BUDGET_MS = 1, // Normal time limit for timer processing + BGITER_CYCLE_BUDGET_MAX_MS = 10 // Maximum time limit when starvation seen +}; + +// dbEntry metadata +typedef struct { + uint32_t iterator_epoch; // iterator epoch of last modification +} bgIterationEntryMetadata; +static_assert(sizeof(bgIterationEntryMetadata) == BGITERATION_ENTRY_METADATA_SIZE, ""); + + +// These can be tweaked by unit tests +static int bgiter_max_clone_item_bytes = BGITER_MAX_CLONE_ITEM_BYTES; +static int bgiter_max_clone_pool_bytes = BGITER_MAX_CLONE_POOL_BYTES; + +void bgIteration_unitTestDisableCloning(void) { + bgiter_max_clone_item_bytes = 0; + bgiter_max_clone_pool_bytes = 0; +} +void bgIteration_unitTestEnableCloning(int item_bytes, int pool_bytes) { + bgiter_max_clone_item_bytes = item_bytes; + bgiter_max_clone_pool_bytes = pool_bytes; +} + +typedef enum { + BGITERATION_TYPE_NONE, + BGITERATION_TYPE_FULLSCAN, + BGITERATION_TYPE_CLUSTERSLOT +} bgIterationType; + + +/* Flag indicates that a consistent iteration is required. This is used to create a point-in-time + * iteration. The iteration client will see all keys AS THEY EXISTED at the time when the iterator + * was created. + * Note: The DBID provided with the DICTENTRY events is the original DBID (at the time of iteration + * start). SWAPDB events are NOT provided during a consistent iteration. */ +#define BGITERATOR_FLAG_CONSISTENT (1 << 0) + +/* Flag indicating that the replication stream for keys which have already been processed should be + * forwarded to the iteration client. Used for non-consistent iteration to track changes + * to keys already processed. By tracking changes, this allows an non-consistent iteration client + * to achieve a consistent view at the END of the iteration. + * NOTE: Replication events will be provided ordered and synchronized with any SWAPDB events. */ +#define BGITERATOR_FLAG_REPLICATION (1 << 1) + + +/* Extensions to bgIteratorItemType. These enumerations are used internally, and are not part of + * the published interface. These allow for extensibility in the internal information-passing + * between the Valkey main thread and the iteration client thread. */ +typedef enum { + /* Indicates that the iteration client has completed use of the bgIterator and that the + * bgIterator should be cleaned up and freed by the Valkey main thread. */ + BGITERATOR_ITEMEXT_ITER_CLOSED = 10 +} bgIteratorItemTypeExtended; + +// Static bgIterator items for items which carry no data +static const bgIteratorItem STATIC_ITEM_TERMINATED = {.type = (bgIteratorItemType)BGITERATOR_ITEM_TERMINATED}; +static const bgIteratorItem STATIC_ITEM_ITER_CLOSED = {.type = (bgIteratorItemType)BGITERATOR_ITEMEXT_ITER_CLOSED}; + + +/* A dictionary with a pointer (itself) as a key (the address pointed to is NOT referenced). + * Nothing is duplicated, this is a very fast dictionary, but potentially unsafe if the original + * items are deleted or moved. + * WARNING: This needs to maintain safety with things that may move the object. + * + In db.c, if the object is reallocated, bgIteration_updateDbEntryPtr() is called. + * + In defrag.c, we don't defrag if there are multiple references (and we incr the refcount). */ + +// Thomas Wang's 64-bit mix +static uint64_t pointerHash(const void *key) { + uint64_t h = (uint64_t)(uintptr_t)key; + h = (~h) + (h << 21); // h = (h << 21) - h - 1; + h = h ^ (h >> 24); + h = (h + (h << 3)) + (h << 8); // h * 265 + h = h ^ (h >> 14); + h = (h + (h << 2)) + (h << 4); // h * 21 + h = h ^ (h >> 28); + h = h + (h << 31); + return h; +} + +static int pointerCompare(const void *key1, const void *key2) { + return key1 == key2; +} + +// This dict grows and shrinks constantly during the iteration. Avoid constant rehashing. +static int onlyAllowExpansion(size_t moreMem, double usedRatio) { + UNUSED(moreMem); + return (usedRatio > 0.5); // Return true only if expanding +} + +static dictType dictEntryPtrDictType = { + .entryGetKey = dictEntryGetKey, + .hashFunction = pointerHash, + .keyCompare = pointerCompare, + .resizeAllowed = onlyAllowExpansion, + .entryDestructor = zfree}; + +static hashtableType dbEntryPtrHashtableType = { + .hashFunction = pointerHash, + .keyCompare = pointerCompare, + .resizeAllowed = onlyAllowExpansion}; + + +// A free list for bgIteratorItem's - avoids churning zmalloc calls +typedef struct itemListNode { + struct itemListNode *next; +} itemListNode; + +static const int FREE_ITEM_MAX = 500; +static itemListNode *freeItemStackHead = NULL; +static int freeItemStackCount = 0; + +static void itemFreeList_returnItemBackToFreeList(bgIteratorItem *item) { + itemListNode *freedNode = (itemListNode *)item; + if (freeItemStackCount < FREE_ITEM_MAX) { + freedNode->next = freeItemStackHead; + freeItemStackHead = freedNode; + freeItemStackCount++; + } else { + zfree(freedNode); + } +} + +// Pop a free node from the free list or allocate if none free +static bgIteratorItem *itemFreeList_getElementOrAllocate(void) { + bgIteratorItem *item; + if (freeItemStackHead) { + item = (bgIteratorItem *)freeItemStackHead; + freeItemStackHead = freeItemStackHead->next; + freeItemStackCount--; + if (freeItemStackHead) valkey_prefetch(freeItemStackHead); + } else { + serverAssert(freeItemStackCount == 0); + // Create new listNode and item + item = zmalloc(sizeof(bgIteratorItem)); + } + return item; +} + +static void itemFreeList_release(void) { + while (freeItemStackHead) { + itemListNode *node = freeItemStackHead; + freeItemStackHead = node->next; + freeItemStackCount--; + zfree(node); + } + serverAssert(freeItemStackCount == 0); +} + + +/* A TEMPORARY set of robj's (of type sds). This is only for temporary sets as the robj's are not + * ref-counted at insertion/deletion. */ +static hashtableType tempKeysetHashtableType = { + .hashFunction = dictObjHash, + .keyCompare = dictObjKeyCompare}; + + +typedef struct genericIterator genericIterator; +typedef void (*iteratorReleaseFunc)(genericIterator *genIt); +typedef fifo *(*iteratorGetEntriesFunc)(genericIterator *genIt, int *orig_dbid, int *cur_dbid); +typedef void (*iteratorSwapDbFunc)(genericIterator *genIt, int db1, int db2); +typedef void (*iteratorFlushDbFunc)(genericIterator *genIt, int cur_dbid); +typedef bool (*iteratorHasPassedItemFunc)(genericIterator *genIt, const_sds key, int cur_dbid); +typedef int (*iteratorOriginalDbFunc)(genericIterator *genIt, int cur_dbid); +typedef bool (*iteratorIsKeyInScopeFunc)(genericIterator *genIt, const_sds key); + +// Function pointers supporting polymorphic iterator implementation +struct genericIterator { + iteratorReleaseFunc release; + iteratorGetEntriesFunc getEntries; + iteratorSwapDbFunc swapDb; + iteratorFlushDbFunc flushDb; + iteratorHasPassedItemFunc hasPassedItem; + iteratorOriginalDbFunc originalDb; + iteratorIsKeyInScopeFunc isKeyInScope; +}; + + +/* This struct is used across threads. Unless otherwise noted, the fields are initialized at + * iterator creation (within the main thread) and are read-only by the client thread. */ +struct bgIterator { + sds name; // Iterator name + bgIteratorReplDoneFunc repldone; // Optional repldone function to be run on the main thread + bgIteratorCleanupFunc cleanup; // Optional cleanup function to be run on main thread + void *privdata; // Client's private data to be passed to cleanup function + + int iteration_flags; // Consistent and/or Replication + int iteration_type; // Full scan or cluster slot + uint32_t consistent_modification_id; // iterator epoch at time of iterator creation + + genericIterator *keyset_iter; // Low-level iterator (polymorphic) + + /* A set of dbEntry, compared by pointer. Used to track items which have already been iterated + * over by out-of-order expedited processing. Ensures a bgIterator does not try to reprocess + * items. Used only by main thread. */ + hashtable *early_iterate_entries; + + mutexQueue *items_for_iterator; // Created/Destroyed in main thread, used in both (threadsafe) + + mutexQueue *return_to_main_thread; // Queue of items to be returned to the Valkey main thread (threadsafe) + + unsigned int item_count_target; // Used only by main thread + + bgIteratorItem *current_item; // Used in client thread, validated in main after iterator complete + + bool client_is_active; // Set to true when client performs 1st read + + /* Set to true in main thread when last item from iteration has been queued to the client. No + * additional items will be enqueued to the client after this has been set. */ + bool completed; + + /* Set to true in main thread when iteration is to be killed. + * Set to true in iteration client when it decides to end early. */ + volatile bool terminated; + + bool cur_cmd_may_replicate; // Used only in main thread during command processing + + // Variables maintaining runtime statistics + unsigned long dbentries_queued; // Updated by main thread + unsigned long dbentries_processed; // Updated by client thread + unsigned long replication_queued; // Updated by main thread + unsigned long replication_processed; // Updated by client thread + unsigned long swapdb_queued; // Updated by main thread + unsigned long swapdb_processed; // Updated by client thread + unsigned long flushdb_queued; // Updated by main thread + unsigned long flushdb_processed; // Updated by client thread + unsigned long dbentry_clones_queued; // Updated by main thread + unsigned long dbentry_clones_processed; // Updated by client thread + monotime monotonic_start_time; // Time iteration started + + /* FLUSHDB and SWAPDB are special in that they affect all keys. When expediting a key, it's + * preferable to put it at the front of the queue. However, if there is a FLUSHDB or SWAPDB in + * the queue, we must maintain strict ordering. + * This value is equivalent to (flushdb_queued-flushdb_processed)+(swapdb_queued-swapdb_processed) */ + int barrier_items; + + /* The item start time is set in the iteration client. It is marked volatile as it can be read + * from the main thread by bgIteratorGetStatus. If 0, this indicates that the iteration client + * is waiting for an item to process. */ + volatile monotime monotonic_item_start_time; +}; + + +// These static values are only accessed from the main Valkey thread. + +static list *allIterators; // list of bgIterator +static dict *nameToIterator; // bgIterator->name -> bgIterator + +// Global, across all iterators, dict contains a dbEntry pointer -> ref count +static dict *inUseEntries; // dbEntry -> ref count + +/* Key values in the current command which don't exist in the DB yet. Needed for determination of + * replication for NON-consistent iterations. */ +static list *curCmdMissingKeys; // list of robj + +/* A counter of the total amount of memory used for buffered replication data. This amount is + * excluded when computing the need for evictions. */ +static ssize_t bufferedReplicationBytes; + +// Memory pool to track current allocated memory of cloned items (in bytes) +static ssize_t bgiteration_current_clone_memory_pool_size; + +/* Snapshot of the last queue size to seed the next queue. We assume all bgIterators consume items + * at roughly the same rate. */ +static int last_item_count_target; + +// Eventloop ID of the timerproc (or AE_DELETED_EVENT_ID) +static long long bgIterator_timeproc_id; + +// Incremented on each new iteration, this is updated in dbEntry metadata whenever an entry is modified. +static uint32_t bgIteration_epoch = 1; + +/* If true, the iterators' cur_cmd_may_replicate flag was determined in the last call to + * blockClientIfRequired. Otherwise, we skipped over computing this flag (maybe because it was a + * READ command). + * If this is true, AND we are in the context of executing a command inside of call(), then we + * should respect the setting of cur_cmd_may_replicate. */ +static bool iteratorReplicationFlagsWereUpdated; + +/* When a key is deleted (expire/evict): + * 1. bgIteration_keyDelete() is called + * 2. the key is physically deleted + * 3. replication is generated + * At the time of replication, we need the (deleted) dbEntry pointer to be able to check + * early_iterated_entries. This variable stores the pointer from the last call of keyDelete() */ +static dbEntry *dbEntryPtrOfLastKeyDelete; + +/* BgIteration debug captures BgIteration activity to a large sds buffer. When an iterator is + * completed, the entire buffer is written to a file in the current working directory. Note that + * memory must be available for the ENTIRE debug in memory. This isn't captured incrementally to + * a file as the file I/O is more likely to affect timing. + * + * Future implementation: the current design is most useful for a single iterator. When items are + * queued to an iterator, the iterator name is not recorded (to save space). + * + * Developer note: using a CONST value here allows the compiler to completely remove all of the + * debugging code at compile time. There is no run-time performance overhead when set to FALSE. + * This is essentially like an IFDEF, however, it's better as it forces the compiler to validate + * syntax. */ +static const bool BGITERATION_DEBUG = false; // DO NOT SUBMIT WITH THIS SYMBOL SET TO TRUE! +static sds debugBuffer; + + +/* ============================================================================================= + * Full Scan Iterator + * ============================================================================================= + * The full scan iterator performs the actual iteration over the Valkey keyset. The iterator is + * only used from within the Valkey main thread. Iteration proceeds one DB at a time, based on + * the DB ordering at the time of iterator creation. Each time the iterator returns items, all + * of the dictionary entries from a single hash bucket are returned. */ + +struct fullScanIterator { + genericIterator callbacks; // (must be first item) + + /* Array of mapping from original DB ID (at the time of iteration start) to that DB's current + * index. So, if the DB which was DB-0 is now at index 6, orig_to_cur_db[0]==6. */ + int *orig_to_cur_db; + + /* The reverse of the above array. This maps a current DB index to its original index (at the + * time of iteration start). */ + int *cur_to_orig_db; + + /* This is the DB we are currently iterating over. This is relative to the ORIGINAL DB + * ordering, at the time of iterator creation. Iteration proceeds from 0..N based on the + * original ordering. */ + int iter_db; + + // Iterator for the DB orig_to_cur_db[iter_db] + kvstore *kvs; // keep track of kvs associated with iter_dbi + int kvs_didx; // hashtable index within the kvstore + size_t ht_cursor; // cursor for scanning hashtable +}; + +static void fullScanIteratorRelease(genericIterator *genIt) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + if (it->kvs) resumeRehashForKvsHashtable(it->kvs, it->kvs_didx); + zfree(it->orig_to_cur_db); + zfree(it->cur_to_orig_db); + zfree(it); +} + +/* Scan callback used by fullScanIteratorGetEntries2 to collect entries into a fifo. */ +static void fullScanIteratorScanCallback(void *privdata, void *entry) { + fifo *dbEntryFifo = (fifo *)privdata; + dbEntry *de = (dbEntry *)entry; + fifoPush(dbEntryFifo, de); +} + +static fifo *fullScanIteratorGetEntries(genericIterator *genIt, int *orig_dbid, int *cur_dbid) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + if (it->iter_db >= server.dbnum) return NULL; // Finished scanning + + fifo *dbEntryFifo = fifoCreate(); + while (fifoLength(dbEntryFifo) == 0) { + while (it->kvs == NULL) { + if (++it->iter_db >= server.dbnum) { + fifoRelease(dbEntryFifo); + return NULL; // Iteration complete + } + serverDb *db = server.db[it->orig_to_cur_db[it->iter_db]]; + if (db != NULL) { + it->kvs = db->keys; + it->kvs_didx = kvstoreGetFirstNonEmptyHashtableIndex(it->kvs); + it->ht_cursor = 0; + if (it->kvs_didx == KVSTORE_INDEX_NOT_FOUND) it->kvs = NULL; + if (it->kvs != NULL) pauseRehashForKvsHashtable(it->kvs, it->kvs_didx); + } + } + + hashtable *ht = kvstoreGetHashtable(it->kvs, it->kvs_didx); + if (ht) { + it->ht_cursor = hashtableScan(ht, it->ht_cursor, fullScanIteratorScanCallback, dbEntryFifo); + } else { + it->ht_cursor = 0; + } + + if (it->ht_cursor == 0) { + /* Done with this hashtable, move to next. */ + resumeRehashForKvsHashtable(it->kvs, it->kvs_didx); + it->kvs_didx = kvstoreGetNextNonEmptyHashtableIndex(it->kvs, it->kvs_didx); + if (it->kvs_didx == KVSTORE_INDEX_NOT_FOUND) it->kvs = NULL; + if (it->kvs != NULL) pauseRehashForKvsHashtable(it->kvs, it->kvs_didx); + } + } + *orig_dbid = it->iter_db; + *cur_dbid = it->orig_to_cur_db[*orig_dbid]; + return dbEntryFifo; +} + +static void fullScanIteratorSwapDb(genericIterator *genIt, int db1, int db2) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + int temp = it->cur_to_orig_db[db1]; + it->cur_to_orig_db[db1] = it->cur_to_orig_db[db2]; + it->cur_to_orig_db[db2] = temp; + + it->orig_to_cur_db[it->cur_to_orig_db[db1]] = db1; + it->orig_to_cur_db[it->cur_to_orig_db[db2]] = db2; +} + +static void fullScanIteratorFlushDb(genericIterator *genIt, int cur_dbid) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + int orig_db = (cur_dbid == -1) ? it->iter_db : it->cur_to_orig_db[cur_dbid]; + if (orig_db == it->iter_db) { + // We are currently iterating on the DB that's being flushed. + it->kvs = NULL; + // Iteration will continue with the next DB. + } +} + +static bool fullScanIteratorHasPassedItem(genericIterator *genIt, const_sds key, int cur_dbid) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + int orig_dbid = it->cur_to_orig_db[cur_dbid]; + + if (orig_dbid < it->iter_db) return true; // Entire DB has already been processed + if (orig_dbid > it->iter_db) return false; // Haven't started this DB yet + // Now, orig_dbid == it->iter_db + + if (it->kvs == NULL) return true; // just finished this DB + + /* We're in the middle of processing a DB. In cluster-mode, the DB is divided into 1 hashtable + * per slot. In cluster-mode-disabled, we treat all keys as in slot 0. */ + int keySlot = server.cluster_enabled ? getKVStoreIndexForKey((sds)key) : 0; + if (keySlot < it->kvs_didx) return true; + if (keySlot > it->kvs_didx) return false; + + // At this point, we're down to a specific hashtable. + + hashtable *ht = kvstoreGetHashtable(it->kvs, keySlot); + if (hashtableScanHasPassedKey(ht, key, it->ht_cursor)) return true; + + return false; +} + +static int fullScanIteratorOriginalDb(genericIterator *genIt, int cur_dbid) { + struct fullScanIterator *it = (struct fullScanIterator *)genIt; + return it->cur_to_orig_db[cur_dbid]; +} + +static bool fullScanIteratorIsKeyInScope(genericIterator *genIt, const_sds key) { + UNUSED(genIt); + UNUSED(key); + return true; // All keys are in scope +} + +static genericIterator *fullScanIteratorCreate(void) { + struct fullScanIterator *it = zmalloc(sizeof(struct fullScanIterator)); + it->orig_to_cur_db = zmalloc(sizeof(int) * server.dbnum); + it->cur_to_orig_db = zmalloc(sizeof(int) * server.dbnum); + for (int i = 0; i < server.dbnum; i++) { + it->orig_to_cur_db[i] = i; + it->cur_to_orig_db[i] = i; + } + it->iter_db = -1; + it->kvs = NULL; + + it->callbacks.release = fullScanIteratorRelease; + it->callbacks.getEntries = fullScanIteratorGetEntries; + it->callbacks.swapDb = fullScanIteratorSwapDb; + it->callbacks.flushDb = fullScanIteratorFlushDb; + it->callbacks.hasPassedItem = fullScanIteratorHasPassedItem; + it->callbacks.originalDb = fullScanIteratorOriginalDb; + it->callbacks.isKeyInScope = fullScanIteratorIsKeyInScope; + + return (genericIterator *)it; +} + + +/* ============================================================================================= + * Cluster Slot Iterator + * ============================================================================================= + * The cluster slot iterator performs iteration over one cluster slot of the Valkey keyset. The + * iterator is only used from within the Valkey main thread. */ +struct clusterSlotIterator { + genericIterator callbacks; // (must be first item) +}; + +static void clusterSlotIteratorRelease(genericIterator *genIt) { + UNUSED(genIt); + serverAssert(false); // Not yet implemented +} + +static fifo *clusterSlotIteratorGetEntries(genericIterator *genIt, int *orig_dbid, int *cur_dbid) { + UNUSED(genIt); + UNUSED(orig_dbid); + UNUSED(cur_dbid); + serverAssert(false); // Not yet implemented +} + +static void clusterSlotIteratorSwapDb(genericIterator *genIt, int db1, int db2) { + UNUSED(genIt); + UNUSED(db1); + UNUSED(db2); + serverAssert(false); // swap not valid in cluster mode +} + +static void clusterSlotIteratorFlushDb(genericIterator *genIt, int cur_dbid) { + UNUSED(genIt); + UNUSED(cur_dbid); + serverAssert(false); // Not yet implemented +} + +static bool clusterSlotIteratorHasPassedItem(genericIterator *genIt, const_sds key, int cur_dbid) { + UNUSED(genIt); + UNUSED(key); + UNUSED(cur_dbid); + serverAssert(false); // Not yet implemented +} + +static int clusterSlotIteratorOriginalDb(genericIterator *genIt, int cur_dbid) { + UNUSED(genIt); + UNUSED(cur_dbid); + return cur_dbid; // swap not supported in cluster mode +} + +/* When checking if a command is in scope for this iterator, all of its keys should be either in + * scope or not. In cluster mode enabled a command cannot reference keys from different slots, so + * this assumption will always be true. */ +static bool clusterSlotIteratorIsKeyInScope(genericIterator *genIt, const_sds key) { + UNUSED(genIt); + UNUSED(key); + serverAssert(false); // Not yet implemented +} + +static genericIterator *clusterSlotIteratorCreate(const int *slots, size_t slots_count) { + struct clusterSlotIterator *it = zmalloc(sizeof(struct clusterSlotIterator)); + it->callbacks.release = clusterSlotIteratorRelease; + it->callbacks.getEntries = clusterSlotIteratorGetEntries; + it->callbacks.swapDb = clusterSlotIteratorSwapDb; + it->callbacks.flushDb = clusterSlotIteratorFlushDb; + it->callbacks.hasPassedItem = clusterSlotIteratorHasPassedItem; + it->callbacks.originalDb = clusterSlotIteratorOriginalDb; + it->callbacks.isKeyInScope = clusterSlotIteratorIsKeyInScope; + + UNUSED(slots); + UNUSED(slots_count); + serverAssert(false); // Not yet implemented + + return (genericIterator *)it; +} + + +/* ============================================================================================= + * General iteration support (across all iterators) + * ============================================================================================= */ + +/* While an item is potentially in use by a background thread, we can't have rehashing by the main + * thread. Returns true if rehashing was paused. */ +static bool pauseRehashing(dbEntry *de) { + switch (de->encoding) { + case OBJ_ENCODING_HASHTABLE: { // SET or HASH + hashtable *ht = objectGetVal(de); + hashtablePauseRehashing(ht); + return true; + } + case OBJ_ENCODING_BTREE: { // SORTED SET + zset *zs = objectGetVal(de); + hashtablePauseRehashing(zs->ht); + return true; + } + default: + return false; + } +} + +static void resumeRehashing(dbEntry *de) { + switch (de->encoding) { + case OBJ_ENCODING_HASHTABLE: { // SET or HASH + hashtable *ht = objectGetVal(de); + hashtableResumeRehashing(ht); + break; + } + case OBJ_ENCODING_BTREE: { // SORTED SET + zset *zs = objectGetVal(de); + hashtableResumeRehashing(zs->ht); + break; + } + default: + break; + } +} + +// Maintain a list of entries which are currently in-use. These items should not be modified. +static void incrementEntryInuse(dbEntry *de) { + dictEntry *existingEntry; + dictEntry *newEntry = dictAddRaw(inUseEntries, de, &existingEntry); + if (newEntry) { + incrRefCount(de); + dictSetSignedIntegerVal(newEntry, 1); + } else { + dictSetSignedIntegerVal(existingEntry, dictGetSignedIntegerVal(existingEntry) + 1); + } +} + + +static void decrementEntryInuse(dbEntry *de) { + dictEntry *entry = dictFind(inUseEntries, de); + if (dictGetSignedIntegerVal(entry) == 1) { + dictDelete(inUseEntries, de); + decrRefCount(de); + } else { + serverAssert(dictGetSignedIntegerVal(entry) > 1); + dictSetSignedIntegerVal(entry, dictGetSignedIntegerVal(entry) - 1); + } +} + +static bool isEntryInuseBySingleIterator(dbEntry *de) { + dictEntry *entry = dictFind(inUseEntries, de); + return dictGetSignedIntegerVal(entry) == 1; +} + +static bool isEntryInuseByAnyIterator(dbEntry *de) { + return (dictFind(inUseEntries, de) != NULL); +} + + +static ssize_t computeStringDbEntrySize(dbEntry *de) { + sds key = objectGetKey(de); + size_t valueSize = stringObjectLen(de); + + return sdslen(key) + valueSize; // ignore the rest of the overhead, it's minor & transient +} + + +static dbEntry *tryCloneDbEntry(dbEntry *de) { + if (bgiteration_current_clone_memory_pool_size + bgiter_max_clone_item_bytes > + bgiter_max_clone_pool_bytes) return NULL; + + /* Future optimization: Incorporate small ziplists, sorted sets, etc. + * OBJ_ENCODING_INT is omitted only because there isn't a good API for cloning it yet. */ + if (de->type == OBJ_STRING && de->encoding != OBJ_ENCODING_INT) { + ssize_t itemSize = computeStringDbEntrySize(de); + + if (itemSize <= bgiter_max_clone_item_bytes) { + bgiteration_current_clone_memory_pool_size += itemSize; + dbEntry *clone = createStringObjectWithKeyAndExpire((char *)objectGetVal(de), + sdslen(objectGetVal(de)), + objectGetKey(de), + objectGetExpire(de)); + ((bgIterationEntryMetadata *)objectGetMetadata(clone))->iterator_epoch = + ((bgIterationEntryMetadata *)objectGetMetadata(de))->iterator_epoch; + return clone; + } + } + + return NULL; +} + +static void freeClonedDictEntry(dbEntry *clonedEntry) { + serverAssert(clonedEntry->type == OBJ_STRING); + + bgiteration_current_clone_memory_pool_size -= computeStringDbEntrySize(clonedEntry); + + decrRefCount(clonedEntry); +} + + +static bgIteratorItem *makeDbEntryItem(dbEntry *de, int dbid, bool isCloned) { + if (!isCloned) incrementEntryInuse(de); + + bgIteratorItem *item = itemFreeList_getElementOrAllocate(); + item->type = BGITERATOR_ITEM_DBENTRY; + item->dbid = dbid; + item->u.dbe.de = de; + item->u.dbe.is_cloned = isCloned; + item->u.dbe.is_rehashing_paused = pauseRehashing(de); + + return item; +} + +static robj **cloneRobjArray(int argc, robj **argv) { + robj **newarray = zmalloc(sizeof(robj *) * argc); + for (int i = 0; i < argc; i++) { + newarray[i] = argv[i]; + incrRefCount(argv[i]); + } + return newarray; +} + + +static void freeRobjArray(int argc, robj **argv) { + for (int i = 0; i < argc; i++) { + decrRefCount(argv[i]); + } + zfree(argv); +} + + +// Called by iterator thread to release an item. +static void returnCurrentItemToMainThread(bgIterator *it) { + bgIteratorItem *item = it->current_item; + if (item == NULL) return; + + switch (item->type) { + case BGITERATOR_ITEM_DBENTRY: + it->dbentries_processed++; + if (item->u.dbe.is_cloned) it->dbentry_clones_processed++; + mutexQueueAdd(it->return_to_main_thread, item); + break; + case BGITERATOR_ITEM_REPLICATION: + it->replication_processed++; + mutexQueueAdd(it->return_to_main_thread, item); + break; + case BGITERATOR_ITEM_SWAPDB: + it->swapdb_processed++; + mutexQueueAdd(it->return_to_main_thread, item); + break; + case BGITERATOR_ITEM_FLUSHDB: + it->flushdb_processed++; + mutexQueueAdd(it->return_to_main_thread, item); + break; + + case BGITERATOR_ITEM_COMPLETE: + case BGITERATOR_ITEM_TERMINATED: + // These are static and just used to wake the iterator - they should never be returned. + serverAssert(false); + break; + + default: + serverAssert(false); + } + + it->current_item = NULL; +} + + +/* ============================================================================================= + * Background Iterator (private) + * ============================================================================================= */ + +static void bgIteratorRelease(bgIterator *it) { + serverAssert(onValkeyMainThread()); + serverAssert(it->current_item == NULL); + serverAssert(mutexQueueLength(it->items_for_iterator) == 0); + serverAssert(mutexQueueLength(it->return_to_main_thread) == 0); + + dictDelete(nameToIterator, it->name); + listDelNode(allIterators, listSearchKey(allIterators, it)); + + mutexQueueRelease(it->items_for_iterator); + it->items_for_iterator = NULL; + + mutexQueueRelease(it->return_to_main_thread); + it->return_to_main_thread = NULL; + + it->keyset_iter->release(it->keyset_iter); + it->keyset_iter = NULL; + + hashtableRelease(it->early_iterate_entries); + it->early_iterate_entries = NULL; + + sdsfree(it->name); + zfree(it); +} + + +static bool shouldFeedIteratorMore(bgIterator *it) { + return (!it->completed && + !it->terminated && + mutexQueueLength(it->items_for_iterator) < it->item_count_target); +} + + +// Debugging routine +static sds createEntryString(int dbid, dbEntry *de) { + sds key = objectGetKey(de); + + sds entrySds = sdsempty(); + entrySds = sdscatprintf(entrySds, "(%d)'%s'", dbid, key); + if (de->type == OBJ_STRING) { + robj *o = getDecodedObject(de); // might be encoded as int + const unsigned valuePrintLen = 20; + entrySds = sdscatprintf(entrySds, " : '%.*s'", valuePrintLen, (char *)objectGetVal(o)); + if (sdslen((sds)objectGetVal(o)) > valuePrintLen) entrySds = sdscat(entrySds, "..."); + decrRefCount(o); + } else { + entrySds = sdscatprintf(entrySds, " : type(%d)", de->type); + } + return entrySds; +} + + +static void feedIterator(bgIterator *it, monotime end_time_us) { + unsigned int initial_queue_len = mutexQueueLength(it->items_for_iterator); + + /* The queue size dynamically adjusts using an AIMD approach. If we have left over stuff from + * the prior call to feedIterator, reduce by half the remaining size. If the queue ran dry + * and we have time left (at the end of this function), additively increase the queue length. */ + if (initial_queue_len > 2 && it->item_count_target >= initial_queue_len) { + it->item_count_target -= initial_queue_len / 2; + } + + // Now do some feeding + bool have_time = (getMonotonicUs() < end_time_us); + int timeCheckCounter = 0; + while (shouldFeedIteratorMore(it) && have_time) { + int orig_dbid, cur_dbid; + fifo *dbEntryFifo = it->keyset_iter->getEntries(it->keyset_iter, &orig_dbid, &cur_dbid); + + if (dbEntryFifo == NULL) { + // Iteration of items is complete for this iterator + serverAssert(it->dbentries_queued >= it->dbentries_processed); + serverAssert(it->replication_queued >= it->replication_processed); + serverAssert(it->swapdb_queued >= it->swapdb_processed); + serverAssert(it->flushdb_queued >= it->flushdb_processed); + serverAssert(it->dbentry_clones_queued >= it->dbentry_clones_processed); + + // Snapshot queue size to seed next iterator when terminated + last_item_count_target = it->item_count_target; + + if (it->iteration_flags & BGITERATOR_FLAG_REPLICATION) { + if (!it->client_is_active || (it->dbentries_queued > it->dbentries_processed)) { + /* Even though we have sent all of the dbEntries, we continue sending + * replication until the iterator has consumed all of the dbEntries. + * client_is_active prevents race conditions in the case of an empty DB. */ + break; + } + if (it->repldone) { + bool clientWantsMoreReplication = (!it->repldone(it->privdata)); + if (clientWantsMoreReplication) break; + } + } + bgIteratorItem *completionItem = itemFreeList_getElementOrAllocate(); + *completionItem = (bgIteratorItem){.type = BGITERATOR_ITEM_COMPLETE}; + if (it->iteration_flags & BGITERATOR_FLAG_REPLICATION) { + rdbSaveInfo rsi; + completionItem->dbid = (rdbPopulateSaveInfo(&rsi)) ? rsi.repl_stream_db : 0; + completionItem->u.master_repl_offset = server.primary_repl_offset; + if (BGITERATION_DEBUG) { + debugBuffer = sdscat(debugBuffer, "REPLDONE FN\n"); + } + } + + if (BGITERATION_DEBUG) { + debugBuffer = sdscat(debugBuffer, "SENDING COMPLETE\n"); + } + + mutexQueueAdd(it->items_for_iterator, completionItem); + it->completed = true; + break; + } + + int dbid = (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT) ? orig_dbid : cur_dbid; + + fifo *itemsToAdd = fifoCreate(); + while (fifoLength(dbEntryFifo) > 0) { + dbEntry *de; + fifoPop(dbEntryFifo, (void **)&de); + + // Remove new/modified items during consistent iteration. + if (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT && + ((bgIterationEntryMetadata *)objectGetMetadata(de))->iterator_epoch > it->consistent_modification_id) { + continue; + } + + // Remove any items which have been processed early + if (hashtableDelete(it->early_iterate_entries, de)) { + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, de); + debugBuffer = sdscatprintf(debugBuffer, "SKIPPING ITEM(early iterate): %s\n", entryString); + sdsfree(entryString); + } + continue; + } + + // For items which are left, convert them from dbEntry to iteratorItem + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, de); + debugBuffer = sdscatprintf(debugBuffer, "ITEM: %s\n", entryString); + sdsfree(entryString); + } + + bgIteratorItem *item = makeDbEntryItem(de, dbid, false); + fifoPush(itemsToAdd, item); + } + fifoRelease(dbEntryFifo); + + if (fifoLength(itemsToAdd) > 0) { + it->dbentries_queued += fifoLength(itemsToAdd); + mutexQueueAddMultiple(it->items_for_iterator, itemsToAdd); + } + fifoRelease(itemsToAdd); + + // This is a predictably fast loop. We don't need to check the time on every pass. + if (++timeCheckCounter % 32 == 0) { + have_time = (getMonotonicUs() < end_time_us); + } + } + + // Smart logic to dynamically adjust the size of the queue + if (initial_queue_len == 0 && have_time && it->item_count_target < BGITER_QUEUE_MAX_LENGTH) { + it->item_count_target += BGITER_QUEUE_INCREASE_INCR; + } +} + + +static bool addEarlyIterationKey(bgIterator *it, dbEntry *earlyEntry, int cur_dbid) { + bool wasAdded = hashtableAdd(it->early_iterate_entries, earlyEntry); + serverAssert(wasAdded); + + int dbid = (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT) + ? it->keyset_iter->originalDb(it->keyset_iter, cur_dbid) + : cur_dbid; + + dbEntry *cloneEntry = tryCloneDbEntry(earlyEntry); + bool isClonedEntry = (cloneEntry != NULL); + bgIteratorItem *item = makeDbEntryItem(isClonedEntry ? cloneEntry : earlyEntry, dbid, isClonedEntry); + + it->dbentries_queued++; + if (isClonedEntry) it->dbentry_clones_queued++; + + if (it->barrier_items == 0) { + // If there are no barrier items, we can add the key right to the front of the queue. + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, item->u.dbe.de); + debugBuffer = sdscatprintf(debugBuffer, "EARLY_1: %s\n", entryString); + sdsfree(entryString); + } + mutexQueuePushPriority(it->items_for_iterator, item); + } else { + // With barrier items, the key must be added to the end, and processed in order. + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, item->u.dbe.de); + debugBuffer = sdscatprintf(debugBuffer, "EARLY: %s\n", entryString); + sdsfree(entryString); + } + mutexQueueAdd(it->items_for_iterator, item); + } + return !isClonedEntry; // Block if the entry will be used by the background thread +} + + +static bool iteratorHasPassedKey(bgIterator *it, int dbid, const_sds key, dbEntry *de) { + if (it->completed || it->terminated) return true; + + if (it->keyset_iter->hasPassedItem(it->keyset_iter, key, dbid)) return true; + + if (de && hashtableFind(it->early_iterate_entries, de, NULL)) return true; + + return false; +} + + +// This expedites a single key and doesn't attempt to avoid expediting through optimization. +static bool expediteSingleKeyWithoutOptimization(bgIterator *it, + int dbid, + robj *oKey, + hashtable *waitingOnKeys) { + bool mustBlock = false; + + sds key = objectGetVal(oKey); + dbEntry *de = dbFind(server.db[dbid], key); + if (de != NULL) { + if (!iteratorHasPassedKey(it, dbid, key, de)) { + if (addEarlyIterationKey(it, de, dbid)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } else { + if (isEntryInuseByAnyIterator(de)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + } + + return mustBlock; +} + + +// MOVE/COPY are unfortunate special commands. They work on 2 DBs at once. +const int MOVE_COMMAND_DBID_ARG_INDEX = 2; +static bool expediteKeysForMove(bgIterator *it, + int dbid, + int argc, + robj **argv, + hashtable *waitingOnKeys) { + if (argc <= MOVE_COMMAND_DBID_ARG_INDEX) return false; + + int destDbid; + if (!getDbIdFromRobj(argv[MOVE_COMMAND_DBID_ARG_INDEX], &destDbid)) return false; + + bool mustBlock = false; + robj *key = argv[1]; + + /* Not looking for special cases to optimize here. Just try to expedite both src and dest + * keys. Note that the dest key might exist (and need iteration) but could be expired and + * could be overwritten by MOVE. In this case, a DEL would replicate due to the expiry. So + * even if the target is expired, we need to replicate it before executing the command. */ + if (expediteSingleKeyWithoutOptimization(it, dbid, key, waitingOnKeys)) mustBlock = true; + if (expediteSingleKeyWithoutOptimization(it, destDbid, key, waitingOnKeys)) mustBlock = true; + + it->cur_cmd_may_replicate = true; + return mustBlock; +} + + +// MOVE/COPY are unfortunate special commands. They work on 2 DBs at once. +static bool expediteKeysForCopy(bgIterator *it, + int dbid, + int argc, + robj **argv, + hashtable *waitingOnKeys) { + int destDbid; + if (!getTargetDbIdForCopyCommand(argc, argv, dbid, &destDbid)) return false; + + bool mustBlock = false; + robj *srcKey = argv[1]; + robj *destKey = argv[2]; + + /* Not trying to optimize COPY. Just expedite source and destination (if it exists). We + * don't really care if the value is overwritten or not (so no need to parse REPLACE option). */ + if (expediteSingleKeyWithoutOptimization(it, dbid, srcKey, waitingOnKeys)) mustBlock = true; + if (expediteSingleKeyWithoutOptimization(it, destDbid, destKey, waitingOnKeys)) mustBlock = true; + + it->cur_cmd_may_replicate = true; + return mustBlock; +} + + +/* There are several cases where a client must be blocked on write operations. (Clients never need + * to be blocked for read operations.) + * + * Note: The CMD_WRITE_FIRSTKEY_ONLY flag allows us to identify commands where the first key is for + * write and the rest are for read. This allows us to make the following optimizations: + * - For keys which are read only, there's no need to block if the key is in-use by an iterator + * - Without replication, there's no need to immediately queue read keys on a consistent iteration + * + * Iterator: CONSISTENT = NO, REPLICATION = NO + * - Block if any write-key is in use by an iterator + * + * Iterator: CONSISTENT = NO, REPLICATION = YES + * - Block if any write-key is in use by an iterator + * - If ANY key has already been iterated (but some keys have not), then + * - Block and immediately queue any key (read or write) that has not + * already been iterated + * Example: SDIFFSTORE KEY_A KEY_B KEY_C + * In this case, KEY_A is written, KEY_B and KEY_C are read. If KEY_A has already been + * iterated over, the replication stream will contain this command. The receiver of this + * replication will need KEY_B and KEY_C in order to process the replication stream. So + * these need to be iterated and the client blocked. + * + * Iterator: CONSISTENT = YES, REPLICATION = NO + * - Block if any write-key is in use by an iterator + * - Block and immediately queue any WRITE-key that has not already been iterated + * + * Iterator: CONSISTENT = YES, REPLICATION = YES + * (Combination only valid in cluster mode - no SWAPDB possible) + * - Block if any write-key is in use by an iterator + * - Block and immediately queue any key (read or write) that has not already been iterated */ +static bool expediteKeysForWrite(bgIterator *it, + int dbid, + struct serverCommand *cmd, + int argc, + robj **argv, + keyReference *keyrefs, + int numKeys, + hashtable *waitingOnKeys) { + serverAssert(numKeys > 0); + + bool mustBlock = false; + + /* All keys of the command should either be in scope or not since in cluster mode enabled they + * should all be in the same slot. So we just check the first key. */ + robj *oKey = argv[keyrefs[0].pos]; + sds key = objectGetVal(oKey); + /* If it's not in the iteration scope for the current iterator, then we don't need to do + * anything with this command. */ + if (!it->keyset_iter->isKeyInScope(it->keyset_iter, key)) return false; + + if ((cmd->flags & CMD_WRITE_FIRSTKEY_ONLY) && + !(it->iteration_flags & BGITERATOR_FLAG_REPLICATION)) { + /* If this write command only modifies the 1st key, we don't need to expedite others + * unless replication enabled. */ + numKeys = 1; + } + + if (cmd->proc == moveCommand) { + // Special case for MOVE + return expediteKeysForMove(it, dbid, argc, argv, waitingOnKeys); + } + + if (cmd->proc == copyCommand) { + // Similar special case for COPY + return expediteKeysForCopy(it, dbid, argc, argv, waitingOnKeys); + } + + if (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT) { + // CONSISTENT = YES, REPLICATION = YES / NO + for (int i = 0; i < numKeys; i++) { + robj *oKey = argv[keyrefs[i].pos]; + sds key = objectGetVal(oKey); + dbEntry *de = dbFind(server.db[dbid], key); + if (de == NULL) continue; // New key, no need to expedite + if (!iteratorHasPassedKey(it, dbid, key, de) && + ((bgIterationEntryMetadata *)objectGetMetadata(de))->iterator_epoch <= it->consistent_modification_id) { + if (addEarlyIterationKey(it, de, dbid)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } else { + if (isEntryInuseByAnyIterator(de)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + } + it->cur_cmd_may_replicate = true; // Will replicate only if replication enabled + } else { + /* Identification of missing keys is only needed for non-consistent iteration. This only + * needs to be collected once (on the 1st non-consistent iteration). */ + bool collectMissing = (listLength(curCmdMissingKeys) == 0); + + if (it->iteration_flags & BGITERATOR_FLAG_REPLICATION) { + // CONSISTENT = NO, REPLICATION = YES + bool someIterated = false; + /* dict containing the keys that have not been iterated yet. + * Using a dict dedupes the keys in case the command contains duplicated keys. */ + dict *notIteratedKeys = dictCreate(&dictEntryPtrDictType); // dict of dbEntry* -> robj* + + for (int i = 0; i < numKeys; i++) { + robj *oKey = argv[keyrefs[i].pos]; + sds key = objectGetVal(oKey); + dbEntry *de = dbFind(server.db[dbid], key); + if (de == NULL) { + if (collectMissing) { + incrRefCount(oKey); + listAddNodeHead(curCmdMissingKeys, oKey); + } + continue; + } + if (iteratorHasPassedKey(it, dbid, key, de)) { + someIterated = true; + } else { + dictAdd(notIteratedKeys, de, oKey); + } + if (isEntryInuseByAnyIterator(de)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + + /* Since missing keys are considered as already iterated, if there are any missing keys + * we must consider that some keys have been iterated, and make sure all other keys + * will be expedited if needed. */ + if (listLength(curCmdMissingKeys) > 0) someIterated = true; + + /* This command may be executing as part of a larger transaction. If some parts of the + * transaction have already been identified to replicate, we must wait on all keys and + * replicate here as well. (Take care not to set cur_cmd_may_replicate to false.) */ + if (someIterated) { + if (server.in_exec) { + /* We are now executing the commands in a multi-exec block. + * + * Regarding MULTI/EXEC: Remember that this code is executed twice for commands + * within a MULTI/EXEC block. First, we parse all the commands when deciding + * if the EXEC should be blocked. Then, as each command is executed, it's + * re-parsed so that we can maintain the early iterated list as the commands + * execute. In this second pass, as each command is executed, we can't change + * the replication decision which was made earlier (when the EXEC was processed). + * We don't want to get tricked (by a key being removed and recreated) into + * starting to replicate in the middle of a MULTI/EXEC block. */ + } else { + it->cur_cmd_may_replicate = true; + } + } + if (it->cur_cmd_may_replicate) { + dictEntry *de; + dictIterator *di = dictGetIterator(notIteratedKeys); + while ((de = dictNext(di)) != NULL) { + dbEntry *notIteratedEntry = dictGetKey(de); + robj *oKey = dictGetVal(de); + + if (addEarlyIterationKey(it, notIteratedEntry, dbid)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + dictReleaseIterator(di); + } + dictRelease(notIteratedKeys); + } else { + // CONSISTENT = NO, REPLICATION = NO + for (int i = 0; i < numKeys; i++) { + robj *oKey = argv[keyrefs[i].pos]; + sds key = objectGetVal(oKey); + dbEntry *de = dbFind(server.db[dbid], key); + if (de == NULL) { + if (collectMissing) { + incrRefCount(oKey); + listAddNodeHead(curCmdMissingKeys, oKey); + } + continue; + } + if (isEntryInuseByAnyIterator(de)) { + mustBlock = true; + hashtableAdd(waitingOnKeys, oKey); + } + } + } + } + + return mustBlock; +} + + +/* Called when an iterator is terminated. Pulls everything out of the queue + * and returns the items to the main thread (before they hit the iterator). */ +static void returnAllItemsToMainThread(bgIterator *it) { + serverAssert(onValkeyMainThread()); + + fifo *poppedFifo = mutexQueuePopAll(it->items_for_iterator, false); + if (poppedFifo == NULL) return; // Nothing to return + + // Release non-dictentry items first... + fifo *itemsToReturn = fifoCreate(); + while (fifoLength(poppedFifo) > 0) { + bgIteratorItem *item; + fifoPop(poppedFifo, (void **)&item); + switch (item->type) { + // back out the "queued" statistic + case BGITERATOR_ITEM_DBENTRY: + it->dbentries_queued--; + if (item->u.dbe.is_cloned) it->dbentry_clones_queued--; + break; + case BGITERATOR_ITEM_REPLICATION: + it->replication_queued--; + break; + case BGITERATOR_ITEM_SWAPDB: + it->swapdb_queued--; + it->barrier_items--; + break; + case BGITERATOR_ITEM_FLUSHDB: + it->flushdb_queued--; + it->barrier_items--; + break; + + case BGITERATOR_ITEM_COMPLETE: + /* This can only happen if the completion item has been enqueued and + * the iterator is terminated before reaching the completion item. */ + itemFreeList_returnItemBackToFreeList(item); + continue; // Skip pushing this onto itemsToReturn + + case BGITERATOR_ITEM_TERMINATED: + /* This can only happen if there is a race when terminating between + * the iteration client and main thread. */ + serverAssert(item == &STATIC_ITEM_TERMINATED); + continue; // Skip pushing this onto itemsToReturn + + default: + serverAssert(false); + } + + fifoPush(itemsToReturn, item); + } + fifoRelease(poppedFifo); + + // Now release items all at once... + if (fifoLength(itemsToReturn) > 0) { + mutexQueueAddMultiple(it->return_to_main_thread, itemsToReturn); + } + fifoRelease(itemsToReturn); +} + + +/* ============================================================================================= + * Foreground support functions (private) + * ============================================================================================= */ + +static size_t replicationItemSize(bgIteratorItem *item) { + serverAssert(item->type == BGITERATOR_ITEM_REPLICATION); + size_t itemSize = sizeof(bgIteratorItem); + for (int i = 0; i < item->u.repl.argc; i++) { + itemSize += objectComputeSize(NULL, item->u.repl.argv[i], 0, 0); + } + return itemSize; +} + +static void processReturnOfItemToMainThread(bgIterator *it, bgIteratorItem *item) { + serverAssert(onValkeyMainThread()); + switch ((int)item->type) { + case BGITERATOR_ITEM_REPLICATION: + bufferedReplicationBytes -= item->u.repl.replication_size; + freeRobjArray(item->u.repl.argc, item->u.repl.argv); + break; + + case BGITERATOR_ITEM_DBENTRY: + if (item->u.dbe.is_cloned) { + freeClonedDictEntry(item->u.dbe.de); + } else { + if (isEntryInuseBySingleIterator(item->u.dbe.de)) { + /* This blocking mechanism assumes a single DB so if the same key appears in + * multiple DBs, commands might get unblocked only to get blocked again. (This + * would happen only rarely, and with minimal impact.) */ + robj *key = createStringObjectFromSds(objectGetKey(item->u.dbe.de)); + unblockClientsInUseOnKey(key); + decrRefCount(key); + } + // resumeRehashing must be called before decrementEntryInuse, since decrementEntryInuse can free + if (item->u.dbe.is_rehashing_paused) resumeRehashing(item->u.dbe.de); + decrementEntryInuse(item->u.dbe.de); + } + break; + + case BGITERATOR_ITEM_SWAPDB: + case BGITERATOR_ITEM_FLUSHDB: + it->barrier_items--; + break; + + case BGITERATOR_ITEMEXT_ITER_CLOSED: { + if (it->terminated) { + /* Abnormal termination + * Normally the item is TERMINATED, but might be COMPLETE in race */ + serverAssert(it->current_item->type == BGITERATOR_ITEM_TERMINATED || + it->current_item->type == BGITERATOR_ITEM_COMPLETE); + // Release any items stranded on the iterator after early termination + returnAllItemsToMainThread(it); + receiveItemsBackFromOneIterator(it); + } else { + // Normal completion + serverAssert(it->current_item->type == BGITERATOR_ITEM_COMPLETE); + } + if (it->current_item != &STATIC_ITEM_TERMINATED) itemFreeList_returnItemBackToFreeList(it->current_item); + it->current_item = NULL; + + serverAssert(mutexQueueLength(it->items_for_iterator) == 0); + serverAssert(it->barrier_items == 0); + serverAssert(it->dbentries_queued == it->dbentries_processed); + serverAssert(it->replication_queued == it->replication_processed); + serverAssert(it->swapdb_queued == it->swapdb_processed); + serverAssert(it->flushdb_queued == it->flushdb_processed); + serverAssert(it->dbentry_clones_queued == it->dbentry_clones_processed); + + listEmpty(curCmdMissingKeys); // Just in case any remain + + bool terminated = it->terminated; + void *privdata = it->privdata; + bgIteratorCleanupFunc cleanup = it->cleanup; + bgIteratorRelease(it); // Fully release the iterator before calling cleanup + + if (BGITERATION_DEBUG) { + if (cleanup) debugBuffer = sdscatprintf(debugBuffer, "CLEANUP FN (%s)\n", + (terminated) ? "terminated" : "success"); + + sds filename = sdscatprintf(sdsempty(), "bgiteration_debug.%d", getpid()); + FILE *f = fopen(filename, "w"); + sdsfree(filename); + + fputs(debugBuffer, f); + + fclose(f); + sdsfree(debugBuffer); + debugBuffer = sdsempty(); + } + + if (cleanup) cleanup(terminated, privdata); + item = NULL; // Prevent return of static item to free list + } break; + + default: + serverAssert(false); // Not expecting any other type of item! + } + + if (item) itemFreeList_returnItemBackToFreeList(item); +} + +static void prepareAndProcessReturnedItems(bgIterator *it, int n, bgIteratorItem **items) { + for (int i = 0; i < n; i++) valkey_prefetch(items[i]); + for (int i = 0; i < n; i++) { + if (items[i]->type != BGITERATOR_ITEM_DBENTRY) continue; + valkey_prefetch(items[i]->u.dbe.de); + } + for (int i = 0; i < n; i++) { + if (items[i]->type != BGITERATOR_ITEM_DBENTRY) continue; + valkey_prefetch(objectGetKey(items[i]->u.dbe.de)); + } + for (int i = 0; i < n; i++) processReturnOfItemToMainThread(it, items[i]); +} + +#define PREFETCH_BATCH_SIZE 16 + +// Returns true if we process at least one item from a given iterator's return_to_main_thread queue. +static bool receiveItemsBackFromOneIterator(bgIterator *it) { + bgIteratorItem *batchPool[PREFETCH_BATCH_SIZE]; + int n = 0; + fifo *poppedFifo = mutexQueuePopAll(it->return_to_main_thread, false); + if (poppedFifo != NULL) { + while (fifoLength(poppedFifo) > 0) { + fifoPop(poppedFifo, (void **)&batchPool[n++]); + if (n == PREFETCH_BATCH_SIZE) { + prepareAndProcessReturnedItems(it, n, batchPool); + n = 0; + } + } + if (n > 0) { + prepareAndProcessReturnedItems(it, n, batchPool); + } + fifoRelease(poppedFifo); + return true; + } + return false; +} + +/* Process each iterator's return_to_main_thread queue + * If `blocking` is true, continue reading until at least one queue was not empty. */ +static void receiveItemsBackFromIterators(bool blocking) { + serverAssert(onValkeyMainThread()); + listIter li; + listNode *node; + bool processedItems = false; + do { + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + processedItems |= receiveItemsBackFromOneIterator(it); + } + if (blocking && !processedItems) usleep(100); // Short sleep before retry + } while (blocking && !processedItems); +} + + +static long long bgIteration_feedIterators_task(struct aeEventLoop *eventLoop, + long long id, + void *clientData) { + UNUSED(eventLoop); + UNUSED(id); + UNUSED(clientData); + serverAssert(onValkeyMainThread()); + + static monotime lastFeedEndTime; // STATIC: Persists For checking starvation + monotime startTime = getMonotonicUs(); + + if (!bgIteration_iterationActive()) { + // No more iterators exist. Self-check, and terminate the "feed" task. + serverAssert(dictSize(nameToIterator) == 0); + serverAssert(dictSize(inUseEntries) == 0); + serverAssert(bufferedReplicationBytes == 0); + + // Shrink dict back to zero (doesn't normally shrink) + dictRelease(inUseEntries); + inUseEntries = dictCreate(&dictEntryPtrDictType); + + itemFreeList_release(); + + bgIterator_timeproc_id = AE_DELETED_EVENT_ID; + lastFeedEndTime = 0; + return AE_NOMORE; + } + + long dutyTimeUs = BGITER_CYCLE_BUDGET_MS * 1000; + if (lastFeedEndTime > 0) { + /* If the timer was delayed, compute the proportional time we should have had, and increase + * the duty cycle to compensate (up to a limit). */ + long starvationUs = (startTime - lastFeedEndTime) - BGITER_CYCLE_DELAY_MS * 1000; + if (starvationUs > 0) { + long starvationCompensationUs = starvationUs * BGITER_CYCLE_BUDGET_MS / + (BGITER_CYCLE_BUDGET_MS + BGITER_CYCLE_DELAY_MS); + dutyTimeUs += starvationCompensationUs; + dutyTimeUs = MIN(dutyTimeUs, BGITER_CYCLE_BUDGET_MAX_MS * 1000); + } + } + monotime endTime = startTime + dutyTimeUs; + + // Run this part regardless of time limit... + receiveItemsBackFromIterators(false); + + // Feeding iterators (below) respects endTime. The stuff above always runs to completion. + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL && getMonotonicUs() < endTime) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated) continue; + feedIterator(it, endTime); + } + + lastFeedEndTime = getMonotonicUs(); + return BGITER_CYCLE_DELAY_MS; +} + + +// Not static, but not API. Intended for unit tests where the event loop may not be active. +void bgIteration_feedIterators(void) { + /* For unit testing, force the item_count_target to 1 in each call. This ensures that we only + * feed a minimal amount to the iterators rather than a non-deterministic amount. */ + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + it->item_count_target = 1; + } + + // Invoke the feeding task (normally invoked by timer). + bgIteration_feedIterators_task(NULL, 0, NULL); +} + + +static void resetReplicationFlagForIterators(client *c) { + /* For any given command, the command may or may not need to be replicated based on the status + * and flags of each iterator. Furthermore, if a command does need to be replicated, this + * replication must occur for an entire atomic unit; we can't replicate only part of a script + * or multi/exec. + * This function is the only place where the replication flag is cleared. */ + + if (c->flag.multi || c->flag.script) { + /* REGARDING MULTI/EXEC + * -------------------- + * When processing a MULTI/EXEC, blockClientIfRequired is called first for the MULTI. Then, + * all of the commands are queued up in server.c:processCommand(). It's only when EXEC is + * encountered, that server.c:call() is fired to begin execution. + * + * AFTER the EXEC is processed by call(), then each of the commands in the MULTI/EXEC block + * will be processed through call(). + * + * If write commands are present, MULTI & EXEC will be passed to the replication stream + * before/after the transaction commands. Note that MULTI & EXEC are not actually + * "executed" at the time when their replication is passed to the replication stream. + * + * Example: MULTI; SET A B; EXEC + * 1. blockClientIfRequired() called for MULTI. MULTI flag IS NOT set. (Won't block.) + * 2. blockClientIfRequired() called for EXEC. MULTI flag IS set. (Might block.) + * 3. blockClientIfRequired() called for SET. MULTI flag IS set. (Won't block.) + * 4. handleCommandReplication() is called for MULTI. + * 5. handleCommandReplication() is called for SET. + * 6. handleCommandReplication() is called for EXEC. + * + * SO - if the MULTI flag is set, we DON'T clear the flag. It should only be cleared at the + * start of the transaction, when MULTI is received - and the flag isn't set yet. */ + + /* REGARDING SCRIPTS + * ----------------- + * When processing a script, blockClientIfRequired is called first for the EVAL/EVALSHA/FCALL. + * Then, all of the commands are processed using a special script client. The script + * client has the CLIENT_SCRIPT flag set. For scripts, the replication flag is set when + * processing the EVAL/EVALSHA/FCALL and should not be cleared when executing individual + * commands in the script. */ + + /* If it's the EXEC command, we fall through and clear the flag below. But for all other + * commands within the transaction, we don't clear the flag. */ + if (c->cmd->proc != execCommand) return; + } + + /* For most commands, the replication flag is cleared and we determine if replication is needed + * based on the keys being used and their state in each iterator. If a modified key hasn't been + * processed yet, there's no need to expedite the key or send the replication. The key will be + * sent later, when reached by the iterator. + * + * However, for scripts, it is not possible to perform this optimization. There is no way to + * know if an undeclared key might be modified. Since the entire script needs to be replicated + * (or not replicated) atomically, we can't take the chance that an undeclared key might be + * hit which requires replication. */ + bool isScript = isScriptCallWriteCmd(c->cmd); + + sds firstScriptKey = NULL; + if (isScript) { + /* If it's a script, we will normally replicate. But if the keys are out of scope for the + * iteration, we shouldn't. The use-case for this is with slot iteration, when the script + * is acting on keys from a different slot. Here, we just check the first declared key, and + * if it's out of scope for the iteration, we won't replicate it. This might cause issues + * for cross-slot scripts (anti-pattern), but the alternative is replicating all scripts, + * regardless of slot. */ + getKeysResult result; + initGetKeysResult(&result); + getKeysFromCommand(c->cmd, c->argv, c->argc, &result); + if (result.numkeys > 0) firstScriptKey = objectGetVal(c->argv[result.keys[0].pos]); + getKeysFreeResult(&result); + } + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated) { + it->cur_cmd_may_replicate = false; + } else { + /* For normal commands, the flag is initialized to false (not to replicate). For these + * commands, we decide later based on the actual commands. + * + * However, for scripts, we don't know what commands will be executed. So IF it's a + * script, and the keys are in scope (on the right slot) we initialize the replication + * flag to true. */ + it->cur_cmd_may_replicate = isScript && firstScriptKey && + it->keyset_iter->isKeyInScope(it->keyset_iter, firstScriptKey); + } + } +} + + +static void handleSwapdb(int db1, int db2) { + serverAssert(onValkeyMainThread()); + serverAssert(bgIteration_iterationActive()); + serverAssert(!server.cluster_enabled); + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated) continue; + + // Let the iterator internal mechanism know + it->keyset_iter->swapDb(it->keyset_iter, db1, db2); + + // Let the background client know + if (!(it->iteration_flags & BGITERATOR_FLAG_CONSISTENT)) { + if (BGITERATION_DEBUG) { + debugBuffer = sdscatprintf(debugBuffer, "SWAP: %d %d\n", db1, db2); + } + + bgIteratorItem *item = itemFreeList_getElementOrAllocate(); + item->type = BGITERATOR_ITEM_SWAPDB; + item->dbid = db1; + item->u.dbid2 = db2; + it->swapdb_queued++; + it->barrier_items++; + mutexQueueAdd(it->items_for_iterator, item); + } + } +} + + +static bool isDbSignificant(int dbid) { + unsigned long long totalKeys = 0; + for (int i = 0; i < server.dbnum; i++) { + totalKeys += (server.db[i]) ? dbSize(server.db[i]) : 0; + } + return (server.db[dbid]) ? (dbSize(server.db[dbid]) > totalKeys / 2) : false; +} + + +static void handleFlushdb(int dbid) { + // Invoked BEFORE the actual flush. -1 indicates FLUSHALL. + bool should_abort_iterators = (dbid == -1 || isDbSignificant(dbid)); + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + + // Let the low-level iterator know the DB is being flushed + it->keyset_iter->flushDb(it->keyset_iter, dbid); + + if (should_abort_iterators || it->iteration_flags & BGITERATOR_FLAG_CONSISTENT) { + if (!it->terminated) bgIteratorTerminate(it); + } else { + /* In this (limited) case, we're only flushing a single DB that contains < half the + * keys. We don't want to kill a full-sync replication. We will just continue with + * iteration, knowing that a replication client will also receive the FLUSHDB on the + * replication stream. There's no need to worry about the items themselves. Since + * we've incremented the refcount, the items still in queue won't be physically deleted. */ + + // Send a flushdb event to notify the client + if (BGITERATION_DEBUG) { + debugBuffer = sdscatprintf(debugBuffer, "FLUSH: %d\n", dbid); + } + bgIteratorItem *item = itemFreeList_getElementOrAllocate(); + item->type = BGITERATOR_ITEM_FLUSHDB; + item->dbid = dbid; + it->flushdb_queued++; + it->barrier_items++; + mutexQueueAdd(it->items_for_iterator, item); + } + } + receiveItemsBackFromIterators(false); // Receive items back before flushing the items +} + + +static bool expediteKeysForWriteOnAllIterators(int dbid, + struct serverCommand *cmd, + int argc, + robj **argv, + keyReference *keyrefs, + int numKeys, + hashtable *waitingOnKeys) { + bool mustBlock = false; + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (expediteKeysForWrite(it, dbid, cmd, argc, argv, keyrefs, numKeys, waitingOnKeys)) + mustBlock = true; + } + + return mustBlock; +} + + +static bool anIteratorWillReplicateForThisCommand(void) { + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->cur_cmd_may_replicate) return true; + } + return false; +} + + +static bool expediteKeysForMultiExec(client *c, hashtable *waitingOnKeys) { + serverAssert(c->cmd->proc == execCommand); + + /* For MULTI/EXEC, Valkey buffers all of the commands until hitting the EXEC. + * At this point, the client holds all of the commands to be executed. This function searches + * for all of the keys used by any of the buffered write commands. In addition, if SWAPDB or + * SELECT is used, this tracks the DBIDs through various swap/select operations. */ + + /* There's a special concern for a NON-consistent iteration with replication. If the keys are + * all "future" keys (which haven't been processed by the iterator yet), then we don't expedite + * the keys or replicate. However, if some keys have already been processed, we need to + * expedite the remaining keys and replicate everything. + * + * When processing a single command, this is all handled. But in this function, for MULTI/EXEC, + * we process 1 command at a time. There's an issue if the first command modifies a "future" + * key, we don't know (without reading ahead) if a later command will modify a prior key. This + * would require the future key to be expedited. + * + * This COULD be addressed by collecting all of the keys into a single structure and then + * analyzing them all at once. However, this won't share code well with the single commands. + * Also, building this structure is a little complex/time-consuming as we need to track both + * key AND dictID. One way to do this might be with a dict of dicts, where the first dict maps + * a dictID to a dict of keys. + * + * ALTERNATIVELY (and it's the simpler approach that's taken here) we can just check if the + * MULTI will be replicated. If so, we re-process the MULTI, just in case there were commands + * prior to deciding that replication was required that might have missed expediting. If so, + * these will be caught on the 2nd time around. + * + * Checking replication status before/after ensures that there can only be a single recursive + * call. */ + bool initiallyAnIteratorWillReplicate = anIteratorWillReplicateForThisCommand(); + + bool mustBlock = false; + int *cur_to_orig_db = NULL; + + int curDb = c->db->id; + for (int cmdNum = 0; cmdNum < c->mstate->count; cmdNum++) { + struct serverCommand *cmd = c->mstate->commands[cmdNum].cmd; + robj **argv = c->mstate->commands[cmdNum].argv; + int argc = c->mstate->commands[cmdNum].argc; + + if (cmd->proc == swapdbCommand) { + int id1, id2; + if (getParamsForSwapdb(argc, argv, c, &id1, &id2)) { + if (cur_to_orig_db == NULL) { + cur_to_orig_db = zmalloc(sizeof(int) * server.dbnum); + for (int i = 0; i < server.dbnum; i++) cur_to_orig_db[i] = i; + } + int temp = cur_to_orig_db[id1]; + cur_to_orig_db[id1] = cur_to_orig_db[id2]; + cur_to_orig_db[id2] = temp; + } + continue; + } + + if (cmd->proc == selectCommand) { + int id; + if (getParamsForSelect(argc, argv, c, &id)) { + curDb = id; + } + continue; + } + + if (!isWriteCmd(cmd)) continue; + + getKeysResult result; + initGetKeysResult(&result); + int numkeys = getKeysFromCommand(cmd, argv, argc, &result); + keyReference *keyrefs = result.keys; + if (numkeys == 0) { + getKeysFreeResult(&result); + continue; // Write command with no keys - like FLUSHDB + } + + if (expediteKeysForWriteOnAllIterators( + cur_to_orig_db ? cur_to_orig_db[curDb] : curDb, + cmd, argc, argv, keyrefs, numkeys, waitingOnKeys)) { + mustBlock = true; + } + getKeysFreeResult(&result); + } + + zfree(cur_to_orig_db); + + if (!initiallyAnIteratorWillReplicate && anIteratorWillReplicateForThisCommand()) { + /* We've decided to replicate. Re-process the MULTI/EXEC just once more to make sure that + * we didn't miss any keys at the beginning. This can't continue to recurse because + * `initiallyAnIteratorWillReplicate` will be TRUE in the recursive call. Note that the + * recursive call may add additional entries to `waitingOnKeys`. */ + if (expediteKeysForMultiExec(c, waitingOnKeys)) mustBlock = true; + } + + return mustBlock; +} + + +static bgIterator *bgIteratorCreate(const char *name, + bgIteratorConsistency consistency, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata, + bgIterationType iter_type, + genericIterator *keyset_iter) { + serverAssert(server.forkless_options_supported); + serverAssert(onValkeyMainThread()); + serverAssert(server.cluster_enabled || iter_type == BGITERATION_TYPE_FULLSCAN); + + int flags; + switch (consistency) { + case BGITERATOR_CONSISTENCY_NONE: flags = 0; break; + case BGITERATOR_CONSISTENCY_START: flags = BGITERATOR_FLAG_CONSISTENT; break; + case BGITERATOR_CONSISTENCY_EVENTUAL: flags = BGITERATOR_FLAG_REPLICATION; break; + default: serverAssert(false); + } + // Consistent, with replication - doesn't make sense. + serverAssert(!((flags & BGITERATOR_FLAG_CONSISTENT) && (flags & BGITERATOR_FLAG_REPLICATION))); + + bgIterator *it = zmalloc(sizeof(bgIterator)); + it->name = sdsnew(name); + it->repldone = repldone; + it->cleanup = cleanup; + it->privdata = privdata; + it->items_for_iterator = mutexQueueCreate(); + it->return_to_main_thread = mutexQueueCreate(); + + // Floor queue size to bgiteration_queue_increase_incr or use last queue size value + if (last_item_count_target < BGITER_QUEUE_INCREASE_INCR) { + last_item_count_target = BGITER_QUEUE_INCREASE_INCR; + } + it->item_count_target = last_item_count_target; + it->iteration_flags = flags; + it->iteration_type = iter_type; + it->consistent_modification_id = bgIteration_epoch++; + it->keyset_iter = keyset_iter; + it->early_iterate_entries = hashtableCreate(&dbEntryPtrHashtableType); + hashtableExpand(it->early_iterate_entries, BGITER_EARLY_ITERATE_DICT_INITIAL_SIZE); + it->current_item = NULL; + it->client_is_active = false; + it->completed = false; + it->terminated = false; + it->cur_cmd_may_replicate = false; + + it->dbentries_queued = 0; + it->dbentries_processed = 0; + it->replication_queued = 0; + it->replication_processed = 0; + it->swapdb_queued = 0; + it->swapdb_processed = 0; + it->flushdb_queued = 0; + it->flushdb_processed = 0; + it->dbentry_clones_queued = 0; + it->dbentry_clones_processed = 0; + + it->barrier_items = 0; + + elapsedStart(&it->monotonic_start_time); + it->monotonic_item_start_time = 0; + + + if (bgIterator_timeproc_id <= 0) { + // If iteration is not currently active, start the feeding task. (Runs in main thread.) + bgIterator_timeproc_id = aeCreateTimeEvent(server.el, 0, bgIteration_feedIterators_task, NULL, NULL); + serverAssert(bgIterator_timeproc_id != AE_ERR); + } + + if (dictAdd(nameToIterator, it->name, it) != DICT_OK) { + // Can't have 2 iterators with the same name! + serverAssert(false); + } + + listAddNodeTail(allIterators, it); + + dictExpand(inUseEntries, listLength(allIterators) * it->item_count_target); + + return it; +} + + +/* ============================================================================================= + * PUBLIC INTERFACE: Iterator creation and use + * ============================================================================================= */ + +// PUBLIC API +bgIterator *bgIteratorCreateFullScanIter(const char *name, + bgIteratorConsistency consistency, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata) { + return bgIteratorCreate(name, consistency, repldone, cleanup, privdata, + BGITERATION_TYPE_FULLSCAN, fullScanIteratorCreate()); +} + +// PUBLIC API +bgIterator *bgIteratorCreateSlotsIter(const char *name, + bgIteratorConsistency consistency, + const int *slots, + int slots_count, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata) { + return bgIteratorCreate(name, consistency, repldone, cleanup, privdata, + BGITERATION_TYPE_CLUSTERSLOT, clusterSlotIteratorCreate(slots, slots_count)); +} + +// PUBLIC API +bgIterator *bgIteratorFind(const char *name) { + serverAssert(onValkeyMainThread()); + + sds sdsname = sdsnew(name); + bgIterator *it = dictFetchValue(nameToIterator, sdsname); + sdsfree(sdsname); + + return it; +} + + +// PUBLIC API +const char *bgIteratorName(bgIterator *it) { + return it->name; +} + + +// PUBLIC API +void bgIteratorGetStatus(bgIterator *it, bgIteratorStatus *status) { + status->dbentries_queued = it->dbentries_queued; + status->dbentries_processed = it->dbentries_processed; + status->replication_queued = it->replication_queued; + status->replication_processed = it->replication_processed; + status->swapdb_queued = it->swapdb_queued; + status->swapdb_processed = it->swapdb_processed; + status->flushdb_queued = it->flushdb_queued; + status->flushdb_processed = it->flushdb_processed; + status->dbentry_clones_queued = it->dbentry_clones_queued; + status->dbentry_clones_processed = it->dbentry_clones_processed; + + status->queue_length = mutexQueueLength(it->items_for_iterator); + status->queue_length_target = it->item_count_target; + + status->runtime_ms = elapsedMs(it->monotonic_start_time); + + monotime nonvolatile_item_start_time = it->monotonic_item_start_time; + status->current_item_ms = (nonvolatile_item_start_time == 0) + ? 0 + : elapsedMs(nonvolatile_item_start_time); +} + + +// PUBLIC API +void bgIteratorTerminate(bgIterator *it) { + serverAssert(onValkeyMainThread()); + + // Remove any items in the queue, but doesn't affect the 1 item that's being processed. + returnAllItemsToMainThread(it); + + // We have to add an item, just in case the READER is waiting on the mutex. + if (BGITERATION_DEBUG) { + debugBuffer = sdscat(debugBuffer, "SENDING TERMINATE\n"); + } + + mutexQueueAdd(it->items_for_iterator, (void *)&STATIC_ITEM_TERMINATED); + + it->terminated = true; +} + + +// PUBLIC API +bool bgIteratorIsTerminating(bgIterator *it) { + return it->terminated; +} + + +// PUBLIC API +bgIteratorItem *bgIteratorRead(bgIterator *it) { + serverAssert(it->current_item == NULL || + (it->current_item->type != BGITERATOR_ITEM_COMPLETE && + it->current_item->type != BGITERATOR_ITEM_TERMINATED)); + + // First, clean up the previous item read + if (it->current_item != NULL) { + returnCurrentItemToMainThread(it); + + /* To support unit tests. Normal clients call bgIteratorRead from an alternate thread. + * Without this, a unit test could get stuck waiting on the completion event because + * feed won't get invoked. For production, feed is called regularly from the main thread. + * Note - this is checking that the exact same thread is used and shouldn't count modules. */ + if (pthread_equal(server.main_thread_id, pthread_self()) != 0) bgIteration_feedIterators_task(NULL, 0, NULL); + } else { + it->client_is_active = true; + } + + it->monotonic_item_start_time = 0; // idle until blocking pop returns + it->current_item = mutexQueuePop(it->items_for_iterator, true); + it->monotonic_item_start_time = getMonotonicUs(); + + return it->current_item; +} + + +// PUBLIC API +void bgIteratorClose(bgIterator *it) { + if (it->current_item != NULL) { + if (it->current_item->type == BGITERATOR_ITEM_COMPLETE || + it->current_item->type == BGITERATOR_ITEM_TERMINATED) { + // Normal confirmation of background completion + } else { + // Client is initiating the termination + it->terminated = true; + returnCurrentItemToMainThread(it); + + it->current_item = (bgIteratorItem *)&STATIC_ITEM_TERMINATED; + } + } else { + // terminated before first item read + it->terminated = true; + it->current_item = (bgIteratorItem *)&STATIC_ITEM_TERMINATED; + } + + mutexQueueAdd(it->return_to_main_thread, (void *)&STATIC_ITEM_ITER_CLOSED); +} + + +/* ============================================================================================= + * PUBLIC INTERFACE: Valkey main-thread support hooks + * ============================================================================================= */ + +// PUBLIC API +void bgIteration_init(void) { + serverAssert(onValkeyMainThread()); + + /* This should be called once and only once from the Valkey main thread. However to support + * unit tests, this is not validated, and multiple invocations are ignored. */ + if (nameToIterator) return; // If already initialized, ignore (unit tests) + + nameToIterator = dictCreate(&sdsrefToPtrDictType); + serverAssert(nameToIterator != NULL); + + allIterators = listCreate(); + serverAssert(allIterators != NULL); + + inUseEntries = dictCreate(&dictEntryPtrDictType); + serverAssert(inUseEntries != NULL); + + curCmdMissingKeys = listCreate(); + serverAssert(curCmdMissingKeys != NULL); + listSetFreeMethod(curCmdMissingKeys, decrRefCountVoid); + + bufferedReplicationBytes = 0; + + if (BGITERATION_DEBUG) { + debugBuffer = sdsMakeRoomFor(sdsempty(), SDS_MAX_PREALLOC); + } +} + + +// PUBLIC API +bool bgIteration_iterationActive(void) { + return (allIterators != NULL && listLength(allIterators) > 0); +} + + +// PUBLIC API +void bgIteration_beforeSleep(void) { + if (!bgIteration_iterationActive()) return; + receiveItemsBackFromIterators(false); +} + + +// PUBLIC API +void bgIteration_keyDelete(int dbid, const_sds key) { + if (!bgIteration_iterationActive()) return; + serverAssert(onValkeyMainThread()); + + if (BGITERATION_DEBUG) { + debugBuffer = sdscatprintf(debugBuffer, "KEYDEL: (%d)%s\n", dbid, key); + } + + dbEntry *de = dbFind(server.db[dbid], (sds)key); + serverAssert(de != NULL); // This API should be called BEFORE removal from main dict + + dbEntryPtrOfLastKeyDelete = de; // save for check at replication time + + // For consistent iterators, we need to make sure the item gets written before delete + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated || !it->keyset_iter->isKeyInScope(it->keyset_iter, key)) continue; + + if (it->iteration_flags & BGITERATOR_FLAG_CONSISTENT && + ((bgIterationEntryMetadata *)objectGetMetadata(de))->iterator_epoch <= it->consistent_modification_id) { + if (!iteratorHasPassedKey(it, dbid, key, de)) { + addEarlyIterationKey(it, de, dbid); // (may also add to inUseEntries) + } + } + } + + /* We might be within the context of a command execution. This happens if the key is found to + * be expired when attempting to execute the command. In this case, we should treat the key as + * missing. If the key exists after the command executes, we can treat it like a new key. */ + if (server.in_call) { + robj *oKey = createObject(OBJ_STRING, sdsdup(key)); + listAddNodeHead(curCmdMissingKeys, oKey); + } +} + + +// PUBLIC API +void bgIteration_flushall(void) { + handleFlushdb(-1); +} + + +// PUBLIC API +bool bgIteration_blockClientIfRequired(client *c) { + serverAssert(onValkeyMainThread()); + iteratorReplicationFlagsWereUpdated = false; + if (!bgIteration_iterationActive()) return false; + if (!isWriteCmd(c->cmd)) return false; + + if (BGITERATION_DEBUG) { + sds sdsArgv = createSdsFromClientArgv(c->argc, c->argv); + debugBuffer = sdscatprintf(debugBuffer, "BLCK?: (%d)%s\n", c->db->id, sdsArgv); + sdsfree(sdsArgv); + } + + /* Before executing a command or atomic transaction, the replication flag is cleared for each + * iterator. If it's determined that the command should replicate, the flag will be set + * as the command and keys are examined for expedite. */ + resetReplicationFlagForIterators(c); + iteratorReplicationFlagsWereUpdated = true; + + if (c->cmd->proc == flushdbCommand || c->cmd->proc == flushallCommand) { + // Handle flush commands prior to execution + int flags; + if (getFlushCommandFlags(c, &flags) == C_OK) { + // The command parsed ok - we WILL flush + handleFlushdb((c->cmd->proc == flushdbCommand) ? c->db->id : -1); + } + } + + bool mustBlock = false; + hashtable *waitOnKeys = hashtableCreate(&tempKeysetHashtableType); // set of robj(sds) + listEmpty(curCmdMissingKeys); + + if (c->cmd->proc == execCommand) { + mustBlock = expediteKeysForMultiExec(c, waitOnKeys); + } else { + getKeysResult result; + initGetKeysResult(&result); + int numkeys = getKeysFromCommand(c->cmd, c->argv, c->argc, &result); + keyReference *keyrefs = result.keys; + if (numkeys > 0) { + mustBlock = expediteKeysForWriteOnAllIterators( + c->db->id, c->cmd, c->argc, c->argv, keyrefs, numkeys, waitOnKeys); + // We shouldn't need to block on a command within a multi (that's not a script) + serverAssert(!(mustBlock && c->flag.multi && !c->flag.script)); + + if (mustBlock && (c->flag.script)) { + /* For scripts, we will block for keys declared in EVAL/EVALSHA/FCALL. + * However, scripts are NOT required to declare keys. Even if it declares keys, + * it's not declaring the DB for the key. After a SELECT or SWAPDB, we might be on + * a key we haven't blocked for. In this case, there is no option but to execute a + * synchronous block and wait for the iterator(s) to be done with the key(s). + * (Yuck.) */ + static const mstime_t SYNC_BLOCKING_LOG_INTERVAL = 60000; + static mstime_t last_log = 0; // STATIC: persists to prevent log spamming + static int blocked_count = 0; // STATIC: persistent count since last log + blocked_count++; + if (server.mstime - last_log > SYNC_BLOCKING_LOG_INTERVAL) { + serverLog(LL_WARNING, + "Forkless operation synchronously blocked %d times for scripts with undeclared keys", + blocked_count); + last_log = server.mstime; + blocked_count = 0; + } + + while (mustBlock) { + receiveItemsBackFromIterators(true); // Blocking + hashtableEmpty(waitOnKeys, NULL); + mustBlock = expediteKeysForWriteOnAllIterators( + c->db->id, c->cmd, c->argc, c->argv, keyrefs, numkeys, waitOnKeys); + } + } + } else { + // WRITE commands with no keys should always be replicated. SWAPDB, FLUSH, FUNCTION, etc. + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + it->cur_cmd_may_replicate = true; + } + } + getKeysFreeResult(&result); + } + + if (mustBlock) { + serverAssert(hashtableSize(waitOnKeys) > 0); + robj **waitKeysArgv = zmalloc(sizeof(robj *) * hashtableSize(waitOnKeys)); + + robj *key; + hashtableIterator hi; + hashtableInitIterator(&hi, waitOnKeys, 0); + unsigned long argvCount = 0; + while (hashtableNext(&hi, (void **)&key)) { + waitKeysArgv[argvCount++] = key; + } + hashtableCleanupIterator(&hi); + serverAssert(argvCount == hashtableSize(waitOnKeys)); + + blockClientInUseOnKeys(c, argvCount, waitKeysArgv); + + zfree(waitKeysArgv); + } + + hashtableRelease(waitOnKeys); + + if (BGITERATION_DEBUG) { + if (mustBlock) debugBuffer = sdscat(debugBuffer, " (blocked)\n"); + } + + return mustBlock; +} + + +// PUBLIC API +void bgIteration_handleCommandReplication(int dbid, + struct serverCommand *cmd, + int argc, + robj **argv) { + if (BGITERATION_DEBUG) { + // DEBUG - enable this to capture replication not queued because iteration is inactive + if (0 && !bgIteration_iterationActive() && (isWriteCmd(cmd) || cmd->proc == multiCommand)) { + sds sdsArgv = createSdsFromClientArgv(argc, argv); + debugBuffer = sdscatprintf(debugBuffer, "REPL? INACT: (%d)%s\n", dbid, sdsArgv); + sdsfree(sdsArgv); + } + } + + if (!bgIteration_iterationActive()) return; + serverAssert(onValkeyMainThread()); + + /* Some commands are replicated which are not writes (like publish) these can be ignored. + * Be careful with MULTI which is not a write command, but must be replicated. */ + if (!isWriteCmd(cmd) && cmd->proc != multiCommand) return; + + if (BGITERATION_DEBUG) { + sds sdsArgv = createSdsFromClientArgv(argc, argv); + debugBuffer = sdscatprintf(debugBuffer, "REPL?: (%d)%s\n", dbid, sdsArgv); + sdsfree(sdsArgv); + } + + if (cmd->proc == swapdbCommand) { + // All iterators and clients must be informed of swapdb + int id1, id2; + // command has been processed, but Valkey allows "swapdb 0 0" (which can be ignored) + if (getParamsForSwapdb(argc, argv, NULL, &id1, &id2)) + handleSwapdb(id1, id2); + } + + /* In the case that a key is touched in a different DB (COPY/MOVE) the key is recorded as + * a "special" key and than handled below. */ + int special_dbid = 0; + sds special_key = NULL; + dbEntry *special_dbEntry = NULL; + if (cmd->proc == moveCommand) { + /* The MOVE command succeeded. However MOVE requires special handling as it creates a new + * key in a different database. We need to make sure that we don't later try to iterate + * on the key as it would be a duplicate key at that point. So, instead, we will mark the + * newly created key as "early iterated". */ + bool success = getDbIdFromRobj(argv[MOVE_COMMAND_DBID_ARG_INDEX], &special_dbid); + serverAssert(success); // the command already succeeded, so this should work! + + robj *oKey = argv[1]; + special_key = (sds)objectGetVal(oKey); + + special_dbEntry = dbFind(server.db[special_dbid], special_key); + } + if (cmd->proc == copyCommand) { + // The COPY command succeeded. However COPY requires special handling (like MOVE). + bool success = getTargetDbIdForCopyCommand(argc, argv, dbid, &special_dbid); + serverAssert(success); // the command already succeeded, so this should work! + + // Find the newly created entry. + robj *oKey = argv[2]; + special_key = (sds)objectGetVal(oKey); + + special_dbEntry = dbFind(server.db[special_dbid], special_key); + } + + /* Implementation note regarding LUA and MULTI: LUA scripts and MULTI-EXEC blocks must be + * treated atomically. We need to ensure that either ALL of the replication (or none of the + * replication) for the atomic operation is processed by the iterator(s). This is handled + * naturally as we can only "complete" the iteration during the feeding process - and feeding + * is only performed when handling timer events (after the LUA/MULTI has completed). */ + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (it->completed || it->terminated) continue; + + /* For consistent iteration, we only iterate values based on version. But for + * non-consistent iteration, we don't need to explicitly iterate any values newly created + * during the iteration. So we mark them as expedited. We know we have a new key if it + * was missing before the command, and exists now. */ + + if (!(it->iteration_flags & BGITERATOR_FLAG_CONSISTENT)) { + // Handle the special case of a key moved to a different DB + if (special_dbEntry != NULL) { + if (it->cur_cmd_may_replicate && + !it->keyset_iter->hasPassedItem(it->keyset_iter, special_key, special_dbid)) { + hashtableAdd(it->early_iterate_entries, special_dbEntry); + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(special_dbid, special_dbEntry); + debugBuffer = sdscatprintf(debugBuffer, "EARLY(special): %s\n", entryString); + sdsfree(entryString); + } + } + + /* Note: In the cases where there's a special command, we are copying or moving an + * item to a different DB. In these limited cases, we can only possibly be + * creating a single key. And if we've handled it here, we don't need to + * handle it as a "missing key" below. If we were to try to handle it as a + * standard "missing key", we would get the DBID incorrect. */ + + } else if (listLength(curCmdMissingKeys) > 0) { + listIter missingIt; + listNode *missingNode; + listRewind(curCmdMissingKeys, &missingIt); + while ((missingNode = listNext(&missingIt)) != NULL) { + robj *oKey = listNodeValue(missingNode); + const_sds key = objectGetVal(oKey); + dbEntry *de = dbFind(server.db[dbid], (sds)key); + if (de != NULL) { + // It exists now! + if (it->cur_cmd_may_replicate && + !it->keyset_iter->hasPassedItem(it->keyset_iter, key, dbid)) { + /* If the current command is allowed to replicate, and there is a new + * key which we haven't yet reached in iteration, it needs to be added + * to the set of early iterate entries. (We know that it's not already + * in that set because it's a newly created key!) */ + bool wasAdded = hashtableAdd(it->early_iterate_entries, de); + serverAssert(wasAdded); + if (BGITERATION_DEBUG) { + sds entryString = createEntryString(dbid, de); + debugBuffer = sdscatprintf(debugBuffer, "EARLY(NEW): %s\n", entryString); + sdsfree(entryString); + } + } + } + } + } + } + + /* Deletes (and unlinks) are special. + * Developer context: For most commands, we call bgIteration_blockClientIfRequired before + * the command and then call bgIteration_handleCommandReplication after the command. While + * the "before" logic is determining the need to block, it can also determine (mostly) the + * need for replication (on each iterator). Doing this all in one place saves us from + * performing some of the same logic twice. When we get to this point in the code, we just + * use the previously determined information regarding replication. This works because + * Valkey is single-threaded and only processes one command at a time. + * + * But deletes (and unlinks) happen multiple ways - and occur outside the normal + * before/after logic for commands. These situations must be handled: + * - A normal (client-driven) DEL/UNLINK command will use the standard before/after + * logic. If the key is in use by bgIteration, the command will be blocked. + * - An EVICTION generates a DEL/UNLINK which happens outside of the context of a client + * issued command. The replication flags on the iterators are stale and relate to the + * prior command executed. + * - An EXPIRATION in the context of a client-driven WRITE command occurs when the client + * command attempts to access a key and it is found to be expired. In this case, the + * client-command has already gone through the blocking process, so it should be OK to + * use it->cmd_may_replicate. + * - An EXPIRATION in the context of a client-driven READ command occurs when the client + * command attempts to access a key and it is found to be expired. In this case, the + * client-command has NOT gone through the blocking process. The replication flags on + * the iterators are stale and relate to the prior (write) command executed. + * - An EXPIRATION outside of a client-driven command occurs due to active expiry. In + * this case, the replication flags on the iterator are stale and relate to the prior + * command executed. + * + * In the case of EXPIRE/EVICT occurring outside the context of a write command, this is + * handled. If the key is in-use by bgIterator, increment of robj's refcount prevents the + * key from deletion. In this case the key will be removed from the main dictionary, but + * held by bgIteration until no longer needed. + * Even though the entry is not physically deleted yet, it is logically deleted and it is + * safe to replicate the DEL/UNLINK. Since iterators process items FIFO, the replication + * for DEL/UNLINK won't actually get processed until other queued replication is processed. + * + * In the case of a client driven DEL command, the key will have already been deleted when + * we hit this routine. In the case of EXPIRE/EVICT, they propagate happens before the key + * is deleted. So if the key is missing, we can use the cached replication decision. But + * if the key still exists (indicating EXPIRE/EVICT) we evaluate it specially. */ + bool shouldReplicateDelCommand = false; + bool isDelCommand = isDeleteCmd(cmd); + if (isDelCommand) { + sds key = objectGetVal(argv[1]); + dbEntry *de = dbFind(server.db[dbid], key); + serverAssert(de == NULL); // dbEntry should be removed before replication (self-check) + if (it->keyset_iter->isKeyInScope(it->keyset_iter, key)) { + bool blockClientIfRequiredWasCalled = (server.in_call > 0); + if (blockClientIfRequiredWasCalled && iteratorReplicationFlagsWereUpdated) { + // Here we know that the DEL is related to the running command + shouldReplicateDelCommand = it->cur_cmd_may_replicate; + } else { + // Otherwise, it's something like active expiration or eviction (unrelated) + if (iteratorHasPassedKey(it, dbid, key, dbEntryPtrOfLastKeyDelete)) { + shouldReplicateDelCommand = true; + } + } + hashtableDelete(it->early_iterate_entries, dbEntryPtrOfLastKeyDelete); // just try delete (might not be here) + } + } + + bool replicate = (it->iteration_flags & BGITERATOR_FLAG_REPLICATION && + ((!isDelCommand && it->cur_cmd_may_replicate) || shouldReplicateDelCommand)); + + if (replicate) { + /* We will replicate the command in these cases: + * 1) For consistent iteration - it->cur_cmd_may_replicate is always true + * 2) For non-consistent, if any of the keys have been processed, expediteKeysForWrite + * will ensure that ALL of the keys have been expedited - and we should replicate + * 3) For non-consistent, if NONE of the keys have been processed, no need to replicate */ + if (BGITERATION_DEBUG) { + debugBuffer = sdscat(debugBuffer, " (queued)\n"); + } + + bgIteratorItem *item = itemFreeList_getElementOrAllocate(); + item->type = BGITERATOR_ITEM_REPLICATION; + item->dbid = dbid; + item->u.repl.cmd = cmd; + item->u.repl.argv = cloneRobjArray(argc, argv); + item->u.repl.argc = argc; + item->u.repl.replication_size = replicationItemSize(item); + bufferedReplicationBytes += item->u.repl.replication_size; + it->replication_queued++; + mutexQueueAdd(it->items_for_iterator, item); + } + } // allIterators loop +} + + +// PUBLIC API +size_t bgIteration_memoryInuseForReplication(void) { + return bufferedReplicationBytes; +} + + +// PUBLIC API +bool bgIteration_isEntryInuse(dbEntry *de) { + serverAssert(onValkeyMainThread()); + if (!bgIteration_iterationActive()) return false; + return isEntryInuseByAnyIterator(de); +} + + +// PUBLIC API +void bgIteration_dbEntryModified(dbEntry *de) { + if (bgIteration_iterationActive()) { + bgIterationEntryMetadata *md = (bgIterationEntryMetadata *)objectGetMetadata(de); + if (md) md->iterator_epoch = bgIteration_epoch; + } +} + + +// PUBLIC API +void bgIteration_keyModified(int dbid, const_sds key) { + if (bgIteration_iterationActive()) { + dbEntry *de = dbFind(server.db[dbid], (sds)key); + if (de) bgIteration_dbEntryModified(de); + } +} + + +// PUBLIC API +void bgIteration_updateDbEntryPtr(dbEntry *old, dbEntry *new) { + if (!bgIteration_iterationActive() || old == new) return; + serverAssert(onValkeyMainThread()); + serverAssert(!isEntryInuseByAnyIterator(old)); + + listIter li; + listNode *node; + listRewind(allIterators, &li); + while ((node = listNext(&li)) != NULL) { + bgIterator *it = listNodeValue(node); + if (hashtableDelete(it->early_iterate_entries, old)) { + if (BGITERATION_DEBUG) { + debugBuffer = sdscatprintf(debugBuffer, "EARLY LIST UPDATE %p -> %p\n", (void *)old, (void *)new); + } + bool wasAdded = hashtableAdd(it->early_iterate_entries, new); + serverAssert(wasAdded); + } + } +} diff --git a/src/bgiteration.h b/src/bgiteration.h new file mode 100644 index 00000000000..6aaac8cc7b8 --- /dev/null +++ b/src/bgiteration.h @@ -0,0 +1,365 @@ +/* + * Copyright Valkey Contributors. + * All rights reserved. + * SPDX-License-Identifier: BSD 3-Clause + */ + +#ifndef __BGITERATION_H +#define __BGITERATION_H + +#include +#include "sds.h" + +/* A mechanism for creating iteration clients which iterate over the main dictionary in a + * background thread. + * + * This mechanism passes keys to the iteration client, while blocking the keys from write by the + * Valkey main thread. Once an iteration client is done with a key, it is returned to the Valkey + * main thread and any pending writers are unblocked. + * + * A bgIterator must be created on the main Valkey thread, and then passed to another thread which + * implements the logic of the iteration client. + * + * Iteration clients are expected to read through the keyspace until the iteration is complete or + * terminated. An iteration client may not perform modifications on a key. */ + +/* Avoids dependency on server.h */ +typedef struct serverObject dbEntry; // An object with key/value inserted into main dictionary +typedef struct serverObject robj; // An object with a value used for command parameters +typedef struct client client; + +/* The bgIterator is an opaque structure. */ +typedef struct bgIterator bgIterator; + + +/* Consistency type for iteration. */ +typedef enum { + /* With no consistency requirements, dbEntries are provided to the iteration client as they + * appear at the time of iteration. No replication is provided. The only guarantee is that + * dbEntries which existed at the start of iteration, and remained through the duration of + * iteration, will be provided to the iteration client once (and only once). If a dbEntry is + * modified during iteration, either the old or the new value may be provided. */ + BGITERATOR_CONSISTENCY_NONE = 0, + + /* With consistency at the start of iteration, a point-in-time iteration is performed. The + * iteration client will see all keys AS THEY EXISTED at the time when the iterator was created. + * Note: The DBID provided with the DICTENTRY events is the original DBID (at the time of iteration + * start). SWAPDB events will not be provided. */ + BGITERATOR_CONSISTENCY_START = 1, + + /* With an eventually consistent iteration, dbEntries will be followed by relevant replication. + * This will allow a client to achieve a consistent state at the END of the iteration. Once a + * dbEntry has been provided to the iteration client, any replication related to that entry will + * also be forwarded to the iteration client. With eventual consistency, keys are provided as + * they are at the time of iteration. This mode requires that the iteration client be aware of + * SWAPDB events. If a SWAPDB is performed, the client will receive a SWAPDB event. + * Replication events will be provided ordered and synchronized with any SWAPDB events. */ + BGITERATOR_CONSISTENCY_EVENTUAL = 2 +} bgIteratorConsistency; + + +/* When running an iterator with replication, a replication-done function (callback) may be + * provided. This function will be executed after the last replication item has been fed into the + * queue for the client. This function will be run on the Valkey main thread, and allows a client + * to recognize the point where no additional replication data will be sent for processing. + * + * PRIVDATA: this pointer is for data private to the iteration client. + * + * Returns true when an iterator stops accepting any replication item into the queue for the client. + * If false is returned, replication will continue, and bgiteration will periodically call the callback + * until true is returned. In this context, returning false indicates that the client is not ready to + * stop receiving replication, it is requesting that replication be continued. */ +typedef bool (*bgIteratorReplDoneFunc)(void *privdata); + + +/* When creating a bgIterator, a cleanup function (callback) may be provided. This function will be + * executed once iteration has completed and this will run on the Valkey main thread. + * + * TERMINATED: will be passed as TRUE if the iteration process was terminated early (either by + * the main thread calling bgIteratorTerminate() or the iteration client calling + * bgIteratorClose()). + * PRIVDATA: this pointer is for data private to the iteration client. */ +typedef void (*bgIteratorCleanupFunc)(bool terminated, void *privdata); + + +/* Create a background full-scan iterator (bgIterator). + * This bgIterator will iterate through the entire keyspace (across all DBs). + * + * NAME: a human readable name for the iterator (must be unique) + * FLAGS: creation flags indicate iteration options + * REPLDONE: if provided, called after the last replication item has been queued (on the Valkey main thread) + * CLEANUP: if provided, called at the end of iteration (on the Valkey main thread) + * PRIVDATA: passed to cleanup function + * + * This method creates and initializes the bgIterator. It does not perform any thread management. + * It is expected that the main Valkey thread will call this method, and then start a new thread to + * to implement the iteration client which will read from the returned bgIterator. + * + * There is no need to delete/destroy a bgIterator. It will automatically be cleaned up after the + * last item is read. */ +bgIterator *bgIteratorCreateFullScanIter( + const char *name, + bgIteratorConsistency consistency, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata); + + +/* Create a background slots iterator (bgIterator). + * This bgIterator will iterate through the keys belonging to a set of cluster slots. + * + * NAME: a human readable name for the iterator (must be unique) + * FLAGS: creation flags indicate iteration options + * SLOTS: array of cluster slots to iterate over + * SLOTS_COUNT: size of the array of slots + * REPLDONE: if provided, called after the last replication item has been queued (on the Valkey main thread) + * CLEANUP: if provided, called at the end of iteration (on the Valkey main thread) + * PRIVDATA: passed to cleanup function + * + * This method creates and initializes the bgIterator. It does not perform any thread management. + * It is expected that the main Valkey thread will call this method, and then start a new thread to + * to implement the iteration client which will read from the returned bgIterator. + * + * The caller of this function has the ownership of the `slots` array's memory. This function will + * just copy its data and leave the array untouched. + * + * There is no need to delete/destroy a bgIterator. It will automatically be cleaned up after the + * last item is read. */ +bgIterator *bgIteratorCreateSlotsIter( + const char *name, + bgIteratorConsistency consistency, + const int *slots, + int slots_count, + bgIteratorReplDoneFunc repldone, + bgIteratorCleanupFunc cleanup, + void *privdata); + + +/* Find an existing bgIterator by name. + * Returns NULL if the iterator does not exist (or has completed). */ +bgIterator *bgIteratorFind(const char *name); + + +/* Get the name of an existing iterator. */ +const char *bgIteratorName(bgIterator *iter); + + +/* Struct to retrieve status information for an active iteration client. */ +typedef struct { + unsigned long dbentries_queued; // Cumulative BGITERATOR_ITEM_DBENTRY queued + unsigned long dbentries_processed; // Cumulative BGITERATOR_ITEM_DBENTRY processed + unsigned long replication_queued; // Cumulative BGITERATOR_ITEM_REPLICATION queued + unsigned long replication_processed; // Cumulative BGITERATOR_ITEM_REPLICATION processed + unsigned long swapdb_queued; // Cumulative BGITERATOR_ITEM_SWAPDB queued + unsigned long swapdb_processed; // Cumulative BGITERATOR_ITEM_SWAPDB processed + unsigned long flushdb_queued; // Cumulative BGITERATOR_ITEM_FLUSHDB queued + unsigned long flushdb_processed; // Cumulative BGITERATOR_ITEM_FLUSHDB processed + unsigned long dbentry_clones_queued; // A subset of dbentries_queued for cloned entries + unsigned long dbentry_clones_processed; // A subset of dbentries_processed for cloned entries + unsigned long queue_length; // Current length of queue to iteration client + unsigned long queue_length_target; // Dynamic target length for queue to iteration client + unsigned long runtime_ms; // Time, in milliseconds, that iterator has been running + unsigned long current_item_ms; // Time, in milliseconds, spent processing current item +} bgIteratorStatus; + + +/* Get the status of a background iteration. + * + * The caller-provided bgIteratorStatus will be populated. */ +void bgIteratorGetStatus(bgIterator *iter, bgIteratorStatus *status); + + +/* Terminate a background iteration. + * + * An iteration is terminated by the Valkey main thread. It is expected that the iteration client + * will continue to read, receiving BGITERATOR_ITEM_TERMINATED or BGITERATOR_ITEM_COMPLETE to + * complete the iteration. (This is necessary to ensure proper cleanup.) + * NOTE: If the iteration client wants to terminate iteration, it may call bgIteratorClose(). */ +void bgIteratorTerminate(bgIterator *iter); + + +/* Check if an iterator is being terminated. + * + * This checks if the iterator is in the process of terminating. For the Valkey main thread, this + * can be used to determine if a call has already been made to bgIteratorTerminate. For an + * iteration client, it normally learns about terminate by reading the next item, this allows + * out-of-band detection of termination which can be useful when processing a large key. */ +bool bgIteratorIsTerminating(bgIterator *iter); + + +typedef enum { + /* Indicates that the iteration has completed normally. No more items to read. + * If replication is enabled, on completion, the final replication offset is recorded in + * 'u.master_repl_offset' and 'dbid' is set to the selected replication db. The iteration + * client will have received all *applicable* replication data to this point. */ + BGITERATOR_ITEM_COMPLETE = 1, + + /* Indicates that the iteration has been terminated before completion. No more items to read.*/ + BGITERATOR_ITEM_TERMINATED, + + /* A dbEntry for DB=dbid. + * NOTE: The dbEntry MAY be expired. It is up to the client to decide how to handle + * expired entries. */ + BGITERATOR_ITEM_DBENTRY, + + /* A replication command for DB=dbid. cmd, argv, & argc provided. + * NOTE: The command may have been re-written before replication. */ + BGITERATOR_ITEM_REPLICATION, + + /* A SWAPDB event. dbid swapped with dbid2. + * Note that SWAPDB events are not provided during consistent iteration. */ + BGITERATOR_ITEM_SWAPDB, + + /* A FLUSHDB event. In most cases, iteration will be terminated, and this event will NOT be + * sent. However, in the case of a single minor DB being flushed, non-consistent iteration is + * permitted to continue. */ + BGITERATOR_ITEM_FLUSHDB +} bgIteratorItemType; + + +typedef struct { + dbEntry *de; + bool is_cloned; + bool is_rehashing_paused; +} dbEntryData; + +typedef struct { + struct serverCommand *cmd; + robj **argv; + int argc; + size_t replication_size; +} replicationData; + +typedef struct { + bgIteratorItemType type; + int dbid; // orig DB ID for CONSISTENT, queue-time DB ID for !CONSISTENT. + union { + dbEntryData dbe; // for BGITERATOR_ITEM_DBENTRY + replicationData repl; // for BGITERATOR_ITEM_REPLICATION + long long master_repl_offset; // for BGITERATOR_ITEM_COMPLETE + int dbid2; // for BGITERATOR_ITEM_SWAPDB + } u; +} bgIteratorItem; + + +/* Read the next bgIteratorItem from the bgIterator. + * + * The iteration client is expected to call this function in a loop. After reading + * BGITERATOR_ITEM_COMPLETE or BGITERATOR_ITEM_TERMINATED, the iteration client must call + * bgIteratorClose to finalize the iteration process. + * + * This is a blocking call. If the main Valkey thread has been too busy to send items to the + * iterator, the iteration client's queue may run dry and this call will block until data is + * available. + * + * NOTE: Reading an item returns previously read items to the main thread. It is unsafe to + * reference an item previously read. + * + * (All memory management is the responsibility of the bgIterator - not the reader.) */ +bgIteratorItem *bgIteratorRead(bgIterator *iter); + + +/* Close the bgIterator, allowing the bgIterator to be deallocated. + * + * This must be called by an iteration client to release the bgIterator. + * + * It is required that this is called after receiving BGITERATOR_ITEM_COMPLETE or + * BGITERATOR_ITEM_TERMINATED and signals that the background activity is complete. + * + * This may also be called by the iteration client to force terminate an iteration early. The + * bgIterator will be marked as terminated. */ +void bgIteratorClose(bgIterator *iter); + + +/******************************************************************************************** + * BGITERATION HOOKS REQUIRED TO SUPPORT ITERATION - CALLS INSERTED INTO MAIN VALKEY CODE + ********************************************************************************************/ + +#define BGITERATION_ENTRY_METADATA_SIZE 4 + +/* Must be called once (and only once) at server startup. */ +void bgIteration_init(void); + + +/* Returns true if any iterators are currently active. */ +bool bgIteration_iterationActive(void); + + +/* Called as a beforeSleep action, receives items back from bgIteration. This is just a little + * quicker than waiting for bgIteration's internal timer. */ +void bgIteration_beforeSleep(void); + + +/* Notify bgIteration that a key is about to be deleted. This call must happen before the removal + * from the main dictionary. In Valkey, key deletion can occur in a READ command if the key is + * expired. Note that this notification is more about status than memory. Since the dbEntry is a + * reference counted object, the dbEntry can't be physically deleted if bgIteration is still + * actively using it. */ +void bgIteration_keyDelete(int dbid, const_sds key); + + +/* Iteration needs to know if a FLUSHALL is being performed. For normal clients, this comes through + * the standard "blockClientIfRequired" interface. This interface is for cases where Valkey + * performs the FLUSHALL operation independently of clients (e.g. when syncing with master). */ +void bgIteration_flushall(void); + + +/* Updating value or expiration of an existing key may lead to reallocation of the dbEntry (robj). + * BgIteration keeps track of expedited keys (by pointer) to avoid repeated iteration. BgIteration + * must be notified when dbEntries are reallocated. BgIteration will not dereference the pointers; + * it is safe to have deallocated the old dbEntry before calling this function. + * + * We can't update the dbEntry if the entry is actually in use (bgIteration_isEntryInuse)! + * + * To simplify calling code, this function does nothing if old_entry == new_entry. */ +void bgIteration_updateDbEntryPtr(dbEntry *old_entry, dbEntry *new_entry); + + +/* Before executing any command, the Valkey main thread must call this function. If the key(s) are + * blocked for writes by an iterator, the function returns true and the client is blocked. A + * blocked client will be unblocked once the key becomes available for write. + * + * This should be called for all commands - even commands which are executed as part of a MULTI/EXEC + * or LUA script. + * + * For MULTI/EXEC - This function is called when hitting the EXEC - after all of the commands + * have been queued. This may block the EXEC, but will NOT block individual + * commands as they are executed in the MULTI/EXEC block. + * + * For LUA script - This function is first called for EVAL/EVALSHA. It may block the script while + * waiting on declared keys. However, if the script accesses undeclared keys or + * performs SWAPDB, a synchronous block may be performed (returning false) on + * individual commands within the script. + * + * Note: this function should be called for all commands (not just writes). */ +bool bgIteration_blockClientIfRequired(client *c); + + +/* After execution of a write command, the Valkey main thread must provide the command to iterators + * which are interested in the replication feed. It is required that all commands have been passed + * through bgIteration_blockClientIfRequired(), however, it is permitted that the command can be + * re-written for propagation. */ +void bgIteration_handleCommandReplication( + int dbid, + struct serverCommand *cmd, + int argc, + robj **argv); + + +/* The memory that bgIteration uses while temporarily buffering replication data is not included in + * the maxmemory computation used for eviction. This function provides insight into the current + * amount of memory used for buffered replication data. */ +size_t bgIteration_memoryInuseForReplication(void); + + +/* Check if a dbEntry is currently in-use/locked by bgIteration. */ +bool bgIteration_isEntryInuse(dbEntry *de); + + +/* Notify bgIteration that a dbEntry has been added/modified. + * - If caller has a dbEntry*, dbEntryModified is more efficient + * - If caller has a dbid/key, a lookup is performed to find the dbEntry */ +void bgIteration_dbEntryModified(dbEntry *de); +void bgIteration_keyModified(int dbid, const_sds key); + +#endif diff --git a/src/db.c b/src/db.c index 7c0da08a303..93b6e565a0d 100644 --- a/src/db.c +++ b/src/db.c @@ -39,6 +39,7 @@ #include "vector.h" #include "expire.h" #include "crc16_slottable.h" +#include "bgiteration.h" /*----------------------------------------------------------------------------- * C-level DB API @@ -369,6 +370,7 @@ static void dbSetValue(serverDb *db, robj *key, robj **valref, int overwrite, vo objectSetLRU(val, objectGetLRU(old)); long long expire = objectGetExpire(old); new = objectSetKeyAndExpire(val, objectGetVal(key), expire); + bgIteration_updateDbEntryPtr(old, new); *oldref = new; /* Replace the old value at its location in the expire space. */ if (expire >= 0) { @@ -438,6 +440,7 @@ void setKey(client *c, serverDb *db, robj *key, robj **valref, int flags) { } else { dbSetValue(db, key, valref, 1, NULL); } + bgIteration_dbEntryModified(*valref); if (!(flags & SETKEY_KEEPTTL)) removeExpire(db, key); if (!(flags & SETKEY_NO_SIGNAL)) signalModifiedKey(c, db, key); } @@ -483,6 +486,8 @@ int dbGenericDeleteWithDictIndex(serverDb *db, robj *key, int async, int flags, hashtablePosition pos; void **ref = kvstoreHashtableTwoPhasePopFindRef(db->keys, dict_index, objectGetVal(key), &pos); if (ref != NULL) { + bgIteration_keyDelete(db->id, (sds)objectGetVal(key)); + robj *val = *ref; /* VM_StringDMA may call dbUnshareStringValue which may free val, so we * need to incr to retain val */ @@ -672,6 +677,9 @@ long long emptyData(int dbnum, int flags, void(callback)(hashtable *)) { return -1; } + /* bgIteration must be notified for flushall. */ + if (dbnum == -1) bgIteration_flushall(); + /* Fire the flushdb modules event. */ moduleFireServerEvent(VALKEYMODULE_EVENT_FLUSHDB, VALKEYMODULE_SUBEVENT_FLUSHDB_START, &fi); @@ -762,6 +770,7 @@ long long dbTotalServerKeyCount(void) { void signalModifiedKey(client *c, serverDb *db, robj *key) { touchWatchedKey(db, key); trackingInvalidateKey(c, key, 1); + bgIteration_keyModified(db->id, objectGetVal(key)); } void signalFlushedDb(int dbid, int async) { @@ -2295,7 +2304,7 @@ robj *dbFindExpires(serverDb *db, sds key) { } unsigned long long dbSize(serverDb *db) { - return kvstoreSize(db->keys); + return (db->keys) ? kvstoreSize(db->keys) : 0; } unsigned long long dbScan(serverDb *db, unsigned long long cursor, kvstoreScanFunction scan_cb, void *privdata) { diff --git a/src/defrag.c b/src/defrag.c index 4debda161e8..f0260f43c64 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -44,6 +44,7 @@ #include "eval.h" #include "script.h" #include "module.h" +#include "bgiteration.h" #include #include @@ -658,6 +659,8 @@ static void defragKey(defragKeysCtx *ctx, robj **elemref) { unsigned char *newzl; ob = *elemref; + if (bgIteration_isEntryInuse(ob)) return; + /* Try to defrag robj and/or string value. */ if ((newob = activeDefragStringOb(ob))) { *elemref = newob; @@ -765,6 +768,11 @@ static void defragPubsubScanCallback(void *privdata, void *elemref) { * and 1 if time is up and more work is needed. */ static int defragLaterItem(robj *ob, unsigned long *cursor, monotime endtime, int dbid) { if (ob) { + if (bgIteration_isEntryInuse(ob)) { + *cursor = 0; + return 0; + } + if (ob->type == OBJ_LIST && ob->encoding == OBJ_ENCODING_QUICKLIST) { return scanLaterList(ob, cursor, endtime); } else if (ob->type == OBJ_SET && ob->encoding == OBJ_ENCODING_HASHTABLE) { diff --git a/src/expire.c b/src/expire.c index 3885e4c99ad..3f896344bc1 100644 --- a/src/expire.c +++ b/src/expire.c @@ -39,6 +39,7 @@ #include "cluster.h" #include "cluster_migrateslots.h" #include "util.h" +#include "bgiteration.h" /*----------------------------------------------------------------------------- * Incremental collection of expired keys. @@ -167,13 +168,18 @@ void fieldExpireScanCallback(void *privdata, void *volaKey, int didx) { robj *o = volaKey; serverAssert(o); serverAssert(hashTypeHasVolatileFields(o)); + + data->has_more_expired_entries = false; + data->sampled++; + + if (bgIteration_isEntryInuse(o)) return; + mstime_t now = server.mstime; size_t expired_fields = dbReclaimExpiredFields(o, data->db, now, data->max_entries, didx); if (expired_fields) { data->has_more_expired_entries = (expired_fields == data->max_entries); data->expired++; } - data->sampled++; } static int expireShouldSkipTableForSamplingCb(hashtable *ht) { diff --git a/src/hashtable.c b/src/hashtable.c index 89db564eaf0..0e59da6fd0f 100644 --- a/src/hashtable.c +++ b/src/hashtable.c @@ -344,7 +344,7 @@ typedef struct { } position; static_assert(sizeof(hashtablePosition) >= sizeof(position), - "Opaque iterator size"); + "Opaque position size"); /* State for incremental find. */ typedef struct { @@ -1406,13 +1406,13 @@ void hashtableResumeAutoShrink(hashtable *ht) { * spaces, "holes", in the bucket chains, which wastes memory. Additionally, we * pause auto shrink when rehashing is paused, meaning the hashtable will not * shrink the bucket count. */ -static void hashtablePauseRehashing(hashtable *ht) { +void hashtablePauseRehashing(hashtable *ht) { ht->pause_rehash++; hashtablePauseAutoShrink(ht); } /* Resumes incremental rehashing, after pausing it. */ -static void hashtableResumeRehashing(hashtable *ht) { +void hashtableResumeRehashing(hashtable *ht) { ht->pause_rehash--; assert(ht->pause_rehash >= 0); hashtableResumeAutoShrink(ht); @@ -2054,13 +2054,17 @@ size_t hashtableScan(hashtable *ht, size_t cursor, hashtableScanFunction fn, voi * A cursor of 0 means the scan has not started, so no keys have been passed. */ bool hashtableScanHasPassedKey(hashtable *ht, const void *key, size_t cursor) { if (cursor == 0) return false; - size_t mask = expToMask(ht->bucket_exp[0]); - uint64_t hash = hashKey(ht, key); - size_t bucket_idx = hash & mask; - size_t cursor_idx = cursor & mask; - /* In reverse-bit-increment order, a bucket has been visited if its - * reversed index is less than the reversed cursor index. */ - return rev(bucket_idx) < rev(cursor_idx); + if (hashtableSize(ht) == 0) return true; + + /* The scan visits buckets in reverse-binary order based on the smallest + * table. During rehashing, a small-table bucket and its corresponding + * large-table buckets are processed together, so the small-table mask + * determines ordering in both cases. */ + int exp = ht->bucket_exp[0]; + if (hashtableIsRehashing(ht) && ht->bucket_exp[1] < exp) exp = ht->bucket_exp[1]; + size_t mask = expToMask(exp); + size_t bucket_idx = hashKey(ht, key) & mask; + return rev(bucket_idx) < rev(cursor & mask); } /* Like hashtableScan, but additionally reallocates the memory used by the dict diff --git a/src/hashtable.h b/src/hashtable.h index 4af1ad0dcf7..289bc183db1 100644 --- a/src/hashtable.h +++ b/src/hashtable.h @@ -129,6 +129,8 @@ size_t hashtableMemUsage(const hashtable *ht); void hashtablePauseAutoShrink(hashtable *ht); void hashtableResumeAutoShrink(hashtable *ht); bool hashtableIsRehashing(hashtable *ht); +void hashtablePauseRehashing(hashtable *ht); +void hashtableResumeRehashing(hashtable *ht); bool hashtableIsRehashingPaused(hashtable *ht); ssize_t hashtableGetRehashingIndex(hashtable *ht); void hashtableRehashingInfo(hashtable *ht, size_t *from_size, size_t *to_size); diff --git a/src/module.c b/src/module.c index d7eb68db9df..86ddcf45e93 100644 --- a/src/module.c +++ b/src/module.c @@ -2440,7 +2440,7 @@ void VM_SetModuleAttribs(ValkeyModuleCtx *ctx, const char *name, int ver, int ap module->apiver = apiver; module->types = listCreate(); module->usedby = listCreate(); - module->using = listCreate(); + module->uses = listCreate(); module->filters = listCreate(); module->module_configs = listCreate(); listSetMatchMethod(module->module_configs, moduleListConfigMatch); @@ -11455,7 +11455,7 @@ void *VM_GetSharedAPI(ValkeyModuleCtx *ctx, const char *apiname) { ValkeyModuleSharedAPI *sapi = dictGetVal(de); if (listSearchKey(sapi->module->usedby, ctx->module) == NULL) { listAddNodeTail(sapi->module->usedby, ctx->module); - listAddNodeTail(ctx->module->using, sapi->module); + listAddNodeTail(ctx->module->uses, sapi->module); } return sapi->func; } @@ -11492,7 +11492,7 @@ int moduleUnregisterUsedAPI(ValkeyModule *module) { listNode *ln; int count = 0; - listRewind(module->using, &li); + listRewind(module->uses, &li); while ((ln = listNext(&li))) { ValkeyModule *used = ln->value; listNode *ln = listSearchKey(used->usedby, module); @@ -13245,7 +13245,7 @@ void moduleFreeModuleStructure(struct ValkeyModule *module) { listRelease(module->types); listRelease(module->filters); listRelease(module->usedby); - listRelease(module->using); + listRelease(module->uses); listRelease(module->module_configs); sdsfree(module->name); moduleLoadQueueEntryFree(module->loadmod); @@ -13890,7 +13890,7 @@ sds genModulesInfoString(sds info) { struct ValkeyModule *module = listNodeValue(ln); sds usedby = genModulesInfoStringRenderModulesList(module->usedby); - sds using = genModulesInfoStringRenderModulesList(module->using); + sds using = genModulesInfoStringRenderModulesList(module->uses); sds options = genModulesInfoStringRenderModuleOptions(module); info = sdscatfmt(info, "module:name=%S,ver=%i,api=%i,filters=%i," diff --git a/src/module.h b/src/module.h index b411cf6a53e..e216a3b68cd 100644 --- a/src/module.h +++ b/src/module.h @@ -107,7 +107,7 @@ typedef struct ValkeyModule { int apiver; /* Module API version as requested during initialization.*/ list *types; /* Module data types. */ list *usedby; /* List of modules using APIs from this one. */ - list *using; /* List of modules we use some APIs of. */ + list *uses; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ list *module_configs; /* List of configurations the module has registered */ int configs_initialized; /* Have the module configurations been initialized? */ diff --git a/src/object.c b/src/object.c index fb5727e4a0e..b5e3de5993b 100644 --- a/src/object.c +++ b/src/object.c @@ -39,6 +39,7 @@ #include "zmalloc.h" #include "sds.h" #include "module.h" +#include "bgiteration.h" #include #include @@ -371,7 +372,7 @@ robj *createStringObjectFromSds(const_sds s) { return createStringObject(s, sdslen(s)); } -static robj *createStringObjectWithKeyAndExpire(const char *ptr, size_t len, const_sds key, long long expire) { +robj *createStringObjectWithKeyAndExpire(const char *ptr, size_t len, const_sds key, long long expire) { if (shouldEmbedStringObject(len, key, expire)) { return createEmbeddedStringObjectWithKeyAndExpire(ptr, len, key, expire); } else { @@ -480,6 +481,7 @@ robj *objectSetKeyAndExpire(robj *o, const_sds key, long long expire) { if (objectGetType(o) == OBJ_STRING && objectGetEncoding(o) == OBJ_ENCODING_EMBSTR) { robj *new = createStringObjectWithKeyAndExpire(objectGetVal(o), sdslen(objectGetVal(o)), key, expire); objectSetLRU(new, objectGetLRU(o)); + bgIteration_updateDbEntryPtr(o, new); decrRefCount(o); return new; } @@ -505,6 +507,7 @@ robj *objectSetKeyAndExpire(robj *o, const_sds key, long long expire) { robj *new = createUnembeddedObjectWithKeyAndExpire(objectGetType(o), ptr, key, expire); objectSetEncoding(new, objectGetEncoding(o)); objectSetLRU(new, objectGetLRU(o)); + bgIteration_updateDbEntryPtr(o, new); decrRefCount(o); return new; } diff --git a/src/server.c b/src/server.c index 4665cbeb77c..7af2ee2369c 100644 --- a/src/server.c +++ b/src/server.c @@ -55,6 +55,7 @@ #include "util.h" #include "eval.h" +#include "bgiteration.h" #include "trace/trace_commands.h" @@ -1944,6 +1945,8 @@ void beforeSleep(struct aeEventLoop *eventLoop) { * later in this function, must be done before blockedBeforeSleep. */ if (server.cluster_enabled) clusterBeforeSleep(); + /* Release keys from bgIteration before processing unblocked clients. */ + bgIteration_beforeSleep(); /* Handle blocked clients. * must be done before flushAppendOnlyFile, in case of appendfsync=always, * since the unblocked clients may write data. */ @@ -2089,7 +2092,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) { /* Before we are going to sleep, let the threads access the dataset by * releasing the GIL. The server main thread will not touch anything at this * time. */ - if (moduleCount()) moduleReleaseGIL(); + if (moduleCount()) { + atomic_store_explicit(&server.module_gil_acquired, 0, memory_order_relaxed); + moduleReleaseGIL(); + } /********************* WARNING ******************** * Do NOT add anything below moduleReleaseGIL !!! * ***************************** ********************/ @@ -2111,6 +2117,7 @@ void afterSleep(struct aeEventLoop *eventLoop, int numevents) { atomic_store_explicit(&server.module_gil_acquiring, 1, memory_order_relaxed); moduleAcquireGIL(); atomic_store_explicit(&server.module_gil_acquiring, 0, memory_order_relaxed); + atomic_store_explicit(&server.module_gil_acquired, 1, memory_order_relaxed); moduleFireServerEvent(VALKEYMODULE_EVENT_EVENTLOOP, VALKEYMODULE_SUBEVENT_EVENTLOOP_AFTER_SLEEP, NULL); latencyEndMonitor(latency); latencyAddSampleIfNeeded("module-acquire-GIL", latency); @@ -3070,8 +3077,11 @@ void initServer(void) { /* Set object metadata size before creating any database key objects */ if (server.forkless_options_supported) { - objectSetMetadataSize(sizeof(uint32_t)); /* This is a placeholder until Threadsave defines a metadata structure */ - /* 4 bytes for iterator_epoch for now*/ + /* NOTE: At this time, there is only one reason for dbEntry metadata: bgIteration. However, + * if/when new metadata options are added, we will need to compute the size of a variable + * size metadata, and provide appropriate accessors to access the specific portion of the + * metadata (each of which may/may not exist, based on immutable startup parameters). */ + objectSetMetadataSize(BGITERATION_ENTRY_METADATA_SIZE); } createDatabaseIfNeeded(0); /* The default database should always exist */ @@ -3088,6 +3098,7 @@ void initServer(void) { server.watching_clients = 0; server.cronloops = 0; server.in_exec = 0; + server.in_call = 0; server.busy_module_yield_flags = BUSY_MODULE_YIELD_NONE; server.busy_module_yield_reply = NULL; server.client_pause_in_transaction = 0; @@ -3185,6 +3196,7 @@ void initServer(void) { commandlogInit(); latencyMonitorInit(); initSharedQueryBuf(); + bgIteration_init(); /* Initialize ACL default password if it exists */ ACLUpdateDefaultUserPassword(server.requirepass); @@ -3746,6 +3758,58 @@ static void propagateNow(int dbid, robj **argv, int argc, int target, int slot) if (propagate_to_slot_migration) clusterFeedSlotExportJobs(dbid, argv, argc, slot); } +/* BgIteration requires that replication is sent after each command, however the + * alsoPropagate mechanism queues replication until the end of the transaction + * (when propagatePendingCommands is invoked). Also, the propagation mechanism + * strips out multi/exec, adding them back during propagatePendingCommands (if + * necessary). This function ensures that replication, including multi/exec are + * sequenced with the commands for bgIteration. + * + * Called from alsoPropagate with regular params. + * Called from propagatePendingCommands with dbid = -1 (to close multi/exec). */ +static void propagateToBgIteration(int dbid, int argc, robj **argv, int target) { + /* STATIC indicates that we have sent the MULTI, and need to match it with + * an EXEC during propagatePendingCommands. */ + static bool sentMultiToBgIterator = false; + /* STATIC indicates that last DBID that was sent, so that we can use the + * same DBID when sending a generated EXEC. */ + static int lastDbidSentToBgIterator; + + if (dbid >= 0) { + // Called from alsoPropagate() to replicate a command + if (target & PROPAGATE_REPL && bgIteration_iterationActive()) { + if (!sentMultiToBgIterator && (scriptIsRunning() || server.in_exec)) { + /* For a script or multi/exec, we should be sending the MULTI at + * the beginning of the execution unit. There shouldn't be any + * commands in the propagation queue yet. */ + serverAssert(server.also_propagate.numops == 0); + /* If this is the first propagated command of a script or multi, + * make it a transaction. It may turn out that there is only 1 + * command in the MULTI block, but we can't know that now. + * Unlike regular replication, we can't defer all of the + * replication until we know for sure. We must call bgIteration + * after each command. */ + static struct serverCommand *cmd_multi = NULL; // STATIC + if (cmd_multi == NULL) cmd_multi = lookupCommandOrOriginal(&shared.multi, 1); + bgIteration_handleCommandReplication(dbid, cmd_multi, 1, &shared.multi); + sentMultiToBgIterator = true; + } + struct serverCommand *cmd = lookupCommandOrOriginal(argv, argc); + bgIteration_handleCommandReplication(dbid, cmd, argc, argv); + lastDbidSentToBgIterator = dbid; + } + } else { + // Called from propagatePendingCommands() to finalize a transaction + if (sentMultiToBgIterator) { + // If a MULTI was sent to bgIterator via alsoPropagate(), then send the matching EXEC. + static struct serverCommand *cmd_exec = NULL; // STATIC + if (cmd_exec == NULL) cmd_exec = lookupCommandOrOriginal(&shared.exec, 1); + bgIteration_handleCommandReplication(lastDbidSentToBgIterator, cmd_exec, 1, &shared.exec); + sentMultiToBgIterator = false; + } + } +} + /* Used inside commands to schedule the propagation of additional commands * after the current command is propagated to AOF / Replication. * @@ -3758,6 +3822,8 @@ static void propagateNow(int dbid, robj **argv, int argc, int target, int slot) * stack allocated). The function automatically increments ref count of * passed objects, so the caller does not need to. */ void alsoPropagate(int dbid, robj **argv, int argc, int target, int slot) { + propagateToBgIteration(dbid, argc, argv, target); + robj **argvcopy; int j; @@ -3824,6 +3890,12 @@ void updateCommandLatencyHistogram(struct hdr_histogram **latency_histogram, int * multiple separated commands. Note that alsoPropagate() is not affected * by CLIENT_PREVENT_PROP flag. */ static void propagatePendingCommands(void) { + /* This is done before the check on server.also_propagate.numops. Numops + * might be zero if there is no replica but we might be running bgIteration + * for something other than replication. If we sent the multi (to + * bgIteration), we need to send the matching exec. */ + propagateToBgIteration(-1, 0, NULL, 0); + if (server.also_propagate.numops == 0) return; int j; @@ -3953,6 +4025,10 @@ int incrCommandStatsOnError(struct serverCommand *cmd, int flags) { * */ void call(client *c, int flags) { + if (bgIteration_blockClientIfRequired(c)) return; + + server.in_call++; + long long dirty; struct ClientFlags client_old_flags = c->flag; @@ -4219,6 +4295,7 @@ void call(client *c, int flags) { } server.executing_client = prev_client; + server.in_call--; } /* Used when a command that is ready for execution needs to be rejected, due to diff --git a/src/server.h b/src/server.h index e1253c15beb..04819ef6256 100644 --- a/src/server.h +++ b/src/server.h @@ -103,7 +103,19 @@ static_assert(sizeof(off_t) >= 8, "off_t must be 64-bit; ensure _FILE_OFFSET_BIT #define dismissMemory zmadvise_dontneed #define VALKEYMODULE_CORE 1 -typedef struct serverObject robj; + +/* serverObject (aka robj) is currently overloaded for 2 purposes. This is a legacy artifact. + * 1. It's carries a reference counted STRING (a keyless value) during parsing and command execution. + * 2. It's also used to carry a key/value pair which is inserted into the DB. In this form, the + * value is not limited to being a string. + * + * The typedef "dbEntry" is used to explicitly connote the latter form. It indicates a key/value + * pair which is suitable to exist in the DB. It might be active in the DB, or may be unlinked from + * the DB (but still contains a key/value). The value may be any of the Valkey data types/encodings. + */ +typedef struct serverObject robj; // A keyless string OR a key/value pair +typedef struct serverObject dbEntry; // Explicitly a key/value pair + #include "valkeymodule.h" /* Modules API defines. */ /* Following includes allow test functions to be called from main() */ @@ -1793,6 +1805,7 @@ struct valkeyServer { size_t initial_memory_usage; /* Bytes used after initialization. */ int always_show_logo; /* Show logo even for non-stdout logging. */ int in_exec; /* Are we inside EXEC? */ + int in_call; /* Nesting level within the call() function. */ int busy_module_yield_flags; /* Are we inside a busy module? (triggered by RM_Yield). see BUSY_MODULE_YIELD_ flags. */ const char *busy_module_yield_reply; /* When non-null, we are inside RM_Yield. */ char *ignore_warnings; /* Config: warnings that should be ignored. */ @@ -1811,6 +1824,7 @@ struct valkeyServer { pid_t child_pid; /* PID of current child */ int child_type; /* Type of current child */ _Atomic(int) module_gil_acquiring; /* Indicates whether the GIL is being acquiring by the main thread. */ + _Atomic(int) module_gil_acquired; /* Indicates if the main thread has the GIL acquired. */ /* Networking */ int port; /* TCP listening port */ int tls_port; /* TLS listening port */ @@ -3558,6 +3572,8 @@ void resetServerStats(void); void monitorActiveDefrag(void); void defragWhileBlocked(void); const char *evictPolicyToString(void); +size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid); +robj *createStringObjectWithKeyAndExpire(const char *ptr, size_t len, const_sds key, long long expire); struct serverMemOverhead *getMemoryOverheadData(void); void freeMemoryOverheadData(struct serverMemOverhead *mh); void checkChildrenDone(void); @@ -3817,6 +3833,7 @@ typedef int(emptyDataHashtableFilter)(int didx); long long emptyData(int dbnum, int flags, void(callback)(hashtable *)); long long emptyDbStructure(serverDb **dbarray, int dbnum, int async, void(callback)(hashtable *)); void resetDbExpiryState(serverDb *db); +int getFlushCommandFlags(client *c, int *flags); void flushAllDataAndResetRDB(int flags); long long dbTotalServerKeyCount(void); serverDb *initTempDb(int id); @@ -3962,11 +3979,13 @@ void startEvictionTimeProc(void); uint8_t *getConfigurableHashSeed(void); uint64_t dictSdsHash(const void *key); uint64_t dictSdsCaseHash(const void *key); +uint64_t dictObjHash(const void *key); uint64_t dictCStrHash(const void *key); uint64_t dictCStrCaseHash(const void *key); uint64_t dictEncObjHash(const void *key); int dictSdsKeyCompare(const void *key1, const void *key2); int dictSdsKeyCaseCompare(const void *key1, const void *key2); +int dictObjKeyCompare(const void *key1, const void *key2); int dictCStrKeyCompare(const void *key1, const void *key2); int dictCStrKeyCaseCompare(const void *key1, const void *key2); int dictEncObjKeyCompare(const void *key1, const void *key2); diff --git a/src/unit/custom_matchers.hpp b/src/unit/custom_matchers.hpp index 1e0e80cf499..b5e7ca21e35 100644 --- a/src/unit/custom_matchers.hpp +++ b/src/unit/custom_matchers.hpp @@ -15,7 +15,11 @@ MATCHER_P(robjEqualsStr, str, "robj string matcher") { assert(arg->type == OBJ_STRING); assert(sdsEncodedObject(arg)); - return strcmp(static_cast(objectGetVal(arg)), str) == 0; + + if (strcmp(static_cast(objectGetVal(arg)), str) == 0) return true; + + *result_listener << "robj(\"" << (char *)objectGetVal(arg) << "\") doesn't match \"" << str << "\""; + return false; } #endif // _CUSTOM_MATCHERS_HPP_ diff --git a/src/unit/test_bgiteration.cpp b/src/unit/test_bgiteration.cpp new file mode 100644 index 00000000000..ca43c895e8d --- /dev/null +++ b/src/unit/test_bgiteration.cpp @@ -0,0 +1,3172 @@ +/* + * Copyright Valkey Contributors. + * All rights reserved. + * SPDX-License-Identifier: BSD 3-Clause + */ + +#include "generated_wrappers.hpp" +#include + +using namespace ::testing; + +extern "C" { +#include "bgiteration.h" +#include "module.h" +#include "server.h" +#include "stdlib.h" +extern hashtableType commandSetType; +extern dictType keylistDictType; +void bgIteration_feedIterators(void); +void createSharedObjects(void); +void hashtableDump(hashtable *ht); +void bgIteration_unitTestDisableCloning(void); +void bgIteration_unitTestEnableCloning(int item_bytes, int pool_bytes); +static size_t mockHashtableScan(hashtable *ht, size_t cursor, hashtableScanFunction fn, void *privdata); +size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid); +} + + +// The private data is a pointer to arbitrary data. This value is used just to +// test that the correct value is passed through. +#define PRIVDATA reinterpret_cast(12345) + +typedef int32_t bgIterationEntryMetadata; // opaque 4 bytes +static_assert(sizeof(bgIterationEntryMetadata) == BGITERATION_ENTRY_METADATA_SIZE); + +// A bgIteration cleanup function used for testing. +static int cleanupCount; +static bool cleanupTerminated; +static void iteratorCleanupFn(bool terminated, void *privdata) { + EXPECT_EQ(privdata, PRIVDATA); + cleanupCount++; + cleanupTerminated = terminated; +} + +// A bgIteration repldone function used for testing. +static int replDoneConfirmed; +static bool iteratorRepldoneFn(void *privdata) { + EXPECT_EQ(privdata, PRIVDATA); + replDoneConfirmed++; + return true; +} + +// A more complicated repldone function that can delay the replcation done condition. +static int replDoneRejected; +static bool iteratorRepldoneFnNotBeingReadyInitially(void *privdata) { + EXPECT_EQ(privdata, PRIVDATA); + // This is to test the behavior when Repl Done function is not ready to be executed. + if (replDoneRejected == 0) { + replDoneRejected++; + return false; + } + replDoneConfirmed++; + return true; +} + + +/* This mock for hashtableScan will return the items in lexical order. It assumes that the entries + * are robjs containing an sds string for the key. The key is expected to begin with a capital + * letter [A-Z]. The caller passes 0 as the cursor to start the iteration. The returned cursor + * value will indicate the prior letter returned (1=A, ...). After entries starting with 'Z' have + * been returned, the cursor of 0 will indicate that the scan is complete. Note that all entries + * starting with the same letter will be returned in a single call. */ +static size_t mockHashtableScan(hashtable *ht, size_t cursor, hashtableScanFunction fn, void *privdata) { + // Just in case, if it's not one of our hashtables, use the unmocked function + bool our_ht = (server.db[0]->keys && ht == kvstoreGetHashtable(server.db[0]->keys, 0)) || + (server.db[1]->keys && ht == kvstoreGetHashtable(server.db[1]->keys, 0)); + if (!our_ht) return __real_hashtableScan(ht, cursor, fn, privdata); + + // Collect all entries from the hashtable + std::vector entries; + hashtableIterator *iter = hashtableCreateIterator(ht, 0); + dbEntry *entry; + while (hashtableNext(iter, (void **)&entry)) { + char first = objectGetKey(entry)[0]; + assert(first >= 'A' && first <= 'Z'); + entries.push_back(entry); + } + hashtableReleaseIterator(iter); + + // Sort by key lexicographically + std::sort(entries.begin(), entries.end(), [](dbEntry *a, dbEntry *b) { + return strcmp(objectGetKey(a), objectGetKey(b)) < 0; + }); + + // cursor 0 means start at 'A', otherwise start after the cursor letter + char startLetter = (char)('A' + cursor); + + // Find the first letter to emit + char emitLetter = 0; + for (dbEntry *e : entries) { + char first = objectGetKey(e)[0]; + if (first >= startLetter) { + emitLetter = first; + break; + } + } + + if (emitLetter == 0) return 0; + + // Call fn for all entries starting with emitLetter + for (dbEntry *e : entries) { + char first = objectGetKey(e)[0]; + if (first == emitLetter) fn(privdata, (void *)e); + } + + size_t nextCursor = (size_t)(emitLetter - 'A' + 1); + return (nextCursor > 25) ? 0 : nextCursor; +} + + +static bool mockHashtableScanHasPassedKey(hashtable *ht, const void *key, size_t cursor) { + // If it's one of our tables, use the mock logic + bool itsOurs = false; + if (server.db[0]->keys && ht == kvstoreGetHashtable(server.db[0]->keys, 0)) itsOurs = true; + if (server.db[1]->keys && ht == kvstoreGetHashtable(server.db[1]->keys, 0)) itsOurs = true; + + // Mock logic uses a lexicographic cursor + if (itsOurs) return ((const char *)key)[0] < (char)('A' + cursor); + + // Otherwise, use the real logic for other hashtables + return __real_hashtableScanHasPassedKey(ht, key, cursor); +} + + +static const char *logfile = ""; + +/* Most of the bgIteration unit tests are based on a CMD instance with 2 DBs. There are 8 keys in + * each DB. The hashtableScan function is mocked to return the keys in a predictable order. + * + * There are a number of helper functions to simulate certain key modification actions within our + * test configuration. Note that this is isolated from the actual call to processCommand. + * + * Because most of bgIteration is based on an ordered processing of keys, it doesn't matter if we + * are simulating CMD or CME, full scan, or slot-based. The majority of tests are independent of + * these concerns. + * + * However, there are some tests which are are unique to these configurations and use a specialized + * derived class to handle the differences. We do not want to duplicate all of the tests for + * the different configurations, but we do want to ensure that each configuration works properly. + * - bgIterationTestCluster - handles tests unique to full scan in cluster mode + * - bgIterationTestClusterSlots - handles tests unique to cluster slot-based iteration */ +class BgIterationTest : public ::testing::Test { + protected: + static const int DB_COUNT = 2; + static const int ITEMS_PER_DB = 8; + + private: + /* With the mock hashtableScan, we get keys in a predictable order. DB0 works with buckets + * containing groups of keys (which hashtableScan returns in a single call). DB1 returns + * each key individually, as more separate buckets. Convention (for test readability) is + * that keys beginning [A-M] would be in DB0 and keys beginning [N-Z] in DB1. Letters are + * intentionally skipped to allow for possible insertions. */ + const char *keys[DB_COUNT][ITEMS_PER_DB] = {{"B0", "B1", "B2", "E0", "E1", "H0", "H1", "H2"}, + {"N0", "O0", "Q0", "R0", "T0", "U0", "W0", "Y0"}}; + + protected: + static const int TOTAL_ITEMS = DB_COUNT * ITEMS_PER_DB; + static const int LAST_ITEM = TOTAL_ITEMS - 1; + + MockValkey mock; + RealValkey real; + client *c = nullptr; // for general use in the tests (with common cleanup) + robj **orig_argv = nullptr; // Used when simulating multi + int orig_argc = 0; // Used when simulating multi + + + struct serverCommand dummy_cmd = {0}; + + // Helper functions for accessing the keys. We can access by db(0..1) and seq(0..7) + // or by item number (0..15). + // NOTE: These virtual functions can be overridden in subclasses which may have different item layout. + virtual const char *getKeyAtDbSeq(int db, int seq) { + assert(db < DB_COUNT); + assert(seq < ITEMS_PER_DB); + return keys[db][seq]; + } + + virtual int getDbFromItemNum(int itemNum) { + assert(itemNum < DB_COUNT * ITEMS_PER_DB); + return itemNum / ITEMS_PER_DB; + } + + virtual int getSeqFromItemNum(int itemNum) { + assert(itemNum < DB_COUNT * ITEMS_PER_DB); + return itemNum % ITEMS_PER_DB; + } + + const char *keyStr(int itemNum) { + return getKeyAtDbSeq(getDbFromItemNum(itemNum), getSeqFromItemNum(itemNum)); + } + + int itemNumFromKey(const char *key) { + for (int itemNum = 0; itemNum < DB_COUNT * ITEMS_PER_DB; itemNum++) { + if (strcmp(key, keyStr(itemNum)) == 0) return itemNum; + } + return -1; + } + + + // Do some general initialization before starting the suite. Normally, the tests are run in + // isolation - and this isn't much different than SetUp(). But if running the + // entire test suite together (just manually running the test executable), this gets called + // only once. + static void SetUpTestSuite() { + monotonicInit(); + + bzero(&server, sizeof(server)); + server.hz = 100; + server.logfile = const_cast(logfile); + createSharedObjects(); + + moduleInitModulesSystem(); + + server.commands = hashtableCreate(&commandSetType); + server.orig_commands = hashtableCreate(&commandSetType); + populateCommandTable(); + } + + + static void TearDownTestSuite() { + hashtableRelease(server.commands); + hashtableRelease(server.orig_commands); + } + + + void initializeServerDb(int dbid, int slot_count_bits = 0) { + server.db[dbid] = static_cast(zcalloc(sizeof(serverDb))); + server.db[dbid]->id = dbid; + server.db[dbid]->keys = kvstoreCreate(&kvstoreKeysHashtableType, slot_count_bits, 0); + server.db[dbid]->expires = kvstoreCreate(&kvstoreExpiresHashtableType, slot_count_bits, 0); + server.db[dbid]->watched_keys = dictCreate(&keylistDictType); + } + + + robj *createStringObjectFromCString(const char *s) { + return createStringObject(s, strlen(s)); + } + + + void addKeyToDb(int dbid, const char *key, const char *val) { + robj *key_obj = createStringObjectFromCString(key); + robj *val_obj = createStringObjectFromCString(val); + dbAdd(server.db[dbid], key_obj, &val_obj); + decrRefCount(key_obj); + } + + + virtual void setupDatabase() { + /* For these unit tests, a standard database is constructed. But we will use our own + * mocked scan function to ensure a consistent iteration order */ + + server.dbnum = DB_COUNT; + server.cluster_enabled = false; + server.db = static_cast(zcalloc(sizeof(serverDb *) * server.dbnum)); + + for (int dbid = 0; dbid < server.dbnum; dbid++) { + initializeServerDb(dbid); + for (int keynum = 0; keynum < ITEMS_PER_DB; keynum++) { + addKeyToDb(dbid, keys[dbid][keynum], keys[dbid][keynum]); + } + } + + EXPECT_CALL(mock, hashtableScan(_, _, _, _)) + .WillRepeatedly(Invoke(mockHashtableScan)); + EXPECT_CALL(mock, hashtableScanHasPassedKey(_, _, _)) + .WillRepeatedly(Invoke(mockHashtableScanHasPassedKey)); + + if (0) debugPrintBucketInfo(); + } + + + void SetUp() override { + server.main_thread_id = pthread_self(); + server.forkless_options_supported = 1; + objectSetMetadataSize(BGITERATION_ENTRY_METADATA_SIZE); + + bgIteration_unitTestDisableCloning(); + + setupDatabase(); + + EXPECT_CALL(mock, aeCreateTimeEvent(_, _, _, _, _)).WillRepeatedly(Return(0)); + bgIteration_init(); + + cleanupCount = 0; + replDoneConfirmed = 0; + replDoneRejected = 0; + + // By default, do nothing for these + EXPECT_CALL(mock, blockClientInUseOnKeys(_, _, _)).WillRepeatedly(Return()); + EXPECT_CALL(mock, unblockClientsInUseOnKey(_)).WillRepeatedly(Return()); + + // By default, expect no permission issues + EXPECT_CALL(mock, ACLCheckAllUserCommandPerm(_, _, _, _, _, _)) + .WillRepeatedly(Return(ACL_OK)); + } + + + void TearDown() override { + bgIteration_feedIterators(); // process returning stuff before deleting DB + bgIteration_feedIterators(); // in case an iterator was closed there might be more + for (int i = 0; i < server.dbnum; i++) { + if (server.db[i]->keys) kvstoreRelease(server.db[i]->keys); + if (server.db[i]->expires) kvstoreRelease(server.db[i]->expires); + dictRelease(server.db[i]->watched_keys); + zfree(server.db[i]); + } + zfree(server.db); + + if (c != NULL) freeTestClient(c); + EXPECT_EQ(server.in_call, 0); // make sure tests are handling this properly + } + + + // Deletes an item from the DB (often at the start of a test) - but does NOT notify + // bgIteration. bgIteration_keyDelete() should be explicitly called where needed. + void simpleDelItem(int itemNum) { + int db = getDbFromItemNum(itemNum); + + sds delKey = sdsnew(keyStr(itemNum)); + int rc = kvstoreHashtableDelete(server.db[db]->keys, 0, delKey); + ASSERT_EQ(rc, 1); + sdsfree(delKey); + } + + + // Find the actual dbEntry object by itemNum + dbEntry *getItem(int itemNum) { + int db = getDbFromItemNum(itemNum); + sds key = sdsnew(keyStr(itemNum)); + dbEntry *de = dbFind(server.db[db], key); + sdsfree(key); + return de; + } + + + // The test expects that the next item read will be BGITERATOR_ITEM_COMPLETE + void expectReadComplete(bgIterator *iter) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + EXPECT_EQ(item->type, BGITERATOR_ITEM_COMPLETE); + bgIteratorClose(iter); + + int oldCleanupCount = cleanupCount; + bgIteration_feedIterators(); + EXPECT_EQ(cleanupCount, oldCleanupCount + 1); + } + + + // The test is cleaning up and isn't validating the remaining cleanup + void expectAnythingCleanup(bgIterator *iter) { + while (true) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + if ((item->type == BGITERATOR_ITEM_COMPLETE || + item->type == BGITERATOR_ITEM_TERMINATED)) { + bgIteratorClose(iter); + break; + } + } + bgIteration_feedIterators(); // Recognize the closed iterator + EXPECT_EQ(cleanupCount, 1); + } + + + void expectDictEntryMetadataMatch(dbEntry *de1, dbEntry *de2) { + bgIterationEntryMetadata *dm1 = static_cast(objectGetMetadata(de1)); + bgIterationEntryMetadata *dm2 = static_cast(objectGetMetadata(de2)); + + EXPECT_NE(dm1, nullptr); + EXPECT_NE(dm2, nullptr); + EXPECT_EQ(*dm1, *dm2); + } + + + // Useful when debugging new tests. It reads/prints all remaining items then crashes. + void cleanupIteratorDebugPrint(bgIterator *iter) { + bool done = false; + printf("[DEBUG] Printing bgIterator '%s' items:\n", bgIteratorName(iter)); + while (!done) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + switch (item->type) { + case BGITERATOR_ITEM_DBENTRY: { + auto obj = item->u.dbe.de; + const char *keyStr = objectGetKey(obj); + printf("Entry: %s -> %s [itemNum: %i]\n", + keyStr, + static_cast(objectGetVal(obj)), + itemNumFromKey(keyStr)); + break; + } + case BGITERATOR_ITEM_REPLICATION: + printf("Repl: DB=%d : ", item->dbid); + for (int i = 0; i < item->u.repl.argc; i++) + printf("%s ", static_cast(objectGetVal(item->u.repl.argv[i]))); + printf("\n"); + break; + case BGITERATOR_ITEM_COMPLETE: + case BGITERATOR_ITEM_TERMINATED: + bgIteratorClose(iter); + done = true; + break; + default: + printf("unhandled: %d\n", item->type); + } + } + bgIteration_feedIterators(); // Recognize the closed iterator + ASSERT_TRUE(false); // Halt the test here + } + + + // Make a copy of the metadata + void *cloneMetadata(dbEntry *de) { + int size = objectGetMetadataSize(de); + void *metadata = zmalloc(size); + memcpy(metadata, objectGetMetadata(de), size); + return metadata; + } + + + // Compare a previous metadata copy to an existing entry + void compareAndFreeClonedMetadata(dbEntry *de, void *metadata) { + EXPECT_EQ(memcmp(objectGetMetadata(de), metadata, objectGetMetadataSize(de)), 0); + zfree(metadata); + } + + + // The test expects the next item will be a specific key + // The item value is verified against the default unless provided as a parameter. + void expectReadKey(bgIterator *iter, int itemNum, const char *value = nullptr) { + int db = getDbFromItemNum(itemNum); + + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_DBENTRY); + EXPECT_EQ(item->dbid, db); + EXPECT_FALSE(item->u.dbe.is_cloned); + EXPECT_STREQ(objectGetKey(item->u.dbe.de), keyStr(itemNum)); + if (value) { + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(value)); + } else { + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(keyStr(itemNum))); + } + } + + + // The test expects the next item will be a specific key amd that the item is cloned. + // Metadata is tested (to make sure the clone includes the proper metadata). + // The item value is verified against the default unless provided as a parameter. + void expectReadClonedKey(bgIterator *iter, int itemNum, void *metadata, const char *value = nullptr) { + int db = getDbFromItemNum(itemNum); + + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_DBENTRY); + EXPECT_EQ(item->dbid, db); + EXPECT_TRUE(item->u.dbe.is_cloned); + compareAndFreeClonedMetadata(item->u.dbe.de, metadata); + EXPECT_STREQ(objectGetKey(item->u.dbe.de), keyStr(itemNum)); + if (value) { + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(value)); + } else { + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(keyStr(itemNum))); + } + } + + + // Test expects the next key, but specified by key name, not itemNum. + void expectReadDbKeyValue(bgIterator *iter, int db, const char *key, const char *value) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_DBENTRY); + EXPECT_EQ(item->dbid, db); + EXPECT_STREQ(objectGetKey(item->u.dbe.de), key); + EXPECT_THAT(item->u.dbe.de, robjEqualsStr(value)); + } + + + // Test expect to read a sequence of key items + void expectReadKeySequence(bgIterator *iter, int startItem, int endItem) { + for (int i = startItem; i <= endItem; i++) expectReadKey(iter, i); + } + + + // Just like expectReadKey, but also tests that a previous item is becoming unblocked. + void expectReadKeyWithUnblock(bgIterator *iter, int itemNum, int unblockItem, const char *value = nullptr) { + bool blocked = true; + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(unblockItem)))) + .WillOnce(Assign(&blocked, false)); + expectReadKey(iter, itemNum, value); + EXPECT_FALSE(blocked); + } + + + // Test expects to read a replication item matching the command help by client 'c' + void expectReadReplication(bgIterator *iter, client *c) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->dbid, c->db->id); + EXPECT_EQ(item->u.repl.cmd, c->cmd); + EXPECT_EQ(item->u.repl.argc, c->argc); + for (int i = 0; i < c->argc; i++) { + EXPECT_STREQ(static_cast(objectGetVal(item->u.repl.argv[i])), + static_cast(objectGetVal(c->argv[i]))); + } + } + + + // We expect to read a MULTI command which should have been inserted. + void expectReadMultiReplication(bgIterator *iter) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->u.repl.cmd, lookupCommandByCString("multi")); + } + + + // We expect to read an EXEC command which should have been inserted. + void expectReadExecReplication(bgIterator *iter) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->u.repl.cmd, lookupCommandByCString("exec")); + } + + + // Expecting that a DEL command should have been replicated. + void expectReadReplicationDel(bgIterator *iter, int itemNum) { + int db = getDbFromItemNum(itemNum); + + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->dbid, db); + EXPECT_EQ(item->u.repl.cmd, lookupCommandByCString("DEL")); + EXPECT_EQ(item->u.repl.argc, 2); + EXPECT_THAT(item->u.repl.argv[0], robjEqualsStr("DEL")); + EXPECT_THAT(item->u.repl.argv[1], robjEqualsStr(keyStr(itemNum))); + } + + + // Expecting that a special SWAPDB item has been inserted. + void expectReadSwapDB(bgIterator *iter, int db1, int db2) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_SWAPDB); + EXPECT_EQ(item->dbid, db1); + EXPECT_EQ(item->u.dbid2, db2); + } + + + // Expecting that a special FLUSHDB item has been inserted. + void expectReadFlushDB(bgIterator *iter, int db, bool withReplication = false) { + bgIteration_feedIterators(); + bgIteratorItem *item = bgIteratorRead(iter); + bgIteration_feedIterators(); + + ASSERT_EQ(item->type, BGITERATOR_ITEM_FLUSHDB); + EXPECT_EQ(item->dbid, db); + + if (withReplication) { + item = bgIteratorRead(iter); + bgIteration_feedIterators(); + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + EXPECT_EQ(item->dbid, db); + EXPECT_EQ(item->u.repl.cmd, lookupCommandByCString("FLUSHDB")); + EXPECT_EQ(item->u.repl.argc, 1); + EXPECT_THAT(item->u.repl.argv[0], robjEqualsStr("flushdb")); + } + } + + + static void debugPrintBucketInfoCb(void *privdata, void *entry) { + UNUSED(privdata); + dbEntry *de = (dbEntry *)entry; + printf("--- %s\n", objectGetKey(de)); + } + + void debugPrintBucketInfo() { + printf("*******DEBUG*******\n"); + for (int db = 0; db < server.dbnum; db++) { + int num_ht = kvstoreNumHashtables(server.db[db]->keys); + for (int slot = 0; slot < num_ht; slot++) { + hashtable *ht = kvstoreGetHashtable(server.db[db]->keys, slot); + if (!ht) continue; + + printf("DB: %d, slot: %d\n", db, slot); + size_t cursor = 0; + do { + cursor = hashtableScan(ht, cursor, debugPrintBucketInfoCb, NULL); + printf("-----------\n"); + } while (cursor != 0); + } + } + ASSERT_TRUE(false); + } + + + // Creates a client with a write command (SET) for the given itemNum + client *getWriteClient(int itemNum, const char *value) { + int db = getDbFromItemNum(itemNum); + + client *c = static_cast(zcalloc(sizeof(client))); + + c->cmd = lookupCommandByCString("set"); + c->db = server.db[db]; + + c->argc = 3; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + c->argv[1] = createStringObjectFromCString(keyStr(itemNum)); + c->argv[2] = createStringObjectFromCString(value); + + return c; + } + + + // Create a client with a write command that touches multiple keys + client *getWriteMultiKeysClient(const char *cmdName, + int dstItemNum, + const std::vector &srcItemsNum) { + assert(!srcItemsNum.empty()); + + const int db = getDbFromItemNum(dstItemNum); + std::for_each(srcItemsNum.cbegin(), srcItemsNum.cend(), [&db, this](int srcItemNum) { + assert(db == getDbFromItemNum(srcItemNum)); + }); + + client *c = static_cast(zcalloc(sizeof(client))); + + c->cmd = lookupCommandByCString(cmdName); + assert(c->cmd != nullptr); + c->db = server.db[db]; + + c->argc = 2 + srcItemsNum.size(); + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + c->argv[1] = createStringObjectFromCString(keyStr(dstItemNum)); + for (unsigned int i = 0; i < srcItemsNum.size(); i++) { + c->argv[2 + i] = createStringObjectFromCString(keyStr(srcItemsNum[i])); + } + + return c; + } + + + client *getWrite2KeysClient(const char *cmdName, int dstItemNum, int srcItemNum) { + return getWriteMultiKeysClient(cmdName, dstItemNum, {srcItemNum}); + } + + + client *getWrite3KeysClient(const char *cmdName, int dstItemNum, int src1ItemNum, int src2ItemNum) { + return getWriteMultiKeysClient(cmdName, dstItemNum, {src1ItemNum, src2ItemNum}); + } + + + // Create a client with a MULTI/EXEC block. + // This parses a series of commands separated by ';' + // Example: getMultiClient("SET A0 xxx; SELECT 1; SET A1 xxx; SET B1 xxx") + client *getMultiClient(const char *commands, int dbid = 0) { + char *commandsCopy = zstrdup(commands); // a mutable copy + char *commandStr, *commandStrSave; + char *token, *tokenSave; + + client *c = static_cast(zcalloc(sizeof(client))); + c->db = server.db[dbid]; + initClientMultiState(c); + c->flag.multi = 1; + c->mstate->cmd_flags |= CMD_WRITE; + + commandStr = strtok_r(commandsCopy, ";", &commandStrSave); + while (commandStr != NULL) { + token = strtok_r(commandStr, " ", &tokenSave); + c->cmd = lookupCommandByCString(token); + + c->argv = static_cast(zcalloc(sizeof(robj *) * 5)); // command + 4 args + + for (int i = 0; token != NULL; i++) { + c->argv[i] = createStringObjectFromCString(token); + c->argc = i + 1; + token = strtok_r(NULL, " ", &tokenSave); + } + + queueMultiCommand(c, 0); + freeClientArgv(c); + + commandStr = strtok_r(NULL, ";", &commandStrSave); + } + + c->cmd = lookupCommandByCString("exec"); + c->argc = 1; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString("EXEC"); + + zfree(commandsCopy); + return c; + } + + + // Initially, a MULTI client is set up to execute the EXEC command (which examines the + // contents of the multi/exec block). This function advances the client to begin executing + // the individual commands within the multi/exec block. + void advanceMultiClientToCommand(client *c, int cmdNum) { + assert(cmdNum >= 0 && cmdNum < c->mstate->count); + if (cmdNum == 0) { + // Save off the EXEC + orig_argc = c->argc; + orig_argv = c->argv; + } + c->argc = c->mstate->commands[cmdNum].argc; + c->argv = c->mstate->commands[cmdNum].argv; + c->argv_len = c->mstate->commands[cmdNum].argv_len; + c->cmd = c->realcmd = c->mstate->commands[cmdNum].cmd; + } + + + // A client with a fictional command: + // SETGET + // - writes a value to the first key (making this CMD_WRITE | CMD_WRITE_FIRSTKEY_ONLY) + // - reads a second key + client *getSetGetClient(int itemNum1, const char *value1, int itemNum2) { + // Fictional command which writes to 1st key and reads the 2nd + int db = getDbFromItemNum(itemNum1); + assert(db == getDbFromItemNum(itemNum2)); // (this would be a testcase error) + + client *c = static_cast(zcalloc(sizeof(client))); + struct serverCommand *cmd = static_cast(zcalloc(sizeof(struct serverCommand))); + + cmd->fullname = sdsnew("SETGET"); + cmd->arity = 4; + cmd->flags = CMD_WRITE | CMD_WRITE_FIRSTKEY_ONLY; + + cmd->legacy_range_key_spec.begin_search_type = KSPEC_BS_INDEX; + cmd->legacy_range_key_spec.bs.index.pos = 1; // firstkey + cmd->legacy_range_key_spec.fk.range.lastkey = -1; + cmd->legacy_range_key_spec.fk.range.keystep = 2; + + c->cmd = cmd; + c->db = server.db[db]; + + c->argc = 4; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(cmd->fullname); + c->argv[1] = createStringObjectFromCString(keyStr(itemNum1)); + c->argv[2] = createStringObjectFromCString(value1); + c->argv[3] = createStringObjectFromCString(keyStr(itemNum2)); + + return c; + } + + + // Client with a fictional write command with no keys specified + client *getNoKeysWriteClient() { + // Fictional command which is marked WRITE, but has no keys. + client *c = static_cast(zcalloc(sizeof(client))); + struct serverCommand *cmd = static_cast(zcalloc(sizeof(struct serverCommand))); + + cmd->fullname = sdsnew("NOKEYSWRITE"); + cmd->arity = 1; + cmd->flags = CMD_WRITE; + + cmd->legacy_range_key_spec.begin_search_type = KSPEC_BS_INVALID; // No keys + + c->cmd = cmd; + c->db = server.db[0]; + + c->argc = 1; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(cmd->fullname); + + return c; + } + + + void freeClientArgv(client *c) { + for (int i = 0; i < c->argc; i++) decrRefCount(c->argv[i]); + zfree(c->argv); + c->argv = NULL; + c->argc = 0; + } + + + // During testing, we create some fake commands. This checks if the command is real or fake. + // A fake command is dynamically allocated and can be freed. Real commands are static. + bool isRealValkeyCommand(struct serverCommand *cmd) { + return lookupCommandByCString(cmd->declared_name); + } + + + void freeTestClient(client *c) { + // If the current command references one of the multi commands, set it back to the EXEC + if (c->mstate != NULL) { + for (int i = 0; i < c->mstate->count; i++) { + if (c->argv == c->mstate->commands[i].argv) { + c->argc = orig_argc; + c->argv = orig_argv; + orig_argc = 0; + orig_argv = nullptr; + break; + } + } + } + freeClientMultiState(c); + freeClientArgv(c); + + if (!isRealValkeyCommand(c->cmd)) { + sdsfree(c->cmd->fullname); + zfree(c->cmd); + } + + zfree(c); + } + + + // Simulate what happens when a write command is blocked + void simulateBlockedWrite(client *c, int expectedNumberBlockedKeys = 1) { + EXPECT_CALL(mock, blockClientInUseOnKeys(c, expectedNumberBlockedKeys, _)).Times(1); + bool blocked = bgIteration_blockClientIfRequired(c); + EXPECT_TRUE(blocked); + } + + + // Simulate what happens when a write command isn't blocked + void simulateUnblockedWrite_inCall(client *c) { + EXPECT_CALL(mock, blockClientInUseOnKeys(c, _, _)).Times(0); + bool blocked = bgIteration_blockClientIfRequired(c); + EXPECT_FALSE(blocked); + server.in_call++; + } + + + // Simulates what happens when a write command (SET) actually executes. This requires a + // scenario where we would NOT be blocked on the write. It actually alters the value of + // the key and updates the metadata. + void simulateUnblockedWriteWithModification(client *c) { + simulateUnblockedWrite_inCall(c); + + // Fake execution of the command - touch the iterator_epoch counter and swap the value + // We need to duplicate the value because setKey() can reallocate it. + robj *value = dupStringObject(c->argv[2]); + setKey(c, c->db, c->argv[1], &value, SETKEY_ADD_OR_UPDATE); + + // Let's make sure that setKey updated the iteration epoch (as it should have) + dbEntry *de = dbFind(c->db, static_cast(objectGetVal(c->argv[1]))); + bgIterationEntryMetadata *md = static_cast(objectGetMetadata(de)); + bgIterationEntryMetadata md_after_setkey = *md; + // Now update the md again, and it should still match + bgIteration_dbEntryModified(de); + EXPECT_EQ(md, objectGetMetadata(de)); // the md location shouldn't have changed + EXPECT_EQ(md_after_setkey, *md); // the md value should still be the same + + bgIteration_handleCommandReplication(c->db->id, c->cmd, c->argc, c->argv); + server.in_call--; + } + + + // Simulate what happens when a write command is NOT blocked, because the key can be cloned + // and expedited. This requires a scenario where we would normally need to block the + // client so that bgIteration can process the item. + void simulateClonedWriteWithModification(bgIterator *it, client *c) { + bgIteratorStatus status; + bgIteratorGetStatus(it, &status); + unsigned long initialClones = status.dbentry_clones_queued; + + // Client should not get blocked + simulateUnblockedWriteWithModification(c); + + // Ensure that cloning took place + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, (initialClones + 1)); + + // Ensure that the real item isn't inuse (because we cloned it instead) + dbEntry *de = dbFind(c->db, static_cast(objectGetVal(c->argv[1]))); + ASSERT_FALSE(bgIteration_isEntryInuse(de)); + } + + + // Simulate the expiration (active expiration) of a key. This is independent of command execution. + void simulateExpiration(int itemNum) { + ASSERT_NE(getItem(itemNum), nullptr); // Should be there before expire + + // Send bgIteration the DEL + int db = getDbFromItemNum(itemNum); + robj *argv[2]; + argv[0] = createStringObjectFromCString("DEL"); + argv[1] = createStringObjectFromCString(keyStr(itemNum)); + serverCommand *cmd = lookupCommandByCString("DEL"); + // KeyDelete should be called before the deletion occurs + bgIteration_keyDelete(db, static_cast(objectGetVal(argv[1]))); + + simpleDelItem(itemNum); // Simulate the actual del + + // Replication happens after the deletion occurs + ASSERT_EQ(server.in_call, 0); // test sanity check + bgIteration_handleCommandReplication(db, cmd, 2, argv); + decrRefCount(argv[0]); + decrRefCount(argv[1]); + + EXPECT_EQ(getItem(itemNum), nullptr); + } + + + // Simulates an expiration, but validates behavior for an item inuse by bgIteration. + void simulateExpirationOfInuse(int itemNum) { + // An inuse item will have a refcount > 1. BgIteration should have incremented the + // refcount while it is inuse. + dbEntry *de = getItem(itemNum); + ASSERT_NE(de, nullptr); // Should be there before expire + EXPECT_TRUE(bgIteration_isEntryInuse(de)); + EXPECT_EQ(de->refcount, 2u); + + simulateExpiration(itemNum); + + // At this point, the item is removed from the DB, but still exists, and the refcount + // has been reduced to 1. This allows a background thread to continue using the item. + EXPECT_EQ(de->refcount, 1u); + } + + + // Simulates an expiration, but the item is a future item which will be expedited. + void simulateExpirationWithExpedite(int itemNum) { + // An inuse item will have a refcount > 1. BgIteration should have incremented the + // refcount while it is inuse. + dbEntry *de = getItem(itemNum); + ASSERT_NE(de, nullptr); // Should be there before expire + EXPECT_FALSE(bgIteration_isEntryInuse(de)); // Not yet inuse + EXPECT_EQ(de->refcount, 1u); + + simulateExpiration(itemNum); + + // At this point, the item is removed from the DB, but still exists, and the refcount + // has been reduced to 1. This allows a background thread to continue using the item. + EXPECT_TRUE(bgIteration_isEntryInuse(de)); // It's inuse now + EXPECT_EQ(getItem(itemNum), nullptr); // but it's not in the DB anymore + EXPECT_EQ(de->refcount, 1u); + } + + + // Simulate execution of a SWAPDB command + void simulateSwapDB(int dbid0, int dbid1) { + char dbStr[2] = {0}; + + client *c = static_cast(zcalloc(sizeof(client))); + + c->cmd = lookupCommandByCString("swapdb"); + c->db = server.db[0]; + + c->argc = 3; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + dbStr[0] = '0' + dbid0; + c->argv[1] = createStringObjectFromCString(dbStr); + dbStr[0] = '0' + dbid1; + c->argv[2] = createStringObjectFromCString(dbStr); + + simulateUnblockedWrite_inCall(c); // SWAPDB should never block + + // The real SWAP does more than this, but this is enough for unit tests + serverDb *aux = server.db[dbid0]; + server.db[dbid0] = server.db[dbid1]; + server.db[dbid1] = aux; + + bgIteration_handleCommandReplication(0, c->cmd, c->argc, c->argv); + server.in_call--; + + freeTestClient(c); + } + + + // Simulate execution of a FLUSHDB or FLUSHALL command + void simulateFlushDB(int db, int anInUseItem = -1) { + client *c = static_cast(zcalloc(sizeof(client))); + + if (db == -1) { + c->cmd = lookupCommandByCString("flushall"); + c->db = server.db[0]; + } else { + c->cmd = lookupCommandByCString("flushdb"); + c->db = server.db[db]; + } + + c->argc = 1; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + + dbEntry *de_in_use; + if (anInUseItem >= 0) { + de_in_use = getItem(anInUseItem); + EXPECT_EQ(de_in_use->refcount, 2u); + } + + simulateUnblockedWrite_inCall(c); // FLUSHDB should never block + + // The real FLUSH does more than this, but this is enough for unit tests + + // Now flush the items + for (int d = 0; d < server.dbnum; d++) { + if (db == -1 || db == d) { + kvstoreRelease(server.db[d]->keys); + server.db[d]->keys = NULL; + } + } + + if (anInUseItem >= 0) { + EXPECT_EQ(de_in_use->refcount, 1u); + } + + // and replicate + + bgIteration_handleCommandReplication(0, c->cmd, c->argc, c->argv); + server.in_call--; + + freeTestClient(c); + } +}; + + +TEST_F(BgIterationTest, dbIsOK) { + // Just run the setup/teardown code to make sure the DB is OK. +} + + +///////////////////////////////////////////////////// +// Simple Full-scan iterator tests +///////////////////////////////////////////////////// + +// A simple full scan that just checks basic flow. +TEST_F(BgIterationTest, createAndCleanup) { + bgIterator *it = bgIteratorCreateFullScanIter("simple", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + EXPECT_EQ(bgIteratorFind("simple"), it); + EXPECT_STREQ(bgIteratorName(it), "simple"); + + bgIteratorStatus status; + bgIteratorGetStatus(it, &status); + + EXPECT_EQ(status.dbentries_queued, 0u); + EXPECT_EQ(status.dbentries_processed, 0u); + EXPECT_EQ(status.replication_queued, 0u); + EXPECT_EQ(status.replication_processed, 0u); + EXPECT_EQ(status.swapdb_queued, 0u); + EXPECT_EQ(status.swapdb_processed, 0u); + EXPECT_EQ(status.flushdb_queued, 0u); + EXPECT_EQ(status.flushdb_processed, 0u); + + EXPECT_EQ(status.queue_length, 0u); + EXPECT_GT(status.queue_length_target, 0u); + + EXPECT_LT(status.runtime_ms, 5u); + EXPECT_EQ(status.current_item_ms, 0u); + + expectAnythingCleanup(it); + + EXPECT_EQ(bgIteratorFind("simple"), nullptr); +} + + +// Close client before reading anything +TEST_F(BgIterationTest, testClientCloseBeforeRead) { + bgIterator *it = bgIteratorCreateFullScanIter("simple", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + bgIteration_feedIterators(); + + bgIteratorClose(it); // Immediately close before reading + + bgIteration_feedIterators(); // Recognize the closed iterator + + // Check that the cleanup callback was executed properly + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + + +// Test that the full scan hits each item in the expected sequence. +TEST_F(BgIterationTest, orderedIteration) { + bgIterator *it = bgIteratorCreateFullScanIter("simple", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, LAST_ITEM); + + // Quick status check. At this point, the final item hasn't been returned yet. + bgIteratorStatus status; + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentries_queued, static_cast(TOTAL_ITEMS)); + EXPECT_EQ(status.dbentries_processed, static_cast(TOTAL_ITEMS) - 1); + + expectReadComplete(it); // Returns the final item, and reads the completion item + + // Check that the cleanup callback was executed properly + EXPECT_EQ(cleanupCount, 1); + EXPECT_FALSE(cleanupTerminated); +} + + +// Test that two simultaneous iterations work properly. +TEST_F(BgIterationTest, twoOrderedIterations) { + bgIterator *it1 = bgIteratorCreateFullScanIter("simple1", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + bgIterator *it2 = bgIteratorCreateFullScanIter("simple2", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + EXPECT_EQ(bgIteratorFind("simple1"), it1); + EXPECT_EQ(bgIteratorFind("simple2"), it2); + + int it1Count = 0; + int it2Count = 0; + while (it1Count < TOTAL_ITEMS || it2Count < TOTAL_ITEMS) { + // Randomly read from either iterator + if ((rand() % 2) == 0) { + if (it1Count < TOTAL_ITEMS) expectReadKey(it1, it1Count++); + } else { + if (it2Count < TOTAL_ITEMS) expectReadKey(it2, it2Count++); + } + } + + // Nothing left but to read the final completions + expectReadComplete(it1); + EXPECT_EQ(cleanupCount, 1); + EXPECT_FALSE(cleanupTerminated); + expectReadComplete(it2); + EXPECT_EQ(cleanupCount, 2); + EXPECT_FALSE(cleanupTerminated); +} + + +///////////////////////////////////////////////////// +// MODIFY A FUTURE ITEM +// The next tests validate the basic pattern when a key, not yet iterated, is modified. +// Each variation of iteration flags is tested. +// Note that these tests execute without cloning (cloning is tested elsewhere). +///////////////////////////////////////////////////// + +// Modify a future item, without replication or consistency. +// Our expectation for this case is that the modification should proceed without blocking, the item +// shouldn't be expedited, and we will see the modified item once the iterator reaches it. +TEST_F(BgIterationTest, modFutureItem) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + c = getWriteClient(6, "xxx"); + + // We DONT expect the client to be blocked - not consistent + simulateUnblockedWriteWithModification(c); + + // Now continue reading, 1, 2, 3, 4, 5 + expectReadKeySequence(it, 1, 5); + + // Let's validate that key 6 shows the new value + expectReadKey(it, 6, "xxx"); + + // Continue... + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a future item, without replication but with consistency. (Like a SAVE operation) +// Our expectation for this case is that the modification SHOULD be blocked, as we have to save the +// the item in it's state before the modification. To reduce blocking time, the item should be +// moved to the head of the queue - there's no replication in this case, so out-of-order processing +// isn't a concern. +TEST_F(BgIterationTest, modFutureItem_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + c = getWriteClient(6, "xxx"); + // Since this is consistent, we will block the client, disallowing the write. + simulateBlockedWrite(c); + + // On a consistent iterator, the event is expedited in-front of items already in queue! + // Read key 6 out of order. + expectReadKey(it, 6); + + // Now, when we read key 1, key 6 is released back to Valkey, and the client will be unblocked. + expectReadKeyWithUnblock(it, 1, 6); + simulateUnblockedWriteWithModification(c); // Now the write can proceed + + // Continue... + expectReadKeySequence(it, 2, 5); + // 6 has already been processed + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a future item, with replication but without consistency. (Like a Threadsave Full Sync operation) +// Our expectation for this case is that the modification should proceed without blocking, as the +// mode is inconsistent. We don't expect replication, as we haven't reached the item yet. We'll +// see the modified item later. +TEST_F(BgIterationTest, modFutureItem_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + c = getWriteClient(6, "xxx"); + + // We DONT expect the client to be blocked - not consistent + simulateUnblockedWriteWithModification(c); + + // NOTE: Since we haven't reached this item yet, and consistency is not required, there's no + // need to replicate this command. So everything should wrap up just fine - we will see + // the new value when we get to it. + + // Now continue reading, 1, 2, 3, 4, 5 + expectReadKeySequence(it, 1, 5); + + // Let's validate that key 6 shows the new value + expectReadKey(it, 6, "xxx"); + + // Continue... + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// MODIFY A CURRENT ITEM +// The next tests validate the basic pattern when a key, currently in use, is modified. +// Each variation of iteration flags is tested. +// Note that these tests execute without cloning (cloning is tested elsewhere). +///////////////////////////////////////////////////// + +// Modify a current item, without replication or consistency. +// Our expectation for this case is that the modification SHOULD be blocked, the item shouldn't +// be expedited (it's already in use). +TEST_F(BgIterationTest, modCurrentItem) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + c = getWriteClient(2, "xxx"); + + // Must be blocked since key is queued + simulateBlockedWrite(c); + + // Now continue reading + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKeyWithUnblock(it, 3, 2); + simulateUnblockedWriteWithModification(c); // the actual write won't affect anything (past key, no replication) + + // Continue... + expectReadKeySequence(it, 4, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a current item, without replication but with consistency. (Like a SAVE operation) +// Our expectation for this case is that the modification SHOULD be blocked, the item shouldn't +// be expedited (it's already in use). +TEST_F(BgIterationTest, modCurrentItem_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + c = getWriteClient(2, "xxx"); + + // Must be blocked since key is queued + simulateBlockedWrite(c); + + // Now continue reading + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKeyWithUnblock(it, 3, 2); + simulateUnblockedWriteWithModification(c); // the actual write won't affect anything (past key, no replication) + + // Continue... + expectReadKeySequence(it, 4, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a current item, with replication but without consistency. (Like a Threadsave Full Sync operation) +// Our expectation for this case is that the modification SHOULD be blocked. After the key is processed, +// the write will proceed, and the replication will be sent. +TEST_F(BgIterationTest, modCurrentItem_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + c = getWriteClient(2, "xxx"); + + // Must be blocked since key is queued + simulateBlockedWrite(c); + + // Now continue reading + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKeyWithUnblock(it, 3, 2); + simulateUnblockedWriteWithModification(c); // the actual write will cause replication + + expectReadKey(it, 4); // 4 got put in queue when 3 was read + + expectReadReplication(it, c); + + // Continue... + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// MODIFY A PAST ITEM +// The next tests validate the basic pattern when a key, not yet iterated on, is modified. +// Each variation of iteration flags is tested. +// Note that these tests execute without cloning (cloning is tested elsewhere). +///////////////////////////////////////////////////// + +// Modify a past item, without replication or consistency. +// Our expectation for this case is that the modification should proceed without blocking. +// No replication is generated and keys are processed similar to no modification. +TEST_F(BgIterationTest, modPastItem) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // This read returns key 0 (making it a past item) + expectReadKey(it, 1); + + // At this point, key 0 is returned. + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + // Continue... + expectReadKeySequence(it, 2, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a past item, without replication but with consistency. (Like a SAVE operation) +// Our expectation for this case is that the modification should proceed without blocking. +// No replication is generated and keys are processed similar to no modification. +TEST_F(BgIterationTest, modPastItem_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // This read returns key 0 (making it a past item) + expectReadKey(it, 1); + + // At this point, key 0 is returned. + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + // Continue... + expectReadKeySequence(it, 2, LAST_ITEM); + expectReadComplete(it); +} + + +// Modify a past item, with replication but without consistency. (Like a Threadsave Full Sync operation) +// Our expectation for this case is that the modification should proceed without blocking. +// Replication will be sent. +TEST_F(BgIterationTest, modPastItem_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // This read returns key 0 (making it a past item) + expectReadKey(it, 1); + + // At this point, key 0 is returned. + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + // Key 2 was already in queue (same bucket as key 1). The replication will follow. + expectReadKey(it, 2); + expectReadReplication(it, c); + + // Continue... + expectReadKeySequence(it, 3, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// TESTS FOR ITEM CLONING +///////////////////////////////////////////////////// + +// In a consistent iteration, verify that a simple string is properly cloned, and that a write can +// occur without blocking. Validate the cloned item and metadata. +TEST_F(BgIterationTest, modFutureItem_start_CloneExpeditedItem) { + // Initialize cloning configurations. + bgIteration_unitTestEnableCloning(50, 100); + + bgIteratorStatus status; + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + c = getWriteClient(6, "xxx"); + + // Quick status check. At this point, no clones exist yet. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 0u); + + // Since item 6 should be cloned, it will not block the client, allowing the write. + void *de6_md = cloneMetadata(getItem(6)); + // This doesn't block, queues a cloned item, and modifies the item (touching metadata) + simulateClonedWriteWithModification(it, c); + + // At this point, one clone is in the queue. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 1u); + + // On a consistent iterator, the event is expedited in-front of items already in queue! + // Read key 6 (which is cloned) out of order. The value will still match the key. + expectReadClonedKey(it, 6, de6_md); // Also validates and frees the metadata + + // Quick status check. At this point, cloned items have not been marked as processed yet. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_processed, 0u); + + // Reading key 1 will release key 6, and the clone will finish processing. + expectReadKey(it, 1); + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_processed, 1u); + + // Now, when we read key 2 should not have an impact on number of processed clones. + expectReadKey(it, 2); + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_processed, 1u); + + // Continue... + expectReadKeySequence(it, 3, 5); + // 6 has already been processed + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +// Check that cloning for simple strings is respecting the size limits and pool size. On a +// consistent iteration, we expect to block or clone on all future keys. We validate that we can +// clone if the item is small enough and the cloning pool has more space left. +TEST_F(BgIterationTest, modFutureItem_start_LargeItemOrClonePoolFull) { + // Initialize cloning configurations to test the clone pool functionality first. + bgIteration_unitTestEnableCloning(50, 50); + + bgIteratorStatus status; + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // Fake a modification to a later key so that we can see if it gets processed out of order. + client *c6 = getWriteClient(6, "xxx"); + client *c7 = getWriteClient(7, "xxx"); + client *c8 = getWriteClient(8, "xxx"); + + // Quick status check. At this point, no clones exist yet. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 0u); + + // Since item 6 should be cloned, it will not block the client, allowing the write. + void *de6_md = cloneMetadata(getItem(6)); + // This doesn't block, queues a cloned item, and modifies the item (touching metadata) + simulateClonedWriteWithModification(it, c6); + + // At this point, one clone is in the queue. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 1u); + + // Now that cloning pool is full, item 7 will not be cloned and the client will be blocked. + simulateBlockedWrite(c7); + ASSERT_TRUE(bgIteration_isEntryInuse(getItem(7))); + + // There is still only one cloned item in the queue. + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_queued, 1u); + + // Now change cloning configurations to test that large items will not be cloned. We adjust + // the clone pool size to allow two items, but set the maximum item size to be smaller than + // the size of item 8. The clone pool size must be larger than the total size of the existing + // clones plus the maximum item clone size. + bgIteration_unitTestEnableCloning(1, 101); + + // This write will pass the clone pool check but fail the item size check, blocking the client. + simulateBlockedWrite(c8); + ASSERT_TRUE(bgIteration_isEntryInuse(getItem(8))); + + // On a consistent iterator, the expedited item in-front of items already in queue! + // Read key 6 out of order. + expectReadClonedKey(it, 6, de6_md); + + // Now, when we expect to read key 7, which was expedited, key 6 will be released back to Valkey + // and the clone will be deallocated here. + expectReadKey(it, 7); + + // Now, when we read key 8, which was expedited, key 7 is released back to Valkey, and the client + // will be unblocked. + // (actually, unblock is called after every key [just in case] - but functionally we only care + // about this one) + expectReadKeyWithUnblock(it, 8, 7); + simulateUnblockedWriteWithModification(c7); + + // Now, when we read key 1, key 8 is released back to Valkey, and the client will be unblocked. + expectReadKeyWithUnblock(it, 1, 8); + simulateUnblockedWriteWithModification(c8); + + // Since only one item was cloned, there should be one clone processed + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.dbentry_clones_processed, 1u); + + // Continue... + expectReadKeySequence(it, 2, 5); + // 6, 7, and 8 have already been processed + expectReadKeySequence(it, 9, LAST_ITEM); + expectReadComplete(it); + freeTestClient(c6); + freeTestClient(c7); + freeTestClient(c8); +} + + +///////////////////////////////////////////////////// +// TESTS RELATED TO MODIFICATION OF TWO ITEMS +// When 2 keys are modified, we need to ensure that both keys have been sent before we can send +// replication. This means that if replication is present, we may have to block/expedite for +// future keys, even in the inconsistent scenario. +///////////////////////////////////////////////////// + +// Replication enabled, but NOT consistent. In this case, if ANY of the keys have been iterated, +// ALL of the keys must be replicated so that the command can be processed properly on the replica. +TEST_F(BgIterationTest, modPastFutureItem_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // In this test, we need a past and future key IN THE SAME DB (they're used in the same command). + // DB1 has lots of buckets. After reading item 9, + // 8 will be past, 10 will be in queue, 11-15 will be future. + expectReadKeySequence(it, 0, 9); + + // We're going to write to key 8 (past) and read from key 12 (future) + // Even though key 12 is for READ in this command, it must be expedited so that it exists before + // the associated replication is sent. + c = getSetGetClient(8, "xxx", 12); + simulateBlockedWrite(c); + + // Key 12 will be expedited, to the front, because there are no barrier items in the queue. + + expectReadKey(it, 12); // expedited + + expectReadKeyWithUnblock(it, 10, 12); // reading key 10 (was in queue already) unblocks 12 + + simulateUnblockedWriteWithModification(c); + + // Continue... + expectReadKey(it, 11); + expectReadReplication(it, c); + + expectReadKeySequence(it, 13, LAST_ITEM); + expectReadComplete(it); +} + +// Replication enabled, but NOT consistent. In this case, if ANY of the keys have been iterated, +// ALL of the keys must be replicated so that the command can be processed properly on the replica. +// With a past and future item, the future item will be expedited. But in this case, we will ensure +// that there's a barrier item (flushdb) in the queue preventing expedite to front of line. +TEST_F(BgIterationTest, modPastFutureItemBarrier_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + // In this test, we need a past and future key IN THE SAME DB (they're used in the same command). + // DB1 has lots of buckets. After reading item 9, + // 8 will be past, 10 will be in queue, 11-15 will be future. + expectReadKeySequence(it, 0, 9); + + // Insert a FLUSHDB (barrier item) into the queue. + simulateFlushDB(0); + + // We're going to write to key 8 (past) and read from key 12 (future) + // Even though key 12 is for READ in this command, it must be expedited so that it exists before + // the associated replication is sent. + c = getSetGetClient(8, "xxx", 12); + simulateBlockedWrite(c); + + // Key 12 will be expedited, BUT NOT TO THE FRONT - because the FLUSHDB item is a barrier item + + expectReadKey(it, 10); // was already in queue + expectReadFlushDB(it, 0, true); // and now the flush (with replication) + expectReadKey(it, 12); // and then the expedited key + + expectReadKeyWithUnblock(it, 11, 12); // reading key 11 unblocks 12 + + simulateUnblockedWriteWithModification(c); + + // Continue... + expectReadKey(it, 13); + expectReadReplication(it, c); + + expectReadKeySequence(it, 14, LAST_ITEM); + expectReadComplete(it); +} + +// Replication NOT enabled. A read-only key doesn't need to be expedited, even if other keys have +// been processed already. (This should work identically for both consistent/non-consistent. +TEST_F(BgIterationTest, modPastFutureItem_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter1", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // In this test, we need a past and future key IN THE SAME DB (they're used in the same command). + // DB1 has lots of buckets. After reading item 9, + // 8 will be past, 10 will be in queue, 11-15 will be future. + expectReadKeySequence(it, 0, 9); + + // We're going to write to key 8 (past) and read from key 12 (future) + // Since there's no replication, we don't have to worry about expediting 12. The write will + // proceed without blocking. + c = getSetGetClient(8, "xxx", 12); + simulateUnblockedWriteWithModification(c); + + // Key 12 will not be expedited. Remaining keys should be received in normal order. + expectReadKeySequence(it, 10, LAST_ITEM); + expectReadComplete(it); +} + + +TEST_F(BgIterationTest, modPastFutureItem) { + bgIterator *it = bgIteratorCreateFullScanIter("iter2", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + + // In this test, we need a past and future key IN THE SAME DB (they're used in the same command). + // DB1 has lots of buckets. After reading item 9, + // 8 will be past, 10 will be in queue, 11-15 will be future. + expectReadKeySequence(it, 0, 9); + + // We're going to write to key 8 (past) and read from key 12 (future) + // Since there's no replication, we don't have to worry about expediting 12. The write will + // proceed without blocking. + c = getSetGetClient(8, "xxx", 12); + simulateUnblockedWriteWithModification(c); + + // Key 9 will not be expedited. Remaining keys should be received in normal order. + expectReadKeySequence(it, 10, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// TESTS RELATED TO MISSING ITEMS +// Missing items are tricky. A missing item might be logically located in the past or future, in +// relation to the current iteration position. The command may (or may not) create the "missing" +// key. Some general considerations: +// * In a consistent iteration, a missing key didn't exist at the time of consistency, or it was +// already processed (saved) at the time of the deletion. If the missing key gets created, we +// must be sure to skip it if we later iterate over it. +// * In a non-consistent iteration with replication: +// * If the key location is already passed, the replication is sent, allowing the key to be +// created (or not) based on the replication. +// * If the key location is in the future, we can allow the command to proceed, without +// replication. If the key is created, we will process it when the iterator gets to it. +// +// We expect: +// no-repl, no-consist: past items are ignored - future items are processed when iterated +// no-repl, yes-consist: past items are ignored - future items are ignored +// yes-repl, no-consist: past item skipped, but replicated - future items are created by replication and skipped later +// yes-repl, yes-consist: past item skipped, but replicated - future items are processed when iterated +///////////////////////////////////////////////////// + +// no-repl, no-consist: creation of PAST item has no impact +TEST_F(BgIterationTest, missingPastItem) { + simpleDelItem(0); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 1); + expectReadKey(it, 2); + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 3, LAST_ITEM); + expectReadComplete(it); +} + + +// no-repl, yes-consist: creation of PAST item has no impact +TEST_F(BgIterationTest, missingPastItem_start) { + simpleDelItem(0); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 1); + expectReadKey(it, 2); + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 3, LAST_ITEM); + expectReadComplete(it); +} + + +// yes-repl, no-consist: creation of a PAST item will be replicated +TEST_F(BgIterationTest, missingPastItem_eventual) { + simpleDelItem(0); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKey(it, 3); + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // replication will be added after item 4 (3,4 in same bucket) + + expectReadKey(it, 4); + + expectReadReplication(it, c); + + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +// no-repl, no-consist: creation of FUTURE item is seen when reached by the iteration. +TEST_F(BgIterationTest, missingFutureItem) { + // Using DB1 so we have lots of buckets + simpleDelItem(14); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + const char *newValue = "xxx"; + c = getWriteClient(14, newValue); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 1, 13); + + // We expect to see item 14. + // Note that for an inconsistent DB view, it is logically undefined if this value is seen (or not). + // But as implemented, we should see it and the test is helpful to understand if/when the + // functionality changes. + expectReadKey(it, 14, newValue); + + expectReadKey(it, LAST_ITEM); + expectReadComplete(it); +} + + +// no-repl, yes-consist: creation of FUTURE item is ignored by consistent iteration. +TEST_F(BgIterationTest, missingFutureItem_start) { + // Using DB1 so we have lots of buckets + simpleDelItem(14); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + c = getWriteClient(14, "xxx"); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 1, 13); + // Key 14 is missing - it didn't exist at start of consistent iteration + expectReadKey(it, LAST_ITEM); + expectReadComplete(it); +} + + +// yes-repl, no-consist: creation of FUTURE item is handled by the replication, and then the key is +// later skipped (treated like an early iteration case). +TEST_F(BgIterationTest, missingFutureItem_eventual) { + // Using DB1 so we have lots of buckets + simpleDelItem(14); // Delete the item before iterator creation + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKey(it, 0); // Items 1 & 2 are in queue (same bucket) + + c = getWriteClient(14, "xxx"); + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 1, 2); + + expectReadReplication(it, c); // Here's the replication creating item 14 + + expectReadKeySequence(it, 3, 13); + // We expect item 14 to be skipped, because it was created by the earlier replication + expectReadKey(it, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// TESTS RELATED TO EXPIRATION +// Expiration can be tricky. When pre-evaluating a command with bgIteration_blockClientIfRequired, +// a key might exist, but be ready for expiration. Then, as the command executes, the key expires +// and gets deleted before the write operation. Consider SET K V. +// In the unexpired case, this appears to bgIteration as a single SET command (which replaces the value). +// In the expired case, bgIteration will receive a DEL followed by a SET. +// +// Another case is a READ command. A read command won't cause the client to be blocked. However, +// if the key is expired, this will cause a DEL. For consistent processing, this key might need to +// be expedited so that it can be processed before it gets deleted. In this case, the key is +// unlinked from the main Valkey dictionary, but the actual deletion is deferred. +///////////////////////////////////////////////////// + +TEST_F(BgIterationTest, expireKeys) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); + + // At this point, key 1 is active, key 2 is in queue. + + simulateExpiration(0); // Past - we no longer care + simulateExpirationOfInuse(2); // Current - it's inuse + simulateExpiration(5); // Future - we don't care (non-consistent) + + expectReadKeySequence(it, 2, 4); + // key 5 has been deleted + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +TEST_F(BgIterationTest, expireKeys_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); + + // At this point, key 1 is active, key 2 is in queue. + + simulateExpiration(0); // Past - we expect replication + simulateExpirationOfInuse(2); // Current - it's inuse, but we expect replication + simulateExpiration(5); // Future - we don't care (non-consistent) + + expectReadKey(it, 2); // this was already queued + + expectReadReplicationDel(it, 0); // Past item should replicate + expectReadReplicationDel(it, 2); // Current item should replicate + // Item 5 is a future item and doesn't need to replicate + + expectReadKeySequence(it, 3, 4); + // Item 5 has been deleted + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +TEST_F(BgIterationTest, expireKeys_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); + + // At this point, key 1 is active, key 2 is in queue. + + simulateExpiration(0); // Past - we no longer care + simulateExpirationOfInuse(2); // Current - we must defer + simulateExpirationWithExpedite(5); // Future - will become inuse and expedited for consistency + + expectReadKey(it, 5); // Expedited to front + + expectReadKeySequence(it, 2, 4); + // Item 5 has been deleted + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +// Special case during a non-consistent iteration with replication and expiration. +// 1. A future key is created (and processed by its replication) - considered early iterated +// 2. Later the key is expired and deleted during command processing (causes DEL to be sent) - no longer early iterated +// 3. The key is recreated as part of the command processing (and this command was replicated) - again early iterated +// 4. Finally, when we iterate to the key, it shouldn't be sent, because it was replicated in step 3. +TEST_F(BgIterationTest, expireKeys_eventual_FutureKeyCreatedThenExpiredDuringSet) { + simpleDelItem(8); // Start with a missing future item + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKey(it, 0); // Get the iterator started + + c = getWriteClient(8, "xxx"); + simulateUnblockedWriteWithModification(c); // Not blocked because this is a future key (but we expect repl) + + // Now do it again, but break out the steps so that we can simulate an expiration + simulateUnblockedWrite_inCall(c); // Shouldn't be blocked because this is a future key + + // Now, as the SET command tries to execute, simulate that the key is expired. + // First, the key should be physically removed and bgIteration_keyDelete called + bgIteration_keyDelete(getDbFromItemNum(8), static_cast(objectGetVal(c->argv[1]))); + simpleDelItem(8); // Simulate the actual del (after bgIteration_keyDelete called) + // Then the replication for the delete occurs + robj *argv[2]; + argv[0] = createStringObjectFromCString("DEL"); + argv[1] = c->argv[1]; + serverCommand *cmd = lookupCommandByCString("DEL"); + bgIteration_handleCommandReplication(getDbFromItemNum(8), cmd, 2, argv); + decrRefCount(argv[0]); + + // Now the SET will run, re-creating the item (which is still a future item) + // We need to duplicate the value because setKey() can reallocate it. + robj *value = dupStringObject(c->argv[2]); + setKey(c, c->db, c->argv[1], &(value), SETKEY_ADD_OR_UPDATE); + + // Finally, replication will be sent because this is creating a new key + bgIteration_handleCommandReplication(getDbFromItemNum(8), c->cmd, c->argc, c->argv); + server.in_call--; + + // Test that everything comes as expected + expectReadKeySequence(it, 1, 2); // All one bucket - queued after key 0 read + + expectReadReplication(it, c); // Repl from the first SET command + expectReadReplicationDel(it, 8); // This is the expected replication of the DEL from expire + expectReadReplication(it, c); // Repl from the second SET command (recreating deleted key) + + expectReadKeySequence(it, 3, 7); // continue with normal iteration + // KEY 8 SHOULD BE OMITTED - This was already replicated + expectReadKeySequence(it, 9, LAST_ITEM); + + expectReadComplete(it); +} + + +// In this test, a future key is expedited. Then it is expired by normal expiration processing. +// We expect to see replication of the delete, since it was early iterated. +TEST_F(BgIterationTest, expireKeys_eventual_ExpeditedKeyExpired) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); // Also queues 1 & 2 + + // This will be blocked, and key 7 expedited + c = getWrite2KeysClient("sunionstore", 0, 7); + simulateBlockedWrite(c, 2); // blocked on both 0 and 7 + + expectReadKeyWithUnblock(it, 7, 0); // 7 expedited to front (unblocks 0) + expectReadKeyWithUnblock(it, 1, 7); // 1 was already in queue (unblocks 7) + + simulateUnblockedWriteWithModification(c); + + // At this point, + // * item 2 is still in the queue + // * replication for the sunionstore is queued + // * item 7 is in an early iterated state + + // Now expire key 7. We expect we will see replication (since 7 has been expedited) + simulateExpiration(7); + + // Check queue... + expectReadKey(it, 2); + expectReadReplication(it, c); + expectReadReplicationDel(it, 7); + + // and the rest + expectReadKeySequence(it, 3, 6); + // 7 is missing (expired) + expectReadKeySequence(it, 8, LAST_ITEM); + expectReadComplete(it); +} + + +///////////////////////////////////////////////////// +// THE REMAINING TESTS ARE GENERAL / UNCATEGORIZED +///////////////////////////////////////////////////// + +// Iteration can be terminated from the main thread or from the child client. +// This tests termination driven from the main thread. +TEST_F(BgIterationTest, earlyTerminationFromMain) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + // At this point, keys 1 & 2 are in queue. A termination should release those keys. + bool blocked1 = true; + bool blocked2 = true; + // We expect no general unblocks, we account for each specific unblock below. + EXPECT_CALL(mock, unblockClientsInUseOnKey(_)).Times(0); + // We should expect to see unblock called for items 1 & 2, as they are released from the queue. + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(1)))) + .WillOnce(Assign(&blocked1, false)); + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(2)))) + .WillOnce(Assign(&blocked2, false)); + bgIteratorTerminate(it); // queues the items for release + EXPECT_TRUE(bgIteratorIsTerminating(it)); + bgIteration_feedIterators(); // actually performs the release + EXPECT_FALSE(blocked1); + EXPECT_FALSE(blocked2); + + bool blocked0 = true; + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(0)))) + .WillOnce(Assign(&blocked0, false)); + bgIteratorItem *item = bgIteratorRead(it); + EXPECT_FALSE(blocked0); + EXPECT_EQ(item->type, BGITERATOR_ITEM_TERMINATED); + + bgIteratorClose(it); // background thread completes the termination + + EXPECT_EQ(cleanupCount, 0); + bgIteration_feedIterators(); // main thread, cleans up iterator and calls cleanup function + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + + +// Iteration can be terminated from the main thread or from the child client. +// This tests termination driven from the child client (the background thread). +TEST_F(BgIterationTest, earlyTerminationFromChild) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + // At this point, keys 1 & 2 are in queue. A termination should release those keys. + bgIteratorClose(it); // background thread initiates the termination + EXPECT_TRUE(bgIteratorIsTerminating(it)); + + bool blocked0 = true; + bool blocked1 = true; + bool blocked2 = true; + // Expecting no extra unblocks + EXPECT_CALL(mock, unblockClientsInUseOnKey(_)).Times(0); + // We expect item 0 (the in progress item) to be released + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(0)))) + .WillOnce(Assign(&blocked0, false)); + // We expect items 1-4 (the queued items) to be released + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(1)))) + .WillOnce(Assign(&blocked1, false)); + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(2)))) + .WillOnce(Assign(&blocked2, false)); + bgIteration_feedIterators(); + EXPECT_FALSE(blocked0); + EXPECT_FALSE(blocked1); + EXPECT_FALSE(blocked2); + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + + +// Edge case. Executing a command (like SUNIONSTORE) which REPLACES the first key and reads the +// second key. In this case, bgIteration will get notified of the key deletion during execution of +// SETUNIONSTORE. Given that both keys are in the future (not iterated yet), we'll allow the +// command to execute, unblocked. We won't replicate as we'll pick up the key when we get to it. +TEST_F(BgIterationTest, writeWith2Keys_eventual_keyDeletedDuringSetReplace) { + // Using DB1 so we have lots of buckets + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, 8); // 9 is in queue + + // Write command that has 2 keys. 1 existing key that we write to and 1 dependant future key. + c = getWrite2KeysClient("sunionstore", 12, 13); + + simulateUnblockedWrite_inCall(c); + + // Now the call to keyDelete happens + sds sdskey = sdsnew(keyStr(12)); + bgIteration_keyDelete(getDbFromItemNum(12), sdskey); + sdsfree(sdskey); + simpleDelItem(12); // So simulate the actual del + + // Now the write will run, re-creating the item (which is still a future item) + const char *const newValueStr = "new value"; + robj *newValueRobj = createStringObjectFromCString(newValueStr); + setKey(c, c->db, c->argv[1], &newValueRobj, SETKEY_ADD_OR_UPDATE); + + // Finally, we are letting bgIteration know that the write command was executed + bgIteration_handleCommandReplication(getDbFromItemNum(12), c->cmd, c->argc, c->argv); + server.in_call--; + + // Since the write command was not replicated, we expect all the keys to be read in the normal + // order from the dictionary. + expectReadKeySequence(it, 9, 11); + expectReadKey(it, 12, newValueStr); + expectReadKeySequence(it, 13, LAST_ITEM); + + expectReadComplete(it); +} + + +// Edge case. When we have a new key which is created by a command, AND replication is enabled, we +// expect that we will replicate the command rather than serializing the key/value later. As an +// example, consider SUNIONSTORE A B. We want to create A by replicating the command. We don't +// want to have to process A as a key later on. But in this case, we can't run the command until +// B has been sent. We expect the command to be blocked while we send B. +TEST_F(BgIterationTest, writeWith2Keys_eventual_setNewKey_DependantFuture) { + // Using DB1 so we have lots of buckets + simpleDelItem(12); // Deleting key 12 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, 8); // 9 is in queue + + // Write command that has 2 keys. 1 new key and 1 dependant future key. + c = getWrite2KeysClient("sunionstore", 12, 13); + + // We are simulating a new key in the dict. This command should block on the dependant key. + // This adds key 13 in the queue since the command depends on it. + simulateBlockedWrite(c); + + // Key 13 is processed out of order since the write depends on it. It was expedited to the + // front because there are no barrier events in the queue. + expectReadKey(it, 13); + + // Key 9 was already in the queue. Reading key 9 will unblock key 13, allowing us to write. + expectReadKey(it, 9); + + // Now that key 13 was processed and released by the iterator, the write command can be executed. + simulateUnblockedWriteWithModification(c); + + // Key 10 was queued when we read key 9 + expectReadKey(it, 10); + + // The replication of the write command was enqueued after key 11 + expectReadReplication(it, c); + + expectReadKey(it, 11); + + // We shouldn't see key 12 - as that was processed via replication. + // We shouldn't see key 13 - as that was expedited earlier + + // Now resuming processing of dict entries + expectReadKeySequence(it, 14, LAST_ITEM); + + expectReadComplete(it); +} + + +// A new key is being created, but is dependent on another key which has already been processed. +// In this case, the command shouldn't be blocked. +TEST_F(BgIterationTest, writeWith2Keys_eventual_setNewKey_DependantPast) { + // Using DB1 so we have lots of buckets + simpleDelItem(12); // Deleting key 12 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKeySequence(it, 0, 9); // 10 is in queue, done with 8 + + // Write command that has 2 keys. 1 new key and 1 dependant past key. + c = getWrite2KeysClient("sunionstore", 12, 8); + + // We are simulating a new key in the dict. + // This command should not block since the dependant key has already been processed. + simulateUnblockedWriteWithModification(c); + + // Key 10 was put in the queue before the write + expectReadKey(it, 10); + + expectReadReplication(it, c); + + expectReadKey(it, 11); + + // Key 12 should be missing - it was processed by replication + + expectReadKeySequence(it, 13, LAST_ITEM); + expectReadComplete(it); +} + + +// A new key is being created, and has dependencies on 2 other keys - one already processed, one not. +// In this case, the command should be blocked so that the future key can be sent first. +TEST_F(BgIterationTest, writeWith3Keys_eventual_setNewKey_1DependantPast1DependantFuture) { + // Using DB1 so we have lots of buckets + simpleDelItem(12); // Deleting key 12 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKeySequence(it, 0, 9); // 8 has been returned, 9 is active, 10 is in queue + + // Write command that has 1 new key and 2 dependencies (past/future) + c = getWrite3KeysClient("sunionstore", 12, 8, 13); + + // The write should be blocked, so that item 13 can be processed. + simulateBlockedWrite(c); + + expectReadKey(it, 13); // 13 was expedited to the front (no barrier events in queue) + + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(13)))).Times(1); + expectReadKey(it, 10); // 10 was already in queue (releases 13) + + simulateUnblockedWriteWithModification(c); + + expectReadKey(it, 11); + + expectReadReplication(it, c); + + expectReadKeySequence(it, 14, LAST_ITEM); + expectReadComplete(it); +} + + +// Test an edge case with the same (future) key being repeated in the command, like: +// SUNIONSTORE A B B +// In this test, A is a previously handled key, and B is a future key. We expect the future key B to +// be expedited (once). +TEST_F(BgIterationTest, writeWith3Keys_eventual_repeatedKey_1DependantPast1RepeatedFuture) { + // Using DB1 so we have lots of buckets + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKeySequence(it, 0, 9); // We're done with 8, and 10 is in queue + + // Write command that has 3 keys. 1 past key and 1 repeated key in the future. + c = getWrite3KeysClient("sunionstore", 8, 12, 12); + + // This command should block because 12 needs to be expedited. + simulateBlockedWrite(c); + + expectReadKey(it, 12); // expedited to the front (no barrier events) + + expectReadKey(it, 10); // was already in queue, releases 12 (unblocking the command) + + // Now that key 12 was processed and released by the iterator, the write command can be executed. + simulateUnblockedWriteWithModification(c); + + expectReadKey(it, 11); // was already in queue since reading key 10 + + expectReadReplication(it, c); + + // Now resuming processing of dict entries. + expectReadKeySequence(it, 13, LAST_ITEM); + expectReadComplete(it); +} + + +/* Tests the replication of a write command that creates a new key and depends on a + * future key which is duplicated in the command. */ +TEST_F(BgIterationTest, writeWith3Keys_eventual_repeatedKey_1newKey1RepeatedFuture) { + simpleDelItem(3); // Deleting key 3 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // At this point, keys 1 & 2 are in queue. + + // Write command that has 3 keys. 1 new key and 1 repeated key in the future. + c = getWrite3KeysClient("sunionstore", 3, 5, 5); + + // This command should block on key 5. + // This adds key 5 in the queue because: + // - the command depends on key 5 which hasn't been processed yet + // - the command creates a new key (key 3). + simulateBlockedWrite(c); + + // Key 5 is expedited to the front because there are no barrier events in queue + expectReadKey(it, 5); + + expectReadKey(it, 1); // was already in queue - releases the expedited key 5 + + // Now that key 5 was processed and released by the iterator, the write command can be executed. + simulateUnblockedWriteWithModification(c); + + expectReadKey(it, 2); // was already in queue + + expectReadReplication(it, c); + + // Now resuming processing of dict entries. + expectReadKey(it, 4); + // Key 5 was handled earlier + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +/* A command modifying an in-progress key, but dependent on a future (repeated) key. */ +TEST_F(BgIterationTest, writeWith3Keys_start_repeatedKey_1DependantPast1RepeatedFuture) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // At this point, keys 1 & 2 are in queue. + + // Write command that has 3 keys. 0 is in progress. 4 is still future. + // How BLPOP works exactly is not relevant to bgIterator, we just chose BLPOP because it's a + // multi-key command that (potentially) modifies all of its keys (ie is not CMD_WRITE_FIRSTKEY_ONLY). + c = getWriteMultiKeysClient("blpop", 0, {4, 4, 0}); + + // This command should block on 2 keys (0 and 4), since: + // - key 0 is in use by the iterator (still in the queue since it has not been processed by the consumer yet) + // - key 4 is in the future + // This adds key 4 in the queue since the command depends on it and it hasn't been processed yet. + simulateBlockedWrite(c, 2); + + // Key 4 is processed out of order since the write depends on it. + // Key 4 is processed before key 1 even though key 1 was already in the queue + // because key 4 was enqueued as a priority item with a no-replication iterator. + // Reading key 4 will release key 0 - releasing that lock on the command + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(0)))).Times(1); + expectReadKey(it, 4); // This unblocks key 0 + + EXPECT_CALL(mock, unblockClientsInUseOnKey(robjEqualsStr(keyStr(4)))).Times(1); + expectReadKey(it, 1); // this was already in queue (releases key 4) + + // Now that keys 4 and 0 were processed and released by the iterator, the write command can be executed. + simulateUnblockedWriteWithModification(c); + + expectReadKeySequence(it, 2, 3); + + // 4 is skipped because it was already expedited + + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +/* Test that creates a new key, repeating the future key in the command. */ +TEST_F(BgIterationTest, writeWith3Keys_repeatedKey_1repeatedNewKey) { + simpleDelItem(6); // Deleting key 6 to then create it with a write command + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + // Getting started + expectReadKeySequence(it, 0, 3); + // Now, 0,1,2 are in the past. 3 is being processed, and 4 is in queue. + + // Write command that has 3 keys. 1 new repeated key and 1 key in the past. + // How BLPOP works exactly is not relevant to bgIterator, we just chose BLPOP because it's a + // multi-key command that (potentially) modifies all of its keys (ie is not CMD_WRITE_FIRSTKEY_ONLY). + c = getWriteMultiKeysClient("blpop", 6, {0, 6, 0}); + + // The write command is not blocked since key 0 & 6 are not in use, and no consistency requirements + simulateUnblockedWriteWithModification(c); + + // Keys 2, 3 are next in the queue (it was put in the queue at the same time as key 1). + expectReadKeySequence(it, 4, 5); + + // There are no consistency requirements - so the new key should just be iterated. + // Key 6 is now in the dict with the value of key 0. + expectReadKey(it, 6, keyStr(0)); + + // Processing the rest of the dict entries. + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +/* In this test, the COPY command is copying from one DB to another. We will create the + * same key in both DBs. We make sure that the proper key is created via replication, and + * the proper key is created by iteration. */ +TEST_F(BgIterationTest, copyHandlesProperDb_eventual) { + // NOTE: Adding H0 to dict 1. Now there is a H0 in both dict 0 and dict 1. + addKeyToDb(1, "H0", "H0"); + + // The test: + // We will simulate (with DB0 selected): COPY B1 H0 DB 1 REPLACE + // This will overwrite DB1:H0 that was created above. + // Since DB0:B1 is already in queue, we need to expedite the target (DB1:H0) as well + // After DB1:H0 is "overwritten", it should be marked early iterate. + // We expect DB0:H0 to NOT be marked early iterate, and should get processed normally. + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); // B0 + // At this point, keys 1(B1) & 2(B2) are in queue. + + // COPY B1 H0 DB 1 REPLACE + c = static_cast(zcalloc(sizeof(client))); + c->cmd = lookupCommandByCString("copy"); + c->db = server.db[0]; + c->argc = 6; + c->argv = static_cast(zcalloc(sizeof(robj *) * c->argc)); + c->argv[0] = createStringObjectFromCString(c->cmd->fullname); + c->argv[1] = createStringObjectFromCString("B1"); + c->argv[2] = createStringObjectFromCString("H0"); + c->argv[3] = createStringObjectFromCString("DB"); + c->argv[4] = createStringObjectFromCString("1"); + c->argv[5] = createStringObjectFromCString("REPLACE"); + + // This should block on 2 keys. DB0:B1 is in queue. DB1:H0 needs to be expedited. + simulateBlockedWrite(c, 2); + + // With no barrier events in queue, DB1:H0 gets moved to the front + // Queue is now 0:B0 (in progress), 1:H0 (expedited to front), 0:B1 (was in queue), 0:B2 (was in queue) + expectReadDbKeyValue(it, 1, "H0", "H0"); + + expectReadKey(it, 1); // DB0:B1 (was already in queue) + expectReadKey(it, 2); // DB0:B2 (was already in queue) - releases B1, unblocking the command (queues key 3 & 4) + + simulateUnblockedWrite_inCall(c); // We shouldn't be blocked this time + + // Now, we'll simulate the actual activity of the COPY. DB1:H0 will be deleted in order to + // be overwritten. + sds sdskey = sdsnew("H0"); + bgIteration_keyDelete(1, sdskey); // bgIteration would be signaled about the deletion + sdsfree(sdskey); + // At this point the key would actually be deleted and recreated by COPY (no need to actually do this) + + // And finally the replication (this should queue replication) + bgIteration_handleCommandReplication(c->db->id, c->cmd, c->argc, c->argv); + server.in_call--; + + expectReadKey(it, 3); + expectReadKey(it, 4); // Queued along with key 3 + + expectReadReplication(it, c); // This is the new replication (creating DB1:H0) + + // The rest should be normal. We shouldn't see DB1:E0 as it was recreated by replication + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +// Check that termination with replication in queue works OK. +TEST_F(BgIterationTest, terminateWithReplication) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); // makes sure we are done with key 0 (don't want to block) + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // Should replicate + + bgIteratorTerminate(it); + + bgIteratorItem *item = bgIteratorRead(it); + ASSERT_EQ(item->type, BGITERATOR_ITEM_TERMINATED); + + bgIteratorClose(it); // background thread completes the termination + + bgIteration_feedIterators(); // main thread, cleans up iterator and calls cleanup function + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + + +// SWAPDB tests - Get ready for the mind-bend... + +/* In the non-consistent iterator (without replication), items are identified with the DBID at + * the time they are placed into the queue. The SWAPDB event signals the change to the + * iterating process - and this is properly sequenced with the DB info for each item. */ +TEST_F(BgIterationTest, swapDB) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + bgIteratorStatus status; + + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + simulateSwapDB(0, 1); // The swap event will be queued after item 2 + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 1u); + EXPECT_EQ(status.swapdb_processed, 0u); + + expectReadKey(it, 1); // These were already in queue, + expectReadKey(it, 2); // ... and the iteration client hasn't seen the swap yet + + expectReadSwapDB(it, 0, 1); + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 1u); + EXPECT_EQ(status.swapdb_processed, 0u); // still processing it... + + // Since we've seen the swap event, items now have the new DBID + + expectReadDbKeyValue(it, 1, keyStr(3), keyStr(3)); // item 3 should show in DB1 + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 1u); + EXPECT_EQ(status.swapdb_processed, 1u); // done processing the swapdb + + // Keys 4 is in the queue - let's swap back! + simulateSwapDB(1, 0); // The swap event will be queued after item 4 + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 2u); // 2nd one queued + EXPECT_EQ(status.swapdb_processed, 1u); + + expectReadDbKeyValue(it, 1, keyStr(4), keyStr(4)); // item 4 should still show in DB1 + + expectReadSwapDB(it, 1, 0); // Now the iterator knows about the 2nd swap + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 2u); + EXPECT_EQ(status.swapdb_processed, 1u); // still processing it... + + // Since we've seen the second swap, items should now show with their original DB + + expectReadKey(it, 5); + bgIteratorGetStatus(it, &status); + EXPECT_EQ(status.swapdb_queued, 2u); + EXPECT_EQ(status.swapdb_processed, 2u); // done processing all swaps + + expectReadKeySequence(it, 6, LAST_ITEM); + expectReadComplete(it); +} + + +/* In the consistent iterator (without replication) all items are presented to the iterating + * process using the DBID at the time of the iterator creation. No changes are evident. + * Swap events are not presented to the iteration client. */ +TEST_F(BgIterationTest, swapDB_start) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + simulateSwapDB(0, 1); // The swap occurs, but the iterator sees no change + + expectReadKey(it, 1); + expectReadKey(it, 2); + expectReadKey(it, 3); + + // Heck, let's go crazy with those swaps... + for (int itemNum = 4; itemNum <= LAST_ITEM; itemNum++) { + simulateSwapDB(0, 1); + expectReadKey(it, itemNum); + } + + expectReadComplete(it); +} + + +/* In the non-consistent iterator WITH replication, items are identified with the DBID at the + * time they are placed into the queue. The SWAPDB event signals the change to the iterating + * process - and this is properly sequenced with the DB info for each item. */ +TEST_F(BgIterationTest, swapDB_eventual) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + simulateSwapDB(0, 1); // The swap event will be queued after item 2 + + expectReadKey(it, 1); // These were already in queue, + expectReadKey(it, 2); // ... and the iteration client hasn't seen the swap yet + + expectReadSwapDB(it, 0, 1); // We should see a SWAPDB event + bgIteratorItem *item = bgIteratorRead(it); // followed by the associated replication + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + bgIteration_feedIterators(); + + // Since we've seen the swap event, items now have the new DBID + expectReadDbKeyValue(it, 1, keyStr(3), keyStr(3)); // item 3 is now in DB1 + + // Key 4 is in the queue - let's swap back! + simulateSwapDB(1, 0); // The swap event will be queued after item 4 + + expectReadDbKeyValue(it, 1, keyStr(4), keyStr(4)); // Still appears as DB1 + + expectReadSwapDB(it, 1, 0); // Now the iterator knows about the 2nd swap + item = bgIteratorRead(it); + ASSERT_EQ(item->type, BGITERATOR_ITEM_REPLICATION); + bgIteration_feedIterators(); + + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + +// There is no test for swapDB_YesReplication_YesConsistent because this configuration is not +// permitted with multiple DBs (not permitted with swaps). + + +// FLUSHDB & FLUSHALL Tests + +TEST_F(BgIterationTest, flushDB_flushAll) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + expectReadKey(it, 1); + + // key 1 is active in the iterator - this key won't be deallocated because of the refcount. + // keys 2 is in queue - but will be returned to Valkey before the flush. It is yanked + // back by Valkey and will not be seen by iterator. + simulateFlushDB(-1, 1); + + bgIteratorItem *item = bgIteratorRead(it); + ASSERT_EQ(item->type, BGITERATOR_ITEM_TERMINATED); + + bgIteratorClose(it); // background thread completes the termination + + bgIteration_feedIterators(); // main thread, cleans up iterator and calls cleanup function + EXPECT_EQ(cleanupCount, 1); + EXPECT_TRUE(cleanupTerminated); +} + +TEST_F(BgIterationTest, flushDB_flushOne) { + bgIterator *it1 = bgIteratorCreateFullScanIter("iter1", BGITERATOR_CONSISTENCY_NONE, NULL, + iteratorCleanupFn, PRIVDATA); + bgIterator *it2 = bgIteratorCreateFullScanIter("iter2", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + bgIteratorStatus status; + + // The test flushes DB0. This is half the data. Since <= half, a non-consistent iterator is + // allowed to proceed. But the consistent iterator will be terminated. + + expectReadKey(it1, 0); + expectReadKey(it2, 0); + expectReadKey(it1, 1); + expectReadKey(it2, 1); + + // key 1 is active in the iterator - this key won't be deallocated because of the refcount. + // keys 2 is in queue - but will be returned to Valkey before the flush. These are yanked + // back by Valkey and will not be seen by iterator. + simulateFlushDB(0, 1); + bgIteratorGetStatus(it1, &status); + EXPECT_EQ(status.flushdb_queued, 1u); + EXPECT_EQ(status.flushdb_processed, 0u); + + // Testing the non-consistent one continues... + // Everything already on the iterator queue should be preserved (deleted from the DB). + // Keys 2 is already queued (and preserved). + expectReadKey(it1, 2); + + // Read the flushdb item on iterator 1. + bgIteratorItem *item = bgIteratorRead(it1); + ASSERT_EQ(item->type, BGITERATOR_ITEM_FLUSHDB); + ASSERT_EQ(item->dbid, 0); + bgIteratorGetStatus(it1, &status); + EXPECT_EQ(status.flushdb_queued, 1u); + EXPECT_EQ(status.flushdb_processed, 0u); // still processing it + + // And iterator 1 keeps processing with the 2nd DB + expectReadKey(it1, ITEMS_PER_DB); + bgIteratorGetStatus(it1, &status); + EXPECT_EQ(status.flushdb_queued, 1u); + EXPECT_EQ(status.flushdb_processed, 1u); // done with all flushdb's + + expectReadKeySequence(it1, ITEMS_PER_DB + 1, LAST_ITEM); + expectReadComplete(it1); + EXPECT_EQ(cleanupCount, 1); + EXPECT_FALSE(cleanupTerminated); + + // But the consistent iterator should be terminated + item = bgIteratorRead(it2); + ASSERT_EQ(item->type, BGITERATOR_ITEM_TERMINATED); + bgIteratorClose(it2); // background thread completes the termination + bgIteration_feedIterators(); // main thread, cleans up iterator and calls cleanup function + EXPECT_EQ(cleanupCount, 2); + EXPECT_TRUE(cleanupTerminated); +} + + +/* A multi with one future and one past key must expedite and replicate. */ +TEST_F(BgIterationTest, multiTwoKeysFirstFuture) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + + expectReadKey(it, 0); // Causes keys 1 & 2 to be queued (same bucket) + expectReadKey(it, 1); // Causes key 0 to be released + + // Now, B0(0) is in the past. H0(5) is in the future. R0(11) [in DB1] is also future. + + /* For a non-consistent iteration, with replication... + * Normally, H0 (future) wouldn't need to expedite - we'd just modify it in place (without + * replication and iterate on it later. But, in this case, since it's wrapped in a multi, with + * B0 (past) - we need to expedite H0 so that the multi can all be handled in the same way. + * Key R0(11) [DB1] just makes thing a little trickier. */ + c = getMultiClient("SET B0 xxx; SET H0 xxx; SELECT 1; SET R0 xxx"); + + // The EXEC should block on 2 keys, because H0(5) & R0(11) should be expedited + simulateBlockedWrite(c, 2); + + // Since there were no barrier events in the queue, these 2 get moved to the front. + // Note - it would be logically OK if these 2 were reversed, but this is how the current algorithm works. + expectReadKey(it, 5); // Key 5 (H0) was expedited + expectReadKey(it, 11); // Key 11 (R0) was expedited + + expectReadKey(it, 2); // (was already in queue) + + // We don't need to actually simulate the multi. Just checking that the keys were expedited. + + // and clean up the rest... + expectReadKeySequence(it, 3, 4); + // Key 5 was already read above (expedited) + expectReadKeySequence(it, 6, 10); + // Key 11 was already read above (expedited) + expectReadKeySequence(it, 12, LAST_ITEM); + expectReadComplete(it); +} + +// Multi blocking on future items. Consistent. +TEST_F(BgIterationTest, multiBlocksOnFutureKey) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + // Since there's no replication, an expedited key will be moved to the front of the queue. + // Let's fake a modification to key 6 (H1) + // Dummy up a MULTI... + c = getMultiClient("SET H1 xxx"); + + // Since this is consistent, we will block the client, disallowing the write. + simulateBlockedWrite(c); + + // H1 (key 6) will be expedited to the front of the queue (because no replication) + expectReadKey(it, 6); + + // Now that we've read key 6, key 0 (B0) is passed and should not block + freeTestClient(c); + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + + // and clean up the rest... + expectReadKeySequence(it, 1, 5); + expectReadKeySequence(it, 7, LAST_ITEM); + expectReadComplete(it); +} + + +// Scenario. We have a multi that doesn't need to be replicated because all of the keys exist +// but are all future keys. Note that missing keys are considered already-iterated, so all +// must exist for this test. Then: +// - we delete a key +// - we re-create the deleted (future) key - normally this would be replicated +// - we access another (future) key - we don't expect to get blocked! +TEST_F(BgIterationTest, multiNotReplicatedButDelRecreateAccess) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + // Keys 1 & 2 are in queue + + c = getMultiClient("DEL H1; SET H1 xxx; SET H2 yyy"); + // Now let's process the multi. Since H1 & H2 are both future (existing) items, we shouldn't + // block or replicate. + simulateUnblockedWrite_inCall(c); // the EXEC + + // Simulate the DEL H1 + server.in_exec = 1; // Simulate actual execution of the MULTI/EXEC + + advanceMultiClientToCommand(c, 0); // DEL H1 + EXPECT_CALL(mock, blockClientInUseOnKeys(c, _, _)).Times(0); + bool blocked = bgIteration_blockClientIfRequired(c); + EXPECT_FALSE(blocked); + + sds delKey = sdsnew(keyStr(6)); + bgIteration_keyDelete(0, delKey); + sdsfree(delKey); + simpleDelItem(6); // H1 + + bgIteration_handleCommandReplication(c->db->id, c->cmd, c->argc, c->argv); // shouldn't replicate + + // Simulate SET H1 - the key doesn't exist, and would normally replicate and mark early iterate, + // but this is in a transaction, and we are not replicating this transaction. + advanceMultiClientToCommand(c, 1); // SET H1 xxx + simulateUnblockedWriteWithModification(c); + + // Now write to another existing future key - this should work if we weren't confused by the DEL + advanceMultiClientToCommand(c, 2); // SET H2 yyy + simulateUnblockedWriteWithModification(c); + server.in_exec = 0; + server.in_call--; + + // Now we can continue iterating, and we should pick up keys 1... (and no replication!) + expectReadKeySequence(it, 1, 5); + expectReadKey(it, 6, "xxx"); + expectReadKey(it, 7, "yyy"); + expectReadKeySequence(it, 8, LAST_ITEM); + expectReadComplete(it); +} + + +// For this test, B0 is added into DB1 - so it exists in both DB 0 and 1. We will process it +// in DB0, but it will be unprocessed in DB1. See if we track SELECT properly. +TEST_F(BgIterationTest, multiHandlesSelectProperly) { + addKeyToDb(1, "B0", "B0"); + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + // Read the 1st key - B0 in DB0. + expectReadKey(it, 0); + // Now, we are done with B0 in DB0, but not in DB1 + expectReadKey(it, 1); // Reads B1, and releases B0 in DB0 + + // These cases should NOT block... (they access B0 in DB0) + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 0; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 1; SELECT 0; SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + + // These cases SHOULD block... (they access B0 in DB1) + c = getMultiClient("SET B0 xxx"); + c->db = server.db[1]; + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SELECT 1; SET B0 xxx"); + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SELECT 1; SET B0 xxx; SELECT 0"); + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SELECT 0; SELECT 1; SET B0 xxx; SELECT 1"); + simulateBlockedWrite(c); + + expectAnythingCleanup(it); +} + +// For this test, B0 is added into DB1 - so it exists in both DB0 and DB1. We will process it +// in DB0, but it will be unprocessed in DB1. See if we track select properly - WHEN WE HAVE NO +// PERMISSION TO EXECUTE SELECT! +TEST_F(BgIterationTest, multiHandlesSelectNoPermissionProperly) { + addKeyToDb(1, "B0", "B0"); + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + // Read the 1st key - B0 in DB0. + expectReadKey(it, 0); + // Now, we are done with B0 in DB0, but not in DB1 + expectReadKey(it, 1); // Reads B1, and releases B0 in DB0 + + // No permission for any commands (specifically select/swapdb) + EXPECT_CALL(mock, ACLCheckAllUserCommandPerm(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(ACL_DENIED_CMD)); + + // These cases should NOT block... (they access B0 in DB0) + // The SELECTs below are inconsequential - with/without select, same result. + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 0; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 1; SELECT 0; SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + + // These cases SHOULD block IF SELECT IS WORKING... (they access B0 in DB1) + c = getMultiClient("SET B0 xxx"); + c->db = server.db[1]; // already starting on DB1 + simulateBlockedWrite(c); // will block, no select + freeTestClient(c); + c = getMultiClient("SELECT 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (select fails) + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 1; SET B0 xxx; SELECT 0"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (select fails) + server.in_call--; + freeTestClient(c); + c = getMultiClient("SELECT 0; SELECT 1; SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (select fails) + server.in_call--; + + expectAnythingCleanup(it); +} + +// For this test, B0 is added into DB1 - so it exists in both DB0 and DB1. We will process it +// in DB0, but it will be unprocessed in DB1. See if we track SWAPDB properly. +TEST_F(BgIterationTest, multiHandlesSwapdbProperly) { + addKeyToDb(1, "B0", "B0"); + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + // Read the 1st key - B0 in DB0. + expectReadKey(it, 0); + // Now, we are done with B0 in DB0, but not in DB1 + expectReadKey(it, 1); // Reads B1, and releases B0 in DB0 + + // These cases should NOT block... (they access B0 in DB0) + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SWAPDB 0 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SWAPDB 0 1; SWAPDB 0 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SWAPDB 0 1; SELECT 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + + // These cases SHOULD block... (they access B0 in DB1) + c = getMultiClient("SET B0 xxx"); + c->db = server.db[1]; + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SET B0 xxx; SWAPDB 0 1"); + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SELECT 0; SET B0 xxx; SWAPDB 0 1"); + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SWAPDB 1 0; SELECT 1; SET B0 xxx; SELECT 1"); + simulateBlockedWrite(c); + + expectAnythingCleanup(it); +} + +// For this test, B0 is added into DB1 - so it exists in both DB0 and DB1. We will process it +// in DB0, but it will be unprocessed in DB1. See if we track select properly - WHEN WE HAVE NO +// PERMISSION TO EXECUTE SWAPDB! +TEST_F(BgIterationTest, multiHandlesSwapdbNoPermissionProperly) { + addKeyToDb(1, "B0", "B0"); + + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + // Read the 1st key - B0 in DB0. + expectReadKey(it, 0); + // Now, we are done with B0 in DB0, but not in DB1 + expectReadKey(it, 1); // Reads B1, and releases B0 in DB0 + + // No permission for any commands (specifically select/swapdb) + EXPECT_CALL(mock, ACLCheckAllUserCommandPerm(_, _, _, _, _, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(ACL_DENIED_CMD)); + + // These cases should NOT block... (they access B0 in DB0) + // The SELECTs & SWAPDBs below are inconsequential - with/without select/swapdb, same result. + c = getMultiClient("SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SWAPDB 0 1"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SET B0 xxx; SWAPDB 0 1; SWAPDB 0 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + c = getMultiClient("SWAPDB 0 1; SELECT 1; SET B0 xxx"); + simulateUnblockedWrite_inCall(c); + server.in_call--; + freeTestClient(c); + + // These cases SHOULD block IF SELECT/SWAPDB IS WORKING... (they access B0 in DB1) + c = getMultiClient("SET B0 xxx"); + c->db = server.db[1]; + simulateBlockedWrite(c); + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SET B0 xxx; SWAPDB 0 1"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (swapdb fails) + server.in_call--; + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SELECT 0; SET B0 xxx; SWAPDB 0 1"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (swapdb/select fails) + server.in_call--; + freeTestClient(c); + c = getMultiClient("SWAPDB 1 0; SWAPDB 1 0; SELECT 1; SET B0 xxx; SELECT 1"); + simulateUnblockedWrite_inCall(c); // will not block because accessing DB0 (swapdb/select fails) + server.in_call--; + + expectAnythingCleanup(it); +} + + +static void *pthreadWait200msAndReadTwoKeys(void *arg) { + bgIterator *it = static_cast(arg); + + usleep(200000); + bgIteratorRead(it); + bgIteratorRead(it); + return nullptr; +} + +static void asyncWait200msAndReadTwoKeys(bgIterator *it) { + int rc; + pthread_attr_t attr; + pthread_t thread; + + rc = pthread_attr_init(&attr); + assert(rc == 0); + rc = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + assert(rc == 0); + + rc = pthread_create(&thread, &attr, pthreadWait200msAndReadTwoKeys, it); + assert(rc == 0); + + rc = pthread_attr_destroy(&attr); + assert(rc == 0); +} + +TEST_F(BgIterationTest, testLuaWithUndeclaredKey) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_START, NULL, + iteratorCleanupFn, PRIVDATA); + + // Read the 1st key - let's get the party started + expectReadKey(it, 0); + + // At this point, key 0 is read. Keys 1 & 2 are queued (they are all in the same bucket). + // If we fake a modification to key 3, we won't know if it's handled out of order. + // So we fake a modification to key 4 + c = getWriteClient(4, "xxx"); + c->flag.script = 1; + + // Now for a LUA script, we have already blocked (on the eval/evalsha) for any declared keys + // But here, we're about to modify an undeclared key. We can't actually block in the middle + // of the LUA script. So this will behave as unblocked, but incur a synchronous wait. + + // Key 4 will get expedited when we simulate the write. After reading key 4, key 1 will need + // to be read to return key 4 to Valkey, unblocking the synchronous wait. + asyncWait200msAndReadTwoKeys(it); + + monotime blockTimer; + elapsedStart(&blockTimer); + simulateUnblockedWrite_inCall(c); // Not blocked, but delays internally + server.in_call--; + // Must have delayed at least 150ms (some time may have passed before timer start) + EXPECT_GT(elapsedMs(blockTimer), 150u); + + // Continue... + expectReadKeySequence(it, 2, 3); + // 4 has already been processed + expectReadKeySequence(it, 5, LAST_ITEM); + expectReadComplete(it); +} + + +// Make sure that replication received while processing the last key is sent +TEST_F(BgIterationTest, replicationReceivedWhileProcessingLastKey) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, LAST_ITEM); + + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // Wouldn't be blocked because done with key 0 + + expectReadReplication(it, c); // Replication happened while processing the last item, should be here. + + simulateUnblockedWriteWithModification(c); // This won't replicate because we are done processing + + expectReadComplete(it); // We expect to see the completion instead +} + +TEST_F(BgIterationTest, repldoneFunctionCalled) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, + iteratorRepldoneFn, iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, LAST_ITEM); + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // Wouldn't be blocked because done with key 0 + + // Since in testing, we are only feeding one item at a time, and synchronously, we won't call + // the repldone function until after we release the last item. + EXPECT_EQ(replDoneConfirmed, 0); + expectReadReplication(it, c); // Replication happened while processing the last item, should be here. + EXPECT_EQ(replDoneConfirmed, 1); // Last key released, now done feeding replication + + simulateUnblockedWriteWithModification(c); // This won't replicate because we are done processing + + expectReadComplete(it); // We expect to see the completion instead +} + +TEST_F(BgIterationTest, repldoneFunctionCalledTwice) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, + iteratorRepldoneFnNotBeingReadyInitially, iteratorCleanupFn, PRIVDATA); + expectReadKeySequence(it, 0, LAST_ITEM); + c = getWriteClient(0, "xxx"); + simulateUnblockedWriteWithModification(c); // Wouldn't be blocked because done with key 0 + + // Won't signal replDone until we've released the final item (which happens when reading the replication) + EXPECT_EQ(replDoneRejected, 0); + EXPECT_EQ(replDoneConfirmed, 0); + expectReadReplication(it, c); // Releases the final item + EXPECT_EQ(replDoneRejected, 1); // replDone called once (and rejected by client) + EXPECT_EQ(replDoneConfirmed, 0); + simulateUnblockedWriteWithModification(c); // This will replicate (because replDone returned false) + + expectReadReplication(it, c); // ReplDone gets called again (and accepted this time) + EXPECT_EQ(replDoneConfirmed, 1); + + simulateUnblockedWriteWithModification(c); // This won't replicate because replication is done + + expectReadComplete(it); // We expect to see the completion instead +} + +// Check that the memory reported for replication is correct +TEST_F(BgIterationTest, checkReplicationByteCount) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, + iteratorRepldoneFn, iteratorCleanupFn, PRIVDATA); + c = getWriteClient(0, "xxx"); + size_t expectedReplicationSize = sizeof(bgIteratorItem); + for (int i = 0; i < c->argc; i++) { + expectedReplicationSize += objectComputeSize(NULL, c->argv[i], 0, 0); + } + + expectReadKey(it, 0); + expectReadKey(it, 1); // Releases and unblocks 0 + EXPECT_EQ(bgIteration_memoryInuseForReplication(), 0u); + + simulateUnblockedWriteWithModification(c); // Wouldn't be blocked because done with key 0 + EXPECT_EQ(bgIteration_memoryInuseForReplication(), expectedReplicationSize); + simulateUnblockedWriteWithModification(c); // and write again (2nd replication) + EXPECT_EQ(bgIteration_memoryInuseForReplication(), 2 * expectedReplicationSize); + + expectReadKey(it, 2); // Keys 0..2 all in same bucket + + expectReadReplication(it, c); + // After reading the 1st replication, it hasn't been returned yet (it's the active item) + EXPECT_EQ(bgIteration_memoryInuseForReplication(), 2 * expectedReplicationSize); + expectReadReplication(it, c); + // After reading the 2nd replication, the 1st has been returned + EXPECT_EQ(bgIteration_memoryInuseForReplication(), expectedReplicationSize); + + expectReadKey(it, 3); + // Now all replication has been returned/freed + EXPECT_EQ(bgIteration_memoryInuseForReplication(), 0u); + + expectReadKeySequence(it, 4, LAST_ITEM); + expectReadComplete(it); +} + +// Test that for an arbitrary write command having no keys, replication should occur. +TEST_F(BgIterationTest, checkNoKeysWriteIsReplicated) { + bgIterator *it = bgIteratorCreateFullScanIter("iter", BGITERATOR_CONSISTENCY_EVENTUAL, NULL, + iteratorCleanupFn, PRIVDATA); + expectReadKey(it, 0); + + c = getNoKeysWriteClient(); + simulateUnblockedWrite_inCall(c); + bgIteration_handleCommandReplication(c->db->id, c->cmd, c->argc, c->argv); + server.in_call--; + + expectReadKeySequence(it, 1, 2); // These were already in queue + + expectReadReplication(it, c); + + expectReadKeySequence(it, 3, LAST_ITEM); + expectReadComplete(it); +} diff --git a/src/unit/wrappers.h b/src/unit/wrappers.h index 0d5be26effd..5c07cd2aa99 100644 --- a/src/unit/wrappers.h +++ b/src/unit/wrappers.h @@ -65,6 +65,15 @@ ssize_t __wrap_streamDecompressorFeed(streamDecompressor *decompressor, uint8_t void __wrap_zmadvise_dontneed(void *ptr, size_t size_hint); int __wrap_processPendingCommandAndInputBuffer(client *c); void __wrap_beforeNextClient(client *c); + +void __wrap_blockClientInUseOnKeys(client *c, int nKeys, robj **keys); +void __wrap_unblockClientsInUseOnKey(robj *key); + +int __wrap_ACLCheckAllUserCommandPerm(user *u, struct serverCommand *cmd, robj **argv, int argc, int dbid, int *idxptr); + +size_t __wrap_hashtableScan(hashtable *ht, size_t cursor, hashtableScanFunction fn, void *privdata); +bool __wrap_hashtableScanHasPassedKey(hashtable *ht, const void *key, size_t cursor); + #undef protected #undef _Bool #undef typename From c5b3d16e2b0e606b915a0aeb29f11e59794323fc Mon Sep 17 00:00:00 2001 From: Jim Brunner Date: Tue, 4 Aug 2026 11:38:22 -0700 Subject: [PATCH 07/18] bgIteration: unpause rehashing before flush (#4333) Make sure that rehashing is unpaused after a flushdb. https://github.com/valkey-io/valkey/pull/3648#issuecomment-5160134468 Signed-off-by: Jim Brunner --- src/bgiteration.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bgiteration.c b/src/bgiteration.c index e0ef2a26c61..a3d406c0527 100644 --- a/src/bgiteration.c +++ b/src/bgiteration.c @@ -592,7 +592,11 @@ static void fullScanIteratorFlushDb(genericIterator *genIt, int cur_dbid) { int orig_db = (cur_dbid == -1) ? it->iter_db : it->cur_to_orig_db[cur_dbid]; if (orig_db == it->iter_db) { // We are currently iterating on the DB that's being flushed. - it->kvs = NULL; + if (it->kvs) { + // If it->kvs is set, we're actively scanning and have paused rehash + resumeRehashForKvsHashtable(it->kvs, it->kvs_didx); + it->kvs = NULL; + } // Iteration will continue with the next DB. } } From f1d5a83ccf9fd2c41e3410cfbdcc0f3961a495a2 Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:35:21 -0700 Subject: [PATCH 08/18] Merge forkless-pre-threadsave into forkless (#4408) Signed-off-by: harrylin98 Signed-off-by: Jim Brunner Signed-off-by: Nitai Caro Signed-off-by: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Co-authored-by: Jim Brunner Co-authored-by: Harry Lin <49881386+harrylin98@users.noreply.github.com> Co-authored-by: Nitai Caro Signed-off-by: Jim Brunner --- cmake/Modules/SourceFiles.cmake | 1 + src/Makefile | 1 + src/aof.c | 4 +- src/bgiteration.c | 37 +- src/bgiteration.h | 2 +- src/commands.def | 15 +- src/commands/bgsave.json | 35 +- src/config.c | 5 + src/db.c | 5 +- src/forkless.c | 410 +++++++++ src/forkless.h | 14 + src/module.c | 29 +- src/module.h | 1 + src/rdb.c | 311 +++++-- src/rdb.h | 22 +- src/replication.c | 20 +- src/rio.h | 7 + src/scripting_engine.c | 7 +- src/server.c | 84 +- src/server.h | 33 +- src/unit/test_bgiteration.cpp | 2 +- src/valkeymodule.h | 7 +- tests/integration/rdb.tcl | 1289 ++++++++++++++++++++++++++--- tests/integration/replication.tcl | 84 ++ tests/support/util.tcl | 36 + tests/unit/info.tcl | 220 +++++ tests/unit/introspection.tcl | 65 +- tests/unit/moduleapi/testrdb.tcl | 27 + tests/unit/other.tcl | 61 +- valkey.conf | 15 +- 30 files changed, 2504 insertions(+), 345 deletions(-) create mode 100644 src/forkless.c create mode 100644 src/forkless.h diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake index c735d9d32a9..dc2c5bc4ca9 100644 --- a/cmake/Modules/SourceFiles.cmake +++ b/cmake/Modules/SourceFiles.cmake @@ -5,6 +5,7 @@ # valkey-server source files set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/threads_mngr.c + ${CMAKE_SOURCE_DIR}/src/forkless.c ${CMAKE_SOURCE_DIR}/src/adlist.c ${CMAKE_SOURCE_DIR}/src/vector.c ${CMAKE_SOURCE_DIR}/src/quicklist.c diff --git a/src/Makefile b/src/Makefile index 23549fa49c2..2dfe01dd9cc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -508,6 +508,7 @@ ENGINE_SERVER_OBJ = \ expire.o \ fbtree.o \ fifo.o \ + forkless.o \ functions.o \ geo.o \ geohash.o \ diff --git a/src/aof.c b/src/aof.c index f38384d1136..a907069ec38 100644 --- a/src/aof.c +++ b/src/aof.c @@ -2594,7 +2594,7 @@ int rewriteAppendOnlyFile(char *filename) { int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - if (hasActiveChildProcess()) return C_ERR; + if (hasActiveSaveOrChild()) return C_ERR; if (dirCreateIfMissing(server.aof_dirname) == -1) { serverLog(LL_WARNING, "Can't open or create append-only dir %s: %s", server.aof_dirname, strerror(errno)); @@ -2664,7 +2664,7 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (server.child_type == CHILD_TYPE_AOF) { addReplyError(c, "Background append only file rewriting already in progress"); - } else if (hasActiveChildProcess() || server.in_exec) { + } else if (hasActiveSaveOrChild() || server.in_exec) { server.aof_rewrite_scheduled = 1; /* When manually triggering AOFRW we reset the count * so that it can be executed immediately. */ diff --git a/src/bgiteration.c b/src/bgiteration.c index a3d406c0527..d722a6db60d 100644 --- a/src/bgiteration.c +++ b/src/bgiteration.c @@ -1,7 +1,7 @@ /* * Copyright Valkey Contributors. * All rights reserved. - * SPDX-License-Identifier: BSD 3-Clause + * SPDX-License-Identifier: BSD-3-Clause */ #include "fmacros.h" @@ -33,15 +33,14 @@ static bool isDeleteCmd(struct serverCommand *cmd) { return ((cmd->proc == delCommand) || (cmd->proc == unlinkCommand)); } - /* This utility utilizes the main thread and background threads for processing. The API is split, * with some of the functions intended for the main thread and others intended for the background * clients. This sanity check ensures that we maintain thread safety, calling the API as intended. */ -static bool onValkeyMainThread(void) { +static bool hasMainThreadExclusivity(void) { /* Modules interact with the main thread using a mutex. If a module owns the mutex, consider * that equivalent to being on the main thread. */ bool mightBeInModule = (atomic_load_explicit(&server.module_gil_acquired, memory_order_relaxed) == 0); - return (mightBeInModule || pthread_equal(server.main_thread_id, pthread_self()) != 0); + return onValkeyMainThread() || mightBeInModule; } @@ -922,7 +921,7 @@ static void returnCurrentItemToMainThread(bgIterator *it) { * ============================================================================================= */ static void bgIteratorRelease(bgIterator *it) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); serverAssert(it->current_item == NULL); serverAssert(mutexQueueLength(it->items_for_iterator) == 0); serverAssert(mutexQueueLength(it->return_to_main_thread) == 0); @@ -1400,7 +1399,7 @@ static bool expediteKeysForWrite(bgIterator *it, /* Called when an iterator is terminated. Pulls everything out of the queue * and returns the items to the main thread (before they hit the iterator). */ static void returnAllItemsToMainThread(bgIterator *it) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); fifo *poppedFifo = mutexQueuePopAll(it->items_for_iterator, false); if (poppedFifo == NULL) return; // Nothing to return @@ -1470,7 +1469,7 @@ static size_t replicationItemSize(bgIteratorItem *item) { } static void processReturnOfItemToMainThread(bgIterator *it, bgIteratorItem *item) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); switch ((int)item->type) { case BGITERATOR_ITEM_REPLICATION: bufferedReplicationBytes -= item->u.repl.replication_size; @@ -1597,7 +1596,7 @@ static bool receiveItemsBackFromOneIterator(bgIterator *it) { /* Process each iterator's return_to_main_thread queue * If `blocking` is true, continue reading until at least one queue was not empty. */ static void receiveItemsBackFromIterators(bool blocking) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); listIter li; listNode *node; bool processedItems = false; @@ -1618,7 +1617,7 @@ static long long bgIteration_feedIterators_task(struct aeEventLoop *eventLoop, UNUSED(eventLoop); UNUSED(id); UNUSED(clientData); - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); static monotime lastFeedEndTime; // STATIC: Persists For checking starvation monotime startTime = getMonotonicUs(); @@ -1783,7 +1782,7 @@ static void resetReplicationFlagForIterators(client *c) { static void handleSwapdb(int db1, int db2) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); serverAssert(bgIteration_iterationActive()); serverAssert(!server.cluster_enabled); @@ -2001,7 +2000,7 @@ static bgIterator *bgIteratorCreate(const char *name, bgIterationType iter_type, genericIterator *keyset_iter) { serverAssert(server.forkless_options_supported); - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); serverAssert(server.cluster_enabled || iter_type == BGITERATION_TYPE_FULLSCAN); int flags; @@ -2103,7 +2102,7 @@ bgIterator *bgIteratorCreateSlotsIter(const char *name, // PUBLIC API bgIterator *bgIteratorFind(const char *name) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); sds sdsname = sdsnew(name); bgIterator *it = dictFetchValue(nameToIterator, sdsname); @@ -2146,7 +2145,7 @@ void bgIteratorGetStatus(bgIterator *it, bgIteratorStatus *status) { // PUBLIC API void bgIteratorTerminate(bgIterator *it) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); // Remove any items in the queue, but doesn't affect the 1 item that's being processed. returnAllItemsToMainThread(it); @@ -2182,7 +2181,7 @@ bgIteratorItem *bgIteratorRead(bgIterator *it) { * Without this, a unit test could get stuck waiting on the completion event because * feed won't get invoked. For production, feed is called regularly from the main thread. * Note - this is checking that the exact same thread is used and shouldn't count modules. */ - if (pthread_equal(server.main_thread_id, pthread_self()) != 0) bgIteration_feedIterators_task(NULL, 0, NULL); + if (onValkeyMainThread()) bgIteration_feedIterators_task(NULL, 0, NULL); } else { it->client_is_active = true; } @@ -2224,7 +2223,7 @@ void bgIteratorClose(bgIterator *it) { // PUBLIC API void bgIteration_init(void) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); /* This should be called once and only once from the Valkey main thread. However to support * unit tests, this is not validated, and multiple invocations are ignored. */ @@ -2267,7 +2266,7 @@ void bgIteration_beforeSleep(void) { // PUBLIC API void bgIteration_keyDelete(int dbid, const_sds key) { if (!bgIteration_iterationActive()) return; - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); if (BGITERATION_DEBUG) { debugBuffer = sdscatprintf(debugBuffer, "KEYDEL: (%d)%s\n", dbid, key); @@ -2312,7 +2311,7 @@ void bgIteration_flushall(void) { // PUBLIC API bool bgIteration_blockClientIfRequired(client *c) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); iteratorReplicationFlagsWereUpdated = false; if (!bgIteration_iterationActive()) return false; if (!isWriteCmd(c->cmd)) return false; @@ -2651,7 +2650,7 @@ size_t bgIteration_memoryInuseForReplication(void) { // PUBLIC API bool bgIteration_isEntryInuse(dbEntry *de) { - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); if (!bgIteration_iterationActive()) return false; return isEntryInuseByAnyIterator(de); } @@ -2678,7 +2677,7 @@ void bgIteration_keyModified(int dbid, const_sds key) { // PUBLIC API void bgIteration_updateDbEntryPtr(dbEntry *old, dbEntry *new) { if (!bgIteration_iterationActive() || old == new) return; - serverAssert(onValkeyMainThread()); + serverAssert(hasMainThreadExclusivity()); serverAssert(!isEntryInuseByAnyIterator(old)); listIter li; diff --git a/src/bgiteration.h b/src/bgiteration.h index 6aaac8cc7b8..c9219b46ea6 100644 --- a/src/bgiteration.h +++ b/src/bgiteration.h @@ -1,7 +1,7 @@ /* * Copyright Valkey Contributors. * All rights reserved. - * SPDX-License-Identifier: BSD 3-Clause + * SPDX-License-Identifier: BSD-3-Clause */ #ifndef __BGITERATION_H diff --git a/src/commands.def b/src/commands.def index 942c7e8a5bb..61a37fd905a 100644 --- a/src/commands.def +++ b/src/commands.def @@ -7173,6 +7173,7 @@ struct COMMAND_STRUCT ACL_Subcommands[] = { commandHistory BGSAVE_History[] = { {"3.2.2","Added the `SCHEDULE` option."}, {"8.1.0","Added the `CANCEL` option."}, +{"9.2.0","Added the `FORK` and `FORKLESS` options. `SCHEDULE` can be combined with `FORK` or `FORKLESS`."}, }; #endif @@ -7186,15 +7187,17 @@ commandHistory BGSAVE_History[] = { #define BGSAVE_Keyspecs NULL #endif -/* BGSAVE operation argument table */ -struct COMMAND_ARG BGSAVE_operation_Subargs[] = { -{MAKE_ARG("schedule",ARG_TYPE_PURE_TOKEN,-1,"SCHEDULE",NULL,"3.2.2",CMD_ARG_NONE,0,NULL)}, -{MAKE_ARG("cancel",ARG_TYPE_PURE_TOKEN,-1,"CANCEL",NULL,"8.1.0",CMD_ARG_NONE,0,NULL)}, +/* BGSAVE save_type argument table */ +struct COMMAND_ARG BGSAVE_save_type_Subargs[] = { +{MAKE_ARG("fork",ARG_TYPE_PURE_TOKEN,-1,"FORK",NULL,NULL,CMD_ARG_NONE,0,NULL)}, +{MAKE_ARG("forkless",ARG_TYPE_PURE_TOKEN,-1,"FORKLESS",NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; /* BGSAVE argument table */ struct COMMAND_ARG BGSAVE_Args[] = { -{MAKE_ARG("operation",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=BGSAVE_operation_Subargs}, +{MAKE_ARG("schedule",ARG_TYPE_PURE_TOKEN,-1,"SCHEDULE",NULL,"3.2.2",CMD_ARG_OPTIONAL,0,NULL)}, +{MAKE_ARG("save-type",ARG_TYPE_ONEOF,-1,NULL,NULL,"9.2.0",CMD_ARG_OPTIONAL,2,NULL),.subargs=BGSAVE_save_type_Subargs}, +{MAKE_ARG("cancel",ARG_TYPE_PURE_TOKEN,-1,"CANCEL",NULL,"8.1.0",CMD_ARG_OPTIONAL,0,NULL)}, }; /********** COMMAND COUNT ********************/ @@ -12033,7 +12036,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { /* server */ {MAKE_CMD("acl","A container for Access List Control commands.","Depends on subcommand.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_History,0,ACL_Tips,0,NULL,-2,CMD_SENTINEL,ACL_CATEGORY_SLOW,NULL,ACL_Keyspecs,0,NULL,0),.subcommands=ACL_Subcommands}, {MAKE_CMD("bgrewriteaof","Asynchronously rewrites the append-only file to disk.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,BGREWRITEAOF_History,0,BGREWRITEAOF_Tips,0,bgrewriteaofCommand,1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,BGREWRITEAOF_Keyspecs,0,NULL,0)}, -{MAKE_CMD("bgsave","Asynchronously saves the database(s) to disk.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,BGSAVE_History,2,BGSAVE_Tips,0,bgsaveCommand,-1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,BGSAVE_Keyspecs,0,NULL,1),.args=BGSAVE_Args}, +{MAKE_CMD("bgsave","Asynchronously saves the database(s) to disk.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,BGSAVE_History,3,BGSAVE_Tips,0,bgsaveCommand,-1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,BGSAVE_Keyspecs,0,NULL,3),.args=BGSAVE_Args}, {MAKE_CMD("command","Returns detailed information about all commands.","O(N) where N is the total number of commands","2.8.13",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,COMMAND_History,0,COMMAND_Tips,1,commandCommand,-1,CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,COMMAND_Keyspecs,0,NULL,0),.subcommands=COMMAND_Subcommands}, {MAKE_CMD("commandlog","A container for command log commands.","Depends on subcommand.","8.1.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,COMMANDLOG_History,0,COMMANDLOG_Tips,0,NULL,-2,0,ACL_CATEGORY_SLOW,NULL,COMMANDLOG_Keyspecs,0,NULL,0),.subcommands=COMMANDLOG_Subcommands}, {MAKE_CMD("config","A container for server configuration commands.","Depends on subcommand.","2.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,CONFIG_History,0,CONFIG_Tips,0,NULL,-2,0,ACL_CATEGORY_SLOW,NULL,CONFIG_Keyspecs,0,NULL,0),.subcommands=CONFIG_Subcommands}, diff --git a/src/commands/bgsave.json b/src/commands/bgsave.json index ef7b8740a0a..445cdcb6a27 100644 --- a/src/commands/bgsave.json +++ b/src/commands/bgsave.json @@ -14,6 +14,10 @@ [ "8.1.0", "Added the `CANCEL` option." + ], + [ + "9.2.0", + "Added the `FORK` and `FORKLESS` options. `SCHEDULE` can be combined with `FORK` or `FORKLESS`." ] ], "command_flags": [ @@ -23,23 +27,36 @@ ], "arguments": [ { - "name": "operation", + "name": "schedule", + "token": "SCHEDULE", + "type": "pure-token", + "optional": true, + "since": "3.2.2" + }, + { + "name": "save-type", "type": "oneof", "optional": true, + "since": "9.2.0", "arguments": [ { - "name": "schedule", - "token": "SCHEDULE", - "type": "pure-token", - "since": "3.2.2" + "name": "fork", + "token": "FORK", + "type": "pure-token" }, { - "name": "cancel", - "token": "CANCEL", - "type": "pure-token", - "since": "8.1.0" + "name": "forkless", + "token": "FORKLESS", + "type": "pure-token" } ] + }, + { + "name": "cancel", + "token": "CANCEL", + "type": "pure-token", + "optional": true, + "since": "8.1.0" } ], "reply_schema": { diff --git a/src/config.c b/src/config.c index 80135ea266a..516a568fc83 100644 --- a/src/config.c +++ b/src/config.c @@ -182,6 +182,10 @@ configEnum rdb_compression_enum[] = {{"no", RDB_COMPRESSION_NO}, {"lz4", RDB_COMPRESSION_LZ4}, {NULL, 0}}; +configEnum bgsave_method_enum[] = {{"fork", RDB_BGSAVE_TYPE_FORK}, + {"forkless", RDB_BGSAVE_TYPE_FORKLESS}, + {NULL, 0}}; + /* Output buffer limits presets. */ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {0, 0, 0}, /* normal */ @@ -3357,6 +3361,7 @@ standardConfig static_configs[] = { createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL), createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL), createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL), + createEnumConfig("default-bgsave-method", NULL, MODIFIABLE_CONFIG, bgsave_method_enum, server.default_bgsave_method, RDB_BGSAVE_TYPE_FORK, NULL, NULL), createBoolConfig("set-proc-title", NULL, IMMUTABLE_CONFIG, server.set_proc_title, 1, NULL, NULL), /* Should setproctitle be used? */ createBoolConfig("lazyfree-lazy-eviction", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 1, NULL, NULL), createBoolConfig("lazyfree-lazy-expire", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 1, NULL, NULL), diff --git a/src/db.c b/src/db.c index 93b6e565a0d..0812e9d3a10 100644 --- a/src/db.c +++ b/src/db.c @@ -38,6 +38,8 @@ #include "module.h" #include "vector.h" #include "expire.h" +#include "bgiteration.h" +#include "forkless.h" #include "crc16_slottable.h" #include "bgiteration.h" @@ -826,7 +828,8 @@ int getFlushCommandFlags(client *c, int *flags) { /* Flushes the whole server data set. */ void flushAllDataAndResetRDB(int flags) { server.dirty += emptyData(-1, flags, NULL); - if (server.child_type == CHILD_TYPE_RDB) killRDBChild(); + if (isForkBgsaveInProgress()) killRDBChild(); + if (isForklessSaveInProgress()) forklessSaveCancel(); if (server.child_type == CHILD_TYPE_SLOT_MIGRATION) killSlotMigrationChild(); if (server.saveparamslen > 0) { rdbSaveInfo rsi, *rsiptr; diff --git a/src/forkless.c b/src/forkless.c new file mode 100644 index 00000000000..40c4e9154b1 --- /dev/null +++ b/src/forkless.c @@ -0,0 +1,410 @@ +#include "forkless.h" +#include "server.h" +#include "bgiteration.h" +#include "mutexqueue.h" +#include "rdb.h" +#include "bio.h" + +static const void *PROCESS_COMPLETE_ITEM = (void *)-1; +static const int SNAPSHOT_FILE_CLOSE_MONITOR_INTERVAL_MS = 200; + +typedef struct { + rio save_rio; /* Must be 1st to permit cast from rio back to forklessSaveInfo */ + int cur_db; /* Last selectDb issued */ + bgIterator *iterator; + uint64_t bytes_written; + int err_code; + mutexQueue *foreground_queue; + bool terminated; + sds temp_file; + sds final_file; +} forklessSaveInfo; + +/* Keep a global indicator of the current iterator (for cancellation purposes). */ +static forklessSaveInfo *currentForklessSave = NULL; + +/* rio check_abort_between_writes callback: checks if the forkless save iterator is being terminated. */ +static int forklessSaveShouldAbort(rio *r) { + static_assert(offsetof(forklessSaveInfo, save_rio) == 0, "rio must be castable to forklessSaveInfo"); + forklessSaveInfo *saveInfo = (forklessSaveInfo *)r; + return saveInfo->iterator && bgIteratorIsTerminating(saveInfo->iterator); +} + +static int writeSelectDb(forklessSaveInfo *saveInfo, int new_db) { + if (new_db == saveInfo->cur_db) return C_OK; + + if (rdbSaveType(&saveInfo->save_rio, RDB_OPCODE_SELECTDB) == -1) { + serverLog(LL_WARNING, "forkless-save: error while writing OPCODE_SELECTDB"); + return C_ERR; + } + if (rdbSaveLen(&saveInfo->save_rio, new_db) == -1) { + serverLog(LL_WARNING, "forkless-save: error while writing selectDb value"); + return C_ERR; + } + saveInfo->cur_db = new_db; + return C_OK; +} + +static int writeDbSizeHints(forklessSaveInfo *saveInfo) { + for (int dbid = 0; dbid < server.dbnum; dbid++) { + serverDb *db = server.db[dbid]; + if (db == NULL || dbSize(db) == 0) continue; + if (writeSelectDb(saveInfo, dbid) != C_OK) return C_ERR; + if (rdbSaveDbSizeHints(&saveInfo->save_rio, db, 0) < 0) return C_ERR; + } + return C_OK; +} + +/* Entry point for background thread. + * Upon entering: + * - The RDB header has been written (magic, aux fields, functions) + * - The DB size hints have been written + * This function is responsible for writing all of the dictionary entries. */ +static void *forklessSaveProcessor(void *arg) { + serverAssert(!onValkeyMainThread()); + forklessSaveInfo *saveInfo = arg; + + serverLog(LL_NOTICE, "forkless-save: background processor started"); + int err = C_OK; + + saveInfo->save_rio.check_abort_between_writes = forklessSaveShouldAbort; + + const unsigned statsIntervalMs = 1000; + monotime lastStatsTime; + elapsedStart(&lastStatsTime); + + bool done = false; + bool terminated = false; + long items = 0; + while (!done && err == C_OK) { + bgIteratorItem *item = bgIteratorRead(saveInfo->iterator); + + switch (item->type) { + case BGITERATOR_ITEM_COMPLETE: + done = true; + break; + + case BGITERATOR_ITEM_TERMINATED: + terminated = true; + done = true; + break; + + case BGITERATOR_ITEM_DBENTRY: + if ((err = writeSelectDb(saveInfo, item->dbid)) == C_ERR) break; + items++; + + robj key; + initStaticStringObject(key, objectGetKey(item->u.dbe.de)); + robj *o = item->u.dbe.de; + + long long expire = objectGetExpire(item->u.dbe.de); + if (rdbSaveKeyValuePair(&saveInfo->save_rio, &key, o, expire, item->dbid, RDB_VERSION) == -1) { + serverLog(LL_WARNING, "forkless-save: error writing KV pair"); + err = C_ERR; + } + break; + default: + /* bgIteration may deliver item types that are not necessarily relevant to us. + * New types may also be added in the future. It is the client's responsibility + * to filter out irrelevant types, so we simply ignore them here. */ + break; + } + + if (elapsedMs(lastStatsTime) >= statsIntervalMs) { + elapsedStart(&lastStatsTime); + atomic_store_explicit(&server.stat_current_save_keys_processed, items, memory_order_relaxed); + } + } + + if (err != C_OK && bgIteratorIsTerminating(saveInfo->iterator)) { + /* We recognized termination before receiving the TERMINATED event */ + terminated = true; + } + + char *message = ""; + if (terminated) + message = "TERMINATED"; + else if (err != C_OK) + message = "***ERROR***"; + serverLog(LL_NOTICE, "forkless-save: background processor finished. %ld items processed. %s", + items, message); + + currentForklessSave = NULL; + saveInfo->err_code = err; + bgIteratorClose(saveInfo->iterator); + return NULL; +} + +static void cleanupSaveInfoAndEmitEndMetrics(forklessSaveInfo *saveInfo) { + if (saveInfo->terminated && saveInfo->err_code == C_OK) saveInfo->err_code = C_ERR; + rdbRecordEndMetrics(RDB_BGSAVE_TYPE_FORKLESS, saveInfo->err_code, time(NULL)); + rdbClearSaveState(time(NULL)); + if (saveInfo->err_code == C_OK) { + serverLog(LL_NOTICE, "forkless-save: forkless save complete. %lld seconds.", (long long)server.rdb_save_time_last); + } else if (saveInfo->terminated) { + serverLog(LL_WARNING, "forkless-save: forkless save terminated. %lld seconds.", (long long)server.rdb_save_time_last); + } else { + serverLog(LL_WARNING, "forkless-save: forkless save failed. %lld seconds.", (long long)server.rdb_save_time_last); + } + stopSaving(saveInfo->err_code == C_OK); + currentForklessSave = NULL; + atomic_store_explicit(&server.stat_current_save_keys_processed, 0, memory_order_relaxed); + atomic_store_explicit(&server.stat_current_save_keys_total, 0, memory_order_relaxed); + + serverAssert(saveInfo->temp_file == NULL); + zfree(saveInfo); +} + +/* Routine for background thread to close and rename the forkless save snapshot file. + * Closing the file requires synchronously flushing the content to disk, which can + * take some time. */ +static void forklessSaveCloseSnapshotFile(void *args[]) { + serverAssert(!onValkeyMainThread()); + forklessSaveInfo *saveInfo = (forklessSaveInfo *)args[0]; + /* Error or not, close the file... */ + if (fsync(fileno(saveInfo->save_rio.io.file.fp)) != 0) { + serverLog(LL_WARNING, "forkless-save: error fsyncing temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + saveInfo->err_code = C_ERR; + } + if (fclose(saveInfo->save_rio.io.file.fp) != 0) { + serverLog(LL_WARNING, "forkless-save: error closing temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + saveInfo->err_code = C_ERR; + } + + if (!saveInfo->terminated && saveInfo->err_code == C_OK) { + if (rename(saveInfo->temp_file, saveInfo->final_file) != 0) { + serverLog(LL_WARNING, "forkless-save: error moving temp file [%s] to destination [%s]: %s", + saveInfo->temp_file, saveInfo->final_file, strerror(errno)); + saveInfo->err_code = C_ERR; + } + } + + if (saveInfo->terminated || saveInfo->err_code != C_OK) { + rdbRemoveTempFile(getpid(), 0); + } + sdsfree(saveInfo->temp_file); + sdsfree(saveInfo->final_file); + saveInfo->temp_file = NULL; + saveInfo->final_file = NULL; + /* Notify the main thread that I am done closing the file. */ + mutexQueueAdd(saveInfo->foreground_queue, (void *)PROCESS_COMPLETE_ITEM); +} + +/* Timer proc which runs in the main valkey event loop. It monitors to see when the background thread + * completes the action to close and rename the snapshot file at the end of disk based forkless save, + * and performs the final clean-up actions. */ +static long long snapshotEndMonitorTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData) { + UNUSED(eventLoop); + UNUSED(id); + serverAssert(onValkeyMainThread()); + + forklessSaveInfo *saveInfo = (forklessSaveInfo *)clientData; + + /* I own this mutex queue from the main thread, check to see if the background + job is done or not. Note we only expect a single notification event here. */ + if (mutexQueuePop(saveInfo->foreground_queue, false) != NULL) { + mutexQueueRelease(saveInfo->foreground_queue); + saveInfo->foreground_queue = NULL; + cleanupSaveInfoAndEmitEndMetrics(saveInfo); + return AE_NOMORE; + } + return SNAPSHOT_FILE_CLOSE_MONITOR_INTERVAL_MS; +} + +void forklessSaveComplete(bool terminated, void *privdata) { + serverAssert(onValkeyMainThread()); + serverLog(LL_NOTICE, "forkless-save: completion proc - %s", (terminated) ? "terminated" : "ok"); + + forklessSaveInfo *saveInfo = privdata; + saveInfo->terminated = terminated; + /* The save iterator should be terminated and freed at this point in time. */ + saveInfo->iterator = NULL; + /* For file based forkless save, we need to generate the RDB end marker. and complete the save */ + if (!saveInfo->terminated && saveInfo->err_code == C_OK) { + saveInfo->err_code = rdbWriteFooter(&saveInfo->save_rio, REPLICA_REQ_NONE) == C_ERR ? C_ERR : C_OK; + } + + /* Done writing, capture bytes written (regardless of pass/fail) */ + saveInfo->bytes_written = saveInfo->save_rio.processed_bytes; + + /* Start a cron job to check for the background job completion */ + aeCreateTimeEvent(server.el, SNAPSHOT_FILE_CLOSE_MONITOR_INTERVAL_MS, snapshotEndMonitorTimeProc, saveInfo, NULL); + /* Submit a background job to close and rename the snapshot file */ + saveInfo->foreground_queue = mutexQueueCreate(); // The monitor proc will delete this + bioCreateLazyFreeJob(forklessSaveCloseSnapshotFile, 1, saveInfo); + serverLog(LL_NOTICE, "forkless-save: created background thread to perform snapshot file close and rename"); + /* We will now wait for the background closeSnapshotFile job to complete. + * The remainder of the cleanup will be performed in the snapshotEndMonitorTimeProc. */ +} + +static int forklessSaveCommonStart(forklessSaveInfo *saveInfo) { + serverAssert(onValkeyMainThread()); + + saveInfo->cur_db = -1; + + serverLog(LL_NOTICE, "Using forkless save for next backup"); + rdbRecordStartMetrics(RDB_BGSAVE_TYPE_FORKLESS); + startSaving(RDBFLAGS_FORKLESS_SAVE); + + rdbSaveInfo rsi, *rsiptr = rdbPopulateSaveInfo(&rsi); + if (rdbWriteHeader(&saveInfo->save_rio, REPLICA_REQ_NONE, RDB_VERSION, RDBFLAGS_NONE, rsiptr) == C_ERR) return C_ERR; + + if (writeDbSizeHints(saveInfo) == C_ERR) return C_ERR; + + return C_OK; +} + +static void startBackgroundThread(forklessSaveInfo *saveInfo) { + serverAssert(onValkeyMainThread()); + + pthread_t thread_id; + pthread_attr_t attr; + int pthread_rc; + pthread_rc = pthread_attr_init(&attr); + serverAssert(pthread_rc == 0); + pthread_rc = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + serverAssert(pthread_rc == 0); + pthread_rc = pthread_create(&thread_id, &attr, &forklessSaveProcessor, saveInfo); + serverAssert(pthread_rc == 0); + pthread_rc = pthread_attr_destroy(&attr); + serverAssert(pthread_rc == 0); +} + +/* Save a point-in-time snapshot to the given filename. + * The filename must be under the server's current working directory. + * Writes to a temp file and renames to the final filename on completion. */ +int forklessSaveToDisk(const char *filename) { + serverAssert(onValkeyMainThread()); + serverAssert(currentForklessSave == NULL); + serverAssert(!isSaveInProgress()); + serverAssert(filename); + serverLog(LL_NOTICE, "Beginning forklessSaveToDisk"); + + server.stat_rdb_saves++; + + char tmpfile[256]; + snprintf(tmpfile, sizeof(tmpfile), "temp-%d.rdb", (int)getpid()); + + FILE *file = fopen(tmpfile, "wb"); + if (file == NULL) { + serverLog(LL_WARNING, "forkless-save: failed to open temp file [%s] for forkless save: %s", + tmpfile, strerror(errno)); + return C_ERR; + } + + forklessSaveInfo *saveInfo = zcalloc(sizeof(forklessSaveInfo)); + saveInfo->temp_file = sdsnew(tmpfile); + saveInfo->final_file = sdsnew(filename); + + rioInitWithFile(&saveInfo->save_rio, file); + if (server.rdb_save_incremental_fsync) { + rioSetAutoSync(&saveInfo->save_rio, REDIS_AUTOSYNC_BYTES); + rioSetReclaimCache(&saveInfo->save_rio, 1); + } + + int rc = forklessSaveCommonStart(saveInfo); + if (rc != C_OK) goto werr; + + /* Saving to a file indicates a consistent snapshot (a backup at a point in time) */ + saveInfo->iterator = bgIteratorCreateFullScanIter(FORKLESS_SAVE_FILE_ITER_NAME, + BGITERATOR_CONSISTENCY_START, NULL, forklessSaveComplete, saveInfo); + if (saveInfo->iterator == NULL) { + serverLog(LL_WARNING, "forkless-save: error creating iterator"); + goto werr; + } + currentForklessSave = saveInfo; + + atomic_store_explicit(&server.stat_current_save_keys_total, dbTotalServerKeyCount(), memory_order_relaxed); + atomic_store_explicit(&server.stat_current_save_keys_processed, 0, memory_order_relaxed); + + startBackgroundThread(saveInfo); + + /* at this point, background iteration has started (saveInfo will be freed later) */ + return C_OK; + +werr: + saveInfo->err_code = C_ERR; + rdbRecordEndMetrics(RDB_BGSAVE_TYPE_FORKLESS, C_ERR, time(NULL)); + rdbClearSaveState(time(NULL)); + serverLog(LL_WARNING, "forkless-save: forkless save failed. %lld seconds.", (long long)server.rdb_save_time_last); + stopSaving(0); + currentForklessSave = NULL; + + if (file != NULL) { + if (fclose(file) != 0) { + serverLog(LL_WARNING, "forkless-save: Could not close temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + } + if (unlink(saveInfo->temp_file) != 0) { + serverLog(LL_WARNING, "forkless-save: Could not delete temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + } + } + sdsfree(saveInfo->temp_file); + sdsfree(saveInfo->final_file); + zfree(saveInfo); + return C_ERR; +} + +/* Cancels the currently running forkless save, if one is in progress. */ +void forklessSaveCancel(void) { + serverAssert(onValkeyMainThread()); + if (currentForklessSave == NULL) return; + bgIteratorTerminate(currentForklessSave->iterator); +} + +int isForklessSaveInProgress(void) { + return server.cur_bgsave_type == RDB_BGSAVE_TYPE_FORKLESS; +} + +/* Appends forkless save INFO metrics to the provided sds string. */ +sds forkless_catInfo(sds info) { + long long estimated_seconds_remaining = -1; + long long current_item_millis = -1; + + if (onValkeyMainThread()) { + bgIterator *iter = bgIteratorFind(FORKLESS_SAVE_FILE_ITER_NAME); + if (iter != NULL) { + bgIteratorStatus status = {0}; + bgIteratorGetStatus(iter, &status); + current_item_millis = status.current_item_ms; + + if (status.dbentries_processed > 0) { + long long total_keys = 0; + for (int i = 0; i < server.dbnum; i++) { + total_keys += server.db[i] ? dbSize(server.db[i]) : 0; + } + estimated_seconds_remaining = (total_keys - status.dbentries_processed) * + status.runtime_ms / status.dbentries_processed / 1000; + } + } + } + + return sdscatprintf(info, + "forkless_current_item_millis:%lld\r\n" + "forkless_estimated_seconds_remaining:%lld\r\n", + current_item_millis, + estimated_seconds_remaining); +} + +/* Appends forkless debug metrics to the provided sds string. */ +sds forkless_catDebugInfo(sds info) { + bgIteratorStatus status = {0}; + + if (onValkeyMainThread()) { + bgIterator *iter = bgIteratorFind(FORKLESS_SAVE_FILE_ITER_NAME); + if (iter != NULL) bgIteratorGetStatus(iter, &status); + } + + return sdscatprintf(info, + "forkless_current_queue_length:%lu\r\n" + "forkless_queue_length_target:%lu\r\n" + "forkless_dbentries_queued:%lu\r\n" + "forkless_dbentries_processed:%lu\r\n", + status.queue_length, + status.queue_length_target, + status.dbentries_queued, + status.dbentries_processed); +} diff --git a/src/forkless.h b/src/forkless.h new file mode 100644 index 00000000000..e5393e52f4d --- /dev/null +++ b/src/forkless.h @@ -0,0 +1,14 @@ +#ifndef __FORKLESS_H__ +#define __FORKLESS_H__ + +#include "server.h" + +#define FORKLESS_SAVE_FILE_ITER_NAME "forkless_save_file" + +int forklessSaveToDisk(const char *filename); +void forklessSaveCancel(void); +int isForklessSaveInProgress(void); +sds forkless_catInfo(sds info); +sds forkless_catDebugInfo(sds info); + +#endif diff --git a/src/module.c b/src/module.c index 86ddcf45e93..6258dfcf120 100644 --- a/src/module.c +++ b/src/module.c @@ -70,6 +70,7 @@ #include "io_threads.h" #include "scripting_engine.h" #include "cluster_migrateslots.h" +#include "forkless.h" #include #include #include @@ -2565,7 +2566,7 @@ void VM_Yield(ValkeyModuleCtx *ctx, int flags, const char *busy_reply) { if (flags & VALKEYMODULE_YIELD_FLAG_CLIENTS) server.busy_module_yield_flags |= BUSY_MODULE_YIELD_CLIENTS; /* Let the server process events */ - if (!pthread_equal(server.main_thread_id, pthread_self())) { + if (!onValkeyMainThread()) { /* If we are not in the main thread, we defer event loop processing to the main thread * after the main thread enters acquiring GIL state in order to protect the event * loop (ae.c) and avoid potential race conditions. */ @@ -7687,6 +7688,23 @@ int moduleVerifyAllAllowAtomicSlotMigrationOrReply(client *c) { return C_OK; } +/* Returns 0 if any module with registered data types did not declare + * VALKEYMODULE_OPTIONS_HANDLE_FORKLESS_SAVE, in which case forkless save should be + * blocked because the module's RDB save callbacks may not be thread-safe. */ +int moduleAllDatatypesHandleForklessSave(void) { + listIter li; + listNode *ln; + + listRewind(modules, &li); + while ((ln = listNext(&li)) != NULL) { + struct ValkeyModule *module = listNodeValue(ln); + if (listLength(module->types) && !(module->options & VALKEYMODULE_OPTIONS_HANDLE_FORKLESS_SAVE)) { + return 0; + } + } + return 1; +} + /* Returns true if any previous IO API failed. * for `Load*` APIs the VALKEYMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with * ValkeyModule_SetModuleOptions first. */ @@ -13533,6 +13551,12 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa ModuleLoadFunc onload; void *handle; + if (isForklessSaveInProgress()) { + serverLog(LL_WARNING, "Module %s failed to load: cannot load during forkless save.", path); + if (errmsg) *errmsg = "cannot load module during forkless save"; + return C_ERR; + } + if (server.async_loading) { serverLog(LL_WARNING, "Module %s failed to load: cannot load during async replication.", path); if (errmsg) *errmsg = "cannot load module during async replication"; @@ -14485,7 +14509,8 @@ int VM_RdbLoad(ValkeyModuleCtx *ctx, ValkeyModuleRdbStream *stream, int flags) { /* Kill existing RDB fork as it is saving outdated data. Also killing it * will prevent COW memory issue. */ - if (server.child_type == CHILD_TYPE_RDB) killRDBChild(); + if (isForkBgsaveInProgress()) killRDBChild(); + if (isForklessSaveInProgress()) forklessSaveCancel(); /* Kill existing slot migration fork as it is saving outdated data. Also killing it * will prevent COW memory issue. */ diff --git a/src/module.h b/src/module.h index e216a3b68cd..7042e8c9103 100644 --- a/src/module.h +++ b/src/module.h @@ -227,6 +227,7 @@ int TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(void); int moduleAllModulesHandleReplAsyncLoad(void); +int moduleAllDatatypesHandleForklessSave(void); int moduleVerifyAllAllowAtomicSlotMigrationOrReply(client *c); sds modulesCollectInfo(sds info, dict *sections_dict, int for_crash_report, int sections); void moduleFireServerEvent(uint64_t eid, int subid, void *data); diff --git a/src/rdb.c b/src/rdb.c index 951321db4ca..faab3f22660 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -49,6 +49,7 @@ #include "cluster_migrateslots.h" #include "compression.h" #include "compression_stream.h" +#include "forkless.h" #include #include @@ -1430,36 +1431,15 @@ ssize_t rdbSaveDb(rio *rdb, int dbid, int rdbflags, int rdbver, long *key_counte if ((res = rdbSaveLen(rdb, dbid)) < 0) goto werr; written += res; - /* Write the RESIZE DB opcode. */ - unsigned long long expires_size = kvstoreSize(db->expires) + kvstoreImportingSize(db->expires); - if ((res = rdbSaveType(rdb, RDB_OPCODE_RESIZEDB)) < 0) goto werr; - written += res; - if ((res = rdbSaveLen(rdb, db_size)) < 0) goto werr; - written += res; - if ((res = rdbSaveLen(rdb, expires_size)) < 0) goto werr; + /* Write the RESIZE DB opcode and slot-info hints. */ + if ((res = rdbSaveDbSizeHints(rdb, db, 1)) < 0) goto werr; written += res; kvs_it = kvstoreIteratorInit(db->keys, HASHTABLE_ITER_SAFE | HASHTABLE_ITER_PREFETCH_VALUES | HASHTABLE_ITER_INCLUDE_IMPORTING); - int last_slot = -1; /* Iterate this DB writing every entry */ void *next; while (kvstoreIteratorNext(kvs_it, &next)) { robj *o = next; - int curr_slot = kvstoreIteratorGetCurrentHashtableIndex(kvs_it); - /* Save slot info. */ - if (server.cluster_enabled && curr_slot != last_slot) { - sds slot_info = sdscatprintf(sdsempty(), "%i,%lu,%lu,%lu", curr_slot, - kvstoreHashtableSize(db->keys, curr_slot), - kvstoreHashtableSize(db->expires, curr_slot), - kvstoreHashtableSize(db->keys_with_volatile_items, curr_slot)); - if ((res = rdbSaveAuxFieldStrStr(rdb, "slot-info", slot_info)) < 0) { - sdsfree(slot_info); - goto werr; - } - written += res; - last_slot = curr_slot; - sdsfree(slot_info); - } sds keystr = objectGetKey(o); robj key; long long expire; @@ -1504,22 +1484,10 @@ ssize_t rdbSaveDb(rio *rdb, int dbid, int rdbflags, int rdbver, long *key_counte * integer pointed by 'error' is set to the value of errno just after the I/O * error. */ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) { - char magic[10]; - uint64_t cksum; long key_counter = 0; int j; - if (server.rdb_checksum && !(rdb->flags & RIO_FLAG_SKIP_RDB_CHECKSUM)) - rdb->update_cksum = rioGenericUpdateChecksum; - const char *magic_prefix = rdbUseValkeyMagic(rdbver) ? "VALKEY" : "REDIS0"; - serverAssert(rdbver >= 0 && rdbver <= RDB_VERSION); - snprintf(magic, sizeof(magic), "%s%03d", magic_prefix, rdbver); - if (rdbWriteRaw(rdb, magic, 9) == -1) goto werr; - if (rdbSaveInfoAuxFields(rdb, rdbflags, rsi) == -1) goto werr; - if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, VALKEYMODULE_AUX_BEFORE_RDB) == -1) goto werr; - - /* save functions */ - if (!(req & REPLICA_REQ_RDB_EXCLUDE_FUNCTIONS) && rdbSaveFunctions(rdb) == -1) goto werr; + if (rdbWriteHeader(rdb, req, rdbver, rdbflags, rsi) == C_ERR) goto werr; /* save all databases, skip this if we're in functions-only mode */ if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA)) { @@ -1531,16 +1499,7 @@ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveI } } - if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, VALKEYMODULE_AUX_AFTER_RDB) == -1) goto werr; - - /* EOF opcode */ - if (rdbSaveType(rdb, RDB_OPCODE_EOF) == -1) goto werr; - - /* RDB checksum field. It will be zero if checksum computation is disabled, the - * loading code skips the check in this case. */ - cksum = rdb->cksum; - memrev64ifbe(&cksum); - if (rioWrite(rdb, &cksum, 8) == 0) goto werr; + if (rdbWriteFooter(rdb, req) == C_ERR) goto werr; return C_OK; werr: @@ -1764,15 +1723,31 @@ int rdbSave(int req, char *filename, rdbSaveInfo *rsi, int rdbflags) { return C_OK; } +int isForkBgsaveInProgress(void) { + return server.child_type == CHILD_TYPE_RDB; +} + +int isSaveInProgress(void) { + return isForkBgsaveInProgress() || isForklessSaveInProgress(); +} + +/* Start a background save, choosing fork or forkless based on bgsave_type. */ +int rdbStartBgsave(int bgsave_type) { + if (bgsave_type == RDB_BGSAVE_TYPE_FORKLESS) { + return forklessSaveToDisk(server.rdb_filename); + } else { + rdbSaveInfo rsi, *rsiptr; + rsiptr = rdbPopulateSaveInfo(&rsi); + return rdbSaveBackground(REPLICA_REQ_NONE, server.rdb_filename, rsiptr, RDBFLAGS_NONE); + } +} + int rdbSaveBackground(int req, char *filename, rdbSaveInfo *rsi, int rdbflags) { pid_t childpid; if (hasActiveChildProcess()) return C_ERR; server.stat_rdb_saves++; - server.dirty_before_bgsave = server.dirty; - server.lastbgsave_try = time(NULL); - if ((childpid = serverFork(CHILD_TYPE_RDB)) == 0) { int retval; @@ -1792,12 +1767,12 @@ int rdbSaveBackground(int req, char *filename, rdbSaveInfo *rsi, int rdbflags) { /* Parent */ if (childpid == -1) { server.lastbgsave_status = C_ERR; + server.lastbgsave_try = time(NULL); serverLog(LL_WARNING, "Can't save in background: fork: %s", strerror(errno)); return C_ERR; } serverLog(LL_NOTICE, "Background saving started by pid %ld", (long)childpid); - server.rdb_save_time_start = time(NULL); - server.rdb_child_type = RDB_CHILD_TYPE_DISK; + rdbRecordStartMetrics(RDB_BGSAVE_TYPE_FORK); return C_OK; } return C_OK; /* unreached */ @@ -3161,14 +3136,19 @@ void stopLoading(int success) { void startSaving(int rdbflags) { /* Fire the persistence modules start event. */ int subevent; - if (rdbflags & RDBFLAGS_AOF_PREAMBLE && getpid() != server.pid) - subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_AOF_START; - else if (rdbflags & RDBFLAGS_AOF_PREAMBLE) - subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START; - else if (getpid() != server.pid) - subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_RDB_START; - else - subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START; + if (rdbflags & RDBFLAGS_AOF_PREAMBLE) { + if (getpid() != server.pid) { + subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_AOF_START; + } else { + subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START; + } + } else { + if (getpid() != server.pid || (rdbflags & RDBFLAGS_FORKLESS_SAVE)) { + subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_RDB_START; + } else { + subevent = VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START; + } + } moduleFireServerEvent(VALKEYMODULE_EVENT_PERSISTENCE, subevent, NULL); } @@ -3905,14 +3885,13 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) { /* A background saving child (BGSAVE) terminated its work. Handle this. * This function covers the case of actual BGSAVEs. */ static void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal, time_t save_end) { - if (!bysignal && exitcode == 0) { - serverLog(LL_NOTICE, "Background saving terminated with success"); - server.dirty = server.dirty - server.dirty_before_bgsave; - server.lastsave = save_end; - server.lastbgsave_status = C_OK; - } else if (!bysignal && exitcode != 0) { - serverLog(LL_WARNING, "Background saving error"); - server.lastbgsave_status = C_ERR; + if (!bysignal) { + if (exitcode == 0) { + serverLog(LL_NOTICE, "Background saving terminated with success"); + } else { + serverLog(LL_WARNING, "Background saving error"); + } + rdbRecordEndMetrics(RDB_BGSAVE_TYPE_FORK, (exitcode == 0) ? C_OK : C_ERR, save_end); } else { mstime_t latency; @@ -3924,7 +3903,7 @@ static void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal, time_t sav latencyTraceIfNeeded(rdb, rdb_unlink_temp_file, latency); /* SIGUSR1 is whitelisted, so we have a way to kill a child without * triggering an error condition. */ - if (bysignal != SIGUSR1) server.lastbgsave_status = C_ERR; + if (bysignal != SIGUSR1) rdbRecordEndMetrics(RDB_BGSAVE_TYPE_FORK, C_ERR, save_end); } } @@ -3957,18 +3936,17 @@ static void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { /* When a background RDB saving/transfer terminates, call the right handler. */ void backgroundSaveDoneHandler(int exitcode, int bysignal) { - int type = server.rdb_child_type; + int type = server.rdb_write_target; time_t save_end = time(NULL); - switch (server.rdb_child_type) { - case RDB_CHILD_TYPE_DISK: backgroundSaveDoneHandlerDisk(exitcode, bysignal, save_end); break; - case RDB_CHILD_TYPE_SOCKET: backgroundSaveDoneHandlerSocket(exitcode, bysignal); break; + switch (server.rdb_write_target) { + case RDB_WRITE_TARGET_DISK: backgroundSaveDoneHandlerDisk(exitcode, bysignal, save_end); break; + case RDB_WRITE_TARGET_SOCKET: backgroundSaveDoneHandlerSocket(exitcode, bysignal); break; default: serverPanic("Unknown RDB child type."); break; } - server.rdb_child_type = RDB_CHILD_TYPE_NONE; - server.rdb_save_time_last = save_end - server.rdb_save_time_start; - server.rdb_save_time_start = -1; + rdbClearSaveState(save_end); + /* Possibly there are replicas waiting for a BGSAVE in order to be served * (the first stage of SYNC is a bulk transfer of dump.rdb) */ updateReplicasWaitingBgsave((!bysignal && exitcode == 0) ? C_OK : C_ERR, type); @@ -4152,7 +4130,7 @@ int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi) { skip_rdb_checksum ? " while skipping RDB checksum for this transfer" : ""); server.rdb_save_time_start = time(NULL); - server.rdb_child_type = RDB_CHILD_TYPE_SOCKET; + server.rdb_write_target = RDB_WRITE_TARGET_SOCKET; if (dual_channel) { /* For dual channel sync, the main process no longer requires these RDB connections. */ zfree(conns); @@ -4171,7 +4149,7 @@ int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi) { } void saveCommand(client *c) { - if (server.child_type == CHILD_TYPE_RDB) { + if (isSaveInProgress()) { addReplyError(c, "Background save already in progress"); return; } @@ -4187,44 +4165,80 @@ void saveCommand(client *c) { } } -/* BGSAVE [SCHEDULE] */ +/* BGSAVE [SCHEDULE [FORK|FORKLESS]] | BGSAVE [FORK|FORKLESS] | BGSAVE CANCEL */ void bgsaveCommand(client *c) { int schedule = 0; - - /* The SCHEDULE option changes the behavior of BGSAVE when an AOF rewrite - * is in progress. Instead of returning an error a BGSAVE gets scheduled. */ - if (c->argc > 1) { - if (c->argc == 2 && !strcasecmp(objectGetVal(c->argv[1]), "schedule")) { + int chosen_save_type = RDB_BGSAVE_TYPE_NONE; + + /* BGSAVE can be invoked with the following options: + * - CANCEL: terminates an in-progress or scheduled BGSAVE (standalone only) + * - SCHEDULE: schedules a BGSAVE when an AOF rewrite is in progress. + * Instead of returning an error, the BGSAVE is scheduled to run + * when the AOF rewrite completes. + * - FORK: uses fork-based save (default) + * - FORKLESS: uses forkless save + * SCHEDULE can be combined with FORK or FORKLESS to specify the save method. */ + for (int i = 1; i < c->argc; i++) { + char *arg = objectGetVal(c->argv[i]); + if (!strcasecmp(arg, "schedule")) { schedule = 1; - } else if (c->argc == 2 && !strcasecmp(objectGetVal(c->argv[1]), "cancel")) { + } else if (!strcasecmp(arg, "cancel")) { + if (c->argc != 2) { + addReplyError(c, "Cancel cannot be combined with other options"); + return; + } /* Terminates an in progress BGSAVE */ - if (server.child_type == CHILD_TYPE_RDB) { - /* There is an ongoing bgsave */ - serverLog(LL_NOTICE, "Background saving will be aborted due to user request"); + if (isForkBgsaveInProgress()) { + /* There is an ongoing fork-based bgsave */ + serverLog(LL_NOTICE, "Background saving (fork) will be aborted due to user request"); killRDBChild(); addReplyStatus(c, "Background saving cancelled"); - } else if (server.rdb_bgsave_scheduled == 1) { + } else if (isForklessSaveInProgress()) { + /* There is an ongoing forkless save */ + serverLog(LL_NOTICE, "Background saving (forkless) will be aborted due to user request"); + forklessSaveCancel(); + addReplyStatus(c, "Background saving cancelled"); + } else if (server.rdb_bgsave_scheduled != RDB_BGSAVE_TYPE_NONE) { serverLog(LL_NOTICE, "Scheduled background saving will be cancelled due to user request"); - server.rdb_bgsave_scheduled = 0; + server.rdb_bgsave_scheduled = RDB_BGSAVE_TYPE_NONE; addReplyStatus(c, "Scheduled background saving cancelled"); } else { addReplyError(c, "Background saving is currently not in progress or scheduled"); } return; + } else if (!strcasecmp(arg, "fork")) { + chosen_save_type = RDB_BGSAVE_TYPE_FORK; + } else if (!strcasecmp(arg, "forkless")) { + if (!server.forkless_options_supported) { + addReplyError(c, "BGSAVE FORKLESS requires starting the server with forkless-options-supported enabled"); + return; + } + chosen_save_type = RDB_BGSAVE_TYPE_FORKLESS; } else { addReplyErrorObject(c, shared.syntaxerr); return; } } + /* If user didn't explicitly specify save type, let the system choose */ + if (chosen_save_type == RDB_BGSAVE_TYPE_NONE) { + chosen_save_type = (server.default_bgsave_method == RDB_BGSAVE_TYPE_FORKLESS && server.forkless_options_supported && moduleAllDatatypesHandleForklessSave()) + ? RDB_BGSAVE_TYPE_FORKLESS + : RDB_BGSAVE_TYPE_FORK; + } else if (chosen_save_type == RDB_BGSAVE_TYPE_FORKLESS && !moduleAllDatatypesHandleForklessSave()) { + addReplyError(c, "Can't use forkless save: one or more loaded modules have not declared " + "VALKEYMODULE_OPTIONS_HANDLE_FORKLESS_SAVE"); + return; + } + rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); - if (server.child_type == CHILD_TYPE_RDB) { + if (isSaveInProgress()) { addReplyError(c, "Background save already in progress"); } else if (hasActiveChildProcess() || server.in_exec) { if (schedule || server.in_exec) { - server.rdb_bgsave_scheduled = 1; + server.rdb_bgsave_scheduled = chosen_save_type; if (schedule) { serverLog(LL_NOTICE, "Background saving scheduled due to user request"); } else { @@ -4236,6 +4250,12 @@ void bgsaveCommand(client *c) { "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " "possible."); } + } else if (chosen_save_type == RDB_BGSAVE_TYPE_FORKLESS) { + if (forklessSaveToDisk(server.rdb_filename) == C_OK) { + addReplyStatus(c, "Background saving started"); + } else { + addReplyErrorObject(c, shared.err); + } } else if (rdbSaveBackground(REPLICA_REQ_NONE, server.rdb_filename, rsiptr, RDBFLAGS_NONE) == C_OK) { addReplyStatus(c, "Background saving started"); } else { @@ -4291,3 +4311,112 @@ rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi) { } return NULL; } + + +/* Write RESIZEDB and slot-info size hints for a single database. + * If include_importing is set, importing slot sizes are included (for fork-based save during migration). + * Returns bytes written on success, -1 on error. */ +ssize_t rdbSaveDbSizeHints(rio *rdb, serverDb *db, int include_importing) { + ssize_t res, written = 0; + + unsigned long long db_size = kvstoreSize(db->keys); + unsigned long long expires_size = kvstoreSize(db->expires); + if (include_importing) { + db_size += kvstoreImportingSize(db->keys); + expires_size += kvstoreImportingSize(db->expires); + } + + if ((res = rdbSaveType(rdb, RDB_OPCODE_RESIZEDB)) < 0) { + serverLog(LL_WARNING, "rdbSaveDbSizeHints: error writing OPCODE_RESIZEDB"); + return -1; + } + written += res; + if ((res = rdbSaveLen(rdb, db_size)) < 0) { + serverLog(LL_WARNING, "rdbSaveDbSizeHints: error writing db_size"); + return -1; + } + written += res; + if ((res = rdbSaveLen(rdb, expires_size)) < 0) { + serverLog(LL_WARNING, "rdbSaveDbSizeHints: error writing expires_size"); + return -1; + } + written += res; + + if (server.cluster_enabled) { + int slot = kvstoreGetFirstNonEmptyHashtableIndex(db->keys); + while (slot != -1) { + sds slot_info = sdscatprintf(sdsempty(), "%i,%lu,%lu,%lu", slot, + kvstoreHashtableSize(db->keys, slot), + kvstoreHashtableSize(db->expires, slot), + kvstoreHashtableSize(db->keys_with_volatile_items, slot)); + if ((res = rdbSaveAuxFieldStrStr(rdb, "slot-info", slot_info)) < 0) { + serverLog(LL_WARNING, "rdbSaveDbSizeHints: error writing slot-info for slot %d", slot); + sdsfree(slot_info); + return -1; + } + written += res; + sdsfree(slot_info); + slot = kvstoreGetNextNonEmptyHashtableIndex(db->keys, slot); + } + } + + return written; +} + +/* Write the RDB header: magic string, aux fields, module aux (before RDB), and functions. + * Returns C_OK on success, C_ERR on error. */ +int rdbWriteHeader(rio *rdb, int req, int rdbver, int rdbflags, rdbSaveInfo *rsi) { + char magic[10]; + if (server.rdb_checksum && !(rdb->flags & RIO_FLAG_SKIP_RDB_CHECKSUM)) { + rdb->update_cksum = rioGenericUpdateChecksum; + } + + const char *magic_prefix = rdbUseValkeyMagic(rdbver) ? "VALKEY" : "REDIS0"; + serverAssert(rdbver >= 0 && rdbver <= RDB_VERSION); + snprintf(magic, sizeof(magic), "%s%03d", magic_prefix, rdbver); + if (rdbWriteRaw(rdb, magic, 9) == -1) return C_ERR; + if (rdbSaveInfoAuxFields(rdb, rdbflags, rsi) == -1) return C_ERR; + if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, VALKEYMODULE_AUX_BEFORE_RDB) == -1) return C_ERR; + /* Save functions */ + if (!(req & REPLICA_REQ_RDB_EXCLUDE_FUNCTIONS) && rdbSaveFunctions(rdb) == -1) return C_ERR; + return C_OK; +} + +/* Write the RDB footer: module aux (after RDB), EOF opcode, and checksum. + * Returns C_OK on success, C_ERR on error. */ +int rdbWriteFooter(rio *rdb, int req) { + if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, VALKEYMODULE_AUX_AFTER_RDB) == -1) return C_ERR; + if (rdbSaveType(rdb, RDB_OPCODE_EOF) == -1) return C_ERR; + uint64_t cksum = rdb->cksum; + memrev64ifbe(&cksum); + if (rioWrite(rdb, &cksum, 8) == 0) return C_ERR; + return C_OK; +} + +/* Common state updates when a background save starts. */ +void rdbRecordStartMetrics(int bgsave_type) { + server.dirty_before_bgsave = server.dirty; + server.lastbgsave_try = time(NULL); + server.rdb_save_time_start = time(NULL); + server.rdb_write_target = RDB_WRITE_TARGET_DISK; + server.cur_bgsave_type = bgsave_type; +} + +/* Reset save timing and target state. Called after any background save or + * transfer completes, regardless of whether it was a persistence event. */ +void rdbClearSaveState(time_t save_end) { + server.rdb_save_time_last = save_end - server.rdb_save_time_start; + server.rdb_save_time_start = -1; + server.rdb_write_target = RDB_WRITE_TARGET_NONE; + server.cur_bgsave_type = RDB_BGSAVE_TYPE_NONE; +} + +/* Record persistence metrics when a background save completes. */ +void rdbRecordEndMetrics(int bgsave_type, int status, time_t save_end) { + server.lastbgsave_status = status; + server.lastbgsave_type = bgsave_type; + if (status == C_OK) { + server.dirty = server.dirty - server.dirty_before_bgsave; + server.lastsave = save_end; + } +} diff --git a/src/rdb.h b/src/rdb.h index 7bceb4cc906..9b6df0f3047 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -172,13 +172,14 @@ enum RdbType { #define RDB_LOAD_SDS (1 << 2) /* flags on the purpose of rdb save or load */ -#define RDBFLAGS_NONE 0 /* No special RDB loading or saving. */ -#define RDBFLAGS_AOF_PREAMBLE (1 << 0) /* Load/save the RDB as AOF preamble. */ -#define RDBFLAGS_REPLICATION (1 << 1) /* Load/save for SYNC. */ -#define RDBFLAGS_ALLOW_DUP (1 << 2) /* Allow duplicated keys when loading.*/ -#define RDBFLAGS_FEED_REPL (1 << 3) /* Feed replication stream when loading.*/ -#define RDBFLAGS_KEEP_CACHE (1 << 4) /* Don't reclaim cache after rdb file is generated */ -#define RDBFLAGS_EMPTY_DATA (1 << 5) /* Flush the database after validating magic and rdb version*/ +#define RDBFLAGS_NONE 0 /* No special RDB loading or saving. */ +#define RDBFLAGS_AOF_PREAMBLE (1 << 0) /* Load/save the RDB as AOF preamble. */ +#define RDBFLAGS_REPLICATION (1 << 1) /* Load/save for SYNC. */ +#define RDBFLAGS_ALLOW_DUP (1 << 2) /* Allow duplicated keys when loading.*/ +#define RDBFLAGS_FEED_REPL (1 << 3) /* Feed replication stream when loading.*/ +#define RDBFLAGS_KEEP_CACHE (1 << 4) /* Don't reclaim cache after rdb file is generated */ +#define RDBFLAGS_EMPTY_DATA (1 << 5) /* Flush the database after validating magic and rdb version*/ +#define RDBFLAGS_FORKLESS_SAVE (1 << 6) /* Save is performed by forkless save (background thread). */ /* When rdbLoadObject() returns NULL, the err flag is * set to hold the type of error that occurred */ @@ -201,6 +202,7 @@ int rdbGetObjectType(robj *o, int rdbver); int rdbLoadObjectType(rio *rdb); int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags); int rdbSaveBackground(int req, char *filename, rdbSaveInfo *rsi, int rdbflags); +int rdbStartBgsave(int bgsave_type); int rdbSaveToReplicasSockets(int req, int rdbver, rdbSaveInfo *rsi); void rdbRemoveTempFile(pid_t childpid, int from_signal); int rdbSaveToFile(const char *filename); @@ -245,5 +247,11 @@ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveI ssize_t rdbSaveFunctions(rio *rdb); rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi); void replicationEmptyDbCallback(hashtable *ht); +ssize_t rdbSaveDbSizeHints(rio *rdb, serverDb *db, int include_importing); +int rdbWriteHeader(rio *rdb, int req, int rdbver, int rdbflags, rdbSaveInfo *rsi); +int rdbWriteFooter(rio *rdb, int req); +void rdbRecordStartMetrics(int bgsave_type); +void rdbRecordEndMetrics(int bgsave_type, int status, time_t save_end); +void rdbClearSaveState(time_t save_end); #endif diff --git a/src/replication.c b/src/replication.c index ddde359f0f2..adfa8e8a620 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1244,7 +1244,7 @@ void syncCommand(client *c) { } /* CASE 1: BGSAVE is in progress, with disk target. */ - if (server.child_type == CHILD_TYPE_RDB && server.rdb_child_type == RDB_CHILD_TYPE_DISK) { + if (server.rdb_write_target == RDB_WRITE_TARGET_DISK) { /* Ok a background save is in progress. Let's check if it is a good * one for replication, i.e. if there is another replica that is * registering differences since the server forked to save. */ @@ -1279,7 +1279,7 @@ void syncCommand(client *c) { } /* CASE 2: BGSAVE is in progress, with socket target. */ - } else if (server.child_type == CHILD_TYPE_RDB && server.rdb_child_type == RDB_CHILD_TYPE_SOCKET) { + } else if (server.rdb_write_target == RDB_WRITE_TARGET_SOCKET) { /* There is an RDB child process but it is writing directly to * children sockets. We need to wait for the next BGSAVE * in order to synchronize. */ @@ -1344,7 +1344,7 @@ void freeClientReplicationData(client *c) { * should not remove directly since that means RDB is important for users * to keep data safe and we may delay configured 'save' for full sync. */ if (server.saveparamslen == 0 && c->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_END && - server.child_type == CHILD_TYPE_RDB && server.rdb_child_type == RDB_CHILD_TYPE_DISK && + server.child_type == CHILD_TYPE_RDB && server.rdb_write_target == RDB_WRITE_TARGET_DISK && anyOtherReplicaWaitRdb(c) == 0) { serverLog(LL_NOTICE, "Background saving, persistence disabled, last replica dropped, killing fork child."); killRDBChild(); @@ -2056,7 +2056,7 @@ void updateReplicasWaitingBgsave(int bgsaveerr, int type) { * already an RDB -> Replicas socket transfer, used in the case of * diskless replication, our work is trivial, we can just put * the replica online. */ - if (type == RDB_CHILD_TYPE_SOCKET) { + if (type == RDB_WRITE_TARGET_SOCKET) { serverLog(LL_NOTICE, "Streamed RDB transfer with replica %s succeeded (socket). Waiting for REPLCONF ACK from " "replica to enable streaming", @@ -5193,9 +5193,9 @@ void waitaofCommand(client *c) { return; } - /* Otherwise, block the client and put it into our list of clients - * waiting for ack from replicas. WAITAOF handles its own reply in - * pgit add rocessClientsWaitingReplicas, so clear pending_command to avoid + /* Otherwise block the client and put it into our list of clients + * waiting for ack from replicas. WAIT handles its own reply in + * processClientsWaitingReplicas, so clear pending_command to avoid * being mistaken for a command that needs re-execution. */ c->flag.pending_command = 0; blockClientForReplicaAck(c, timeout, offset, numreplicas, numlocal); @@ -5446,7 +5446,7 @@ void replicationCron(void) { int is_presync = (replica->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_START || - (replica->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_END && server.rdb_child_type != RDB_CHILD_TYPE_SOCKET)); + (replica->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_END && server.rdb_write_target != RDB_WRITE_TARGET_SOCKET)); if (is_presync) { connWrite(replica->conn, "\n", 1); @@ -5475,7 +5475,7 @@ void replicationCron(void) { * by the fork child so if a disk-based replica is stuck it doesn't prevent the fork child * from terminating. */ if (replica->repl_data->repl_state == REPLICA_STATE_WAIT_BGSAVE_END && - server.rdb_child_type == RDB_CHILD_TYPE_SOCKET) { + server.rdb_write_target == RDB_WRITE_TARGET_SOCKET) { if (replica->repl_data->repl_last_partial_write != 0 && (server.unixtime - replica->repl_data->repl_last_partial_write) > server.repl_timeout) { serverLog(LL_WARNING, "Disconnecting timedout replica (full sync): %s", @@ -5554,7 +5554,7 @@ int shouldStartChildReplication(int *mincapa_out, int *req_out, int *rdbver_out) * In case of diskless replication, we make sure to wait the specified * number of seconds (according to configuration) so that other replicas * have the time to arrive before we start streaming. */ - if (!hasActiveChildProcess()) { + if (!hasActiveSaveOrChild()) { time_t idle, max_idle = 0; int replicas_waiting = 0; int mincapa; diff --git a/src/rio.h b/src/rio.h index 1512a27a431..d3d73273efb 100644 --- a/src/rio.h +++ b/src/rio.h @@ -72,6 +72,12 @@ struct _rio { * computation. */ void (*update_cksum)(struct _rio *, const void *buf, size_t len); + /* Optional callback invoked between write chunks. If it returns + * non-zero, the write is aborted (rioWrite returns 0 to caller). + * Allows long-running operations that issue many writes (e.g. + * serializing a large collection) to be interrupted. */ + int (*check_abort_between_writes)(struct _rio *); + /* The current checksum and flags (see RIO_FLAG_*) */ uint64_t cksum, flags; @@ -168,6 +174,7 @@ static inline size_t rioWriteRaw(rio *r, const void *buf, size_t len) { static inline size_t rioWrite(rio *r, const void *buf, size_t len) { if (r->flags & RIO_FLAG_WRITE_ERROR || r->flags & RIO_FLAG_CLOSE_ASAP) return 0; while (len) { + if (r->check_abort_between_writes && r->check_abort_between_writes(r)) return 0; size_t bytes_to_write = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; if (r->update_cksum) r->update_cksum(r, buf, bytes_to_write); diff --git a/src/scripting_engine.c b/src/scripting_engine.c index 91b3b6d05dd..96f927f87f0 100644 --- a/src/scripting_engine.c +++ b/src/scripting_engine.c @@ -73,11 +73,6 @@ dictType engineDictType = { .entryDestructor = zfree, }; -static int isCalledFromAsyncThread(void) { - pthread_t curr_thread = pthread_self(); - return !pthread_equal(server.main_thread_id, curr_thread); -} - /* Initializes the scripting engine manager. * The engine manager is responsible for managing the several scripting engines * that are loaded in the server and implemented by Valkey Modules. @@ -313,7 +308,7 @@ void scriptingEngineCallFreeFunction(scriptingEngine *engine, subsystemType type, compiledFunction *compiled_func) { serverAssert(type == VMSE_EVAL || type == VMSE_FUNCTION); - int is_async = isCalledFromAsyncThread(); + int is_async = !onValkeyMainThread(); /* We need to acquire the module GIL when running from an async thread while * flushing the script functions. */ diff --git a/src/server.c b/src/server.c index 7af2ee2369c..0d592cdb5aa 100644 --- a/src/server.c +++ b/src/server.c @@ -53,6 +53,7 @@ #include "module.h" #include "scripting_engine.h" #include "util.h" +#include "forkless.h" #include "eval.h" #include "bgiteration.h" @@ -905,6 +906,12 @@ int hasActiveChildProcess(void) { return server.child_pid != -1; } +/* Returns true if a background save (fork or forkless) or child process is + * active. */ +int hasActiveSaveOrChild(void) { + return hasActiveChildProcess() || isSaveInProgress(); +} + void resetChildState(void) { server.child_type = CHILD_TYPE_NONE; server.child_pid = -1; @@ -1673,8 +1680,10 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa databasesCron(); /* Start a scheduled AOF rewrite if this was requested by the user while - * a BGSAVE was in progress. */ - if (!hasActiveChildProcess() && server.aof_rewrite_scheduled && !aofRewriteLimited()) { + * a BGSAVE was in progress. We don't start the rewrite if there is an + * active child process (to avoid multiple concurrent fork children) or if + * a forkless save is in progress (to avoid potential copy-on-write). */ + if (!hasActiveSaveOrChild() && server.aof_rewrite_scheduled && !aofRewriteLimited()) { rewriteAppendOnlyFileBackground(); } @@ -1682,7 +1691,7 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa if (hasActiveChildProcess() || scriptingEngineDebuggerPendingChildren()) { run_with_period(1000) receiveChildInfo(); checkChildrenDone(); - } else { + } else if (!isSaveInProgress()) { /* If there is not a background saving/rewrite in progress check if * we have to save/rewrite now. */ for (j = 0; j < server.saveparamslen; j++) { @@ -1696,15 +1705,19 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa (server.unixtime - server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) { serverLog(LL_NOTICE, "%d changes in %d seconds. Saving...", sp->changes, (int)sp->seconds); - rdbSaveInfo rsi, *rsiptr; - rsiptr = rdbPopulateSaveInfo(&rsi); - rdbSaveBackground(REPLICA_REQ_NONE, server.rdb_filename, rsiptr, RDBFLAGS_NONE); + int type = (server.default_bgsave_method == RDB_BGSAVE_TYPE_FORKLESS && + server.forkless_options_supported && + moduleAllDatatypesHandleForklessSave()) + ? RDB_BGSAVE_TYPE_FORKLESS + : RDB_BGSAVE_TYPE_FORK; + rdbStartBgsave(type); break; } } - /* Trigger an AOF rewrite if needed. */ - if (server.aof_state == AOF_ON && !hasActiveChildProcess() && server.aof_rewrite_perc && + /* Trigger an AOF rewrite if needed. Avoid starting while another child process + * is active. Also avoid when forkless save is in progress to prevent potential copy-on-write. */ + if (server.aof_state == AOF_ON && !hasActiveSaveOrChild() && server.aof_rewrite_perc && server.aof_current_size > server.aof_rewrite_min_size) { long long base = server.aof_rewrite_base_size ? server.aof_rewrite_base_size : 1; long long growth = (server.aof_current_size * 100 / base) - 100; @@ -1775,12 +1788,9 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa * Note: this code must be after the replicationCron() call above so * make sure when refactoring this file to keep this order. This is useful * because we want to give priority to RDB savings for replication. */ - if (!hasActiveChildProcess() && server.rdb_bgsave_scheduled && + if (!hasActiveSaveOrChild() && server.rdb_bgsave_scheduled && (server.unixtime - server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) { - rdbSaveInfo rsi, *rsiptr; - rsiptr = rdbPopulateSaveInfo(&rsi); - if (rdbSaveBackground(REPLICA_REQ_NONE, server.rdb_filename, rsiptr, RDBFLAGS_NONE) == C_OK) - server.rdb_bgsave_scheduled = 0; + if (rdbStartBgsave(server.rdb_bgsave_scheduled) == C_OK) server.rdb_bgsave_scheduled = RDB_BGSAVE_TYPE_NONE; } /* TLS auto-reload if enabled (only when TLS is built-in). */ @@ -3104,13 +3114,13 @@ void initServer(void) { server.client_pause_in_transaction = 0; server.child_pid = -1; server.child_type = CHILD_TYPE_NONE; - server.rdb_child_type = RDB_CHILD_TYPE_NONE; + server.rdb_write_target = RDB_WRITE_TARGET_NONE; server.rdb_pipe_conns = NULL; server.rdb_pipe_numconns = 0; server.rdb_pipe_numconns_writing = 0; server.rdb_pipe_buff = NULL; server.rdb_pipe_bufflen = 0; - server.rdb_bgsave_scheduled = 0; + server.rdb_bgsave_scheduled = RDB_BGSAVE_TYPE_NONE; server.child_info_pipe[0] = -1; server.child_info_pipe[1] = -1; server.child_info_nread = 0; @@ -3145,6 +3155,8 @@ void initServer(void) { server.cron_malloc_stats.allocator_active = 0; server.cron_malloc_stats.allocator_resident = 0; server.lastbgsave_status = C_OK; + server.lastbgsave_type = RDB_BGSAVE_TYPE_NONE; + server.cur_bgsave_type = RDB_BGSAVE_TYPE_NONE; server.aof_last_write_status = C_OK; server.aof_last_write_errno = 0; server.repl_good_replicas_count = 0; @@ -5078,7 +5090,7 @@ int finishShutdown(void) { /* Kill the saving child if there is a background saving in progress. We want to avoid race conditions, for instance our saving child may overwrite the synchronous saving did by SHUTDOWN. */ - if (server.child_type == CHILD_TYPE_RDB) { + if (isForkBgsaveInProgress()) { serverLog(LL_WARNING, "There is a child saving an .rdb. Killing it!"); killRDBChild(); /* Note that, in killRDBChild normally has backgroundSaveDoneHandler @@ -5089,6 +5101,10 @@ int finishShutdown(void) { * but OS will close this fd when process exits. */ rdbRemoveTempFile(server.child_pid, 0); } + if (isForklessSaveInProgress()) { + serverLog(LL_WARNING, "There is a thread saving an .rdb. Cancelling it!"); + forklessSaveCancel(); + } /* Kill module child if there is one. */ if (server.child_type == CHILD_TYPE_MODULE) { @@ -6563,14 +6579,30 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { /* Persistence */ if (all_sections || (dictFind(section_dict, "persistence") != NULL)) { if (sections++) info = sdscat(info, "\r\n"); - double fork_perc = 0; + double save_perc = 0; if (server.stat_module_progress) { - fork_perc = server.stat_module_progress * 100; + save_perc = server.stat_module_progress * 100; } else if (server.stat_current_save_keys_total) { - fork_perc = ((double)server.stat_current_save_keys_processed / server.stat_current_save_keys_total) * 100; + save_perc = ((double)server.stat_current_save_keys_processed / server.stat_current_save_keys_total) * 100; } int aof_bio_fsync_status = atomic_load_explicit(&server.aof_bio_fsync_status, memory_order_relaxed); + /* Determine current bgsave type */ + const char *current_bgsave_type; + switch (server.cur_bgsave_type) { + case RDB_BGSAVE_TYPE_FORK: current_bgsave_type = "fork"; break; + case RDB_BGSAVE_TYPE_FORKLESS: current_bgsave_type = "forkless"; break; + default: current_bgsave_type = "none"; break; + } + + /* Determine last bgsave type */ + const char *last_bgsave_type; + switch (server.lastbgsave_type) { + case RDB_BGSAVE_TYPE_FORK: last_bgsave_type = "fork"; break; + case RDB_BGSAVE_TYPE_FORKLESS: last_bgsave_type = "forkless"; break; + default: last_bgsave_type = "none"; break; + } + info = sdscatprintf( info, "# Persistence\r\n" FMTARGS( @@ -6579,15 +6611,17 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "current_cow_peak:%zu\r\n", server.stat_current_cow_peak, "current_cow_size:%zu\r\n", server.stat_current_cow_bytes, "current_cow_size_age:%lu\r\n", (server.stat_current_cow_updated ? (unsigned long)elapsedMs(server.stat_current_cow_updated) / 1000 : 0), - "current_fork_perc:%.2f\r\n", fork_perc, + "current_fork_perc:%.2f\r\n", save_perc, "current_save_keys_processed:%zu\r\n", server.stat_current_save_keys_processed, "current_save_keys_total:%zu\r\n", server.stat_current_save_keys_total, "rdb_changes_since_last_save:%lld\r\n", server.dirty, - "rdb_bgsave_in_progress:%d\r\n", server.child_type == CHILD_TYPE_RDB, + "rdb_bgsave_in_progress:%d\r\n", isSaveInProgress(), + "rdb_current_bgsave_type:%s\r\n", current_bgsave_type, + "rdb_last_bgsave_type:%s\r\n", last_bgsave_type, "rdb_last_save_time:%jd\r\n", (intmax_t)server.lastsave, "rdb_last_bgsave_status:%s\r\n", (server.lastbgsave_status == C_OK) ? "ok" : "err", "rdb_last_bgsave_time_sec:%jd\r\n", (intmax_t)server.rdb_save_time_last, - "rdb_current_bgsave_time_sec:%jd\r\n", (intmax_t)((server.child_type != CHILD_TYPE_RDB) ? -1 : time(NULL) - server.rdb_save_time_start), + "rdb_current_bgsave_time_sec:%jd\r\n", (intmax_t)(isSaveInProgress() ? time(NULL) - server.rdb_save_time_start : -1), "rdb_saves:%lld\r\n", server.stat_rdb_saves, "rdb_last_cow_size:%zu\r\n", server.stat_rdb_cow_bytes, "rdb_last_load_keys_expired:%lld\r\n", server.rdb_last_load_keys_expired, @@ -6652,6 +6686,9 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "loading_loaded_perc:%.2f\r\n", perc, "loading_eta_seconds:%jd\r\n", (intmax_t)eta)); } + + /* Forkless / bgiteration metrics */ + info = forkless_catInfo(info); } /* Stats */ @@ -7005,6 +7042,9 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "eventloop_cmd_per_cycle_max:%lld\r\n", server.el_cmd_cnt_max, "io_threaded_reads_pending:%lld\r\n", server.stat_io_reads_pending, "io_threaded_writes_pending:%lld\r\n", server.stat_io_writes_pending)); + + /* Forkless debug metrics */ + info = forkless_catDebugInfo(info); } return info; diff --git a/src/server.h b/src/server.h index 04819ef6256..d4f85dec095 100644 --- a/src/server.h +++ b/src/server.h @@ -147,6 +147,8 @@ struct ValkeyModule; #define C_ERR -1 #define C_RETRY -2 +#define onValkeyMainThread() (pthread_equal(server.main_thread_id, pthread_self()) != 0) + /* Static server configuration */ #define CONFIG_DEFAULT_HZ 10 /* Time interrupt calls/sec. */ #define CONFIG_MIN_HZ 1 @@ -664,10 +666,19 @@ typedef enum { CLUSTER_CONFIGFILE_SAVE_BEHAVIOR_BEST_EFFORT, /* Attempt to save on a "best-effort" basis, process will not exit if it fails. */ } cluster_persist_config_mode; -/* RDB active child save type. */ -#define RDB_CHILD_TYPE_NONE 0 -#define RDB_CHILD_TYPE_DISK 1 /* RDB is written to disk. */ -#define RDB_CHILD_TYPE_SOCKET 2 /* RDB is written to replica socket. */ +/* RDB write target type. */ +typedef enum { + RDB_WRITE_TARGET_NONE = 0, + RDB_WRITE_TARGET_DISK = 1, /* RDB is written to disk. */ + RDB_WRITE_TARGET_SOCKET = 2 /* RDB is written to replica socket. */ +} rdbWriteTarget; + +/* RDB bgsave type. */ +typedef enum { + RDB_BGSAVE_TYPE_NONE = 0, + RDB_BGSAVE_TYPE_FORK = 1, /* Fork-based bgsave. */ + RDB_BGSAVE_TYPE_FORKLESS = 2 /* Forkless bgsave. */ +} rdbBgsaveType; /* Keyspace changes notification classes. Every class is associated with a * character for configuration purposes. */ @@ -1941,8 +1952,8 @@ struct valkeyServer { size_t stat_current_cow_peak; /* Peak size of copy on write bytes. */ size_t stat_current_cow_bytes; /* Copy on write bytes while child is active. */ monotime stat_current_cow_updated; /* Last update time of stat_current_cow_bytes */ - size_t stat_current_save_keys_processed; /* Processed keys while child is active. */ - size_t stat_current_save_keys_total; /* Number of keys when child started. */ + _Atomic(size_t) stat_current_save_keys_processed; /* Processed keys while save is active. */ + _Atomic(size_t) stat_current_save_keys_total; /* Number of keys when save started. */ size_t stat_rdb_cow_bytes; /* Copy on write bytes during RDB saving. */ size_t stat_aof_cow_bytes; /* Copy on write bytes during AOF rewrite. */ size_t stat_module_cow_bytes; /* Copy on write bytes during module fork. */ @@ -2088,8 +2099,10 @@ struct valkeyServer { time_t lastbgsave_try; /* Unix time of last attempted bgsave */ time_t rdb_save_time_last; /* Time used by last RDB save run. */ time_t rdb_save_time_start; /* Current RDB save start time. */ - int rdb_bgsave_scheduled; /* BGSAVE when possible if true. */ - int rdb_child_type; /* Type of save by active child. */ + rdbBgsaveType rdb_bgsave_scheduled; /* BGSAVE when possible if non-zero. */ + rdbWriteTarget rdb_write_target; /* Type of save by active child. */ + rdbBgsaveType cur_bgsave_type; /* Current bgsave type. */ + rdbBgsaveType lastbgsave_type; /* Last completed bgsave type. */ int lastbgsave_status; /* C_OK or C_ERR */ int stop_writes_on_bgsave_err; /* Don't allow writes if can't BGSAVE */ int rdb_pipe_read; /* RDB pipe used to transfer the rdb data */ @@ -2103,6 +2116,7 @@ struct valkeyServer { int rdb_key_save_delay; /* Delay in microseconds between keys while * writing aof or rdb. (for testings). negative * value means fractions of microseconds (on average). */ + int default_bgsave_method; /* Default bgsave method: RDB_BGSAVE_TYPE_FORK or RDB_BGSAVE_TYPE_FORKLESS */ int key_load_delay; /* Delay in microseconds between keys while * loading aof or rdb. (for testings). negative * value means fractions of microseconds (on average). */ @@ -3367,6 +3381,9 @@ void receiveChildInfo(void); /* Fork helpers */ int serverFork(int purpose); int hasActiveChildProcess(void); +int isSaveInProgress(void); +int hasActiveSaveOrChild(void); +int isForkBgsaveInProgress(void); void resetChildState(void); int isMutuallyExclusiveChildType(int type); diff --git a/src/unit/test_bgiteration.cpp b/src/unit/test_bgiteration.cpp index ca43c895e8d..5d6f06d4adb 100644 --- a/src/unit/test_bgiteration.cpp +++ b/src/unit/test_bgiteration.cpp @@ -1,7 +1,7 @@ /* * Copyright Valkey Contributors. * All rights reserved. - * SPDX-License-Identifier: BSD 3-Clause + * SPDX-License-Identifier: BSD-3-Clause */ #include "generated_wrappers.hpp" diff --git a/src/valkeymodule.h b/src/valkeymodule.h index ed7b2922294..742d5c8c912 100644 --- a/src/valkeymodule.h +++ b/src/valkeymodule.h @@ -342,10 +342,15 @@ typedef uint64_t ValkeyModuleTimerID; * slot migration must be used. */ #define VALKEYMODULE_OPTIONS_HANDLE_ATOMIC_SLOT_MIGRATION (1 << 5) +/* Declare that the module's RDB save callbacks are thread-safe and can be + * invoked from a background thread during forkless save. When not set by any + * module that has registered data types, forkless save will be blocked. */ +#define VALKEYMODULE_OPTIONS_HANDLE_FORKLESS_SAVE (1 << 6) + /* Next option flag, must be updated when adding new module flags above! * This flag should not be used directly by the module. * Use ValkeyModule_GetModuleOptionsAll instead. */ -#define _VALKEYMODULE_OPTIONS_FLAGS_NEXT (1 << 6) +#define _VALKEYMODULE_OPTIONS_FLAGS_NEXT (1 << 7) /* Definitions for ValkeyModule_SetCommandInfo. */ diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index b312965e9bb..7571a23d2cc 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -224,96 +224,1134 @@ start_server_and_kill_it [list "dir" $server_path] { } } -start_server {} { - test {Test FLUSHALL aborts bgsave} { - r config set save "" - # 5000 keys with 1ms sleep per key should take 5 second - r config set rdb-key-save-delay 1000 - populate 5000 - assert_lessthan 999 [s rdb_changes_since_last_save] - r bgsave - assert_equal [s rdb_bgsave_in_progress] 1 - r flushall - # wait a second max (bgsave should take 5) +start_server {overrides {forkless-options-supported yes save ""}} { + foreach bgsave_type {"" "fork" "forkless"} { + test "Test FLUSHALL aborts bgsave $bgsave_type" { + # 5000 keys with 1ms sleep per key should take 5 second + r config set rdb-key-save-delay 1000 + populate 5000 + assert_lessthan 999 [s rdb_changes_since_last_save] + r bgsave {*}$bgsave_type + assert_equal [s rdb_bgsave_in_progress] 1 + + # Verify we're testing the right save type while it's running + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_current_bgsave_type] $expected_type + + r flushall + # wait a second max (bgsave should take 5) + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 0 + } else { + fail "bgsave not aborted" + } + # verify that bgsave failed, by checking that the change counter is still high + assert_lessthan 999 [s rdb_changes_since_last_save] + # make sure the server is still writable + r set x xx + } + } + + foreach bgsave_type {"" "fork" "forkless"} { + test "bgsave $bgsave_type resets the change counter" { + r config set rdb-key-save-delay 0 + r bgsave {*}$bgsave_type + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 0 + } else { + fail "bgsave not done" + } + assert_equal [s rdb_changes_since_last_save] 0 + + # Verify we tested the right save type + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_last_bgsave_type] $expected_type + } + } + + foreach bgsave_type {"fork" "forkless"} { + test "bgsave $bgsave_type metrics are correct after success" { + set saves_before [s rdb_saves] + populate 100 "" 16 + r bgsave $bgsave_type + waitForBgsave r + assert {[s rdb_saves] == $saves_before + 1} + assert {[s rdb_last_bgsave_time_sec] >= 0 && [s rdb_last_bgsave_time_sec] < 3600} + assert_equal [s rdb_last_bgsave_status] "ok" + assert_equal [s rdb_last_bgsave_type] $bgsave_type + assert {[s rdb_bgsave_in_progress] == 0} + assert {[s current_fork_perc] == 0} + assert {[s current_save_keys_processed] == 0} + assert {[s current_save_keys_total] == 0} + } + } + + foreach bgsave_type {"fork" "forkless"} { + test "bgsave $bgsave_type metrics are correct after failure" { + set saves_before [s rdb_saves] + populate 1000 "" 16 + r config set rdb-key-save-delay 10000000 + r bgsave $bgsave_type + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "$bgsave_type bgsave didn't start" + } + if {$bgsave_type eq "fork"} { + set pid [get_child_pid 0] + catch {exec kill -9 $pid} + } else { + r flushdb + } + waitForBgsave r + assert {[s rdb_last_bgsave_time_sec] >= 0 && [s rdb_last_bgsave_time_sec] < 3600} + assert_equal [s rdb_last_bgsave_status] "err" + assert_equal [s rdb_last_bgsave_type] $bgsave_type + assert {[s rdb_bgsave_in_progress] == 0} + assert {[s current_fork_perc] == 0} + assert {[s current_save_keys_processed] == 0} + assert {[s current_save_keys_total] == 0} + r config set rdb-key-save-delay 0 + } + } + + foreach bgsave_type {"" "fork" "forkless"} { + test "bgsave cancel aborts $bgsave_type save" { + # Generating RDB will take some 100 seconds + r config set rdb-key-save-delay 1000000 + populate 100 "" 16 + + r bgsave {*}$bgsave_type + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave did not start in time" + } + + # Verify we're testing the right save type + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_current_bgsave_type] $expected_type + + if {$bgsave_type ne "forkless"} { + set fork_child_pid [get_child_pid 0] + } + + assert {[r bgsave cancel] eq {Background saving cancelled}} + + if {$bgsave_type ne "forkless"} { + set temp_rdb [file join [lindex [r config get dir] 1] temp-${fork_child_pid}.rdb] + # Temp rdb must be deleted + wait_for_condition 50 100 { + ![file exists $temp_rdb] + } else { + fail "bgsave temp file was not deleted after cancel" + } + } + + # Make sure no save is running and that bgsave return an error + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 0 + } else { + fail "bgsave is currently running" + } + assert_error "ERR Background saving is currently not in progress or scheduled" {r bgsave cancel} + } + } +} + +start_server {overrides {forkless-options-supported yes save ""}} { + test "forkless bgsave contains expired keys from when save started" { + + # Set two keys that expire together + r set k1 v1 + r set k2 v2 + set curr_time [clock seconds] + r expireat k1 [expr {$curr_time + 2}] + r expireat k2 [expr {$curr_time + 2}] + + # Start slow forkless save + r config set rdb-key-save-delay 10000000 + r bgsave forkless + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless bgsave did not start" + } + + # Let both keys expire + after 3000 + + # Serialize k1 in the foreground by touching it + r set k1 v11 + + # Complete forkless save so k2 will be serialized in background + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Check both keys are in the RDB + set rdb_path [file join [lindex [r config get dir] 1] [lindex [r config get dbfilename] 1]] + set fd [open $rdb_path rb] + set rdb_content [read $fd] + close $fd + assert {[string first "k1" $rdb_content] != -1} + assert {[string first "k2" $rdb_content] != -1} + } {} {needs:debug} +} + +start_server {overrides {forkless-options-supported yes save ""}} { + test "FLUSHDB during single-db forkless bgsave causes save to fail" { + + # Populate database with complex dataset + createComplexDataset r 100 + + # Get initial key count + set initial_keys [r dbsize] + assert {$initial_keys > 0} + + # Start forkless save with very slow save (high delay per key) + r config set rdb-key-save-delay 10000 + r bgsave forkless + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless bgsave did not start" + } + + # FLUSHDB should cancel the save + r flushdb + assert_equal [r dbsize] 0 + + # Speed up and wait for save to abort + # Note: Cancellation needs to be processed by background thread + r config set rdb-key-save-delay 0 wait_for_condition 50 100 { [s rdb_bgsave_in_progress] == 0 } else { - fail "bgsave not aborted" + fail "forkless bgsave did not abort" } - # verify that bgsave failed, by checking that the change counter is still high - assert_lessthan 999 [s rdb_changes_since_last_save] - # make sure the server is still writable - r set x xx - } + + # Verify save failed + assert_equal [s rdb_last_bgsave_status] err + assert_equal [s rdb_last_bgsave_type] forkless + } {} {needs:debug} +} - test {bgsave resets the change counter} { +start_server {overrides {forkless-options-supported yes save ""}} { + test "FLUSHDB during multi-db forkless bgsave causes save to fail" { + + # Populate multiple databases + for {set i 0} {$i < 100} {incr i} { + r set key$i val$i + } + r select 1 + for {set i 0} {$i < 100} {incr i} { + r set key$i val$i + } + r select 0 + + # Start slow forkless save + r config set rdb-key-save-delay 10000 + r bgsave forkless + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless bgsave did not start" + } + + # Give forkless save time to start iterating + after 100 + + # FLUSHDB on db 1 while save is running - this should terminate the save + r select 1 + r flushdb + assert_equal [r dbsize] 0 + r select 0 + + # Resume save speed and wait for save to abort r config set rdb-key-save-delay 0 - r bgsave wait_for_condition 50 100 { [s rdb_bgsave_in_progress] == 0 } else { - fail "bgsave not done" + fail "forkless bgsave did not abort" } - assert_equal [s rdb_changes_since_last_save] 0 - } + + # Forkless save should have failed + assert_equal [s rdb_last_bgsave_status] err + } {} {needs:debug} +} - test {bgsave cancel aborts save} { - r config set save "" - # Generating RDB will take some 100 seconds - r config set rdb-key-save-delay 1000000 - populate 100 "" 16 +start_server {overrides {forkless-options-supported yes save ""}} { + test "multiple databases modifications during forkless bgsave" { + + # Populate 5 databases with all data types + for {set db 0} {$db < 5} {incr db} { + r select $db + createComplexDatasetForVerification r 20 "db${db}_" + } + r select 0 + + # Start slow forkless save + r config set rdb-key-save-delay 10000 + r bgsave forkless + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless bgsave did not start" + } + + # Modify keys in all databases while save is running + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + r append db${db}_before_$i "value_after_$i" + r incr db${db}_int_$i + r set db${db}_after_$i "VALUE_AFTER_$i" + r lpush db${db}_lst_$i "LL2" "LL1" + r rpush db${db}_lst_$i "RR1" "RR2" + r sadd db${db}_set_$i "BB1" "BB2" + r zadd db${db}_zset_$i 5 "Z2" + r hset db${db}_hash_$i "H1" "c" + r pfadd db${db}_hll_$i "PF2" + r geoadd db${db}_geo_$i -122.1592 47.5976 "bellevue" + r xadd db${db}_stream_$i "*" "D1" "V2" + r xreadgroup GROUP db${db}_group_$i consumer_after_$i COUNT 1 STREAMS db${db}_stream_$i > + r bitfield db${db}_bits_$i SET u4 0 0 INCRBY u4 0 1 + r geosearchstore db${db}_geo_set_$i db${db}_geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi + r geosearchstore db${db}_geo_set_dist_$i db${db}_geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi ASC COUNT 10 STOREDIST + } + } + r select 0 + + # Verify modifications happened in live database + assert {[s rdb_changes_since_last_save] > 0} + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get db${db}_before_$i] "value_before_${i}value_after_$i" + } + } + r select 0 + + # Speed up and complete save + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Verify save completed successfully + assert_equal [s rdb_last_bgsave_status] ok + + # Reload from RDB and verify ORIGINAL values are preserved + # (consistent snapshot should capture state at start of save) + catch {r debug reload nosave} + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + # Strings: original value, not appended + assert_equal [r get db${db}_before_$i] "value_before_$i" + # Ints: original value, not incremented + assert_equal [r get db${db}_int_$i] [expr {42 + $i}] + # New keys should not exist + assert_equal [r exists db${db}_after_$i] 0 + # Lists: original 4 elements, not 8 + assert_equal [r llen db${db}_lst_$i] 4 + assert_equal [r lrange db${db}_lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + # Sets: original 2 members + assert_equal [r scard db${db}_set_$i] 2 + assert_equal [lsort [r smembers db${db}_set_$i]] [list "B1" "B2"] + # Sorted sets: original score + assert_equal [r zscore db${db}_zset_$i "Z2"] 2 + # Hashes: original value + assert_equal [r hget db${db}_hash_$i "H1"] "a" + # HLL: original count + assert_equal [r pfcount db${db}_hll_$i] 1 + # Geo: original 1 member + assert_equal [r zcard db${db}_geo_$i] 1 + assert_equal [r zcard db${db}_geo_set_$i] 1 + } + } + } {} {needs:debug} +} - r bgsave +start_server {overrides {forkless-options-supported yes save ""}} { + test "modify new keys during forkless bgsave" { + + # Populate database with all data types + createComplexDatasetForVerification r 20 + set original_keys [r dbsize] + + # Start forkless save with very slow save (high delay per key) + r config set rdb-key-save-delay 10000 + r bgsave forkless wait_for_condition 50 100 { [s rdb_bgsave_in_progress] == 1 } else { - fail "bgsave did not start in time" + fail "forkless bgsave did not start" + } + + # Create new keys of all data types while save is running + for {set i 0} {$i < 100} {incr i} { + r set after_$i "value_after_$i" + r set after_i_$i 42 + r lpush after_lst_$i "L2" "L1" + r rpush after_lst_$i "R1" "R2" + r sadd after_set_$i "B1" + r sadd after_iset_$i 12 34 + r zadd after_zset_$i 1 "Z1" + r hset after_hash_$i "H1" "a" + r pfadd after_hll_$i "PF1" + r set after_bits_$i "\x0f" + r bitfield after_bits_$i SET u4 0 0 INCRBY u4 0 1 + r geoadd after_geo_$i -122.345 47.775 "costco" + r geosearchstore after_geo_set_$i after_geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi + r geosearchstore after_geo_set_dist_$i after_geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi ASC COUNT 10 STOREDIST + r xadd after_stream_$i "*" "D1" "V2" + r xgroup create after_stream_$i after_group_$i 0 + r hsetex after_hashttl_$i EX 10000 FIELDS 1 HTTL1 a } - set fork_child_pid [get_child_pid 0] - assert {[r bgsave cancel] eq {Background saving cancelled}} - set temp_rdb [file join [lindex [r config get dir] 1] temp-${fork_child_pid}.rdb] - # Temp rdb must be deleted + # Verify new keys were created (14 key types per iteration × 100 iterations) + set expected_keys [expr {$original_keys + 100 * 14}] + assert_equal [r dbsize] $expected_keys + assert_equal [s rdb_bgsave_in_progress] 1 + + # Speed up and complete save + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Verify save completed successfully + assert_equal [s rdb_last_bgsave_status] ok + + # Reload and verify ONLY original keys exist (new keys should NOT be in snapshot) + catch {r debug reload nosave} + assert_equal [r dbsize] $original_keys + + # Verify all original data types preserved + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r llen lst_$i] 4 + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [r scard set_$i] 2 + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + + # Verify new keys do NOT exist in snapshot + for {set i 0} {$i < 100} {incr i} { + assert_equal [r exists after_$i] 0 + assert_equal [r exists after_lst_$i] 0 + assert_equal [r exists after_set_$i] 0 + assert_equal [r exists after_zset_$i] 0 + assert_equal [r exists after_hash_$i] 0 + assert_equal [r exists after_hll_$i] 0 + assert_equal [r exists after_geo_$i] 0 + assert_equal [r exists after_geo_set_$i] 0 + assert_equal [r exists after_geo_set_dist_$i] 0 + assert_equal [r exists after_stream_$i] 0 + assert_equal [r exists after_hashttl_$i] 0 + } + } {} {needs:debug} +} + +start_server {overrides {forkless-options-supported yes save ""}} { + test "SWAPDB during forkless bgsave" { + + # Populate 5 databases with all data types + for {set db 0} {$db < 5} {incr db} { + r select $db + createComplexDatasetForVerification r 20 "db${db}_" + } + r select 0 + + # Start slow forkless save + r config set rdb-key-save-delay 10000 + r bgsave forkless wait_for_condition 50 100 { - ![file exists $temp_rdb] + [s rdb_bgsave_in_progress] == 1 } else { - fail "bgsave temp file was not deleted after cancel" + fail "forkless bgsave did not start" + } + + # Keep swapping databases while save is running + set perm [list 0 1 2 3 4] + set swaps 0 + while {[s rdb_bgsave_in_progress] == 1 && $swaps < 200} { + incr swaps + # Shuffle permutation + for {set i 4} {$i > 0} {incr i -1} { + set j [expr {int(rand() * ($i + 1))}] + set temp [lindex $perm $i] + lset perm $i [lindex $perm $j] + lset perm $j $temp + } + # Swap each database with its permuted target + for {set db 0} {$db < 5} {incr db} { + r swapdb $db [lindex $perm $db] + } + } + + # Speed up save and wait for completion + r config set rdb-key-save-delay 0 + waitForBgsave r + assert {$swaps > 100} + + # Verify save completed successfully + assert_equal [s rdb_last_bgsave_status] ok + + # Reload from RDB and verify keys are in ORIGINAL databases + # (SWAPDB is ignored for consistent snapshots) + r select 0 + catch {r debug reload nosave} + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get db${db}_before_$i] "value_before_$i" + assert_equal [r get db${db}_int_$i] [expr {42 + $i}] + assert_equal [r lrange db${db}_lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers db${db}_set_$i]] [list "B1" "B2"] + assert_equal [r zscore db${db}_zset_$i "Z1"] 1 + assert_equal [r hget db${db}_hash_$i "H1"] "a" + assert_equal [r pfcount db${db}_hll_$i] 1 + assert_equal [r zcard db${db}_geo_$i] 1 + assert_equal [r zcard db${db}_geo_set_$i] 1 + } } + } {} {needs:debug} +} - # Make sure no save is running and that bgsave return an error - wait_for_condition 50 100 { - [s rdb_bgsave_in_progress] == 0 +start_server {overrides {forkless-options-supported yes save ""}} { + test "delete all keys after SWAPDB during forkless bgsave" { + + # Populate 5 databases with all data types + for {set db 0} {$db < 5} {incr db} { + r select $db + createComplexDatasetForVerification r 20 "db${db}_" + } + r select 0 + + # Start slow forkless save + r config set rdb-key-save-delay 10000 + r bgsave forkless + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 } else { - fail "bgsave is currently running" + fail "forkless bgsave did not start" } - assert_error "ERR Background saving is currently not in progress or scheduled" {r bgsave cancel} - } + + # Swap databases with fixed permutation [2, 3, 4, 0, 1] + set perm [list 2 3 4 0 1] + for {set db 0} {$db < 5} {incr db} { + r swapdb $db [lindex $perm $db] + } + + # Delete all keys in all databases + for {set db 4} {$db >= 0} {incr db -1} { + r select $db + set keys [r keys *] + foreach key $keys { + r del $key + } + assert_equal [r dbsize] 0 + } + + # Speed up and complete save + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Reload from RDB and verify ORIGINAL keys still exist + # Consistent snapshot should preserve state before SWAPDB and deletions + r select 0 + catch {r debug reload nosave} + for {set db 0} {$db < 5} {incr db} { + r select $db + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get db${db}_before_$i] "value_before_$i" + assert_equal [r get db${db}_int_$i] [expr {42 + $i}] + assert_equal [r lrange db${db}_lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers db${db}_set_$i]] [list "B1" "B2"] + assert_equal [r zscore db${db}_zset_$i "Z1"] 1 + assert_equal [r hget db${db}_hash_$i "H1"] "a" + assert_equal [r pfcount db${db}_hll_$i] 1 + assert_equal [r zcard db${db}_geo_$i] 1 + assert_equal [r zcard db${db}_geo_set_$i] 1 + } + } + } {} {needs:debug} +} - test {bgsave cancel schedulled request} { - r config set save "" - # Generating RDB will take some 100 seconds - r config set rdb-key-save-delay 1000000 - populate 100 "" 16 +start_server {overrides {forkless-options-supported yes save ""}} { + test "deleting keys during forkless bgsave" { + + # Populate database with all data types + createComplexDatasetForVerification r 20 + + # Start forkless save with very slow save (high delay per key) + r config set rdb-key-save-delay 10000 + r bgsave forkless + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless bgsave did not start" + } + + # Delete all keys in the database + set keys [r keys *] + foreach key $keys { + r del $key + } + + # Verify all keys deleted and save still in progress + assert_equal [r dbsize] 0 + assert_equal [s rdb_bgsave_in_progress] 1 + + # Speed up and complete save + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Reload from RDB and verify ORIGINAL keys still exist + catch {r debug reload nosave} + for {set i 0} {$i < 20} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers set_$i]] [list "B1" "B2"] + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + } {} {needs:debug} +} + +start_server {overrides {forkless-options-supported yes save ""}} { + test "blocking commands during forkless bgsave" { + + # Create initial dataset with 100 keys + createComplexDatasetForVerification r 100 + + # Start blocking commands on nonexistent keys BEFORE save starts + set rd1 [valkey_deferring_client] + set rd2 [valkey_deferring_client] + set rd3 [valkey_deferring_client] + set rd4 [valkey_deferring_client] + set rd5 [valkey_deferring_client] + set rd6 [valkey_deferring_client] + set rd7 [valkey_deferring_client] + + # Consume an item from a nonexistent key + $rd1 blpop new1 0 + + # Set up a cascade of brpoplpush's on nonexistent keys + $rd2 brpoplpush new2 new3 0 + $rd3 brpoplpush new3 new4 0 + + # Nonexistent keys + $rd4 brpoplpush new5 new6 0 + + # Cascade of brpoplpush's onto an existing key + $rd5 brpoplpush new88 new7 0 + $rd6 brpoplpush new7 lst_2 0 + + # Destination exists + $rd7 brpoplpush new8 lst_70 0 + + # Start save with slow speed + r config set rdb-key-save-delay 100000 + r bgsave forkless + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave didn't start" + } + + # Start more blocking commands during save + set rd8 [valkey_deferring_client] + set rd9 [valkey_deferring_client] + set rd10 [valkey_deferring_client] + set rd11 [valkey_deferring_client] + set rd12 [valkey_deferring_client] + + # Existing keys with new destinations, setting off some of the waiters + $rd8 brpoplpush lst_33 new1 0 + $rd9 brpoplpush lst_27 new2 0 + + # Duplicate another brpoplpush above + $rd10 brpoplpush new5 new6 0 + + # New key but existing destination + $rd11 brpoplpush new9 lst_3 0 + + # Consume an item from a nonexistent key + $rd12 brpop new100 0 + + # Set off more waiters + r rpush new5 foobar + r rpush new88 foobar + + assert_equal [s rdb_bgsave_in_progress] 1 + + # Resume save at normal speed + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Wait for blocking commands to complete and read responses + after 1000 + $rd8 read + $rd9 read + $rd1 read + $rd2 read + $rd3 read + $rd5 read + $rd6 read + + # Don't read from rd4, rd7, rd10, rd11, rd12 - they remain blocked or timeout + + # Verify the blocking commands executed correctly + assert_equal [r llen new1] 0 + assert_equal [r llen new2] 0 + assert_equal [r llen new3] 0 + assert_equal [lindex [r lrange new4 -1 -1] 0] "R2" + assert_equal [r llen new5] 0 + assert_equal [lindex [r lrange new6 -1 -1] 0] "foobar" + assert_equal [r llen new88] 0 + assert_equal [r llen new7] 0 + assert_equal [lindex [r lrange lst_2 0 0] 0] "foobar" + assert_equal [r llen new8] 0 + assert_equal [r llen lst_70] 4 + assert_equal [r llen lst_33] 3 + assert_equal [r llen lst_27] 3 + assert_equal [r llen lst_3] 4 + + # Close deferred clients (those that didn't complete will be force-closed) + $rd1 close + $rd2 close + $rd3 close + $rd4 close + $rd5 close + $rd6 close + $rd7 close + $rd8 close + $rd9 close + $rd10 close + $rd11 close + $rd12 close + + # Verify snapshot contains original keys (blocking commands should not affect snapshot) + catch {r debug reload nosave} + + # All original data types should be preserved in snapshot + for {set i 0} {$i < 100} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers set_$i]] [list "B1" "B2"] + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + + # New keys created during save should NOT be in snapshot + assert_equal [r exists new1] 0 + assert_equal [r exists new2] 0 + assert_equal [r exists new3] 0 + assert_equal [r exists new4] 0 + assert_equal [r exists new5] 0 + assert_equal [r exists new6] 0 + assert_equal [r exists new7] 0 + assert_equal [r exists new8] 0 + assert_equal [r exists new88] 0 + assert_equal [r exists new9] 0 + assert_equal [r exists new100] 0 + } {} {needs:debug} +} + +start_server {overrides {forkless-options-supported yes save ""}} { + test "TTL expiration during forkless bgsave" { + + # Create initial dataset with 100 keys + set num_keys 100 + createComplexDatasetForVerification r $num_keys + + # Set TTLs on all keys - key i expires in (i/10 + 1) seconds + set start_time [clock milliseconds] + for {set i 0} {$i < $num_keys} {incr i} { + set ttl [expr {$i/10 + 1}] + foreach prefix {before int lst set zset hash hll bits geo geo_set stream iset} { + r expire ${prefix}_${i} $ttl + } + } + + # Start save and wait for completion + r bgsave forkless + waitForBgsave r + + # Reload from RDB + catch {r debug reload nosave} + + # Check keycount is reasonable + set keycount [r dbsize] + assert {$keycount <= $num_keys * 12} + + # Verify keys based on elapsed time + set verified [list] + for {set i 0} {$i < $num_keys} {incr i} { + lappend verified $i + } + + while {[llength $verified] > 0} { + set elapsed_time [expr {([clock milliseconds] - $start_time) / 1000.0}] + + foreach i $verified { + # If not yet expired, verify all data types exist + if {$elapsed_time < [expr {$i/10.0}]} { + assert_equal [r exists before_${i}] 1 + assert_equal [r exists int_${i}] 1 + assert_equal [r exists lst_${i}] 1 + assert_equal [r exists set_${i}] 1 + assert_equal [r exists zset_${i}] 1 + assert_equal [r exists hash_${i}] 1 + assert_equal [r exists hll_${i}] 1 + assert_equal [r exists bits_${i}] 1 + assert_equal [r exists geo_${i}] 1 + assert_equal [r exists geo_set_${i}] 1 + assert_equal [r exists stream_${i}] 1 + assert_equal [r exists iset_${i}] 1 + } + + # If expired for more than 2 seconds, verify all data types are gone + if {$elapsed_time > [expr {$i/10.0 + 2}]} { + assert_equal [r exists before_${i}] 0 + assert_equal [r exists int_${i}] 0 + assert_equal [r exists lst_${i}] 0 + assert_equal [r exists set_${i}] 0 + assert_equal [r exists zset_${i}] 0 + assert_equal [r exists hash_${i}] 0 + assert_equal [r exists hll_${i}] 0 + assert_equal [r exists bits_${i}] 0 + assert_equal [r exists geo_${i}] 0 + assert_equal [r exists geo_set_${i}] 0 + assert_equal [r exists stream_${i}] 0 + assert_equal [r exists iset_${i}] 0 + set verified [lsearch -all -inline -not -exact $verified $i] + } + } + + after 100 + } + } {} {needs:debug} +} - # start a long AOF child - r bgrewriteaof +start_server {overrides {forkless-options-supported yes save ""}} { + test "evictions during forkless bgsave" { + + # Create initial dataset + createComplexDatasetForVerification r 1000 + + # Start save with stopped speed + r config set rdb-key-save-delay 10000 + r bgsave forkless + wait_for_condition 50 100 { - [s aof_rewrite_in_progress] == 1 + [s rdb_bgsave_in_progress] == 1 } else { - fail "aof not started" + fail "bgsave didn't start" } - # Make sure cancel return valid status - assert {[r bgsave schedule] eq {Background saving scheduled}} + # Trigger evictions by setting maxmemory below current usage + set current_memory [s used_memory] + set target_memory [expr {$current_memory * 3 / 4}] + r config set maxmemory $target_memory + r config set maxmemory-policy allkeys-lru + + # Generate evictions by adding new data + r set foo bar + + # Verify evictions occurred + set evicted_keys [s evicted_keys] + assert {$evicted_keys > 0} + assert_equal [s rdb_bgsave_in_progress] 1 + + # Resume save at normal speed + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Verify snapshot contains original keys + catch {r debug reload nosave} + for {set i 0} {$i < 1000} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers set_$i]] [list "B1" "B2"] + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + } {} {needs:debug} +} - # Cancel the scheduled save - assert {[r bgsave cancel] eq {Scheduled background saving cancelled}} +start_server {overrides {forkless-options-supported yes save ""}} { + test "comprehensive modifications on all data types during forkless bgsave" { + + # Create initial dataset with 1000 keys + createComplexDatasetForVerification r 1000 + + # Start save with slow speed to keep it running during modifications + r config set rdb-key-save-delay 1000000 + r bgsave forkless + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave didn't start" + } + + # Overwrite keys during save - all data types (pipelined to avoid blocking) + set rd [valkey_deferring_client] + for {set i 0} {$i < 1000} {incr i} { + $rd append before_$i "value_after_$i" + $rd incr int_$i + $rd set after_$i "VALUE_AFTER_$i" + $rd lpush lst_$i LL2 LL1 + $rd rpush lst_$i RR1 RR2 + $rd sadd set_$i BB1 BB2 + $rd zadd zset_$i 5 Z2 + $rd hset hash_$i H1 c + $rd pfadd hll_$i PF2 + $rd bitfield bits_$i SET u4 0 0 INCRBY u4 0 1 + $rd geoadd geo_$i -122.1592 47.5976 bellevue + $rd geosearchstore geo_set_$i geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi + $rd geosearchstore geo_set_dist_$i geo_$i FROMLONLAT -122.191729 47.685821 BYRADIUS 10 mi ASC COUNT 10 STOREDIST + $rd xadd stream_$i "*" D1 V2 + $rd xreadgroup GROUP group_$i consumer_after_$i COUNT 1 STREAMS stream_$i > + $rd hsetex hashttl_$i EX 10000 FIELDS 1 HTTL1 a + } + + # Verify changes happened and save still in progress + assert {[s rdb_changes_since_last_save] > 0} + assert_equal [s rdb_bgsave_in_progress] 1 + + # Speed up save and wait for completion + r config set rdb-key-save-delay 0 + waitForBgsave r + # Drain pipelined responses + $rd close + + # Verify snapshot contains original keys + catch {r debug reload nosave} + for {set i 0} {$i < 1000} {incr i} { + assert_equal [r get before_$i] "value_before_$i" + assert_equal [r get int_$i] [expr {42 + $i}] + assert_equal [r lrange lst_$i 0 -1] [list "L1" "L2" "R1" "R2"] + assert_equal [lsort [r smembers set_$i]] [list "B1" "B2"] + assert_equal [r zscore zset_$i "Z1"] 1 + assert_equal [r hget hash_$i "H1"] "a" + assert_equal [r pfcount hll_$i] 1 + assert_equal [r zcard geo_$i] 1 + assert_equal [r zcard geo_set_$i] 1 + } + } {} {needs:debug} +} - # Make sure a second call to bgsave cancel return an error - assert_error "ERR Background saving is currently not in progress or scheduled" {r bgsave cancel} - } +start_server {overrides {forkless-options-supported yes save ""}} { + test "store key deletion by georadius during forkless bgsave" { + + # Create initial dataset with geo data + createComplexDatasetForVerification r 1000 + + # Create additional zsets for georadius STORE operations + r zadd georad_zset_delete_test 1 Z1 2 Z2 + r zadd georadmem_zset_test 1 Z1 2 Z2 3 Z3 + + # Start save with stopped speed + r config set rdb-key-save-delay 10000 + r bgsave forkless + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave didn't start" + } + + # Use GEORADIUS with STORE - deletes georad_zset_delete_test key + r georadius geo_1 -122.191729 47.685821 5 mi STORE georad_zset_delete_test + + # Use GEORADIUSBYMEMBER with STORE - does not delete georadmem_zset_test as it returns 1 member + r georadiusbymember geo_1 seattle 5 mi STORE georadmem_zset_test + + # Verify changes were made + assert {[s rdb_changes_since_last_save] > 0} + assert_equal [s rdb_bgsave_in_progress] 1 + + # Resume save at normal speed + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Verify snapshot contains original keys + catch {r debug reload nosave} + + # Original geo_1 key should be preserved + assert_equal [r zcard geo_1] 1 + + # Original zsets should be preserved (not deleted by STORE operations) + assert_equal [r zcard georad_zset_delete_test] 2 + assert_equal [r zcard georadmem_zset_test] 3 + } {} {needs:debug} +} + +start_server {overrides {forkless-options-supported yes save ""}} { + test "transactions during forkless bgsave" { + + # Populate 5 databases + for {set db 0} {$db < 5} {incr db} { + r select $db + createComplexDatasetForVerification r 100 + } + r select 0 + + # Prepare transactions before save starts + set rd0 [valkey_deferring_client] + set rd1 [valkey_deferring_client] + + $rd0 select 0 + $rd0 multi + $rd0 set int_1 bad + $rd0 incrby int_2 2 + + $rd1 select 1 + $rd1 multi + $rd1 set int_1 bad + $rd1 set int_2 bad1 + $rd1 lpush lst_3 bad1 + $rd1 sadd set_3 bad1 + $rd1 set newkey bad1 + + # Start save with slow speed + r config set rdb-key-save-delay 10000 + r bgsave forkless + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "bgsave didn't start" + } + + # Start more transactions during save + set rd4 [valkey_deferring_client] + set rd3 [valkey_deferring_client] + + $rd4 select 4 + $rd4 multi + $rd4 hset hash_49 bad1 a + $rd4 hset hash_10 H1 b + $rd4 zadd zset_3 2 bad1 + $rd4 set newkey bad1 + $rd4 set another_newkey bad55 + $rd4 xadd newstream * D1 V2 + + $rd3 select 3 + $rd3 multi + $rd3 set aftersave bad1 + $rd3 xadd another_newstream * D1 V3 + + assert_equal [s rdb_bgsave_in_progress] 1 + + # Execute first 3 transactions + $rd0 exec + $rd1 exec + $rd4 exec + + # Read all responses: select, multi, queued commands, exec + # rd0: select(OK) multi(OK) set(QUEUED) incrby(QUEUED) exec(result) + for {set i 0} {$i < 5} {incr i} { $rd0 read } + # rd1: select(OK) multi(OK) set(QUEUED) set(QUEUED) lpush(QUEUED) sadd(QUEUED) set(QUEUED) exec(result) + for {set i 0} {$i < 8} {incr i} { $rd1 read } + # rd4: select(OK) multi(OK) hset(QUEUED) hset(QUEUED) zadd(QUEUED) set(QUEUED) set(QUEUED) xadd(QUEUED) exec(result) + for {set i 0} {$i < 9} {incr i} { $rd4 read } + + # Verify transactions executed + r select 0 + assert_equal [r get int_1] "bad" + r select 1 + assert_equal [r get int_2] "bad1" + r select 4 + assert_equal [r get newkey] "bad1" + r select 3 + assert_equal [r exists aftersave] 0 + r select 4 + assert_equal [r xlen newstream] 1 + r select 3 + assert_equal [r xlen another_newstream] 0 + + assert_equal [s rdb_bgsave_in_progress] 1 + + # Resume save at normal speed + r config set rdb-key-save-delay 0 + waitForBgsave r + + # Execute last transaction after save completes + r select 3 + assert_equal [r exists aftersave] 0 + $rd3 exec + # consume rd3 replies: select(OK) multi(OK) set(QUEUED) xadd(QUEUED) exec(result) + for {set i 0} {$i < 5} {incr i} { $rd3 read } + assert_equal [r get aftersave] "bad1" + assert_equal [r xlen another_newstream] 1 + + # Close deferred clients + $rd0 close + $rd1 close + $rd3 close + $rd4 close + + # Verify snapshot contains original keys + catch {r debug reload nosave} + + # Original keys should be preserved in all databases + r select 0 + assert_equal [r get before_0] "value_before_0" + assert_equal [r get int_1] "43" + r select 1 + assert_equal [r get int_2] "44" + assert_equal [r llen lst_3] 4 + r select 4 + assert_equal [r exists newkey] 0 + assert_equal [r exists another_newkey] 0 + assert_equal [r exists newstream] 0 + r select 3 + assert_equal [r exists aftersave] 0 + assert_equal [r exists another_newstream] 0 + } {} {needs:debug} +} +start_server {overrides {forkless-options-supported yes save ""}} { + foreach first_type {fork forkless} { + foreach second_type {fork forkless} { + test "$first_type bgsave blocks $second_type bgsave" { + r config set rdb-key-save-delay 1000000 + populate 100 "" 16 + + r bgsave $first_type + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "$first_type bgsave did not start" + } + assert_equal [s rdb_current_bgsave_type] $first_type + assert_error "ERR Background save already in progress" {r bgsave $second_type} + + r bgsave cancel + waitForBgsave r + } + } + } } test {client freed during loading} { @@ -512,52 +1550,60 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "scriptbackup } } -start_server {} { - test "failed bgsave prevents writes" { - # Make sure the server saves an RDB on shutdown - r config set save "900 1" - - r config set rdb-key-save-delay 10000000 - populate 1000 - r set x x - r bgsave - set pid1 [get_child_pid 0] - catch {exec kill -9 $pid1} - waitForBgsave r - - # make sure a read command succeeds - assert_equal [r get x] x - - # make sure a write command fails - assert_error {MISCONF *} {r set x y} - - # repeat with script - assert_error {MISCONF *} {r eval { - return redis.call('set','x',1) - } 1 x - } - assert_equal {x} [r eval { - return redis.call('get','x') - } 1 x - ] +start_server {overrides {forkless-options-supported yes save ""}} { + foreach bgsave_type {"" "fork" "forkless"} { + test "failed bgsave $bgsave_type prevents writes" { + # Make sure the server saves an RDB on shutdown + r config set save "900 1" + + r config set rdb-key-save-delay 10000000 + populate 1000 + r set x x + r bgsave {*}$bgsave_type + + if {$bgsave_type ne "forkless"} { + set pid1 [get_child_pid 0] + catch {exec kill -9 $pid1} + } else { + # For forkless save, cancel it to simulate failure + r bgsave cancel + } + waitForBgsave r - # again with script using shebang - assert_error {MISCONF *} {r eval {#!lua - return redis.call('set','x',1) - } 1 x - } - assert_equal {x} [r eval {#!lua flags=no-writes - return redis.call('get','x') - } 1 x - ] + # make sure a read command succeeds + assert_equal [r get x] x - r config set rdb-key-save-delay 0 - r bgsave - waitForBgsave r + # make sure a write command fails + assert_error {MISCONF *} {r set x y} - # server is writable again - r set x y - } {OK} + # repeat with script + assert_error {MISCONF *} {r eval { + return redis.call('set','x',1) + } 1 x + } + assert_equal {x} [r eval { + return redis.call('get','x') + } 1 x + ] + + # again with script using shebang + assert_error {MISCONF *} {r eval {#!lua + return redis.call('set','x',1) + } 1 x + } + assert_equal {x} [r eval {#!lua flags=no-writes + return redis.call('get','x') + } 1 x + ] + + r config set rdb-key-save-delay 0 + r bgsave {*}$bgsave_type + waitForBgsave r + + # server is writable again + r set x y + } {OK} + } } start_server {} { @@ -588,4 +1634,47 @@ start_server {} { } } + +start_server {overrides {forkless-options-supported yes save ""}} { + test {default-bgsave-method can be set to forkless with forkless-options-supported} { + r config set default-bgsave-method forkless + assert_equal [lindex [r config get default-bgsave-method] 1] "forkless" + } +} + +start_server {} { + test {BGSAVE FORKLESS requires forkless-options-supported} { + catch {r bgsave forkless} err + assert_match "*forkless-options-supported*" $err + } +} + +start_server {overrides {forkless-options-supported yes save ""}} { + test {BGSAVE FORKLESS works with forkless-options-supported} { + r bgsave forkless + waitForBgsave r + assert_equal [s rdb_last_bgsave_type] "forkless" + } +} + +start_server {overrides {forkless-options-supported yes default-bgsave-method forkless}} { + test {BGSAVE uses forkless when default-bgsave-method is forkless} { + r set key value + set result [r bgsave] + assert_match "*Background saving started*" $result + waitForBgsave r + assert_equal [s rdb_last_bgsave_type] "forkless" + } +} + +start_server {overrides {default-bgsave-method fork}} { + test {BGSAVE uses fork when default-bgsave-method is fork} { + r set key value + set result [r bgsave] + assert_match "*Background saving started*" $result + waitForBgsave r + assert_equal [s rdb_last_bgsave_type] "fork" + } +} + } ;# tags diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 238c0366faa..1d9592be959 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -1682,6 +1682,90 @@ start_server {tags {"repl external:skip"}} { } } +# Verify that after a diskless (socket) replication sync, save metrics +# are correctly reset and rdb_last_bgsave_time_sec is a plausible duration. +start_server {tags {"repl external:skip"}} { + start_server {} { + test {diskless sync: save metrics are plausible after socket transfer} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set replica [srv 0 client] + + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 0 + $master config set save "" + $replica config set save "" + + $master debug populate 100 + + $replica replicaof $master_host $master_port + + wait_for_condition 100 100 { + [string match {*master_link_status:up*} [$replica info replication]] + } else { + fail "Replica didn't complete sync" + } + + # After diskless sync, master metrics should be sane + set time_sec [$master info persistence] + set bgsave_time [getInfoProperty $time_sec rdb_last_bgsave_time_sec] + assert {$bgsave_time >= 0 && $bgsave_time < 3600} + + # Save state should be cleared + assert_equal [getInfoProperty $time_sec rdb_bgsave_in_progress] "0" + assert_equal [getInfoProperty $time_sec current_save_keys_processed] "0" + assert_equal [getInfoProperty $time_sec current_save_keys_total] "0" + } + } +} + +start_server {tags {"repl external:skip"}} { + start_server {} { + test {diskless sync: save metrics are plausible after failed socket transfer} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set replica [srv 0 client] + + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 0 + $master config set save "" + $replica config set save "" + + $master debug populate 1000 + $master config set rdb-key-save-delay 100000 + + $replica replicaof $master_host $master_port + + # Wait for bgsave to start on master + wait_for_condition 100 100 { + [getInfoProperty [$master info persistence] rdb_bgsave_in_progress] == 1 + } else { + fail "diskless bgsave didn't start" + } + + # Kill the replica connection to abort the transfer + $replica replicaof no one + + # Wait for bgsave to finish on master + wait_for_condition 100 100 { + [getInfoProperty [$master info persistence] rdb_bgsave_in_progress] == 0 + } else { + fail "diskless bgsave didn't stop after replica disconnect" + } + + # Metrics should still be sane after failure + set time_sec [$master info persistence] + set bgsave_time [getInfoProperty $time_sec rdb_last_bgsave_time_sec] + assert {$bgsave_time >= 0 && $bgsave_time < 3600} + assert_equal [getInfoProperty $time_sec current_save_keys_processed] "0" + assert_equal [getInfoProperty $time_sec current_save_keys_total] "0" + + $master config set rdb-key-save-delay 0 + } + } +} start_server {tags {"repl external:skip"}} { set replica [srv 0 client] $replica config set repl-diskless-load disabled diff --git a/tests/support/util.tcl b/tests/support/util.tcl index ca9b0b6af7e..f3622319c0e 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -47,6 +47,42 @@ proc write_binary_file {path data} { close $fd } +# Create keys of all data types with predictable/consistent names for verification +proc createComplexDatasetForVerification {r count {prefix ""}} { + for {set i 0} {$i < $count} {incr i} { + # String keys + {*}$r set ${prefix}before_$i "value_before_$i" + {*}$r set ${prefix}int_$i [expr {42 + $i}] + {*}$r set ${prefix}bits_$i "\x0f" + + # List keys + {*}$r lpush ${prefix}lst_$i "L2" "L1" + {*}$r rpush ${prefix}lst_$i "R1" "R2" + + # Set keys + {*}$r sadd ${prefix}set_$i "B1" "B2" + {*}$r sadd ${prefix}iset_$i 12 34 + + # Sorted set keys + {*}$r zadd ${prefix}zset_$i 1 "Z1" 2 "Z2" + + # Hash keys + {*}$r hset ${prefix}hash_$i "H1" "a" + {*}$r hset ${prefix}hash_$i "H2" 1 + + # HyperLogLog + {*}$r pfadd ${prefix}hll_$i "PF1" + + # Geo + {*}$r geoadd ${prefix}geo_$i -122.335167 47.608013 "seattle" + {*}$r geosearchstore ${prefix}geo_set_$i ${prefix}geo_$i FROMLONLAT -122.335167 47.608013 BYRADIUS 10 mi + + # Stream + {*}$r xadd ${prefix}stream_$i "*" "D1" "V1" + {*}$r xgroup create ${prefix}stream_$i ${prefix}group_$i 0 MKSTREAM + } +} + # Useful for some test proc zlistAlikeSort {a b} { if {[lindex $a 0] > [lindex $b 0]} {return 1} diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl index 5c876b08edf..99188fe7c0c 100644 --- a/tests/unit/info.tcl +++ b/tests/unit/info.tcl @@ -570,3 +570,223 @@ start_server {tags {"info" "external:skip"}} { assert_equal [dict get $mem_stats db.dict.rehashing.count] {1} } } + +start_server {tags {"info" "external:skip"} overrides {save "" forkless-options-supported yes}} { + test {INFO forkless save metrics show default values when no save is running} { + r config set save "" + r flushall + + set info [r info persistence] + + # When no forkless save is running, time metrics should be -1 + assert_match "*forkless_current_item_millis:-1*" $info + assert_match "*forkless_estimated_seconds_remaining:-1*" $info + + # Debug metrics should be 0 + set dbg [r info debug] + assert_match "*forkless_current_queue_length:0*" $dbg + assert_match "*forkless_queue_length_target:0*" $dbg + assert_match "*forkless_dbentries_queued:0*" $dbg + assert_match "*forkless_dbentries_processed:0*" $dbg + } + + test {INFO forkless save metrics are present during active save} { + r config set save "" + r flushall + r debug populate 1000 + + # Start slow forkless save + r config set rdb-key-save-delay 100000 + r bgsave forkless + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + set info [r info persistence] + + # Verify time metrics are present in persistence section + assert_match "*forkless_current_item_millis:*" $info + assert_match "*forkless_estimated_seconds_remaining:*" $info + + # Verify queue metrics are present in debug section + set dbg [r info debug] + assert_match "*forkless_current_queue_length:*" $dbg + assert_match "*forkless_queue_length_target:*" $dbg + assert_match "*forkless_dbentries_queued:*" $dbg + assert_match "*forkless_dbentries_processed:*" $dbg + + # Verify queue_length_target has a reasonable value + set target [getInfoProperty $dbg forkless_queue_length_target] + assert {$target > 0} + + r config set rdb-key-save-delay 0 + r bgsave cancel + waitForBgsave r + } + + test {INFO forkless save cumulative metrics increase during save} { + r config set save "" + r flushall + r debug populate 100 + + # Start slow forkless save + r config set rdb-key-save-delay 50000 + r bgsave forkless + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + # Wait a bit for some processing + after 200 + + set dbg [r info debug] + set queued1 [getInfoProperty $dbg forkless_dbentries_queued] + set processed1 [getInfoProperty $dbg forkless_dbentries_processed] + + # Wait more + after 200 + + set dbg [r info debug] + set queued2 [getInfoProperty $dbg forkless_dbentries_queued] + set processed2 [getInfoProperty $dbg forkless_dbentries_processed] + + # Cumulative metrics should increase or stay same (never decrease) + assert {$queued2 >= $queued1} + assert {$processed2 >= $processed1} + + # At least one should have increased + assert {$queued2 > $queued1 || $processed2 > $processed1} + + r config set rdb-key-save-delay 0 + r bgsave cancel + waitForBgsave r + } + + test {INFO forkless save current_item_millis is counted} { + r config set save "" + r flushall + r debug populate 10 + + # Start very slow forkless save - 2 seconds per key + r config set rdb-key-save-delay 2000000 + r bgsave forkless + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + # Wait until the item has been processing for at least 1 second + wait_for_condition 50 100 { + [s forkless_current_item_millis] > 1000 + } else { + fail "forkless_current_item_millis never exceeded 1000" + } + + set item_time [s forkless_current_item_millis] + + # Should be processing an item for ~1 second (1000+ ms) + assert {$item_time > 1000} + + r config set rdb-key-save-delay 0 + r bgsave cancel + waitForBgsave r + } + + test {INFO forkless save estimated_seconds_remaining is reasonable} { + r config set save "" + r flushall + r debug populate 100 + + # Set 1 second delay per key + r config set rdb-key-save-delay 1000000 + waitForBgsave r + r bgsave forkless + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + # Wait for ~1 key to be processed (1.2 seconds to be safe) + after 1200 + + set dbg [r info debug] + set processed [getInfoProperty $dbg forkless_dbentries_processed] + set estimated [s forkless_estimated_seconds_remaining] + + # Should have processed at least 1 key + assert {$processed >= 1} + + # With 100 keys and ~1 processed at 1sec/key, the estimate should be + # in the tens of seconds range. Use a wide band to avoid timing flakes. + assert {$estimated > 0 && $estimated < 300} + + r config set rdb-key-save-delay 0 + r bgsave cancel + waitForBgsave r + } + + test {INFO forkless save metrics show default values after save completes} { + r config set save "" + r flushall + r debug populate 100 + + # Start and complete a fast forkless save + r config set rdb-key-save-delay 0 + r bgsave forkless + waitForBgsave r + + set info [r info persistence] + + # After save completes, time metrics should be -1 + assert_match "*forkless_current_item_millis:-1*" $info + assert_match "*forkless_estimated_seconds_remaining:-1*" $info + + # Debug metrics should be 0 + set dbg [r info debug] + assert_match "*forkless_current_queue_length:0*" $dbg + assert_match "*forkless_queue_length_target:0*" $dbg + assert_match "*forkless_dbentries_queued:0*" $dbg + assert_match "*forkless_dbentries_processed:0*" $dbg + } + + test {INFO rdb_current_bgsave_time_sec increases during forkless save} { + r config set save "" + r flushall + r debug populate 100 + + # Start slow forkless save + r config set rdb-key-save-delay 100000 + r bgsave forkless + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + set time1 [s rdb_current_bgsave_time_sec] + + # Wait 2 seconds + after 2000 + + set time2 [s rdb_current_bgsave_time_sec] + + # Time should have increased by approximately 2 seconds + assert {$time2 >= $time1 + 1} + assert {$time2 <= $time1 + 3} + + r config set rdb-key-save-delay 0 + r bgsave cancel + waitForBgsave r + } +} diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 25467a7d709..3c467687ac0 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -990,38 +990,7 @@ start_server {tags {"introspection"}} { assert_error "ERR timeout is negative" {r client pause -1} } - test "CLIENT KILL close the client connection during bgsave" { - # Start a slow bgsave, trigger an active fork. - r flushall - r set k v - r config set rdb-key-save-delay 10000000 - r bgsave - wait_for_condition 1000 10 { - [s rdb_bgsave_in_progress] eq 1 - } else { - fail "bgsave did not start in time" - } - - # Kill (close) the connection - r client kill skipme no - - # In the past, client connections needed to wait for bgsave - # to end before actually closing, now they are closed immediately. - assert_error "*I/O error*" {r ping} ;# get the error very quickly - assert_equal "PONG" [r ping] - # Make sure the bgsave is still in progress - assert_equal [s rdb_bgsave_in_progress] 1 - - # Stop the child before we proceed to the next test - r config set rdb-key-save-delay 0 - r flushall - wait_for_condition 1000 10 { - [s rdb_bgsave_in_progress] eq 0 - } else { - fail "bgsave did not stop in time" - } - } {} {needs:save} test "CLIENT REPLY OFF/ON: disable all commands reply" { set rd [valkey_deferring_client] @@ -2116,3 +2085,37 @@ test {CONFIG hash-seed is immutable and settable at startup} { } } } {} {external:skip} + +start_server {overrides {forkless-options-supported yes} tags {"introspection" "external:skip"}} { + foreach bgsave_type {"" "fork" "forkless"} { + test "CLIENT KILL close the client connection during bgsave - $bgsave_type" { + r flushall + r set k v + r config set rdb-key-save-delay 10000000 + r bgsave {*}$bgsave_type + wait_for_condition 1000 10 { + [s rdb_bgsave_in_progress] eq 1 + } else { + fail "bgsave did not start in time" + } + + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_current_bgsave_type] $expected_type + + r client kill skipme no + + assert_error "*I/O error*" {r ping} + assert_equal "PONG" [r ping] + + assert_equal [s rdb_bgsave_in_progress] 1 + + r config set rdb-key-save-delay 0 + r flushall + wait_for_condition 1000 10 { + [s rdb_bgsave_in_progress] eq 0 + } else { + fail "bgsave did not stop in time" + } + } {} {needs:save} + } +} diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl index d8061700899..ee2985f26c7 100644 --- a/tests/unit/moduleapi/testrdb.tcl +++ b/tests/unit/moduleapi/testrdb.tcl @@ -304,3 +304,30 @@ tags "modules" { } } } + +start_server {tags {"modules"} overrides {forkless-options-supported yes save "" enable-debug-command yes enable-module-command yes}} { + test {MODULE LOAD is blocked during forkless save} { + r debug populate 100 + + # Start slow forkless save + r config set rdb-key-save-delay 200000 + r bgsave forkless + + wait_for_condition 50 100 { + [s rdb_bgsave_in_progress] == 1 + } else { + fail "forkless save didn't start" + } + + # Try to load a module - should fail during forkless save + catch {r module load $testmodule} err + assert_match "*Error*" $err + + r config set rdb-key-save-delay 0 + r bgsave cancel + waitForBgsave r + + # After forkless save completes, module load should succeed + assert_equal {OK} [r module load $testmodule] + } +} diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl index 41d5c2354bd..dae22ba28ba 100644 --- a/tests/unit/other.tcl +++ b/tests/unit/other.tcl @@ -240,17 +240,6 @@ start_server {tags {"other"}} { } } - test {BGSAVE} { - # Use FLUSHALL instead of FLUSHDB, FLUSHALL do a foreground save - # and reset the dirty counter to 0, so we won't trigger an unexpected bgsave. - r flushall - r save - r set x 10 - r bgsave - waitForBgsave r - r debug reload - r get x - } {10} {needs:debug needs:save} test {SELECT an out of range DB} { catch {r select 1000000} err @@ -311,21 +300,6 @@ start_server {tags {"other"}} { } {1} {needs:debug} } - test {EXPIRES after a reload (snapshot + append only file rewrite)} { - r flushdb - r set x 10 - r expire x 1000 - r save - r debug reload - set ttl [r ttl x] - set e1 [expr {$ttl > 900 && $ttl <= 1000}] - r bgrewriteaof - waitForBgrewriteaof r - r debug loadaof - set ttl [r ttl x] - set e2 [expr {$ttl > 900 && $ttl <= 1000}] - list $e1 $e2 - } {1 1} {needs:debug needs:save} test {EXPIRES after AOF reload (without rewrite)} { r flushdb @@ -778,3 +752,38 @@ if {$::verbose} { } close $tempFileId file delete $tempFileName + +start_server {overrides {forkless-options-supported yes} tags {"other" "external:skip"}} { + foreach bgsave_type {"" "fork" "forkless"} { + test "BGSAVE $bgsave_type" { + r flushall + r save + r set x 10 + r bgsave {*}$bgsave_type + waitForBgsave r + + set expected_type [expr {$bgsave_type eq "forkless" ? "forkless" : "fork"}] + assert_equal [s rdb_last_bgsave_type] $expected_type + + r debug reload + r get x + } {10} {needs:debug needs:save} + + test "EXPIRES after a reload ($bgsave_type snapshot + append only file rewrite)" { + r flushdb + r set x 10 + r expire x 1000 + r bgsave {*}$bgsave_type + waitForBgsave r + r debug reload + set ttl [r ttl x] + set e1 [expr {$ttl > 900 && $ttl <= 1000}] + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set ttl [r ttl x] + set e2 [expr {$ttl > 900 && $ttl <= 1000}] + list $e1 $e2 + } {1 1} {needs:debug needs:save} + } +} diff --git a/valkey.conf b/valkey.conf index b1bf6a2225e..8015ec2fe16 100644 --- a/valkey.conf +++ b/valkey.conf @@ -556,14 +556,15 @@ locale-collate "" # # hash-seed example-seed-val -# Enable support for forkless save operations by allocating metadata for each key. +# Enable support for forkless snapshot operations by allocating metadata for each key. # This is an immutable configuration that must be set at server startup and # cannot be changed at runtime. # # When enabled, the server allocates 4 additional bytes per key. # # Note: This only enables the infrastructure support. The actual forkless save -# behavior is controlled separately by the 'forkless-enabled' runtime configuration. +# behavior is controlled separately by 'default-bgsave-method forkless' or by +# explicitly using 'BGSAVE FORKLESS'. # # forkless-options-supported no @@ -605,6 +606,16 @@ locale-collate "" # permissions, and so forth. stop-writes-on-bgsave-error yes +# Default method for background saves (BGSAVE and automatic periodic saves). +# +# Setting this to 'forkless' requires 'forkless-options-supported yes' to be +# enabled at startup. If it is not enabled, the server will fall back to fork. +# +# Regardless of this setting, you can always override the method explicitly +# with 'BGSAVE FORKLESS' or 'BGSAVE FORK'. +# +# default-bgsave-method fork + # Control compression when dumping .rdb databases. # Supported values: # yes - use the default compression algorithm (currently lzf) From 0a50265b7c2c8f29d837599fd983257b91b557a5 Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Tue, 18 Aug 2026 07:43:44 -0700 Subject: [PATCH 09/18] Forkless comments cleanup (#4450) Signed-off-by: Nitai Caro Signed-off-by: Jim Brunner Co-authored-by: Nitai Caro Co-authored-by: Jim Brunner --- .config/typos.toml | 4 ++-- src/rdb.c | 2 ++ src/unit/test_bgiteration.cpp | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.config/typos.toml b/.config/typos.toml index c2a9be035db..1aac5925ede 100644 --- a/.config/typos.toml +++ b/.config/typos.toml @@ -18,7 +18,7 @@ Collet = "Collet" # LZ4 author Yann Collet nd = "nd" Ba = "Ba" Addd = "Addd" -threadsave = "threadsave" +forkless = "forkless" [default.extend-identifiers] dbe = "dbe" @@ -64,7 +64,7 @@ seeked = "seeked" [type.c.extend-words] arange = "arange" -Threadsave = "Threadsave" +Forkless = "Forkless" fo = "fo" frst = "frst" limite = "limite" diff --git a/src/rdb.c b/src/rdb.c index faab3f22660..3cbf991e876 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -4387,6 +4387,8 @@ int rdbWriteHeader(rio *rdb, int req, int rdbver, int rdbflags, rdbSaveInfo *rsi int rdbWriteFooter(rio *rdb, int req) { if (!(req & REPLICA_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, VALKEYMODULE_AUX_AFTER_RDB) == -1) return C_ERR; if (rdbSaveType(rdb, RDB_OPCODE_EOF) == -1) return C_ERR; + /* RDB checksum field. It will be zero if checksum computation is disabled, the + * loading code skips the check in this case. */ uint64_t cksum = rdb->cksum; memrev64ifbe(&cksum); if (rioWrite(rdb, &cksum, 8) == 0) return C_ERR; diff --git a/src/unit/test_bgiteration.cpp b/src/unit/test_bgiteration.cpp index 5d6f06d4adb..decf1c32e89 100644 --- a/src/unit/test_bgiteration.cpp +++ b/src/unit/test_bgiteration.cpp @@ -1218,7 +1218,7 @@ TEST_F(BgIterationTest, modFutureItem_start) { } -// Modify a future item, with replication but without consistency. (Like a Threadsave Full Sync operation) +// Modify a future item, with replication but without consistency. (Like a Forkless Full Sync operation) // Our expectation for this case is that the modification should proceed without blocking, as the // mode is inconsistent. We don't expect replication, as we haven't reached the item yet. We'll // see the modified item later. @@ -1315,7 +1315,7 @@ TEST_F(BgIterationTest, modCurrentItem_start) { } -// Modify a current item, with replication but without consistency. (Like a Threadsave Full Sync operation) +// Modify a current item, with replication but without consistency. (Like a Forkless Full Sync operation) // Our expectation for this case is that the modification SHOULD be blocked. After the key is processed, // the write will proceed, and the replication will be sent. TEST_F(BgIterationTest, modCurrentItem_eventual) { @@ -1400,7 +1400,7 @@ TEST_F(BgIterationTest, modPastItem_start) { } -// Modify a past item, with replication but without consistency. (Like a Threadsave Full Sync operation) +// Modify a past item, with replication but without consistency. (Like a Forkless Full Sync operation) // Our expectation for this case is that the modification should proceed without blocking. // Replication will be sent. TEST_F(BgIterationTest, modPastItem_eventual) { From e094e328d4b032adb11addd1973d9ffdc99bd8ab Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:03:19 -0700 Subject: [PATCH 10/18] Match BGSAVE CANCEL command documentation in bgsave.json to code (#4461) Restructured bgsave documentation for a resulting grammar of `BGSAVE [ [SCHEDULE] [FORK|FORKLESS] | CANCEL ]` Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/commands.def | 22 +++++++++++----- src/commands/bgsave.json | 56 ++++++++++++++++++++++++---------------- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/src/commands.def b/src/commands.def index 61a37fd905a..35233f07e80 100644 --- a/src/commands.def +++ b/src/commands.def @@ -7187,17 +7187,27 @@ commandHistory BGSAVE_History[] = { #define BGSAVE_Keyspecs NULL #endif -/* BGSAVE save_type argument table */ -struct COMMAND_ARG BGSAVE_save_type_Subargs[] = { +/* BGSAVE operation save save_type argument table */ +struct COMMAND_ARG BGSAVE_operation_save_save_type_Subargs[] = { {MAKE_ARG("fork",ARG_TYPE_PURE_TOKEN,-1,"FORK",NULL,NULL,CMD_ARG_NONE,0,NULL)}, {MAKE_ARG("forkless",ARG_TYPE_PURE_TOKEN,-1,"FORKLESS",NULL,NULL,CMD_ARG_NONE,0,NULL)}, }; +/* BGSAVE operation save argument table */ +struct COMMAND_ARG BGSAVE_operation_save_Subargs[] = { +{MAKE_ARG("schedule",ARG_TYPE_PURE_TOKEN,-1,"SCHEDULE",NULL,"3.2.2",CMD_ARG_OPTIONAL,0,NULL)}, +{MAKE_ARG("save-type",ARG_TYPE_ONEOF,-1,NULL,NULL,"9.2.0",CMD_ARG_OPTIONAL,2,NULL),.subargs=BGSAVE_operation_save_save_type_Subargs}, +}; + +/* BGSAVE operation argument table */ +struct COMMAND_ARG BGSAVE_operation_Subargs[] = { +{MAKE_ARG("save",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=BGSAVE_operation_save_Subargs}, +{MAKE_ARG("cancel",ARG_TYPE_PURE_TOKEN,-1,"CANCEL",NULL,"8.1.0",CMD_ARG_NONE,0,NULL)}, +}; + /* BGSAVE argument table */ struct COMMAND_ARG BGSAVE_Args[] = { -{MAKE_ARG("schedule",ARG_TYPE_PURE_TOKEN,-1,"SCHEDULE",NULL,"3.2.2",CMD_ARG_OPTIONAL,0,NULL)}, -{MAKE_ARG("save-type",ARG_TYPE_ONEOF,-1,NULL,NULL,"9.2.0",CMD_ARG_OPTIONAL,2,NULL),.subargs=BGSAVE_save_type_Subargs}, -{MAKE_ARG("cancel",ARG_TYPE_PURE_TOKEN,-1,"CANCEL",NULL,"8.1.0",CMD_ARG_OPTIONAL,0,NULL)}, +{MAKE_ARG("operation",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=BGSAVE_operation_Subargs}, }; /********** COMMAND COUNT ********************/ @@ -12036,7 +12046,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { /* server */ {MAKE_CMD("acl","A container for Access List Control commands.","Depends on subcommand.","6.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,ACL_History,0,ACL_Tips,0,NULL,-2,CMD_SENTINEL,ACL_CATEGORY_SLOW,NULL,ACL_Keyspecs,0,NULL,0),.subcommands=ACL_Subcommands}, {MAKE_CMD("bgrewriteaof","Asynchronously rewrites the append-only file to disk.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,BGREWRITEAOF_History,0,BGREWRITEAOF_Tips,0,bgrewriteaofCommand,1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,BGREWRITEAOF_Keyspecs,0,NULL,0)}, -{MAKE_CMD("bgsave","Asynchronously saves the database(s) to disk.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,BGSAVE_History,3,BGSAVE_Tips,0,bgsaveCommand,-1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,BGSAVE_Keyspecs,0,NULL,3),.args=BGSAVE_Args}, +{MAKE_CMD("bgsave","Asynchronously saves the database(s) to disk.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,BGSAVE_History,3,BGSAVE_Tips,0,bgsaveCommand,-1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,BGSAVE_Keyspecs,0,NULL,1),.args=BGSAVE_Args}, {MAKE_CMD("command","Returns detailed information about all commands.","O(N) where N is the total number of commands","2.8.13",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,COMMAND_History,0,COMMAND_Tips,1,commandCommand,-1,CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,COMMAND_Keyspecs,0,NULL,0),.subcommands=COMMAND_Subcommands}, {MAKE_CMD("commandlog","A container for command log commands.","Depends on subcommand.","8.1.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,COMMANDLOG_History,0,COMMANDLOG_Tips,0,NULL,-2,0,ACL_CATEGORY_SLOW,NULL,COMMANDLOG_Keyspecs,0,NULL,0),.subcommands=COMMANDLOG_Subcommands}, {MAKE_CMD("config","A container for server configuration commands.","Depends on subcommand.","2.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,CONFIG_History,0,CONFIG_Tips,0,NULL,-2,0,ACL_CATEGORY_SLOW,NULL,CONFIG_Keyspecs,0,NULL,0),.subcommands=CONFIG_Subcommands}, diff --git a/src/commands/bgsave.json b/src/commands/bgsave.json index 445cdcb6a27..fdecc4a78d6 100644 --- a/src/commands/bgsave.json +++ b/src/commands/bgsave.json @@ -27,36 +27,48 @@ ], "arguments": [ { - "name": "schedule", - "token": "SCHEDULE", - "type": "pure-token", - "optional": true, - "since": "3.2.2" - }, - { - "name": "save-type", + "name": "operation", "type": "oneof", "optional": true, - "since": "9.2.0", "arguments": [ { - "name": "fork", - "token": "FORK", - "type": "pure-token" + "name": "save", + "type": "block", + "arguments": [ + { + "name": "schedule", + "token": "SCHEDULE", + "type": "pure-token", + "optional": true, + "since": "3.2.2" + }, + { + "name": "save-type", + "type": "oneof", + "optional": true, + "since": "9.2.0", + "arguments": [ + { + "name": "fork", + "token": "FORK", + "type": "pure-token" + }, + { + "name": "forkless", + "token": "FORKLESS", + "type": "pure-token" + } + ] + } + ] }, { - "name": "forkless", - "token": "FORKLESS", - "type": "pure-token" + "name": "cancel", + "token": "CANCEL", + "type": "pure-token", + "since": "8.1.0" } ] - }, - { - "name": "cancel", - "token": "CANCEL", - "type": "pure-token", - "optional": true, - "since": "8.1.0" } ], "reply_schema": { From 3419a59b49a577c67b38608bbc40003dded4f1b8 Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:12:42 -0700 Subject: [PATCH 11/18] Schedule AOF rewrite during a forkless save instead of failing (#4462) Treat save/aof conflict like bgsave, scheduling instead of failing. Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/aof.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index a907069ec38..c3e7065218f 100644 --- a/src/aof.c +++ b/src/aof.c @@ -965,7 +965,7 @@ int startAppendOnly(void) { serverAssert(server.aof_state == AOF_OFF); server.aof_state = AOF_WAIT_REWRITE; - if (hasActiveChildProcess() && server.child_type != CHILD_TYPE_AOF) { + if (hasActiveSaveOrChild() && server.child_type != CHILD_TYPE_AOF) { server.aof_rewrite_scheduled = 1; serverLog(LL_NOTICE, "AOF was enabled but there is already another background operation. An AOF background was " "scheduled to start when possible."); From 42f5521b2e62084cd9ef58d75d9730e3b6ca6b30 Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:14:15 -0700 Subject: [PATCH 12/18] Fix parameter name in bgiteration constructor doc blocks (#4463) Comment update for bgiteration.h Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/bgiteration.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bgiteration.h b/src/bgiteration.h index c9219b46ea6..ea9d00e6fd4 100644 --- a/src/bgiteration.h +++ b/src/bgiteration.h @@ -86,7 +86,7 @@ typedef void (*bgIteratorCleanupFunc)(bool terminated, void *privdata); * This bgIterator will iterate through the entire keyspace (across all DBs). * * NAME: a human readable name for the iterator (must be unique) - * FLAGS: creation flags indicate iteration options + * CONSISTENCY: the consistency guarantee for the iteration * REPLDONE: if provided, called after the last replication item has been queued (on the Valkey main thread) * CLEANUP: if provided, called at the end of iteration (on the Valkey main thread) * PRIVDATA: passed to cleanup function @@ -109,7 +109,7 @@ bgIterator *bgIteratorCreateFullScanIter( * This bgIterator will iterate through the keys belonging to a set of cluster slots. * * NAME: a human readable name for the iterator (must be unique) - * FLAGS: creation flags indicate iteration options + * CONSISTENCY: the consistency guarantee for the iteration * SLOTS: array of cluster slots to iterate over * SLOTS_COUNT: size of the array of slots * REPLDONE: if provided, called after the last replication item has been queued (on the Valkey main thread) From 50ec9360ad19ed34bc370fee5c27606c2a5d063b Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:18:48 -0700 Subject: [PATCH 13/18] For in-use blocking, check the client is in the list before removing it (#4464) Minor self-check in blocking code. No functional change. Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/blocked.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/blocked.c b/src/blocked.c index affc733131c..f3e4896fb7b 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -1021,7 +1021,9 @@ static void unlinkBlockInUseClient(client *c) { robj *key = dictGetKey(de); list *clientList = keyToClients_getBlockedClientsList(key); serverAssert(clientList != NULL); - listDelNode(clientList, listSearchKey(clientList, c)); + listNode *ln = listSearchKey(clientList, c); + serverAssert(ln != NULL); + listDelNode(clientList, ln); if (listLength(clientList) == 0) hashtableDelete(inuse_key_to_clients, key); } dictReleaseIterator(di); From 0c675861e88d978b4c9af0dad2747ff490928cdf Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:23:18 -0700 Subject: [PATCH 14/18] Update stale comment on BGSAVE CANCEL (#4479) Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/rdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index 3cbf991e876..96652e8c658 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -4171,7 +4171,7 @@ void bgsaveCommand(client *c) { int chosen_save_type = RDB_BGSAVE_TYPE_NONE; /* BGSAVE can be invoked with the following options: - * - CANCEL: terminates an in-progress or scheduled BGSAVE (standalone only) + * - CANCEL: terminates an in-progress or scheduled BGSAVE * - SCHEDULE: schedules a BGSAVE when an AOF rewrite is in progress. * Instead of returning an error, the BGSAVE is scheduled to run * when the AOF rewrite completes. From 042af21da659a8f1e67d643bff2bd2d7f369c92c Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:15:55 -0700 Subject: [PATCH 15/18] Flush and fsync the snapshot before publishing it (#4466) Forkless save: flush RIO buffer and fsync before close. Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/forkless.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/forkless.c b/src/forkless.c index 40c4e9154b1..70d1f46a8c0 100644 --- a/src/forkless.c +++ b/src/forkless.c @@ -162,7 +162,15 @@ static void forklessSaveCloseSnapshotFile(void *args[]) { serverAssert(!onValkeyMainThread()); forklessSaveInfo *saveInfo = (forklessSaveInfo *)args[0]; /* Error or not, close the file... */ - if (fsync(fileno(saveInfo->save_rio.io.file.fp)) != 0) { + /* Flush the RIO buffer to the OS before fsync, otherwise any bytes still + * buffered (including the tail written after the last autosync boundary and + * the RDB footer) are not covered by the fsync below. */ + if (rioFlush(&saveInfo->save_rio) == 0) { + serverLog(LL_WARNING, "forkless-save: error flushing temp file [%s]: %s", + saveInfo->temp_file, strerror(errno)); + saveInfo->err_code = C_ERR; + } + if (valkey_fsync(fileno(saveInfo->save_rio.io.file.fp)) != 0) { serverLog(LL_WARNING, "forkless-save: error fsyncing temp file [%s]: %s", saveInfo->temp_file, strerror(errno)); saveInfo->err_code = C_ERR; @@ -178,6 +186,11 @@ static void forklessSaveCloseSnapshotFile(void *args[]) { serverLog(LL_WARNING, "forkless-save: error moving temp file [%s] to destination [%s]: %s", saveInfo->temp_file, saveInfo->final_file, strerror(errno)); saveInfo->err_code = C_ERR; + } else if (fsyncFileDir(saveInfo->final_file) != 0) { + /* fsync the directory so the rename itself survives a crash. */ + serverLog(LL_WARNING, "forkless-save: error syncing directory for [%s]: %s", + saveInfo->final_file, strerror(errno)); + saveInfo->err_code = C_ERR; } } From 9e26ccd18ece78d324a4d2973693cecb6919e2db Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:18:03 -0700 Subject: [PATCH 16/18] Only touch currentForklessSave on the main thread (#4467) Eliminate thread contention on internal currentForklessSave variable. --------- Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/forkless.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/forkless.c b/src/forkless.c index 70d1f46a8c0..1d0038cbd1b 100644 --- a/src/forkless.c +++ b/src/forkless.c @@ -129,7 +129,6 @@ static void *forklessSaveProcessor(void *arg) { serverLog(LL_NOTICE, "forkless-save: background processor finished. %ld items processed. %s", items, message); - currentForklessSave = NULL; saveInfo->err_code = err; bgIteratorClose(saveInfo->iterator); return NULL; @@ -234,6 +233,7 @@ void forklessSaveComplete(bool terminated, void *privdata) { saveInfo->terminated = terminated; /* The save iterator should be terminated and freed at this point in time. */ saveInfo->iterator = NULL; + currentForklessSave = NULL; /* For file based forkless save, we need to generate the RDB end marker. and complete the save */ if (!saveInfo->terminated && saveInfo->err_code == C_OK) { saveInfo->err_code = rdbWriteFooter(&saveInfo->save_rio, REPLICA_REQ_NONE) == C_ERR ? C_ERR : C_OK; From 0989e391fbbcb4bdc73ba314d1562ac679c8efd2 Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:20:32 -0700 Subject: [PATCH 17/18] Fix forkless save remaining time estimate (#4468) Forkless Save: minor change to estimated done time. Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/forkless.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/forkless.c b/src/forkless.c index 1d0038cbd1b..2341eae3e3d 100644 --- a/src/forkless.c +++ b/src/forkless.c @@ -385,12 +385,13 @@ sds forkless_catInfo(sds info) { current_item_millis = status.current_item_ms; if (status.dbentries_processed > 0) { - long long total_keys = 0; - for (int i = 0; i < server.dbnum; i++) { - total_keys += server.db[i] ? dbSize(server.db[i]) : 0; - } - estimated_seconds_remaining = (total_keys - status.dbentries_processed) * - status.runtime_ms / status.dbentries_processed / 1000; + long long total_keys = + (long long)atomic_load_explicit(&server.stat_current_save_keys_total, memory_order_relaxed); + /* The ETA is best effort. Clamp at 0 since dbentries_processed + * can exceed total_keys (e.g. a full sync may process more than + * the start-time key count). */ + long long remaining = max(total_keys - (long long)status.dbentries_processed, 0); + estimated_seconds_remaining = remaining * status.runtime_ms / status.dbentries_processed / 1000; } } } From 17b2a5b325a9abfbbf5e0267e558d816c5898f88 Mon Sep 17 00:00:00 2001 From: nitaicaro <42576749+nitaicaro@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:24:28 -0700 Subject: [PATCH 18/18] Preserve object metadata when an object is reallocated (#4475) Minor change to clone metadata when an object is cloned/copied. Signed-off-by: Nitai Caro Co-authored-by: Nitai Caro --- src/object.c | 18 ++++++++++++++++++ src/server.h | 1 + 2 files changed, 19 insertions(+) diff --git a/src/object.c b/src/object.c index b5e3de5993b..534184f1416 100644 --- a/src/object.c +++ b/src/object.c @@ -157,6 +157,22 @@ void *objectGetMetadata(const robj *o) { return (void *)data; } +/* Copy the opaque metadata region from one object to another. Used when an + * object is reallocated (e.g. objectSetKeyAndExpire) so that metadata attached + * by subsystems such as background iteration (forkless save) survives the move. + * + * The copy happens only when both objects actually carry metadata (both have an + * embedded key and the configured metadata size is non-zero). If the source has + * no metadata there is nothing to preserve, and the destination keeps its + * zero-initialized metadata. */ +void objectCopyMetadata(robj *dst, const robj *src) { + if (object_metadata_size == 0) return; + void *src_md = objectGetMetadata(src); + void *dst_md = objectGetMetadata(dst); + if (src_md == NULL || dst_md == NULL) return; + memcpy(dst_md, src_md, object_metadata_size); +} + /* ===================== Creation and parsing of objects ==================== */ /* Creates an object, optionally with embedded key and expire fields. The key @@ -481,6 +497,7 @@ robj *objectSetKeyAndExpire(robj *o, const_sds key, long long expire) { if (objectGetType(o) == OBJ_STRING && objectGetEncoding(o) == OBJ_ENCODING_EMBSTR) { robj *new = createStringObjectWithKeyAndExpire(objectGetVal(o), sdslen(objectGetVal(o)), key, expire); objectSetLRU(new, objectGetLRU(o)); + objectCopyMetadata(new, o); bgIteration_updateDbEntryPtr(o, new); decrRefCount(o); return new; @@ -507,6 +524,7 @@ robj *objectSetKeyAndExpire(robj *o, const_sds key, long long expire) { robj *new = createUnembeddedObjectWithKeyAndExpire(objectGetType(o), ptr, key, expire); objectSetEncoding(new, objectGetEncoding(o)); objectSetLRU(new, objectGetLRU(o)); + objectCopyMetadata(new, o); bgIteration_updateDbEntryPtr(o, new); decrRefCount(o); return new; diff --git a/src/server.h b/src/server.h index d4f85dec095..ff9279c6107 100644 --- a/src/server.h +++ b/src/server.h @@ -3266,6 +3266,7 @@ void objectSetLRU(robj *o, unsigned int lru); void objectSetMetadataSize(size_t size); size_t objectGetMetadataSize(const robj *o); void *objectGetMetadata(const robj *o); +void objectCopyMetadata(robj *dst, const robj *src); /* Synchronous I/O with timeout */ ssize_t syncWrite(int fd, char *ptr, ssize_t size, long long timeout);