diff --git a/score/kvs/kvs.cpp b/score/kvs/kvs.cpp index 1fc39005b..0c4d745ba 100644 --- a/score/kvs/kvs.cpp +++ b/score/kvs/kvs.cpp @@ -17,6 +17,7 @@ #include #include #include +#include // TODO Default Value Handling TBD // TODO Add Score Logging @@ -29,6 +30,12 @@ using namespace std; namespace score::mw::per::kvs { +namespace +{ +/* Size of a snapshot hash file in bytes (adler32 checksum) */ +constexpr size_t HASH_FILE_SIZE = 4U; +} /* namespace */ + /*********************** KVS Implementation *********************/ Kvs::Kvs() : filesystem(std::make_unique( @@ -47,7 +54,8 @@ Kvs::Kvs(Kvs&& other) noexcept object would also be okay*/ , writer(std::move(other.writer)), - logger(std::move(other.logger)) + logger(std::move(other.logger)), + max_storage_bytes(other.max_storage_bytes) { { std::lock_guard lock(other.kvs_mutex); @@ -81,6 +89,7 @@ Kvs& Kvs::operator=(Kvs&& other) noexcept parser = std::move(other.parser); writer = std::move(other.writer); logger = std::move(other.logger); + max_storage_bytes = other.max_storage_bytes; } return *this; } @@ -218,7 +227,8 @@ score::Result> Kvs::open_json(const score:: score::Result Kvs::open(const InstanceId& instance_id, OpenNeedDefaults need_defaults, OpenNeedKvs need_kvs, - const std::string&& dir) + const std::string&& dir, + std::optional max_storage_bytes) { score::Result result = score::MakeUnexpected(ErrorCode::UnmappedError); /* Redundant initialization needed, since Resul would call @@ -230,6 +240,7 @@ score::Result Kvs::open(const InstanceId& instance_id, const score::filesystem::Path filename_kvs = filename_prefix.Native() + "_0"; Kvs kvs; /* Create KVS instance */ + kvs.max_storage_bytes = max_storage_bytes; /* Store maximum storage limit */ auto default_res = kvs.open_json( filename_default, need_defaults == OpenNeedDefaults::Required ? OpenJsonNeedFile::Required : OpenJsonNeedFile::Optional); @@ -472,6 +483,55 @@ score::ResultBlank Kvs::remove_key(const std::string_view key) return result; } +score::Result Kvs::get_file_size(const score::filesystem::Path& file_path) { + std::error_code ec; + const auto size = std::filesystem::file_size(file_path.CStr(), ec); + + if (ec) { + // Check if the error is "file not found" + if (ec == std::errc::no_such_file_or_directory) { + // File does not exist, its size is 0. This is not an error. + return 0; + } + logger->LogError() << "Error: Could not get size of file " << file_path << ": " << ec.message(); + return score::MakeUnexpected(ErrorCode::PhysicalStorageFailure); + } + + return size; +} + +/* Helper Function to get current storage size of all persisted files (defaults and historical snapshots) */ +score::Result Kvs::get_current_storage_size() { + size_t total_size = 0; + const std::array file_extensions = {".json", ".hash"}; + + // Add the size of the default files + const std::string default_suffix = "_default"; + for (const char* extension : file_extensions) { + const score::filesystem::Path file_path = filename_prefix.Native() + default_suffix + extension; + auto size_result = get_file_size(file_path); + if (!size_result) { + return size_result; // Propagate error directly + } + total_size += size_result.value(); + } + + // Add the size of current and historical snapshots that will be kept after rotation (0 to N-1). + for (size_t snapshot_index = 0; snapshot_index < KVS_MAX_SNAPSHOTS; ++snapshot_index) { + const std::string snapshot_suffix = "_" + to_string(snapshot_index); + + for (const char* extension : file_extensions) { + const score::filesystem::Path file_path = filename_prefix.Native() + snapshot_suffix + extension; + auto size_result = get_file_size(file_path); + if (!size_result) { + return size_result; // Propagate error directly + } + total_size += size_result.value(); + } + } + return total_size; +} + /* Helper: write data to a file and ensure it reaches physical storage.*/ score::ResultBlank Kvs::write_and_sync(const std::string& path, const void* data, std::size_t size) { @@ -550,13 +610,10 @@ score::ResultBlank Kvs::write_json_data(const std::string& buf) return result; } -/* Flush the key-value store*/ -score::ResultBlank Kvs::flush() -{ - score::ResultBlank result = score::MakeUnexpected(ErrorCode::UnmappedError); - /* Create JSON Object */ +score::Result Kvs::serialize_and_check() { score::json::Object root_obj; - bool error = false; + + // 1. Serialize the current KVS map to a buffer { std::unique_lock lock(kvs_mutex, std::try_to_lock); if (lock.owns_lock()) @@ -564,52 +621,71 @@ score::ResultBlank Kvs::flush() for (const auto& [key, value] : kvs) { auto conv = kvsvalue_to_any(value); - if (!conv) - { - result = score::MakeUnexpected(static_cast(*conv.error())); - error = true; - break; - } - else - { - root_obj.emplace(key, std::move(conv.value()) /*emplace in map uses move operator*/ - ); + if (!conv) { + return score::MakeUnexpected(conv.error()); } + root_obj.emplace(key, std::move(conv.value())); } - } - else - { - result = score::MakeUnexpected(ErrorCode::MutexLockFailed); - error = true; + } else { + return score::MakeUnexpected(ErrorCode::MutexLockFailed); } } - if (!error) - { - /* Serialize Buffer */ - auto buf_res = writer->ToBuffer(root_obj); - if (!buf_res) - { - result = score::MakeUnexpected(ErrorCode::JsonGeneratorError); - } - else - { - /* Rotate Snapshots */ - auto rotate_result = snapshot_rotate(); - if (!rotate_result) - { - result = rotate_result; - } - else - { - /* Write JSON Data */ - std::string buf = std::move(buf_res.value()); - result = write_json_data(buf); - } - } + auto buf_res = writer->ToBuffer(root_obj); + if (!buf_res) { + return score::MakeUnexpected(ErrorCode::JsonGeneratorError); } + const std::string& buf = buf_res.value(); - return result; + // 2. Get the size of all other persisted files + auto current_size_res = get_current_storage_size(); + if (!current_size_res) { + return score::MakeUnexpected(current_size_res.error()); + } + + // 3. Calculate the potential total size + const size_t total_size = current_size_res.value() + buf.size() + HASH_FILE_SIZE; + + // 4. Check against the limit + if (this->max_storage_bytes.has_value() && total_size > this->max_storage_bytes.value()) { + logger->LogError() << "error: KVS storage limit would be exceeded. total_size:" << total_size + << " max_storage_bytes:" << this->max_storage_bytes.value(); + return score::MakeUnexpected(ErrorCode::OutOfStorageSpace); + } + + return buf; +} + +/* Flush the key-value store*/ +score::ResultBlank Kvs::flush() { + auto result = serialize_and_check(); + if (!result) { + return score::MakeUnexpected(result.error()); + } + + auto rotate_result = snapshot_rotate(); + if (!rotate_result) { + return rotate_result; + } + + return write_json_data(result.value()); +} + +/* Performs a 'dry run' to check the potential storage size */ +score::Result Kvs::calculate_potential_size() { + auto result = serialize_and_check(); + if (!result) { + return score::MakeUnexpected(result.error()); + } + + // Re-calculate size to return it, as serialize_and_check only returns the buffer + const std::string& buf = result.value(); + auto current_size_res = get_current_storage_size(); + if (!current_size_res) { + return score::MakeUnexpected(current_size_res.error()); + } + + return current_size_res.value() + buf.size() + HASH_FILE_SIZE; } /* Retrieve the snapshot count*/ diff --git a/score/kvs/kvs.hpp b/score/kvs/kvs.hpp index cdccf1a09..a62baee36 100644 --- a/score/kvs/kvs.hpp +++ b/score/kvs/kvs.hpp @@ -161,6 +161,8 @@ class Kvs final * - OpenNeedKvs::Optional: An empty KVS will be used if no KVS exists. * @param dir The directory path where the KVS files are located. It is passed as an rvalue * reference to avoid unnecessary copying. Use "" or "." for the current directory. + * @param max_storage_bytes Optional maximum total storage size in bytes. When unset + * (the default), no storage limit is enforced. * @return A Result object containing either: * - A Kvs object if the operation is successful. * - An ErrorCode if an error occurs during the operation. @@ -172,7 +174,8 @@ class Kvs final static score::Result open(const InstanceId& instance_id, OpenNeedDefaults need_defaults, OpenNeedKvs need_kvs, - const std::string&& dir); + const std::string&& dir, + std::optional max_storage_bytes = std::nullopt); /** * @brief Resets a key-value-storage to its initial state @@ -352,6 +355,21 @@ class Kvs final */ score::Result get_hash_filename(const SnapshotId& snapshot_id) const; + /** + * @brief Performs a 'dry run' to check if the current in-memory store would + * exceed the storage limit upon flushing. + * + * This function serializes the current key-value data to a temporary buffer + * and calculates the potential total storage size. It checks this size against + * the configured `max_storage_bytes` limit. + * + * @return A score::Result object containing either: + * - On success: The estimated total size (size_t) that the KVS would occupy after a flush. + * - On failure: An `OutOfStorageSpace` error if the limit would be exceeded, + * or another ErrorCode for other failures (e.g., serialization). + */ + score::Result calculate_potential_size(); + private: /* Private constructor to prevent direct instantiation */ Kvs(); @@ -376,6 +394,9 @@ class Kvs final /* Logging */ std::unique_ptr logger; + /* Maximum storage limit in bytes. Unset means no limit is enforced. */ + std::optional max_storage_bytes; + /* Private Methods */ score::ResultBlank snapshot_rotate(); score::Result> parse_json_data(const std::string& data); @@ -383,6 +404,10 @@ class Kvs final OpenJsonNeedFile need_file); score::ResultBlank write_json_data(const std::string& buf); score::ResultBlank write_and_sync(const std::string& path, const void* data, std::size_t size); + + score::Result serialize_and_check(); + score::Result get_file_size(const score::filesystem::Path& file_path); + score::Result get_current_storage_size(); }; } /* namespace score::mw::per::kvs */ diff --git a/score/kvs/kvsbuilder.cpp b/score/kvs/kvsbuilder.cpp index a248ebfea..21c58ca08 100644 --- a/score/kvs/kvsbuilder.cpp +++ b/score/kvs/kvsbuilder.cpp @@ -20,7 +20,8 @@ KvsBuilder::KvsBuilder(const InstanceId& instance_id) : instance_id(instance_id), need_defaults(false), need_kvs(false), - directory("./data_folder/") /* Default Directory */ + directory("./data_folder/"), /* Default Directory */ + maximum_storage_bytes(std::nullopt) /* No storage limit by default */ { } @@ -42,6 +43,12 @@ KvsBuilder& KvsBuilder::dir(std::string&& dir_path) return *this; } +KvsBuilder& KvsBuilder::max_storage_bytes(std::optional max_storage_bytes) +{ + this->maximum_storage_bytes = max_storage_bytes; + return *this; +} + score::Result KvsBuilder::build() { score::Result result = score::MakeUnexpected(ErrorCode::UnmappedError); @@ -55,7 +62,8 @@ score::Result KvsBuilder::build() result = Kvs::open(instance_id, need_defaults ? OpenNeedDefaults::Required : OpenNeedDefaults::Optional, need_kvs ? OpenNeedKvs::Required : OpenNeedKvs::Optional, - std::move(directory)); + std::move(directory), + maximum_storage_bytes); return result; } diff --git a/score/kvs/kvsbuilder.hpp b/score/kvs/kvsbuilder.hpp index 2f6e1f6ef..480c2b85d 100644 --- a/score/kvs/kvsbuilder.hpp +++ b/score/kvs/kvsbuilder.hpp @@ -15,6 +15,7 @@ #include "kvs.hpp" #include +#include namespace score::mw::per::kvs { @@ -89,6 +90,14 @@ class KvsBuilder final */ KvsBuilder& dir(std::string&& dir_path); + /** + * @brief Configure the maximum storage size for the KVS in bytes. + * @param max_storage_bytes The maximum allowed total storage size in bytes. When this + * option is not used, no storage limit is enforced. + * @return Reference to this builder (for chaining). + */ + KvsBuilder& max_storage_bytes(std::optional max_storage_bytes); + /** * @brief Builds and opens the Kvs instance with the configured options. * @@ -99,10 +108,11 @@ class KvsBuilder final score::Result build(); private: - InstanceId instance_id; ///< ID of the KVS instance - bool need_defaults; ///< Whether default values are required - bool need_kvs; ///< Whether an existing KVS is required - std::string directory; ///< Directory where to store the KVS Files + InstanceId instance_id; ///< ID of the KVS instance + bool need_defaults; ///< Whether default values are required + bool need_kvs; ///< Whether an existing KVS is required + std::string directory; ///< Directory where to store the KVS Files + std::optional maximum_storage_bytes; ///< Maximum total storage size in bytes; unset means no limit }; } /* namespace score::mw::per::kvs */ diff --git a/score/kvs/tests/test_kvs.cpp b/score/kvs/tests/test_kvs.cpp index c3d5d0805..3776ae47d 100644 --- a/score/kvs/tests/test_kvs.cpp +++ b/score/kvs/tests/test_kvs.cpp @@ -1235,3 +1235,111 @@ TEST(kvs_get_filename, get_hashname_failure) cleanup_environment(); } + +/* Storage limit used by the max-size tests. The limit is an explicit test input: + an unconfigured KVS enforces no limit at all. */ +constexpr size_t kTestMaxStorageBytes = 1000U; + +TEST(kvs_max_storage_bytes, flush_succeeds_without_configured_limit) +{ + /* The builder default is an unset optional, meaning no limit is enforced. + Data far larger than kTestMaxStorageBytes must therefore still flush. */ + const std::string test_dir = "./kvs_no_limit_test/"; + std::filesystem::remove_all(test_dir); + + KvsBuilder builder(instance_id); + builder.dir(std::string(test_dir)); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + const std::string large_data(kTestMaxStorageBytes * 4U, 'a'); + auto set_res = kvs.set_value("large_data", KvsValue(large_data.c_str())); + ASSERT_TRUE(set_res); + + auto flush_res = kvs.flush(); + EXPECT_TRUE(flush_res); + + std::filesystem::remove_all(test_dir); +} + +TEST(kvs_max_storage_bytes, flush_fails_when_storage_limit_exceeded) +{ + const std::string test_dir = "./kvs_storage_test/"; + std::filesystem::remove_all(test_dir); + + KvsBuilder builder(instance_id); + builder.dir(std::string(test_dir)); + builder.max_storage_bytes(kTestMaxStorageBytes); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + /* Add data close to the limit. There is overhead for the JSON structure (key, + type info, braces) and the hash file, so keep the payload below the maximum. */ + const size_t overhead_estimate = 100U; + const std::string large_data(kTestMaxStorageBytes - overhead_estimate, 'a'); + + auto set_res1 = kvs.set_value("large_data", KvsValue(large_data.c_str())); + ASSERT_TRUE(set_res1); + + /* The first flush still fits */ + auto flush_res1 = kvs.flush(); + ASSERT_TRUE(flush_res1); + + /* A little more data pushes the total over the limit */ + auto set_res2 = kvs.set_value("extra_data", KvsValue("this should not fit")); + ASSERT_TRUE(set_res2); + + auto flush_res2 = kvs.flush(); + ASSERT_FALSE(flush_res2); + EXPECT_EQ(static_cast(*flush_res2.error()), ErrorCode::OutOfStorageSpace); + + std::filesystem::remove_all(test_dir); +} + +TEST(kvs_check_size, check_size_within_limit_succeeds) +{ + const std::string test_dir = "./kvs_check_size_within_test/"; + std::filesystem::remove_all(test_dir); + + KvsBuilder builder(InstanceId(1)); + builder.dir(std::string(test_dir)); + builder.max_storage_bytes(kTestMaxStorageBytes); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + auto set_res = kvs.set_value("key", KvsValue("some_data")); + ASSERT_TRUE(set_res); + + auto check_res = kvs.calculate_potential_size(); + ASSERT_TRUE(check_res) << "calculate_potential_size should succeed for data within limits"; + EXPECT_GT(check_res.value(), 0U); + EXPECT_LT(check_res.value(), kTestMaxStorageBytes); + + std::filesystem::remove_all(test_dir); +} + +TEST(kvs_check_size, check_size_exceeding_limit_fails) +{ + const std::string test_dir = "./kvs_check_size_exceeding_test/"; + std::filesystem::remove_all(test_dir); + + KvsBuilder builder(InstanceId(2)); + builder.dir(std::string(test_dir)); + builder.max_storage_bytes(kTestMaxStorageBytes); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + const std::string large_data(kTestMaxStorageBytes, 'x'); + auto set_res = kvs.set_value("oversized_key", KvsValue(large_data.c_str())); + ASSERT_TRUE(set_res); + + auto check_res = kvs.calculate_potential_size(); + ASSERT_FALSE(check_res) << "calculate_potential_size should fail when the storage limit is exceeded"; + EXPECT_EQ(static_cast(*check_res.error()), ErrorCode::OutOfStorageSpace); + + std::filesystem::remove_all(test_dir); +}