reject binary subtypes MessagePack and BSON cannot represent

docs/mkdocs/docs/api/basic_json/to_bson.md

@@ -43,6 +43,8 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
 - Throws [`out_of_range.412`](../../home/exceptions.md#jsonexceptionout_of_range412) if the length of a document, array,
   string, or binary value exceeds the range of the 32-bit BSON length field; example:
   `"BSON length 2147483661 exceeds maximum of 2147483647"`
+- Throws [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) if the subtype of a binary value
+  exceeds the range of the 8-bit BSON subtype field; example: `"BSON binary subtype 258 exceeds maximum of 255"`
 
 ## Complexity
 

docs/mkdocs/docs/api/basic_json/to_msgpack.md

@@ -34,6 +34,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 
 Strong guarantee: if an exception is thrown, there are no changes in the JSON value.
 
+## Exceptions
+
+- Throws [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) if the subtype of a binary value
+  exceeds the range of the 8-bit MessagePack ext type; example:
+  `"MessagePack binary subtype 511 exceeds maximum of 255"`
+
 ## Complexity
 
 Linear in the size of the JSON value `j`.

docs/mkdocs/docs/home/exceptions.md

@@ -933,6 +933,28 @@ BSON stores the length of documents, arrays, strings, and binary values in a sig
     [`to_bson`](../api/basic_json/to_bson.md) produced documents with negative length prefixes that
     [`from_bson`](../api/basic_json/from_bson.md) rejected.
 
+### json.exception.out_of_range.413
+
+MessagePack ext types and BSON binary values store the subtype of a binary value in a single byte, whereas
+[`byte_container_with_subtype`](../api/byte_container_with_subtype/index.md) and the
+[CBOR](../features/binary_formats/cbor.md) tags it is read from hold the full 64-bit range. This exception is thrown
+when a subtype is too large to be described by such a subtype field.
+
+!!! failure "Example messages"
+
+    ```
+    MessagePack binary subtype 511 exceeds maximum of 255
+    ```
+    ```
+    BSON binary subtype 258 exceeds maximum of 255
+    ```
+
+!!! note
+
+    This exception was added in version 3.13.0. Before that, the subtype was silently truncated to its lowest byte, so
+    [`to_msgpack`](../api/basic_json/to_msgpack.md) and [`to_bson`](../api/basic_json/to_bson.md) could emit a different
+    subtype than the value carried, including subtypes both formats reserve.
+
 ## Further exceptions
 
 This exception is thrown in case of errors that cannot be classified with the

include/nlohmann/detail/output/binary_writer.hpp

@@ -688,7 +688,7 @@ class binary_writer
                 // step 1.5: if this is an ext type, write the subtype
                 if (use_ext)
                 {
-                    write_number(static_cast<std::int8_t>(j.m_data.m_value.binary->subtype()));
+                    write_number(static_cast<std::int8_t>(to_binary_subtype(j.m_data.m_value.binary->subtype(), "MessagePack")));
                 }
 
                 // step 2: write the byte string
@@ -1178,7 +1178,7 @@ class binary_writer
         write_bson_entry_header(name, 0x05);
 
         write_number<std::int32_t>(to_bson_length(value.size()), true);
-        write_number(value.has_subtype() ? static_cast<std::uint8_t>(value.subtype()) : static_cast<std::uint8_t>(0x00));
+        write_number(value.has_subtype() ? to_binary_subtype(value.subtype(), "BSON") : static_cast<std::uint8_t>(0x00));
 
         oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());
     }
@@ -1799,6 +1799,25 @@ class binary_writer
     // Utility functions //
     ///////////////////////
 
+    /*!
+    @brief Checks that @a subtype fits into the 8-bit subtype field shared by
+           MessagePack ext types and BSON binary values
+    @param[in] subtype  the binary subtype to write
+    @param[in] format   the format name to use in the exception message
+    @return The subtype as an unsigned 8-bit integer
+    @throw out_of_range.413 if @a subtype exceeds the range of std::uint8_t
+    */
+    static std::uint8_t to_binary_subtype(const typename BasicJsonType::binary_t::subtype_type subtype,
+                                          const char* format)
+    {
+        if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::uint8_t>(subtype)))
+        {
+            JSON_THROW(out_of_range::create(413, concat(format, " binary subtype ", std::to_string(subtype), " exceeds maximum of ", std::to_string((std::numeric_limits<std::uint8_t>::max)())), nullptr));
+        }
+
+        return static_cast<std::uint8_t>(subtype);
+    }
+
     /*
     @brief write a number to output input
     @param[in] n number of type @a NumberType

single_include/nlohmann/json.hpp

@@ -17622,7 +17622,7 @@ class binary_writer
                 // step 1.5: if this is an ext type, write the subtype
                 if (use_ext)
                 {
-                    write_number(static_cast<std::int8_t>(j.m_data.m_value.binary->subtype()));
+                    write_number(static_cast<std::int8_t>(to_binary_subtype(j.m_data.m_value.binary->subtype(), "MessagePack")));
                 }
 
                 // step 2: write the byte string
@@ -18112,7 +18112,7 @@ class binary_writer
         write_bson_entry_header(name, 0x05);
 
         write_number<std::int32_t>(to_bson_length(value.size()), true);
-        write_number(value.has_subtype() ? static_cast<std::uint8_t>(value.subtype()) : static_cast<std::uint8_t>(0x00));
+        write_number(value.has_subtype() ? to_binary_subtype(value.subtype(), "BSON") : static_cast<std::uint8_t>(0x00));
 
         oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());
     }
@@ -18733,6 +18733,25 @@ class binary_writer
     // Utility functions //
     ///////////////////////
 
+    /*!
+    @brief Checks that @a subtype fits into the 8-bit subtype field shared by
+           MessagePack ext types and BSON binary values
+    @param[in] subtype  the binary subtype to write
+    @param[in] format   the format name to use in the exception message
+    @return The subtype as an unsigned 8-bit integer
+    @throw out_of_range.413 if @a subtype exceeds the range of std::uint8_t
+    */
+    static std::uint8_t to_binary_subtype(const typename BasicJsonType::binary_t::subtype_type subtype,
+                                          const char* format)
+    {
+        if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::uint8_t>(subtype)))
+        {
+            JSON_THROW(out_of_range::create(413, concat(format, " binary subtype ", std::to_string(subtype), " exceeds maximum of ", std::to_string((std::numeric_limits<std::uint8_t>::max)())), nullptr));
+        }
+
+        return static_cast<std::uint8_t>(subtype);
+    }
+
     /*
     @brief write a number to output input
     @param[in] n number of type @a NumberType

tests/src/unit-bson.cpp

@@ -111,6 +111,19 @@ TEST_CASE("BSON")
         CHECK_THROWS_WITH_AS(huge_binary_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483661 exceeds maximum of 2147483647", huge_binary_json::out_of_range&);
     }
 
+    SECTION("binary subtypes exceeding 0xFF cannot be serialized to BSON")
+    {
+        // the largest subtype the BSON subtype byte can hold still round-trips
+        json j;
+        j["b"] = json::binary(std::vector<std::uint8_t> {0xCA, 0xFE, 0xBA, 0xBE}, 0xFF);
+        CHECK(json::from_bson(json::to_bson(j)) == j);
+
+        // a subtype beyond that would be truncated onto a different (reserved) subtype
+        json k;
+        k["b"] = json::binary(std::vector<std::uint8_t> {0xCA, 0xFE, 0xBA, 0xBE}, 258);
+        CHECK_THROWS_WITH_AS(json::to_bson(k), "[json.exception.out_of_range.413] BSON binary subtype 258 exceeds maximum of 255", json::out_of_range&);
+    }
+
     SECTION("string length must be at least 1")
     {
         // from https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11175

tests/src/unit-msgpack.cpp

@@ -1424,6 +1424,17 @@ TEST_CASE("MessagePack")
         }
     }
 
+    SECTION("binary subtypes exceeding 0xFF cannot be serialized to MessagePack")
+    {
+        // the largest subtype the ext type field can hold still round-trips
+        json const j = json::binary(std::vector<uint8_t> {0xCA, 0xFE, 0xBA, 0xBE}, 0xFF);
+        CHECK(json::from_msgpack(json::to_msgpack(j)) == j);
+
+        // a subtype beyond that would be truncated onto a different (reserved) ext type
+        json const k = json::binary(std::vector<uint8_t> {0xCA, 0xFE, 0xBA, 0xBE}, 511);
+        CHECK_THROWS_WITH_AS(json::to_msgpack(k), "[json.exception.out_of_range.413] MessagePack binary subtype 511 exceeds maximum of 255", json::out_of_range&);
+    }
+
     SECTION("from float32")
     {
         auto given = std::vector<uint8_t>({0xca, 0x41, 0xc8, 0x00, 0x01});