diff --git a/CMakeLists.txt b/CMakeLists.txt index fbe077d..8269481 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,6 +81,7 @@ add_executable(svector_page_dump src/storage/tools/root_page_parser.cc src/storage/tools/data_page_parser.cc src/storage/tools/hnsw_layout.cc + src/storage/tools/hnsw_graph.cc ) target_include_directories(svector_page_dump PRIVATE diff --git a/src/storage/tools/README.md b/src/storage/tools/README.md index 20cd134..b01fb12 100644 --- a/src/storage/tools/README.md +++ b/src/storage/tools/README.md @@ -49,6 +49,14 @@ svector_page_dump [options] metadata as index metadata (level, entry level/points) and decode data page records as HNSW NeighbourEntry/OverflowEntry instead of a plain SVECTOR vector +- `-g, --graph`: Render the HNSW graph reachable from the root page's entry + point as ASCII tree art, walking every level via NID links. Requires `-i` + and a level-0 primary store root page (see "Show the HNSW Graph" below) +- `--max-nodes N`: Cap on distinct nodes rendered by `-g`/`--graph` across + all levels combined (default: 50) +- `--graph-style tree|list`: How `-g`/`--graph` renders each level -- `tree` + (default) draws nested tree art; `list` prints one flat adjacency line per + node instead (see "Show the HNSW Graph" below) - `-h, --help`: Show help message ## Examples @@ -127,6 +135,66 @@ and data page records decode as NeighbourEntry (owner, lower-level link, neighbours, overflow chain) or OverflowEntry (incoming links, overflow chain) fields instead of vector floats. +### 7. Show the HNSW Graph + +```bash +./svector_page_dump table.ibd 4 -i -g +``` + +Renders the graph reachable from ``'s entry point as ASCII +tree art, one level at a time: + +``` +HNSW Graph (M=16, entry level 1, showing up to 50 nodes) + +Level 1 (entry) +•4:0 +├─ •7:2 +└─ •12:5 + └─ •4:0 ... + +Level 0 +•4:0 +├─ •2:9 +│ ├─ •9:2 +│ └─ •15:4 +├─ •7:3 +└─ •22:0 +``` + +Only `` (the level-0 primary store) is needed -- every other +level and page is found by following NID links embedded in each node's own +record, the same way a real search descends the graph. A neighbour already +drawn elsewhere in the same level is shown as a `...` leaf rather +than re-expanded, since HNSW levels are graphs, not trees. `--max-nodes` +(default 50) bounds how many distinct nodes are fetched across the whole +walk; nodes beyond that show as `(truncated)` leaves. Add `-v` to also show +each node's owner VID as `()`. + +For levels with many edges per node, `--graph-style list` prints a flat +adjacency line per node instead of nested tree art: + +```bash +./svector_page_dump table.ibd 4 -i -g --graph-style list +``` + +``` +HNSW Graph (M=16, entry level 1, showing up to 50 nodes) + +Level 1 (entry) +•4:0(10:0) -> [7:2(10:3)] degree=1 + +Level 0 +•4:0(10:0) -> [2:9(10:5), 7:3(10:3)] degree=2 +•2:9(10:5) -> [4:0(10:0), 9:2(10:7), 15:4(10:8)] degree=3 +•7:3(10:3) -> [4:0(10:0)] degree=1 +``` + +Every node reachable from the level's entry point is listed exactly once +(BFS order), each showing its own outgoing neighbours and degree; there is +no `...`/`(seen above)` marker since a node is never re-expanded once +listed. + ## Finding Root Page Number The root page number is stored in the table's metadata. You can find it by: @@ -160,6 +228,8 @@ The tool consists of: - `hnsw_layout.{h,cc}`: Reimplements the HNSW index's on-disk metadata and record layouts (see `../../index/hnsw/storage.h` and `hnsw.h`) for `-i` mode, independent of the HNSW index sources +- `hnsw_graph.{h,cc}`: Walks and renders the HNSW graph as ASCII tree art for + `-g`/`--graph`, following NID links level by level via `hnsw_layout.h` - `svector_page_dump.cc`: Main driver program All components use the format definitions from `../root_page.h` and `../data_page.h` to stay in sync with the server implementation. `hnsw_layout.{h,cc}` must be kept in sync by hand with `../../index/hnsw/storage.h`/`hnsw.h` the same way, since it does not link against them. diff --git a/src/storage/tools/hnsw_graph.cc b/src/storage/tools/hnsw_graph.cc new file mode 100644 index 0000000..e66d0b3 --- /dev/null +++ b/src/storage/tools/hnsw_graph.cc @@ -0,0 +1,289 @@ +// Copyright (c) 2026 VillageSQL Contributors +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License, version 2.0, +// as published by the Free Software Foundation. +// +// This program is designed to work with certain software (including +// but not limited to OpenSSL) that is licensed under separate terms, +// as designated in a particular file or component or in included license +// documentation. The authors of MySQL hereby grant you an additional +// permission to link the program and your derivative works with the +// separately licensed software that they have either included with +// the program or referenced in the documentation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License, version 2.0, for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +#include "hnsw_graph.h" + +#include +#include + +#include "data_page_parser.h" + +namespace svector { +namespace tool { + +namespace { + +// Mirrors LevelStore::max_neighbours (see ../../index/hnsw/storage.h): +// level 0 gets twice the configured degree, every level above it gets M. +uint32_t level_max_neighbours(uint8_t level, uint32_t M) { + return level == 0 ? 2 * M : M; +} + +// Compact "page:slot" form of a NID/VID's Column::Ref, as opposed to +// format_hnsw_ref()'s verbose "Page #

, Slot #" -- graph output prints +// one of these per node (and, in verbose mode, a second for its VID), so +// terseness matters more here than in a single-record dump. +std::string ref_label(uint64_t ref_value) { + HnswColumnRef ref = hnsw_decode_ref(ref_value); + return std::to_string(ref.page_ref) + ":" + std::to_string(ref.slot_index); +} + +std::string node_label(uint64_t nid_ref) { + return "\xE2\x80\xA2" + ref_label(nid_ref); // U+2022 BULLET +} + +// Fetches nid_ref's NeighbourEntry record at level. Returns false (with a +// message already printed at line_prefix + label) if the page or slot +// cannot be read as a live Neighbour record. +bool fetch_record(PageReader &reader, uint64_t nid_ref, uint8_t level, + uint32_t M, const std::string &line_prefix, + const std::string &label, std::ostream &out, + DataPageParser::RecordStatus &rec) { + HnswColumnRef ref = hnsw_decode_ref(nid_ref); + bool has_lower = level > 0; + uint16_t column_size = + hnsw_neighbour_column_size(level_max_neighbours(level, M), has_lower); + + auto page_data = reader.read_page(ref.page_ref); + if (!page_data) { + out << line_prefix << label << " (error reading page " << ref.page_ref + << ": " << reader.get_error() << ")\n"; + return false; + } + + DataPageParser::DataPageInfo info; + std::string error; + if (!DataPageParser::parse(*page_data, column_size, info, error, + HnswRecordKind::Neighbour, has_lower)) { + out << line_prefix << label << " (error decoding page " << ref.page_ref + << ": " << error << ")\n"; + return false; + } + if (ref.slot_index >= info.records.size() || + info.records[ref.slot_index].is_free) { + out << line_prefix << label << " (slot " << ref.slot_index << " on page " + << ref.page_ref << " is not a live record)\n"; + return false; + } + + rec = info.records[ref.slot_index]; + return true; +} + +// Recursively prints nid_ref and its subtree using tree-drawing connectors. +// prefix is the indentation already emitted for this node's own line; +// is_root suppresses the connector (the level's local entry point gets a +// bare line, like `tree`'s root). visited/budget/total_shown are threaded +// through the whole level's walk. When this call is the level's root and it +// has a lower-level link, out_lower_ref/out_has_lower report it back to the +// caller so the next level's walk knows where to start. +void print_node(PageReader &reader, uint8_t level, uint32_t M, bool verbose, + uint64_t nid_ref, const std::string &prefix, bool is_root, + bool is_last, std::unordered_set &visited, + uint32_t &budget, uint32_t &total_shown, std::ostream &out, + uint64_t &out_lower_ref, bool &out_has_lower) { + std::string label = node_label(nid_ref); + std::string line_prefix = + is_root ? prefix + : prefix + (is_last ? "\xE2\x94\x94\xE2\x94\x80 " + : "\xE2\x94\x9C\xE2\x94\x80 "); + // U+2514 U+2500 ("+- ") for the last child, U+251C U+2500 ("|- ") otherwise. + + if (visited.count(nid_ref)) { + out << line_prefix << label << " ...\n"; + return; + } + + if (budget == 0) { + out << line_prefix << label << " (truncated)\n"; + return; + } + + visited.insert(nid_ref); + --budget; + ++total_shown; + + DataPageParser::RecordStatus rec; + if (!fetch_record(reader, nid_ref, level, M, line_prefix, label, out, rec)) + return; + + if (verbose) + out << line_prefix << label << "(" << ref_label(rec.owner_vid) << ")\n"; + else + out << line_prefix << label << "\n"; + + if (level > 0 && rec.lower_level_nid != 0) { + out_lower_ref = rec.lower_level_nid & HNSW_NID_REF_MASK; + out_has_lower = true; + } + + // Only forward (non-incoming) edges are drawn -- see hnsw_graph.h. + std::vector children; + for (const auto &neighbour : rec.neighbours) { + uint64_t nid = neighbour.first; + if (nid == 0 || (nid & HNSW_NID_INCOMING_BIT)) + continue; + children.push_back(nid & HNSW_NID_REF_MASK); + } + + std::string child_prefix = + is_root ? prefix : prefix + (is_last ? " " : "\xE2\x94\x82 "); + // " " under a last child, "| " (U+2502) otherwise. + + for (size_t i = 0; i < children.size(); ++i) { + uint64_t dummy_ref = 0; + bool dummy_has = false; + print_node(reader, level, M, verbose, children[i], child_prefix, + /*is_root=*/false, /*is_last=*/i + 1 == children.size(), visited, + budget, total_shown, out, dummy_ref, dummy_has); + } +} + +// Walks the level's local neighbour graph breadth-first from entry_ref, +// printing one line per node: "