From 1e28958e4af2d3844eac84f99e5312b63e19ff1e Mon Sep 17 00:00:00 2001 From: akamick86 <12109250+akamick86@users.noreply.github.com> Date: Fri, 29 May 2026 15:17:27 -0700 Subject: [PATCH] perf: optionally memory-map the address parser CRF model crf_read deserializes the entire CRF model (address_parser_crf.dat, ~967MB) into private heap on every load. It is the largest file libpostal reads and dominates resident memory in memory-constrained or many-process setups. This adds an optional path that memory-maps the model so it is shared across processes and paged in on demand instead of copied up front. The model is big-endian flat arrays on disk (two double-array tries and two CSR weight matrices) which cannot be mmap'd directly on little-endian hosts, so on first load a host-endian, 8-byte aligned sidecar is written next to the data (.trcache and .spcache) and later loads map it read-only. Any failure (read-only directory, stale or corrupt cache, Windows) falls back to the original reader, so output is identical. Off by default, controlled by LIBPOSTAL_MMAP_CACHE: unset or 0 disables it with no behavior change and no cache files, 1 enables it and validates the cache against the source size, mtime and inode, and trust enables it but skips the freshness check for immutable read-only deployments that ship a pre-built cache. Only crf_read uses the cached readers; other loaders are unchanged. Single process, 800 varied addresses, peak RSS 2489MB -> 1078MB (-57%); steady-state parse latency unchanged. --- src/crf.c | 12 +- src/crf.h | 5 +- src/file_utils.c | 271 +++++++++++++++++++++++++++++++++++++++++ src/file_utils.h | 72 +++++++++++ src/sparse_matrix.c | 148 +++++++++++++++++++++- src/sparse_matrix.h | 10 ++ src/trie.c | 171 +++++++++++++++++++++++++- src/trie.h | 10 ++ test/Makefile.am | 2 +- test/test.c | 2 + test/test_mmap_cache.c | 135 ++++++++++++++++++++ 11 files changed, 828 insertions(+), 10 deletions(-) create mode 100644 test/test_mmap_cache.c diff --git a/src/crf.c b/src/crf.c index e668209e3..2f936367b 100644 --- a/src/crf.c +++ b/src/crf.c @@ -198,7 +198,7 @@ bool crf_save(crf_t *self, char *filename) { } -crf_t *crf_read(FILE *f) { +crf_t *crf_read(FILE *f, const char *path) { if (f == NULL) return NULL; uint32_t signature; @@ -240,22 +240,22 @@ crf_t *crf_read(FILE *f) { goto exit_crf_created; } - crf->state_features = trie_read(f); + crf->state_features = trie_read_cached(f, path); if (crf->state_features == NULL) { goto exit_crf_created; } - crf->weights = sparse_matrix_read(f); + crf->weights = sparse_matrix_read_cached(f, path); if (crf->weights == NULL) { goto exit_crf_created; } - crf->state_trans_features = trie_read(f); + crf->state_trans_features = trie_read_cached(f, path); if (crf->state_trans_features == NULL) { goto exit_crf_created; } - crf->state_trans_weights = sparse_matrix_read(f); + crf->state_trans_weights = sparse_matrix_read_cached(f, path); if (crf->state_trans_weights == NULL) { goto exit_crf_created; } @@ -286,7 +286,7 @@ crf_t *crf_load(char *filename) { if (filename == NULL) return NULL; FILE *f = fopen(filename, "rb"); if (f == NULL) return NULL; - crf_t *crf = crf_read(f); + crf_t *crf = crf_read(f, filename); fclose(f); return crf; } diff --git a/src/crf.h b/src/crf.h index af0dabeb2..48ffac7ba 100644 --- a/src/crf.h +++ b/src/crf.h @@ -45,7 +45,10 @@ bool crf_tagger_predict(crf_t *self, void *tagger, void *context, cstring_array bool crf_write(crf_t *self, FILE *f); bool crf_save(crf_t *self, char *filename); -crf_t *crf_read(FILE *f); +// `path` is the source file path, used for the optional shared mmap cache of the +// model's tries and weight matrices (see LIBPOSTAL_MMAP_CACHE in file_utils.h). +// Pass NULL to disable caching for this read. +crf_t *crf_read(FILE *f, const char *path); crf_t *crf_load(char *filename); void crf_destroy(crf_t *self); diff --git a/src/file_utils.c b/src/file_utils.c index 5fc2dfbe7..993859111 100644 --- a/src/file_utils.c +++ b/src/file_utils.c @@ -282,3 +282,274 @@ bool file_read_chars(FILE *file, char *buf, size_t len) { bool file_write_chars(FILE *file, const char *buf, size_t len) { return (fwrite(buf, sizeof(char), len, file) == len); } + +#ifndef _WIN32 + +#include +#include +#include + +#if defined(__APPLE__) +#define FILE_MMAP_MTIME_NSEC(st) ((uint64_t)(st).st_mtimespec.tv_nsec) +#else +#define FILE_MMAP_MTIME_NSEC(st) ((uint64_t)(st).st_mtim.tv_nsec) +#endif + +#define FILE_MMAP_ENDIAN_CHECK 0x01020304U + +typedef enum { + FILE_MMAP_MODE_OFF = 0, + FILE_MMAP_MODE_ON = 1, + FILE_MMAP_MODE_TRUST = 2 +} file_mmap_mode_t; + +static file_mmap_mode_t file_mmap_cache_mode(void) { + const char *v = getenv("LIBPOSTAL_MMAP_CACHE"); + if (v == NULL || v[0] == '\0') + return FILE_MMAP_MODE_OFF; + if (strcmp(v, "trust") == 0 || strcmp(v, "TRUST") == 0) + return FILE_MMAP_MODE_TRUST; + if (strcmp(v, "0") == 0 || strcmp(v, "off") == 0 || + strcmp(v, "false") == 0 || strcmp(v, "no") == 0) + return FILE_MMAP_MODE_OFF; + return FILE_MMAP_MODE_ON; +} + +bool file_mmap_cache_enabled(void) { + return file_mmap_cache_mode() != FILE_MMAP_MODE_OFF; +} + +bool file_mmap_cache_trusted(void) { + return file_mmap_cache_mode() == FILE_MMAP_MODE_TRUST; +} + +// Common header at the start of every cache file. 72 bytes, a multiple of 8 so +// the type header and (8-aligned) arrays that follow keep their alignment. +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t endian_check; + uint32_t num_arrays; + uint64_t src_size; + uint64_t src_mtime; + uint64_t src_mtime_nsec; + uint64_t src_ino; + uint64_t src_offset; + uint64_t src_disk_size; + uint32_t type_header_len; + uint32_t reserved; +} file_mmap_cache_common_t; + +typedef struct { + uint32_t elem_size; + uint32_t reserved; + uint64_t count; +} file_mmap_array_desc_t; + +static inline uint64_t file_mmap_align8(uint64_t x) { + return (x + 7u) & ~((uint64_t)7u); +} + +file_mmap_src_id_t file_mmap_src_id_from_stat(const struct stat *st) { + file_mmap_src_id_t id; + id.size = (uint64_t)st->st_size; + id.mtime = (uint64_t)st->st_mtime; + id.mtime_nsec = FILE_MMAP_MTIME_NSEC(*st); + id.ino = (uint64_t)st->st_ino; + return id; +} + +char *file_mmap_cache_path(const char *path, uint64_t off, const char *suffix) { + size_t len = strlen(path) + strlen(suffix) + 32; + char *cache_path = malloc(len); + if (cache_path == NULL) + return NULL; + if (off == 0) + snprintf(cache_path, len, "%s%s", path, suffix); + else + snprintf(cache_path, len, "%s.%llu%s", path, (unsigned long long)off, suffix); + return cache_path; +} + +static bool file_mmap_write_all(FILE *f, const void *buf, size_t len) { + return len == 0 || fwrite(buf, 1, len, f) == len; +} + +static bool file_mmap_write_pad(FILE *f, uint64_t *offset) { + static const char zeros[8] = {0}; + uint64_t pad = file_mmap_align8(*offset) - *offset; + if (pad > 0 && fwrite(zeros, 1, (size_t)pad, f) != (size_t)pad) + return false; + *offset += pad; + return true; +} + +bool file_mmap_cache_build(const char *cache_path, uint32_t magic, uint32_t version, + file_mmap_src_id_t src, uint64_t src_offset, uint64_t src_disk_size, + const void *type_header, uint32_t type_header_len, + const file_mmap_array_t *arrays, uint32_t num_arrays) { + if (num_arrays > FILE_MMAP_CACHE_MAX_ARRAYS) + return false; + + size_t path_len = strlen(cache_path); + char *tmp_path = malloc(path_len + 32); + if (tmp_path == NULL) + return false; + snprintf(tmp_path, path_len + 32, "%s.tmp.%ld", cache_path, (long)getpid()); + + FILE *f = fopen(tmp_path, "wb"); + if (f == NULL) { + free(tmp_path); + return false; + } + + bool ok = false; + uint64_t offset = 0; + + file_mmap_cache_common_t header; + memset(&header, 0, sizeof(header)); + header.magic = magic; + header.version = version; + header.endian_check = FILE_MMAP_ENDIAN_CHECK; + header.num_arrays = num_arrays; + header.src_size = src.size; + header.src_mtime = src.mtime; + header.src_mtime_nsec = src.mtime_nsec; + header.src_ino = src.ino; + header.src_offset = src_offset; + header.src_disk_size = src_disk_size; + header.type_header_len = type_header_len; + + if (!file_mmap_write_all(f, &header, sizeof(header))) + goto cleanup; + offset += sizeof(header); + + if (!file_mmap_write_all(f, type_header, type_header_len)) + goto cleanup; + offset += type_header_len; + + if (!file_mmap_write_pad(f, &offset)) + goto cleanup; + + for (uint32_t i = 0; i < num_arrays; i++) { + file_mmap_array_desc_t desc; + memset(&desc, 0, sizeof(desc)); + desc.elem_size = arrays[i].elem_size; + desc.count = arrays[i].count; + if (!file_mmap_write_all(f, &desc, sizeof(desc))) + goto cleanup; + offset += sizeof(desc); + } + + for (uint32_t i = 0; i < num_arrays; i++) { + if (!file_mmap_write_pad(f, &offset)) + goto cleanup; + size_t bytes = (size_t)arrays[i].elem_size * (size_t)arrays[i].count; + if (!file_mmap_write_all(f, arrays[i].data, bytes)) + goto cleanup; + offset += bytes; + } + + if (fflush(f) != 0) + goto cleanup; + // Make the data durable before the rename publishes the cache. + if (fsync(fileno(f)) != 0) + goto cleanup; + + ok = true; + +cleanup: + fclose(f); + if (ok && rename(tmp_path, cache_path) != 0) + ok = false; + if (!ok) + unlink(tmp_path); + free(tmp_path); + return ok; +} + +void *file_mmap_cache_open(const char *cache_path, uint32_t magic, uint32_t version, + file_mmap_src_id_t src, uint64_t src_offset, + const void **type_header, uint32_t type_header_len, + file_mmap_array_ref_t *arrays, uint32_t num_arrays, + uint64_t *src_disk_size, size_t *map_len) { + if (num_arrays > FILE_MMAP_CACHE_MAX_ARRAYS) + return NULL; + + int fd = open(cache_path, O_RDONLY); + if (fd < 0) + return NULL; + + struct stat st; + if (fstat(fd, &st) != 0 || st.st_size < (off_t)sizeof(file_mmap_cache_common_t)) { + close(fd); + return NULL; + } + size_t len = (size_t)st.st_size; + + void *base = mmap(NULL, len, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + if (base == MAP_FAILED) + return NULL; + + const file_mmap_cache_common_t *h = (const file_mmap_cache_common_t *)base; + bool trusted = file_mmap_cache_trusted(); + if (h->magic != magic || + h->version != version || + h->endian_check != FILE_MMAP_ENDIAN_CHECK || + h->num_arrays != num_arrays || + h->type_header_len != type_header_len || + h->src_offset != src_offset || + h->src_size != src.size || + // Freshness fields are skipped in trust mode (read-only/pre-built caches). + (!trusted && (h->src_mtime != src.mtime || + h->src_mtime_nsec != src.mtime_nsec || + h->src_ino != src.ino))) { + munmap(base, len); + return NULL; + } + + uint64_t off = file_mmap_align8(sizeof(file_mmap_cache_common_t) + (uint64_t)type_header_len); + if (off + (uint64_t)num_arrays * sizeof(file_mmap_array_desc_t) > len) { + munmap(base, len); + return NULL; + } + const file_mmap_array_desc_t *descs = (const file_mmap_array_desc_t *)((const char *)base + off); + off += (uint64_t)num_arrays * sizeof(file_mmap_array_desc_t); + + for (uint32_t i = 0; i < num_arrays; i++) { + if (descs[i].elem_size != arrays[i].elem_size || descs[i].elem_size == 0) { + munmap(base, len); + return NULL; + } + off = file_mmap_align8(off); + // Overflow-safe bounds check: compare count against remaining space without + // computing elem_size*count (which could wrap 64-bit for a corrupt cache). + if (off > len || + descs[i].count > (len - off) / descs[i].elem_size || + ((uintptr_t)((char *)base + off) & 7u) != 0) { + munmap(base, len); + return NULL; + } + uint64_t bytes = (uint64_t)descs[i].elem_size * descs[i].count; + arrays[i].base = (char *)base + off; + arrays[i].count = descs[i].count; + off += bytes; + } + + if (type_header != NULL) + *type_header = (type_header_len > 0) ? (const char *)base + sizeof(file_mmap_cache_common_t) : NULL; + if (src_disk_size != NULL) + *src_disk_size = h->src_disk_size; + if (map_len != NULL) + *map_len = len; + + return base; +} + +void file_mmap_cache_close(void *base, size_t map_len) { + if (base != NULL) + munmap(base, map_len); +} + +#endif /* _WIN32 */ diff --git a/src/file_utils.h b/src/file_utils.h index ba9b9511c..3fd87afe3 100644 --- a/src/file_utils.h +++ b/src/file_utils.h @@ -93,5 +93,77 @@ bool file_write_uint8(FILE *file, uint8_t value); bool file_read_chars(FILE *file, char *buf, size_t len); bool file_write_chars(FILE *file, const char *buf, size_t len); +/* + * Optional memory-mapped "flat array" cache (POSIX only) + * ------------------------------------------------------ + * Several libpostal structures are, on disk, a small fixed header plus a few + * flat arrays stored big-endian (e.g. CSR sparse matrices and double-array + * tries). That layout can't be mmap'd directly on little-endian hosts, so these + * helpers build a host-endian, 8-byte-aligned sidecar cache once and then mmap + * it read-only, letting the arrays be shared across processes and demand-paged + * (used and reclaimed on demand) instead of eagerly read into private memory. + * + * Disabled by default; controlled by the LIBPOSTAL_MMAP_CACHE environment var: + * unset / "0" / "off" / "false" -> off (no behavior change, no cache files) + * "1" / "on" / "true" -> on, validate cache freshness via the + * source size + mtime + inode + * "trust" -> on, but skip the mtime/inode freshness + * check (for immutable, pre-built, read-only + * deployments where those differ from build + * time, e.g. a container image) + */ + +#include + +#define FILE_MMAP_CACHE_MAX_ARRAYS 4U + +// Identity of the source file region the cache was built from. +typedef struct { + uint64_t size; + uint64_t mtime; + uint64_t mtime_nsec; + uint64_t ino; +} file_mmap_src_id_t; + +// One array to write into the cache. +typedef struct { + const void *data; + uint32_t elem_size; + uint64_t count; +} file_mmap_array_t; + +// One array resolved from a mapped cache (base points into the mapping). +typedef struct { + uint32_t elem_size; // in: expected element size; validated against the cache + void *base; // out + uint64_t count; // out +} file_mmap_array_ref_t; + +// Whether caching is enabled at all (LIBPOSTAL_MMAP_CACHE not off). +bool file_mmap_cache_enabled(void); +// Whether the freshness check should be skipped (LIBPOSTAL_MMAP_CACHE=trust). +bool file_mmap_cache_trusted(void); + +file_mmap_src_id_t file_mmap_src_id_from_stat(const struct stat *st); +// Returns a malloc'd "" (off==0) or "." path. +char *file_mmap_cache_path(const char *path, uint64_t off, const char *suffix); + +// Build a cache file atomically (temp + fsync + rename). Returns true on success. +bool file_mmap_cache_build(const char *cache_path, uint32_t magic, uint32_t version, + file_mmap_src_id_t src, uint64_t src_offset, uint64_t src_disk_size, + const void *type_header, uint32_t type_header_len, + const file_mmap_array_t *arrays, uint32_t num_arrays); + +// Map + validate a cache. Returns the mapping base (free with file_mmap_cache_close) +// or NULL. On success fills *type_header (into the mapping), *src_disk_size, *map_len, +// and each arrays[i].base/count. arrays[i].elem_size must be set by the caller. +void *file_mmap_cache_open(const char *cache_path, uint32_t magic, uint32_t version, + file_mmap_src_id_t src, uint64_t src_offset, + const void **type_header, uint32_t type_header_len, + file_mmap_array_ref_t *arrays, uint32_t num_arrays, + uint64_t *src_disk_size, size_t *map_len); + +void file_mmap_cache_close(void *base, size_t map_len); + #endif diff --git a/src/sparse_matrix.c b/src/sparse_matrix.c index c765dc364..f5d9c31e3 100644 --- a/src/sparse_matrix.c +++ b/src/sparse_matrix.c @@ -34,9 +34,153 @@ sparse_matrix_t *sparse_matrix_new(void) { } +#define SPARSE_MATRIX_MMAP_MAGIC 0x53504D54U /* "SPMT" */ +#define SPARSE_MATRIX_MMAP_VERSION 1U +#define SPARSE_MATRIX_MMAP_SUFFIX ".spcache" + +// Type-specific cache header (the CSR shape). +typedef struct { + uint32_t m; + uint32_t n; +} sparse_matrix_mmap_header_t; + +#ifndef _WIN32 +// Build a sparse matrix whose three CSR arrays point into a mapped cache. Takes +// ownership of `base`: on failure it unmaps and returns NULL. +static sparse_matrix_t *sparse_matrix_from_mmap(void *base, size_t map_len, + const sparse_matrix_mmap_header_t *th, file_mmap_array_ref_t *refs) { + sparse_matrix_t *sp = calloc(1, sizeof(sparse_matrix_t)); + if (sp == NULL) { + file_mmap_cache_close(base, map_len); + return NULL; + } + sp->m = th->m; + sp->n = th->n; + sp->indptr = uint32_array_new_size(1); + sp->indices = uint32_array_new_size(1); + sp->data = double_array_new_size(1); + if (sp->indptr == NULL || sp->indices == NULL || sp->data == NULL) { + sparse_matrix_destroy(sp); // is_mmap still false: frees owned wrappers + file_mmap_cache_close(base, map_len); + return NULL; + } + free(sp->indptr->a); + sp->indptr->a = (uint32_t *)refs[0].base; + sp->indptr->n = sp->indptr->m = refs[0].count; + free(sp->indices->a); + sp->indices->a = (uint32_t *)refs[1].base; + sp->indices->n = sp->indices->m = refs[1].count; + free(sp->data->a); + sp->data->a = (double *)refs[2].base; + sp->data->n = sp->data->m = refs[2].count; + sp->is_mmap = true; + sp->mmap_base = base; + sp->mmap_len = map_len; + return sp; +} +#endif + +sparse_matrix_t *sparse_matrix_read_cached(FILE *f, const char *path) { +#ifndef _WIN32 + if (path != NULL && file_mmap_cache_enabled()) { + off_t off = ftello(f); + struct stat st; + if (off >= 0 && fstat(fileno(f), &st) == 0) { + file_mmap_src_id_t src = file_mmap_src_id_from_stat(&st); + char *cache_path = file_mmap_cache_path(path, (uint64_t)off, SPARSE_MATRIX_MMAP_SUFFIX); + if (cache_path != NULL) { + file_mmap_array_ref_t refs[3] = { + { .elem_size = sizeof(uint32_t) }, + { .elem_size = sizeof(uint32_t) }, + { .elem_size = sizeof(double) }, + }; + const void *th = NULL; + uint64_t disk = 0; + size_t map_len = 0; + + // Warm path: map a valid cache and skip the matrix in the source. + void *base = file_mmap_cache_open(cache_path, SPARSE_MATRIX_MMAP_MAGIC, + SPARSE_MATRIX_MMAP_VERSION, src, (uint64_t)off, &th, + sizeof(sparse_matrix_mmap_header_t), refs, 3, &disk, &map_len); + if (base != NULL) { + sparse_matrix_t *sp = sparse_matrix_from_mmap(base, map_len, + (const sparse_matrix_mmap_header_t *)th, refs); + if (sp != NULL) { + if ((uint64_t)off + disk <= src.size && + fseeko(f, off + (off_t)disk, SEEK_SET) == 0) { + free(cache_path); + return sp; + } + sparse_matrix_destroy(sp); // munmaps; fall back below + } + fseeko(f, off, SEEK_SET); + } + + // Cold/stale path: read normally, then build + map the cache. + sparse_matrix_t *copy = sparse_matrix_read(f); + if (copy == NULL) { + free(cache_path); + return NULL; + } + off_t end = ftello(f); + if (end > off) { + sparse_matrix_mmap_header_t hdr = { .m = copy->m, .n = copy->n }; + file_mmap_array_t arrays[3] = { + { .data = copy->indptr->a, .elem_size = sizeof(uint32_t), .count = copy->indptr->n }, + { .data = copy->indices->a, .elem_size = sizeof(uint32_t), .count = copy->indices->n }, + { .data = copy->data->a, .elem_size = sizeof(double), .count = copy->data->n }, + }; + if (file_mmap_cache_build(cache_path, SPARSE_MATRIX_MMAP_MAGIC, + SPARSE_MATRIX_MMAP_VERSION, src, (uint64_t)off, (uint64_t)(end - off), + &hdr, sizeof(hdr), arrays, 3)) { + base = file_mmap_cache_open(cache_path, SPARSE_MATRIX_MMAP_MAGIC, + SPARSE_MATRIX_MMAP_VERSION, src, (uint64_t)off, &th, + sizeof(sparse_matrix_mmap_header_t), refs, 3, &disk, &map_len); + if (base != NULL) { + sparse_matrix_t *sp = sparse_matrix_from_mmap(base, map_len, + (const sparse_matrix_mmap_header_t *)th, refs); + if (sp != NULL) { + sparse_matrix_destroy(copy); + free(cache_path); + return sp; // file already positioned past the matrix + } + } + } + } + free(cache_path); + return copy; + } + } + } +#else + (void)path; +#endif + return sparse_matrix_read(f); +} + void sparse_matrix_destroy(sparse_matrix_t *self) { if (self == NULL) return; +#ifndef _WIN32 + if (self->is_mmap) { + if (self->indptr != NULL) { + self->indptr->a = NULL; + uint32_array_destroy(self->indptr); + } + if (self->indices != NULL) { + self->indices->a = NULL; + uint32_array_destroy(self->indices); + } + if (self->data != NULL) { + self->data->a = NULL; + double_array_destroy(self->data); + } + file_mmap_cache_close(self->mmap_base, self->mmap_len); + free(self); + return; + } +#endif + if (self->indptr != NULL) { uint32_array_destroy(self->indptr); } @@ -316,7 +460,9 @@ int sparse_matrix_dot_sparse(sparse_matrix_t *self, sparse_matrix_t *other, doub sparse_matrix_t *sparse_matrix_read(FILE *f) { - sparse_matrix_t *sp = malloc(sizeof(sparse_matrix_t)); + // calloc so is_mmap/mmap_base/mmap_len start zeroed; sparse_matrix_destroy + // branches on is_mmap and would otherwise read uninitialized memory. + sparse_matrix_t *sp = calloc(1, sizeof(sparse_matrix_t)); if (sp == NULL) return NULL; sp->indptr = NULL; diff --git a/src/sparse_matrix.h b/src/sparse_matrix.h index bc23c0686..803a667fc 100644 --- a/src/sparse_matrix.h +++ b/src/sparse_matrix.h @@ -52,6 +52,11 @@ typedef struct { uint32_array *indptr; uint32_array *indices; double_array *data; + // When loaded via the mmap cache, indptr/indices/data point into a shared + // read-only mapping rather than being individually malloc'd. + bool is_mmap; + void *mmap_base; + size_t mmap_len; } sparse_matrix_t; @@ -83,6 +88,11 @@ int sparse_matrix_dot_sparse(sparse_matrix_t *self, sparse_matrix_t *other, doub bool sparse_matrix_write(sparse_matrix_t *self, FILE *f); sparse_matrix_t *sparse_matrix_read(FILE *f); +// Like sparse_matrix_read, but uses the optional shared mmap cache (see +// file_utils.h, LIBPOSTAL_MMAP_CACHE) keyed on the source file `path` and the +// matrix's offset. When caching is disabled or `path` is NULL this is just +// sparse_matrix_read. +sparse_matrix_t *sparse_matrix_read_cached(FILE *f, const char *path); #define sparse_matrix_foreach_row(sp, row_var, index_var, length_var, code) { \ uint32_t _row_start = 0, _row_end = 0; \ diff --git a/src/trie.c b/src/trie.c index 2e11ba6e3..ac45304b1 100644 --- a/src/trie.c +++ b/src/trie.c @@ -1,6 +1,10 @@ #include "trie.h" #include +#ifndef _WIN32 +#include +#endif + /* * Maps the 256 characters (suitable for UTF-8 strings) to array indices * ordered by frequency of usage in Wikipedia titles. @@ -903,6 +907,30 @@ void trie_destroy(trie_t *self) { if (!self) return; +#ifndef _WIN32 + if (self->is_mmap) { + // nodes/data/tail buffers live in the shared mapping: detach them before + // destroying the wrapper structs, then release the single mapping. + if (self->nodes) { + self->nodes->a = NULL; + trie_node_array_destroy(self->nodes); + } + if (self->data) { + self->data->a = NULL; + trie_data_array_destroy(self->data); + } + if (self->tail) { + self->tail->a = NULL; + uchar_array_destroy(self->tail); + } + if (self->alphabet) + free(self->alphabet); + file_mmap_cache_close(self->mmap_base, self->mmap_len); + free(self); + return; + } +#endif + if (self->alphabet) free(self->alphabet); if (self->nodes) @@ -911,7 +939,7 @@ void trie_destroy(trie_t *self) { uchar_array_destroy(self->tail); if (self->data) trie_data_array_destroy(self->data); - + free(self); } @@ -1107,6 +1135,147 @@ trie_t *trie_read(FILE *file) { return NULL; } +#define TRIE_MMAP_MAGIC 0x54524945U /* "TRIE" */ +#define TRIE_MMAP_VERSION 1U +#define TRIE_MMAP_SUFFIX ".trcache" + +// Type-specific cache header: the alphabet (needed to rebuild alpha_map) + counts. +typedef struct { + uint32_t alphabet_size; + uint32_t num_keys; + uint8_t alphabet[NUM_CHARS]; +} trie_mmap_header_t; + +#ifndef _WIN32 +// Build a trie whose nodes/data/tail arrays point into a mapped cache. Takes +// ownership of `base`: on failure it unmaps and returns NULL. +static trie_t *trie_from_mmap(void *base, size_t map_len, + const trie_mmap_header_t *th, file_mmap_array_ref_t *refs) { + if (th->alphabet_size == 0 || th->alphabet_size > NUM_CHARS) { + file_mmap_cache_close(base, map_len); + return NULL; + } + trie_t *trie = calloc(1, sizeof(trie_t)); + if (trie == NULL) { + file_mmap_cache_close(base, map_len); + return NULL; + } + trie->alphabet = malloc(th->alphabet_size); + if (trie->alphabet == NULL) { + free(trie); + file_mmap_cache_close(base, map_len); + return NULL; + } + memcpy(trie->alphabet, th->alphabet, th->alphabet_size); + trie->alphabet_size = th->alphabet_size; + trie->num_keys = th->num_keys; + trie->null_node = NULL_NODE; + for (uint32_t i = 0; i < trie->alphabet_size; i++) { + trie->alpha_map[(uint8_t)trie->alphabet[i]] = (uint8_t)i; + } + + trie->nodes = trie_node_array_new_size(1); + trie->data = trie_data_array_new_size(1); + trie->tail = uchar_array_new_size(1); + if (trie->nodes == NULL || trie->data == NULL || trie->tail == NULL) { + trie_destroy(trie); // is_mmap still false: frees owned wrappers + alphabet + file_mmap_cache_close(base, map_len); + return NULL; + } + free(trie->nodes->a); + trie->nodes->a = (trie_node_t *)refs[0].base; + trie->nodes->n = trie->nodes->m = refs[0].count; + free(trie->data->a); + trie->data->a = (trie_data_node_t *)refs[1].base; + trie->data->n = trie->data->m = refs[1].count; + free(trie->tail->a); + trie->tail->a = (unsigned char *)refs[2].base; + trie->tail->n = trie->tail->m = refs[2].count; + trie->is_mmap = true; + trie->mmap_base = base; + trie->mmap_len = map_len; + return trie; +} +#endif + +trie_t *trie_read_cached(FILE *file, const char *path) { +#ifndef _WIN32 + if (path != NULL && file_mmap_cache_enabled()) { + off_t off = ftello(file); + struct stat st; + if (off >= 0 && fstat(fileno(file), &st) == 0) { + file_mmap_src_id_t src = file_mmap_src_id_from_stat(&st); + char *cache_path = file_mmap_cache_path(path, (uint64_t)off, TRIE_MMAP_SUFFIX); + if (cache_path != NULL) { + file_mmap_array_ref_t refs[3] = { + { .elem_size = sizeof(trie_node_t) }, + { .elem_size = sizeof(trie_data_node_t) }, + { .elem_size = 1 }, + }; + const void *th = NULL; + uint64_t disk = 0; + size_t map_len = 0; + + // Warm path: map a valid cache and skip the trie in the source. + void *base = file_mmap_cache_open(cache_path, TRIE_MMAP_MAGIC, TRIE_MMAP_VERSION, + src, (uint64_t)off, &th, sizeof(trie_mmap_header_t), refs, 3, &disk, &map_len); + if (base != NULL) { + trie_t *trie = trie_from_mmap(base, map_len, (const trie_mmap_header_t *)th, refs); + if (trie != NULL) { + if ((uint64_t)off + disk <= src.size && + fseeko(file, off + (off_t)disk, SEEK_SET) == 0) { + free(cache_path); + return trie; + } + trie_destroy(trie); // munmaps; fall back below + } + fseeko(file, off, SEEK_SET); + } + + // Cold/stale path: read normally, then build + map the cache. + trie_t *copy = trie_read(file); + if (copy == NULL) { + free(cache_path); + return NULL; + } + off_t end = ftello(file); + if (end > off) { + trie_mmap_header_t hdr; + memset(&hdr, 0, sizeof(hdr)); + hdr.alphabet_size = copy->alphabet_size; + hdr.num_keys = copy->num_keys; + if (copy->alphabet_size <= NUM_CHARS) + memcpy(hdr.alphabet, copy->alphabet, copy->alphabet_size); + file_mmap_array_t arrays[3] = { + { .data = copy->nodes->a, .elem_size = sizeof(trie_node_t), .count = copy->nodes->n }, + { .data = copy->data->a, .elem_size = sizeof(trie_data_node_t), .count = copy->data->n }, + { .data = copy->tail->a, .elem_size = 1, .count = copy->tail->n }, + }; + if (file_mmap_cache_build(cache_path, TRIE_MMAP_MAGIC, TRIE_MMAP_VERSION, + src, (uint64_t)off, (uint64_t)(end - off), &hdr, sizeof(hdr), arrays, 3)) { + base = file_mmap_cache_open(cache_path, TRIE_MMAP_MAGIC, TRIE_MMAP_VERSION, + src, (uint64_t)off, &th, sizeof(trie_mmap_header_t), refs, 3, &disk, &map_len); + if (base != NULL) { + trie_t *trie = trie_from_mmap(base, map_len, (const trie_mmap_header_t *)th, refs); + if (trie != NULL) { + trie_destroy(copy); + free(cache_path); + return trie; // file already positioned past the trie + } + } + } + } + free(cache_path); + return copy; + } + } + } +#else + (void)path; +#endif + return trie_read(file); +} + trie_t *trie_load(char *path) { FILE *file; diff --git a/src/trie.h b/src/trie.h index d2f8519ea..8fa00ef3a 100644 --- a/src/trie.h +++ b/src/trie.h @@ -76,6 +76,12 @@ typedef struct trie { uint8_t alpha_map[NUM_CHARS]; uint32_t alphabet_size; uint32_t num_keys; + // When loaded via the mmap cache, nodes/data/tail point into a shared + // read-only mapping rather than being individually malloc'd; trie_destroy + // then munmaps instead of freeing them. + bool is_mmap; + void *mmap_base; + size_t mmap_len; } trie_t; trie_t *trie_new_alphabet(uint8_t *alphabet, uint32_t alphabet_size); @@ -144,6 +150,10 @@ bool trie_write(trie_t *self, FILE *file); bool trie_save(trie_t *self, char *path); trie_t *trie_read(FILE *file); +// Like trie_read, but uses the optional shared mmap cache (see file_utils.h, +// LIBPOSTAL_MMAP_CACHE) keyed on the source file `path` and the trie's offset +// within it. When caching is disabled or `path` is NULL this is just trie_read. +trie_t *trie_read_cached(FILE *file, const char *path); trie_t *trie_load(char *path); void trie_destroy(trie_t *self); diff --git a/test/Makefile.am b/test/Makefile.am index f2e911f2a..c98665414 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -9,6 +9,6 @@ CFLAGS = $(CFLAGS_BASE) TESTS = test_libpostal noinst_PROGRAMS = test_libpostal -test_libpostal_SOURCES = test.c test_expand.c test_parser.c test_transliterate.c test_numex.c test_trie.c test_string_utils.c test_crf_context.c ../src/strndup.c ../src/file_utils.c ../src/string_utils.c ../src/utf8proc/utf8proc.c ../src/trie.c ../src/trie_search.c ../src/transliterate.c ../src/numex.c ../src/features.c +test_libpostal_SOURCES = test.c test_expand.c test_parser.c test_transliterate.c test_numex.c test_trie.c test_mmap_cache.c test_string_utils.c test_crf_context.c ../src/strndup.c ../src/file_utils.c ../src/string_utils.c ../src/utf8proc/utf8proc.c ../src/trie.c ../src/trie_search.c ../src/transliterate.c ../src/numex.c ../src/features.c test_libpostal_LDADD = ../src/libpostal.la ../src/libscanner.la $(CBLAS_LIBS) test_libpostal_CFLAGS = $(CFLAGS_O3) diff --git a/test/test.c b/test/test.c index e75938661..1e68cf385 100644 --- a/test/test.c +++ b/test/test.c @@ -7,6 +7,7 @@ SUITE_EXTERN(libpostal_numex_tests); SUITE_EXTERN(libpostal_string_utils_tests); SUITE_EXTERN(libpostal_trie_tests); SUITE_EXTERN(libpostal_crf_context_tests); +SUITE_EXTERN(libpostal_mmap_cache_tests); GREATEST_MAIN_DEFS(); @@ -21,5 +22,6 @@ int main(int argc, char **argv) { RUN_SUITE(libpostal_string_utils_tests); RUN_SUITE(libpostal_trie_tests); RUN_SUITE(libpostal_crf_context_tests); + RUN_SUITE(libpostal_mmap_cache_tests); GREATEST_MAIN_END(); } diff --git a/test/test_mmap_cache.c b/test/test_mmap_cache.c new file mode 100644 index 000000000..68411f5ee --- /dev/null +++ b/test/test_mmap_cache.c @@ -0,0 +1,135 @@ +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#endif + +#include "greatest.h" +#include "../src/file_utils.h" + +SUITE(libpostal_mmap_cache_tests); + +#ifndef _WIN32 + +#define TEST_MMAP_MAGIC 0x54455354u /* "TEST" */ +#define TEST_MMAP_VERSION 1u + +typedef struct { + uint32_t a; + uint32_t b; +} test_type_header_t; + +static greatest_test_res mmap_cache_roundtrip_at(const char *cache_path, file_mmap_src_id_t src, uint64_t off) { + uint32_t arr0[6] = {10, 20, 30, 40, 50, 60}; + double arr1[4] = {1.5, 2.5, 3.5, 4.5}; + test_type_header_t th = { .a = 0xAABBCCDDu, .b = 7 }; + + file_mmap_array_t build[2] = { + { .data = arr0, .elem_size = sizeof(uint32_t), .count = 6 }, + { .data = arr1, .elem_size = sizeof(double), .count = 4 }, + }; + ASSERT(file_mmap_cache_build(cache_path, TEST_MMAP_MAGIC, TEST_MMAP_VERSION, src, off, 0, + &th, sizeof(th), build, 2)); + + file_mmap_array_ref_t refs[2] = { + { .elem_size = sizeof(uint32_t) }, + { .elem_size = sizeof(double) }, + }; + const void *thp = NULL; + uint64_t disk = 999; + size_t mlen = 0; + void *base = file_mmap_cache_open(cache_path, TEST_MMAP_MAGIC, TEST_MMAP_VERSION, src, off, + &thp, sizeof(th), refs, 2, &disk, &mlen); + ASSERT(base != NULL); + + const test_type_header_t *thr = (const test_type_header_t *)thp; + ASSERT_EQ(0xAABBCCDDu, thr->a); + ASSERT_EQ(7, thr->b); + ASSERT_EQ(6, refs[0].count); + ASSERT_EQ(4, refs[1].count); + ASSERT((((uintptr_t)refs[0].base) & 7u) == 0); + ASSERT((((uintptr_t)refs[1].base) & 7u) == 0); + + uint32_t *r0 = (uint32_t *)refs[0].base; + for (int i = 0; i < 6; i++) ASSERT_EQ(arr0[i], r0[i]); + double *r1 = (double *)refs[1].base; + for (int i = 0; i < 4; i++) ASSERT(memcmp(&r1[i], &arr1[i], sizeof(double)) == 0); + + file_mmap_cache_close(base, mlen); + PASS(); +} + +TEST test_mmap_cache_roundtrip(void) { + char path[256]; + snprintf(path, sizeof(path), "test_mmap_cache_%ld.trcache", (long)getpid()); + file_mmap_src_id_t src = { .size = 1000, .mtime = 123, .mtime_nsec = 456, .ino = 789 }; + + CHECK_CALL(mmap_cache_roundtrip_at(path, src, 0)); + CHECK_CALL(mmap_cache_roundtrip_at(path, src, 4096)); + + remove(path); + PASS(); +} + +TEST test_mmap_cache_validation(void) { + char path[256]; + snprintf(path, sizeof(path), "test_mmap_cache_val_%ld.trcache", (long)getpid()); + file_mmap_src_id_t src = { .size = 1000, .mtime = 123, .mtime_nsec = 456, .ino = 789 }; + + test_type_header_t th = { .a = 1, .b = 2 }; + uint32_t arr[3] = {1, 2, 3}; + file_mmap_array_t build[1] = { { .data = arr, .elem_size = sizeof(uint32_t), .count = 3 } }; + ASSERT(file_mmap_cache_build(path, TEST_MMAP_MAGIC, TEST_MMAP_VERSION, src, 0, 0, + &th, sizeof(th), build, 1)); + + file_mmap_array_ref_t refs[1] = { { .elem_size = sizeof(uint32_t) } }; + const void *thp = NULL; + size_t mlen = 0; + void *base; + + // Wrong magic / version / size / offset / type-header length -> reject. + base = file_mmap_cache_open(path, 0xDEADBEEFu, TEST_MMAP_VERSION, src, 0, &thp, sizeof(th), refs, 1, NULL, &mlen); + ASSERT(base == NULL); + base = file_mmap_cache_open(path, TEST_MMAP_MAGIC, 999, src, 0, &thp, sizeof(th), refs, 1, NULL, &mlen); + ASSERT(base == NULL); + file_mmap_src_id_t bad_size = src; bad_size.size = 2000; + base = file_mmap_cache_open(path, TEST_MMAP_MAGIC, TEST_MMAP_VERSION, bad_size, 0, &thp, sizeof(th), refs, 1, NULL, &mlen); + ASSERT(base == NULL); + base = file_mmap_cache_open(path, TEST_MMAP_MAGIC, TEST_MMAP_VERSION, src, 64, &thp, sizeof(th), refs, 1, NULL, &mlen); + ASSERT(base == NULL); + base = file_mmap_cache_open(path, TEST_MMAP_MAGIC, TEST_MMAP_VERSION, src, 0, &thp, sizeof(th) + 4, refs, 1, NULL, &mlen); + ASSERT(base == NULL); + + // Changed mtime/inode: default (no env / strict) rejects. + file_mmap_src_id_t changed = src; changed.mtime = 999; changed.mtime_nsec = 0; changed.ino = 111; + unsetenv("LIBPOSTAL_MMAP_CACHE"); + base = file_mmap_cache_open(path, TEST_MMAP_MAGIC, TEST_MMAP_VERSION, changed, 0, &thp, sizeof(th), refs, 1, NULL, &mlen); + ASSERT(base == NULL); + + // LIBPOSTAL_MMAP_CACHE=trust: mtime/inode ignored, accepted (size still matches). + setenv("LIBPOSTAL_MMAP_CACHE", "trust", 1); + base = file_mmap_cache_open(path, TEST_MMAP_MAGIC, TEST_MMAP_VERSION, changed, 0, &thp, sizeof(th), refs, 1, NULL, &mlen); + ASSERT(base != NULL); + ASSERT_EQ(3, refs[0].count); + file_mmap_cache_close(base, mlen); + + // Trust still rejects a wrong SIZE (size is always validated). + base = file_mmap_cache_open(path, TEST_MMAP_MAGIC, TEST_MMAP_VERSION, bad_size, 0, &thp, sizeof(th), refs, 1, NULL, &mlen); + unsetenv("LIBPOSTAL_MMAP_CACHE"); + ASSERT(base == NULL); + + remove(path); + PASS(); +} + +#endif + +GREATEST_SUITE(libpostal_mmap_cache_tests) { +#ifndef _WIN32 + RUN_TEST(test_mmap_cache_roundtrip); + RUN_TEST(test_mmap_cache_validation); +#endif +}