diff --git a/ph/Cargo.toml b/ph/Cargo.toml index 2a2ef49e..79b231ce 100644 --- a/ph/Cargo.toml +++ b/ph/Cargo.toml @@ -21,6 +21,8 @@ sip13 = ["seedable_hash/sip13"] # if enabled, makes available BuildSip13 that u fnv = ["seedable_hash/fnv"] sux = ["dep:sux", "mem_dbg"] cacheline-ef = ["dep:cacheline-ef", "mem_dbg"] +epserde = ["dep:epserde", "sux/epserde", "seedable_hash/epserde"] +mem_dbg = ["dep:mem_dbg", "seedable_hash/mem_dbg"] [dependencies] bitm = { version="0.5", path="../bitm" } @@ -31,9 +33,12 @@ seedable_hash = { version="0.2", path="../seedable_hash", default-features = fal arrayvec = "0.7" voracious_radix_sort = { version = "1.2.0", features = ["voracious_multithread"] } cseq = { version = "0.1", path = "../cseq", optional = true } -sux = { version = "0.6", optional = true } -mem_dbg = { version="0.3", optional = true } # required by sux and cacheline-ef +sux = { version = "0.9", optional = true } +mem_dbg = { version = "0.3.2", optional = true } # required by sux and cacheline-ef cacheline-ef = { version = "1.1.0", optional = true } +epserde = { version = "0.11", optional = true } +maligned = "0.2.1" +rdst = "0.20.14" [target.'cfg(target_pointer_width = "32")'.dependencies] aligned-vec = { version="0.6", optional=true } # for 32 bit support @@ -43,4 +48,4 @@ criterion = "0.5" [[bench]] name = "fmph" -harness = false \ No newline at end of file +harness = false diff --git a/ph/src/fmph/gofunction.rs b/ph/src/fmph/gofunction.rs index 1a17bf54..5a5e8a08 100644 --- a/ph/src/fmph/gofunction.rs +++ b/ph/src/fmph/gofunction.rs @@ -759,8 +759,7 @@ mod tests { use std::fmt::{Debug, Display}; use crate::seeds::Bits; - fn test_read_write(h: &GOFunction) - where SS::VecElement: std::cmp::PartialEq + Debug + fn test_read_write>(h: &GOFunction) { let mut buff = Vec::new(); h.write(&mut buff).unwrap(); diff --git a/ph/src/phast/compressed_array.rs b/ph/src/phast/compressed_array.rs index 775e2d4c..575a345d 100644 --- a/ph/src/phast/compressed_array.rs +++ b/ph/src/phast/compressed_array.rs @@ -189,6 +189,8 @@ impl GetSize for LinearRegressionArray { } /// Implementation of `CompressedArray` that stores each value with the same number of bits required to store the largest one. +#[cfg_attr(feature = "epserde", derive(epserde::Epserde))] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] pub struct Compact { pub items: Box<[u64]>, pub item_size: u8, @@ -277,7 +279,10 @@ impl CompressedArray for CompactFast { /// CompressedArray implementation by Elias-Fano from `sux` crate. -#[cfg(feature = "sux")] pub struct SuxEliasFano(sux::dict::elias_fano::EfSeq); +#[cfg(feature = "sux")] +#[cfg_attr(feature = "epserde", derive(epserde::Epserde))] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] +pub struct SuxEliasFano(E); #[cfg(feature = "sux")] impl CompressedBuilder for sux::dict::EliasFanoBuilder { #[inline] fn new(num_of_values: usize, max_value: usize) -> Self { diff --git a/ph/src/phast/conf.rs b/ph/src/phast/conf.rs index 176598c9..cc5202e8 100644 --- a/ph/src/phast/conf.rs +++ b/ph/src/phast/conf.rs @@ -6,6 +6,8 @@ use super::SeedChooser; /// PHast map-or-bump function configuration. #[derive(Clone, Copy)] +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), repr(C), epserde_zero_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] pub struct Conf { pub(crate) buckets_num: usize, // number of buckets, B pub(crate) slice_len_minus_one: u16, // slice length L diff --git a/ph/src/phast/function.rs b/ph/src/phast/function.rs index 7eff43e1..713e9eb4 100644 --- a/ph/src/phast/function.rs +++ b/ph/src/phast/function.rs @@ -9,9 +9,12 @@ use voracious_radix_sort::RadixSort; use rayon::prelude::*; /// Represents map-or-bump function. -pub(crate) struct SeedEx { - pub(crate) seeds: Box<[SSVecElement]>, +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), repr(C))] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] +pub struct SeedEx> { + pub(crate) seeds: B, pub(crate) conf: Conf, + pub(crate) _marker: std::marker::PhantomData } impl SeedEx { @@ -19,7 +22,7 @@ impl SeedEx { pub(crate) fn bucket_for(&self, key: u64) -> usize { self.conf.bucket_for(key) } #[inline(always)] - pub(crate) fn seed_for(&self, seed_size: SS, key: u64) -> u16 where SS: SeedSize { + pub(crate) fn seed_for>(&self, seed_size: SS, key: u64) -> u16 { //self.seeds.get_fragment(self.bucket_for(key), self.conf.bits_per_seed()) as u16 seed_size.get_seed(&self.seeds, self.bucket_for(key)) } @@ -32,9 +35,12 @@ impl GetSize for SeedEx { } -pub(crate) struct Level { - pub(crate) seeds: SeedEx, - pub(crate) shift: usize +#[cfg_attr(feature = "epserde", derive(epserde::Epserde))] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] +pub struct Level> { + pub(crate) seeds: S, + pub(crate) shift: usize, + pub(crate) _marker: std::marker::PhantomData } impl GetSize for Level { @@ -60,7 +66,7 @@ pub(crate) fn build_level_from_slice_st(keys: &[K], params: &Param keys_vec.extend(keys.into_iter().filter(|key| { params.seed_size.get_seed(&seeds, conf.bucket_for(hasher.hash_one(key, level_nr))) == 0 }).cloned()); - (keys_vec, SeedEx::{ seeds, conf }, unassigned_values, unassigned_len) + (keys_vec, SeedEx::{ seeds, conf, _marker: std::marker::PhantomData }, unassigned_values, unassigned_len) } #[inline] @@ -87,7 +93,7 @@ pub(crate) fn build_level_from_slice_mt(keys: &[K], params: &Param keys_vec.par_extend(keys.into_par_iter().filter(|key| { params.seed_size.get_seed(&seeds, conf.bucket_for(hasher.hash_one(key, level_nr))) == 0 }).cloned()); - (keys_vec, SeedEx::{ seeds, conf }, unassigned_values, unassigned_len) + (keys_vec, SeedEx::{ seeds, conf, _marker: std::marker::PhantomData }, unassigned_values, unassigned_len) } #[inline(always)] @@ -105,7 +111,7 @@ pub(crate) fn build_level_st(keys: &mut Vec::, params: &Params< keys.retain(|key| { params.seed_size.get_seed(&seeds, conf.bucket_for(hasher.hash_one(key, level_nr))) == 0 }); - (SeedEx::{ seeds, conf }, unassigned_values, unassigned_len) + (SeedEx::{ seeds, conf, _marker: std::marker::PhantomData }, unassigned_values, unassigned_len) } #[inline] @@ -133,7 +139,7 @@ pub(crate) fn build_level_mt(keys: &mut Vec::, params: &Params< keys.par_extend(result.into_par_iter().filter(|key| { params.seed_size.get_seed(&seeds, conf.bucket_for(hasher.hash_one(key, level_nr))) == 0 })); - (SeedEx::{ seeds, conf }, unassigned_values, unassigned_len) + (SeedEx::{ seeds, conf, _marker: std::marker::PhantomData }, unassigned_values, unassigned_len) } /// PHast (Perfect Hashing made fast) - Minimal Perfect Hash Function @@ -149,8 +155,7 @@ pub(crate) fn build_level_mt(keys: &mut Vec::, params: &Params< /// /// See: /// Piotr Beling, Peter Sanders, *PHast - Perfect Hashing made fast*, 2025, -pub struct Function - where SS: SeedSize +pub struct Function { level0: SeedEx, unassigned: CA, @@ -307,7 +312,7 @@ impl F unassigned.push(last); } } - levels.push(Level { seeds, shift }); + levels.push(Level { seeds, shift, _marker: std::marker::PhantomData }); } debug_assert!(level0_unassigned.next().is_none()); drop(level0_unassigned); diff --git a/ph/src/phast/function2.rs b/ph/src/phast/function2.rs index 99a00094..b554ab7d 100644 --- a/ph/src/phast/function2.rs +++ b/ph/src/phast/function2.rs @@ -24,14 +24,15 @@ use voracious_radix_sort::RadixSort; /// /// See: /// Piotr Beling, Peter Sanders, *PHast - Perfect Hashing made fast*, 2025, -pub struct Function2 - where SS: SeedSize +#[cfg_attr(feature = "epserde", derive(epserde::Epserde))] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] +pub struct Function2::VecElement>, L = Box<[Level<::VecElement>]>, LL = Level<::VecElement>> { - level0: SeedEx, + level0: L0, unassigned: CA, - levels: Box<[Level]>, + levels: L, hasher: S, - last_level: Level<::VecElement>, + last_level: LL, last_level_seed: u64, seed_chooser: SC, seed_size: SS, @@ -184,7 +185,7 @@ impl F unassigned.push(last); } } - levels.push(Level { seeds, shift }); + levels.push(Level { seeds, shift, _marker: std::marker::PhantomData }); } //dbg!(keys.len()); // TODO keys.len()==0 let mut last_seed = levels.len() as u64+1; @@ -192,7 +193,7 @@ impl F let last_seeds = if keys.is_empty() { last_shift = 0; - SeedEx::<::VecElement>{ seeds: Box::default(), conf: Conf { buckets_num: 0, slice_len_minus_one: 0, num_of_slices: 0 } } + SeedEx::<::VecElement>{ seeds: Box::default(), conf: Conf { buckets_num: 0, slice_len_minus_one: 0, num_of_slices: 0 }, _marker: std::marker::PhantomData } } else { let (last_seeds, unassigned_values, _unassigned_len) = Self::build_last_level(keys, &hasher, &mut last_seed); @@ -223,7 +224,7 @@ impl F levels: levels.into_boxed_slice(), hasher, seed_chooser, - last_level: Level { seeds: last_seeds, shift: last_shift }, + last_level: Level { seeds: last_seeds, shift: last_shift, _marker: std::marker::PhantomData }, last_level_seed: last_seed, seed_size, } @@ -245,7 +246,7 @@ impl F if let Some((seeds, unassigned_values, unassigned_len)) = build_last_level(&hashes, conf, bits_per_seed, evaluator.clone()) { - return (SeedEx{ seeds, conf }, unassigned_values, unassigned_len); + return (SeedEx{ seeds, conf, _marker: std::marker::PhantomData }, unassigned_values, unassigned_len); } *seed += 1; //dbg!(*seed); diff --git a/ph/src/phast/partial.rs b/ph/src/phast/partial.rs index 34bed7b9..218e7fbd 100644 --- a/ph/src/phast/partial.rs +++ b/ph/src/phast/partial.rs @@ -11,7 +11,7 @@ use std::hash::{BuildHasher, Hash, RandomState}; /// /// See: /// Piotr Beling, Peter Sanders, *PHast - Perfect Hashing made fast*, 2025, -pub struct Partial where SS: SeedSize { +pub struct Partial { seeds: SeedEx, hasher: S, seed_chooser: SC, @@ -135,7 +135,7 @@ impl Partial { hashes.voracious_sort(); let (seeds, build_conf) = build_st(hashes, conf, seed_size, bucket_evaluator, seed_chooser); (Self { - seeds: SeedEx{ seeds, conf }, + seeds: SeedEx{ seeds, conf, _marker: std::marker::PhantomData }, hasher, seed_chooser, seed_size @@ -150,7 +150,7 @@ impl Partial { hashes.voracious_mt_sort(threads_num); let (seeds, build_conf) = build_mt(hashes, conf, seed_size, WINDOW_SIZE, bucket_evaluator, seed_chooser, threads_num); (Self { - seeds: SeedEx{ seeds, conf }, + seeds: SeedEx{ seeds, conf, _marker: std::marker::PhantomData }, hasher, seed_chooser, seed_size, diff --git a/ph/src/phast/perfect.rs b/ph/src/phast/perfect.rs index c3cbe058..2b510fac 100644 --- a/ph/src/phast/perfect.rs +++ b/ph/src/phast/perfect.rs @@ -131,7 +131,7 @@ impl Perfect { while !keys.is_empty() { let seeds = build_level(&mut keys, levels.len() as u64+1, &hasher); let out_range = seeds.conf.output_range(seed_chooser, seed_size.into()); - levels.push(Level { seeds, shift }); + levels.push(Level { seeds, shift, _marker: std::marker::PhantomData }); shift += out_range; } Self { @@ -159,7 +159,7 @@ impl Perfect { keys_vec.extend(keys.into_iter().filter(|key| { params.seed_size.get_seed(&seeds, conf.bucket_for(hasher.hash_one(key, level_nr))) == 0 }).cloned()); - (keys_vec, SeedEx{ seeds, conf }) + (keys_vec, SeedEx{ seeds, conf, _marker: std::marker::PhantomData }) } #[inline] @@ -182,7 +182,7 @@ impl Perfect { keys_vec.par_extend(keys.into_par_iter().filter(|key| { params.seed_size.get_seed(&seeds, conf.bucket_for(hasher.hash_one(key, level_nr))) == 0 }).cloned()); - (keys_vec, SeedEx{ seeds, conf }) + (keys_vec, SeedEx{ seeds, conf, _marker: std::marker::PhantomData }) } #[inline(always)] @@ -197,7 +197,7 @@ impl Perfect { keys.retain(|key| { params.seed_size.get_seed(&seeds, conf.bucket_for(hasher.hash_one(key, level_nr))) == 0 }); - SeedEx{ seeds, conf } + SeedEx{ seeds, conf, _marker: std::marker::PhantomData } } #[inline] @@ -224,7 +224,7 @@ impl Perfect { keys.par_extend(result.into_par_iter().filter(|key| { params.seed_size.get_seed(&seeds, conf.bucket_for(hasher.hash_one(key, level_nr))) == 0 })); - SeedEx{ seeds, conf } + SeedEx{ seeds, conf, _marker: std::marker::PhantomData } } /// Returns maximum number of keys which can be mapped to the same value by `k`-[`Perfect`] function `self`. diff --git a/ph/src/phast/seed_chooser/mod.rs b/ph/src/phast/seed_chooser/mod.rs index 35a29509..e77401f6 100644 --- a/ph/src/phast/seed_chooser/mod.rs +++ b/ph/src/phast/seed_chooser/mod.rs @@ -156,6 +156,8 @@ const SMALL_BUCKET_LIMIT: usize = 8; /// /// It chooses best seed with quite strong hasher, without shift component, /// which should lead to small size, but long construction time. +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), epserde_deep_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Clone, Copy)] pub struct SeedOnly; diff --git a/ph/src/phast/seed_chooser/shift.rs b/ph/src/phast/seed_chooser/shift.rs index 3d5dadc4..c85bb2db 100644 --- a/ph/src/phast/seed_chooser/shift.rs +++ b/ph/src/phast/seed_chooser/shift.rs @@ -34,6 +34,8 @@ use super::SeedChooser; /// /// It chooses best seed using only shifting without wrapping, /// which leads to very fast construction but the cost of bigger size. +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), repr(C), epserde_zero_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Clone, Copy, Default)] pub struct ShiftOnly; diff --git a/ph/src/seeds.rs b/ph/src/seeds.rs index e9ee8753..686f3582 100644 --- a/ph/src/seeds.rs +++ b/ph/src/seeds.rs @@ -64,6 +64,8 @@ pub trait SeedSize: Copy + Into + Sync + TryFrom { } /// Size in bits. +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), epserde_deep_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Copy, Clone)] pub struct Bits(pub u8); @@ -128,6 +130,8 @@ impl SeedSize for Bits { /// Size in bits. /// /// Uses unaligned reads/writes to access data in SeedSize implementation. +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), epserde_deep_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Copy, Clone)] pub struct BitsFast(pub u8); @@ -197,6 +201,8 @@ impl SeedSize for BitsFast { } /// Seed size of 8 bits. +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), epserde_deep_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Copy, Clone, Default)] pub struct Bits8; @@ -248,6 +254,8 @@ impl SeedSize for Bits8 { } /// Seed size given as a power of two (knowing at compile time). +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), epserde_deep_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Copy, Clone, Default)] pub struct TwoToPowerBitsStatic; diff --git a/seedable_hash/Cargo.toml b/seedable_hash/Cargo.toml index 0673bc55..423db5ba 100644 --- a/seedable_hash/Cargo.toml +++ b/seedable_hash/Cargo.toml @@ -14,10 +14,14 @@ keywords = [ "hashing", "hash", "seed", "reduce", "map" ] [features] default = ["wyhash"] sip13 = [] # if enabled, makes available BuildSip13 that uses Sip13 from compiler internals +epserde = ["dep:epserde"] +mem_dbg = ["dep:mem_dbg"] [dependencies] wyhash = { version="0.5" , optional=true } fnv = { version="1", optional=true } gxhash = { version = "3.5", optional = true } rapidhash = { version = "1.3", optional = true, default-features = false } -xxhash-rust = { version = "0.8", features = ["xxh3"], optional=true } \ No newline at end of file +xxhash-rust = { version = "0.8", features = ["xxh3"], optional=true } +epserde = { version = "0.11.3", optional = true } +mem_dbg = { version = "0.3.0", optional = true } diff --git a/seedable_hash/src/lib.rs b/seedable_hash/src/lib.rs index b9a1b153..5fa4f801 100644 --- a/seedable_hash/src/lib.rs +++ b/seedable_hash/src/lib.rs @@ -23,6 +23,8 @@ pub trait BuildSeededHasher { } /// [`BuildSeededHasher`] that uses standard [`BuildHasher`]. +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), epserde_deep_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Default, Copy, Clone)] pub struct Seedable(pub BH); @@ -39,6 +41,8 @@ impl BuildSeededHasher for Seedable { /// [`BuildSeededHasher`] that uses [`std::hash::SipHasher13`]. #[cfg(feature = "sip13")] +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), repr(C), epserde_zero_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Default, Copy, Clone)] pub struct BuildSip13; @@ -55,6 +59,8 @@ impl BuildSeededHasher for BuildSip13 { /// [`BuildSeededHasher`] that uses `wyhash` crate. #[cfg(feature = "wyhash")] +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), repr(C), epserde_zero_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Default, Copy, Clone)] pub struct BuildWyHash; @@ -70,6 +76,8 @@ impl BuildSeededHasher for BuildWyHash { /// [`BuildSeededHasher`] that uses `Xxh3` from `xxhash_rust` crate. #[cfg(feature = "xxhash-rust")] +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), repr(C), epserde_zero_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Default, Copy, Clone)] pub struct BuildXxh3; @@ -95,6 +103,8 @@ impl BuildSeededHasher for fnv::FnvBuildHasher { /// [`BuildSeededHasher`] that uses `GxHasher` from `gxhash` crate. #[cfg(feature = "gxhash")] +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), repr(C), epserde_zero_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Default, Copy, Clone)] pub struct BuildGxHash; @@ -111,6 +121,8 @@ impl BuildSeededHasher for BuildGxHash { /// [`BuildSeededHasher`] that uses `rapidhash::RapidHasher`. #[cfg(feature = "rapidhash")] +#[cfg_attr(feature = "epserde", derive(epserde::Epserde), repr(C), epserde_zero_copy)] +#[cfg_attr(feature = "mem_dbg", derive(mem_dbg::MemDbg, mem_dbg::MemSize))] #[derive(Default, Copy, Clone)] pub struct BuildRapidHash;