From 79e1e0d599f1b42f1cf99074c4b46189777ccdd8 Mon Sep 17 00:00:00 2001 From: nashzhou Date: Thu, 15 Oct 2026 14:42:26 +0800 Subject: [PATCH] perf(overlaybd): replace B-tree index merge with a k-way sweep ReadOnlyIndex::merge fed every mapping of every layer through MutableIndex, a BTreeSet overlay: each insert paid a node allocation/removal pair plus an overlap range scan, which made merge the dominant allocator-bound cost of image open and create_snapshot_and_restack under high pause/resume concurrency. Layer indexes are sorted and internally disjoint, so the merged index is the upper envelope of k sorted interval lists. Replace the merge with a single sweep: per-layer cursors activate the mapping covering the sweep position, the newest active layer wins each run, and runs split only where a newer layer starts. Older mappings fully covered by newer runs are dropped the way the B-tree removed them; adjacent pieces are not coalesced and piece boundaries (offsets, moffsets, tags, zeroed flags) match the B-tree overlay exactly. On 8 layers x 200k mappings the sweep merges in ~4ms vs ~291ms for the B-tree version with identical output. Equivalence is locked in by a 512-case randomized test comparing the sweep against the historical implementation kept as an oracle, plus targeted coverage for the covered-older-mapping and zeroed-fragment corners. MutableIndex itself is unchanged; it remains the writable-upper index behind ComboIndex. --- storage/overlaybd/src/lsmt/index.rs | 216 ++++++++++++++++++++++++++-- 1 file changed, 207 insertions(+), 9 deletions(-) diff --git a/storage/overlaybd/src/lsmt/index.rs b/storage/overlaybd/src/lsmt/index.rs index 6861c52..dc9ad49 100644 --- a/storage/overlaybd/src/lsmt/index.rs +++ b/storage/overlaybd/src/lsmt/index.rs @@ -206,20 +206,102 @@ impl ReadOnlyIndex { Self { mappings } } + /// Merges layer indexes into one. `indexes[0]` is the newest layer; every + /// output range takes the tag of the newest layer mapping covering it. + /// + /// Each input index must be sorted by offset and internally disjoint — + /// the LSMT on-disk index format guarantees this. The sweep below keeps + /// the exact piece boundaries of the B-tree overlay it replaces (adjacent + /// pieces are not coalesced), but runs in a single pass with no + /// per-mapping allocations: the B-tree version paid a node insert/remove + /// and an overlap scan for every mapping of every layer, which dominated + /// open and restack profiles under high pause/resume concurrency. pub fn merge(indexes: &[&ReadOnlyIndex]) -> Self { - let mut mi = MutableIndex::new(); + let layers = indexes.len(); + // Per-layer cursor into the next unconsumed mapping. + let mut next: Vec = vec![0; layers]; + // Mappings are disjoint within a layer, so at most one mapping per + // layer covers the sweep position `pos`. + let mut active: Vec> = vec![None; layers]; + let total_mappings: usize = indexes.iter().map(|idx| idx.mappings.len()).sum(); + let mut merged: Vec = Vec::with_capacity(total_mappings); + let mut pos = 0u64; + + loop { + for (layer, idx) in indexes.iter().enumerate() { + let mut cursor = next[layer]; + while let Some(m) = idx.mappings.get(cursor) { + if m.offset() > pos { + break; + } + if m.length() > 0 { + let prev = active[layer].replace(*m); + debug_assert!( + prev.is_none_or(|prev| prev.end() <= pos), + "layer {layer} mappings must be sorted and disjoint" + ); + } + cursor += 1; + } + next[layer] = cursor; + } - for (i, idx) in indexes.iter().enumerate().rev() { - for m in &idx.mappings { - let mut m_new = *m; - // tag = 0 means the newest/top layer - // The smaller the tag is, the newer the data is - m_new.tag = i as u8; - mi.insert(m_new); + // A run of a newer layer can sweep past the start of an older + // mapping without the sweep stopping there (older starts never + // truncate a newer run). If the newer coverage consumed the older + // mapping entirely, it must never surface — the B-tree version + // removed such mappings on overlap. + for slot in active.iter_mut() { + if slot.is_some_and(|m| m.end() <= pos) { + *slot = None; + } + } + + let Some(winner_layer) = active.iter().position(Option::is_some) else { + // Nothing covers `pos`; jump to the next mapped start. + let Some(jump) = indexes + .iter() + .enumerate() + .filter_map(|(layer, idx)| { + idx.mappings + .get(next[layer]) + .and_then(|m| (m.length() > 0).then_some(m.offset())) + }) + .min() + else { + break; + }; + pos = jump; + continue; + }; + let mut winner = active[winner_layer].take().expect("winner just checked"); + debug_assert!(winner.offset() <= pos && pos < winner.end()); + + // Only a newer layer's next mapping can override the winner before + // it ends; older layers never truncate a newer run. + let mut run_end = winner.end(); + for layer in 0..winner_layer { + if let Some(m) = indexes[layer].mappings.get(next[layer]) { + if m.length() > 0 && m.offset() < run_end { + run_end = m.offset(); + } + } + } + + let mut piece = winner; + piece.forward_offset_to(pos); + piece.backward_end_to(run_end); + piece.tag = winner_layer as u8; + merged.push(piece); + + if run_end < winner.end() { + winner.forward_offset_to(run_end); + active[winner_layer] = Some(winner); } + pos = run_end; } - ReadOnlyIndex::new(mi.dump()) + ReadOnlyIndex::new(merged) } pub fn mappings(&self) -> &[SegmentMapping] { @@ -1273,4 +1355,120 @@ mod tests { let n = compress_raw_index(&mut mappings); assert_eq!(n, 2); // Cannot merge due to MAX_LENGTH constraint } + + // ── merge: k-way sweep vs the historical B-tree overlay ──────────────── + + /// The merge implementation the sweep replaces, kept as the behavioral + /// oracle: per-mapping newest-layer override through `MutableIndex`. + fn merge_reference(indexes: &[&ReadOnlyIndex]) -> ReadOnlyIndex { + let mut mi = MutableIndex::new(); + for (i, idx) in indexes.iter().enumerate().rev() { + for m in &idx.mappings { + let mut m_new = *m; + m_new.tag = i as u8; + mi.insert(m_new); + } + } + ReadOnlyIndex::new(mi.dump()) + } + + /// Deterministic LCG so failures reproduce exactly. + struct Lcg(u64); + impl Lcg { + fn below(&mut self, n: u64) -> u64 { + self.0 = self + .0 + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (self.0 >> 33) % n + } + } + + /// A random, sorted, internally disjoint layer over `0..space`. + fn random_layer(rng: &mut Lcg, space: u64) -> ReadOnlyIndex { + let mut mappings = Vec::new(); + let mut cursor = 0u64; + while cursor < space { + cursor += rng.below(4); + if cursor >= space { + break; + } + let length = 1 + rng.below(16.min(space - cursor)); + let zeroed = rng.below(4) == 0; + let moffset = if zeroed { + NO_PHYSICAL_OFFSET + } else { + 1 + rng.below(4096) + }; + mappings.push(SegmentMapping::new( + cursor, + length as u32, + moffset, + zeroed, + 0, + )); + cursor += length; + } + ReadOnlyIndex::new(mappings) + } + + #[test] + fn test_readonly_merge_sweep_matches_btree_overlay() { + let mut rng = Lcg(0x5eed); + for case in 0..512u32 { + let layers = rng.below(7) as usize; + let space = 1 + rng.below(256); + let indexes: Vec = + (0..layers).map(|_| random_layer(&mut rng, space)).collect(); + let refs: Vec<&ReadOnlyIndex> = indexes.iter().collect(); + + let sweep = ReadOnlyIndex::merge(&refs); + let oracle = merge_reference(&refs); + assert_eq!( + sweep.mappings, oracle.mappings, + "case {case}: layers {layers}, space {space}" + ); + } + } + + #[test] + fn test_readonly_merge_drops_older_mappings_covered_by_newer_runs() { + // The newer run [5, 100) sweeps past the older [50, 60) start; the + // covered mapping must not resurface after the sweep. + let older = ReadOnlyIndex::new(vec![sm(0, 10, 1), sm(50, 10, 2)]); + let newer = ReadOnlyIndex::new(vec![sm_tag(5, 95, 9, 0)]); + let merged = ReadOnlyIndex::merge(&[&newer, &older]); + assert_mappings_eq(&merged.mappings, &[sm_tag(0, 5, 1, 1), sm_tag(5, 95, 9, 0)]); + + // Partial newer coverage leaves the older tail. + let newer = ReadOnlyIndex::new(vec![sm_tag(5, 50, 9, 0)]); + let merged = ReadOnlyIndex::merge(&[&newer, &older]); + assert_mappings_eq( + &merged.mappings, + &[sm_tag(0, 5, 1, 1), sm_tag(5, 50, 9, 0), sm_tag(55, 5, 7, 1)], + ); + } + + #[test] + fn test_readonly_merge_zeroed_and_mid_fragment() { + let base = ReadOnlyIndex::new(vec![SegmentMapping::new(0, 100, 7, false, 0)]); + let mid = ReadOnlyIndex::new(vec![SegmentMapping::new( + 40, + 20, + NO_PHYSICAL_OFFSET, + true, + 0, + )]); + let top = ReadOnlyIndex::new(vec![sm_tag(50, 10, 99, 0)]); + let merged = ReadOnlyIndex::merge(&[&top, &mid, &base]); + assert_mappings_eq( + &merged.mappings, + &[ + sm_tag(0, 40, 7, 2), + SegmentMapping::new(40, 10, NO_PHYSICAL_OFFSET, true, 1), + sm_tag(50, 10, 99, 0), + sm_tag(60, 40, 67, 2), + ], + ); + } } -- 2.43.0