bp_core/storage/
fragment.rs

1//! In-memory index of fragments held by a Pouch.
2//!
3//! The `FragmentIndex` is rebuilt at daemon startup by scanning the on-disk
4//! `fragments/` directory and is kept in sync with every store/remove operation.
5
6use serde::{Deserialize, Serialize};
7use std::collections::HashMap;
8
9/// Lightweight record of a single fragment stored on disk.
10///
11/// Does not hold the fragment data — only enough information for the quality
12/// monitor to issue proof-of-storage challenges and for the storage manager
13/// to locate the file.
14#[derive(Debug, Clone, Serialize, Deserialize)]
15pub struct FragmentMeta {
16    /// UUID of the fragment (also the filename without extension).
17    pub fragment_id: String,
18    /// BLAKE3 chunk hash prefix this fragment belongs to.
19    pub chunk_id: String,
20    /// Number of source symbols `k` for this fragment's chunk.
21    pub k: usize,
22    /// On-disk size in bytes (full binary blob including header).
23    pub size_bytes: u64,
24}
25
26/// In-memory index: `chunk_id → Vec<FragmentMeta>`.
27///
28/// Keyed by chunk_id so that the storage manager can quickly find all
29/// fragments of a given chunk (for recoding or proof-of-storage responses).
30#[derive(Debug, Default)]
31pub struct FragmentIndex {
32    /// chunk_id → list of fragment metadata records.
33    inner: HashMap<String, Vec<FragmentMeta>>,
34    /// Total bytes tracked by this index (sum of fragment sizes).
35    total_bytes: u64,
36}
37
38impl FragmentIndex {
39    /// Create an empty index.
40    pub fn new() -> Self {
41        Self::default()
42    }
43
44    /// Register a new fragment.
45    pub fn insert(&mut self, meta: FragmentMeta) {
46        self.total_bytes += meta.size_bytes;
47        self.inner
48            .entry(meta.chunk_id.clone())
49            .or_default()
50            .push(meta);
51    }
52
53    /// Remove a fragment by `chunk_id` + `fragment_id`.
54    /// Returns the removed entry if found.
55    pub fn remove(&mut self, chunk_id: &str, fragment_id: &str) -> Option<FragmentMeta> {
56        let entries = self.inner.get_mut(chunk_id)?;
57        let pos = entries.iter().position(|m| m.fragment_id == fragment_id)?;
58        let removed = entries.swap_remove(pos);
59        self.total_bytes = self.total_bytes.saturating_sub(removed.size_bytes);
60        if entries.is_empty() {
61            self.inner.remove(chunk_id);
62        }
63        Some(removed)
64    }
65
66    /// All fragment records for a given chunk (for recoding / PoS challenges).
67    pub fn fragments_for_chunk(&self, chunk_id: &str) -> &[FragmentMeta] {
68        self.inner
69            .get(chunk_id)
70            .map(Vec::as_slice)
71            .unwrap_or_default()
72    }
73
74    /// All chunk IDs tracked.
75    pub fn chunk_ids(&self) -> impl Iterator<Item = &str> {
76        self.inner.keys().map(String::as_str)
77    }
78
79    /// Total number of fragments across all chunks.
80    pub fn fragment_count(&self) -> usize {
81        self.inner.values().map(Vec::len).sum()
82    }
83
84    /// Total bytes occupied by all stored fragments.
85    pub fn total_bytes(&self) -> u64 {
86        self.total_bytes
87    }
88
89    /// Whether the index holds any fragment for the given chunk.
90    pub fn contains_chunk(&self, chunk_id: &str) -> bool {
91        self.inner.contains_key(chunk_id)
92    }
93}
94
95#[cfg(test)]
96mod tests {
97    use super::*;
98
99    fn make_meta(chunk: &str, frag: &str, size: u64) -> FragmentMeta {
100        FragmentMeta {
101            fragment_id: frag.into(),
102            chunk_id: chunk.into(),
103            k: 4,
104            size_bytes: size,
105        }
106    }
107
108    #[test]
109    fn insert_and_query() {
110        let mut idx = FragmentIndex::new();
111        idx.insert(make_meta("chunk1", "frag-a", 1024));
112        idx.insert(make_meta("chunk1", "frag-b", 1024));
113        idx.insert(make_meta("chunk2", "frag-c", 2048));
114
115        assert_eq!(idx.fragment_count(), 3);
116        assert_eq!(idx.total_bytes(), 4096);
117        assert_eq!(idx.fragments_for_chunk("chunk1").len(), 2);
118        assert_eq!(idx.fragments_for_chunk("chunk2").len(), 1);
119        assert_eq!(idx.fragments_for_chunk("chunk3").len(), 0);
120    }
121
122    #[test]
123    fn remove_existing() {
124        let mut idx = FragmentIndex::new();
125        idx.insert(make_meta("c1", "f1", 500));
126        idx.insert(make_meta("c1", "f2", 500));
127
128        let removed = idx.remove("c1", "f1");
129        assert!(removed.is_some());
130        assert_eq!(removed.unwrap().fragment_id, "f1");
131        assert_eq!(idx.fragment_count(), 1);
132        assert_eq!(idx.total_bytes(), 500);
133    }
134
135    #[test]
136    fn remove_last_fragment_cleans_chunk_key() {
137        let mut idx = FragmentIndex::new();
138        idx.insert(make_meta("c1", "f1", 100));
139        idx.remove("c1", "f1");
140        assert!(!idx.contains_chunk("c1"));
141        assert_eq!(idx.fragment_count(), 0);
142    }
143
144    #[test]
145    fn remove_nonexistent_returns_none() {
146        let mut idx = FragmentIndex::new();
147        assert!(idx.remove("no-chunk", "no-frag").is_none());
148    }
149}
bp_core/storage/fragment.rs

bp_core/storage/
fragment.rs
