Thanks to visit codestin.com
Credit goes to github.com

Skip to content

[storage] support associating metadata w/ variable-store commit ops #1450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Aug 22, 2025
Merged

[storage] support associating metadata w/ variable-store commit ops #1450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
eaea334
allow metadata to be associated with commit ops in variable stores
roberto-bayardo Aug 21, 2025
692c757
update examples
roberto-bayardo Aug 21, 2025
9b0c2a5
fix comment
roberto-bayardo Aug 21, 2025
f4278e4
hide raise_inactivity_floor from crate-level API
roberto-bayardo Aug 21, 2025
5483c76
copilot nit
roberto-bayardo Aug 21, 2025
ef5f3a3
beef up test
roberto-bayardo Aug 21, 2025
3dcba93
add get_metadata to keyless
roberto-bayardo Aug 21, 2025
64fc1d3
fix Display implementation
roberto-bayardo Aug 21, 2025
220f911
change commit metadata to be an Option
roberto-bayardo Aug 21, 2025
b07670a
test get() on commit ops
roberto-bayardo Aug 21, 2025
e1e5be2
use checked_sub
roberto-bayardo Aug 21, 2025
5508fb2
fix doc error
roberto-bayardo Aug 21, 2025
4511d51
fix example
roberto-bayardo Aug 21, 2025
094bc60
make get_metadata return the location
roberto-bayardo Aug 22, 2025
2807cfa
fix comment
roberto-bayardo Aug 22, 2025
f36b829
fix restart benchmark
patrick-ogrady Aug 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions examples/sync/src/databases/immutable.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Immutable database types and helpers for the sync example.

use crate::{Hasher, Key, Translator, Value};
use commonware_cryptography::Hasher as CryptoHasher;
use commonware_cryptography::{Hasher as CryptoHasher, Sha256};
use commonware_runtime::{Clock, Metrics, Storage};
use commonware_storage::{
adb::{
Expand Down Expand Up @@ -62,12 +62,12 @@ pub fn create_test_operations(count: usize, seed: u64) -> Vec<Operation> {
operations.push(Operation::Set(key, value));

if (i + 1) % 10 == 0 {
operations.push(Operation::Commit());
operations.push(Operation::Commit(None));
}
}

// Always end with a commit
operations.push(Operation::Commit());
operations.push(Operation::Commit(Some(Sha256::fill(1))));
operations
}

Expand All @@ -90,8 +90,8 @@ where
Operation::Set(key, value) => {
database.set(key, value).await?;
}
Operation::Commit() => {
database.commit().await?;
Operation::Commit(metadata) => {
database.commit(metadata).await?;
}
_ => {}
}
Expand All @@ -100,7 +100,7 @@ where
}

async fn commit(&mut self) -> Result<(), commonware_storage::adb::Error> {
self.commit().await
self.commit(None).await
}

fn root(&self, hasher: &mut Standard<commonware_cryptography::Sha256>) -> Key {
Expand Down
2 changes: 1 addition & 1 deletion storage/src/adb/any/fixed/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@ impl<
///
/// This method does not change the state of the db's snapshot, but it always changes the root
/// since it applies at least one operation.
pub(crate) async fn raise_inactivity_floor(&mut self, max_steps: u64) -> Result<(), Error> {
async fn raise_inactivity_floor(&mut self, max_steps: u64) -> Result<(), Error> {
for _ in 0..max_steps {
if self.inactivity_floor_loc == self.op_count() {
break;
Expand Down
77 changes: 51 additions & 26 deletions storage/src/adb/any/variable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ impl<E: RStorage + Clock + Metrics, K: Array, V: Codec, H: CHasher, T: Translato
uncommitted_ops.insert(key, (None, Some(loc)));
}
}
Operation::CommitFloor(loc) => {
Operation::CommitFloor(_, loc) => {
self.inactivity_floor_loc = loc;

// Apply all uncommitted operations.
Expand Down Expand Up @@ -590,11 +590,11 @@ impl<E: RStorage + Clock + Metrics, K: Array, V: Codec, H: CHasher, T: Translato
/// Commit any pending operations to the db, ensuring they are persisted to disk & recoverable
/// upon return from this function. Also raises the inactivity floor according to the schedule,
/// and prunes those operations below it. Batch operations will be parallelized if a thread pool
/// is provided.
pub async fn commit(&mut self) -> Result<(), Error> {
/// is provided. Caller can associate an arbitrary `metadata` value with the commit.
pub async fn commit(&mut self, metadata: Option<V>) -> Result<(), Error> {
// Raise the inactivity floor by the # of uncommitted operations, plus 1 to account for the
// commit op that will be appended.
self.raise_inactivity_floor(self.uncommitted_ops + 1)
self.raise_inactivity_floor(metadata, self.uncommitted_ops + 1)
.await?;
self.uncommitted_ops = 0;
self.sync().await?;
Expand All @@ -608,6 +608,22 @@ impl<E: RStorage + Clock + Metrics, K: Array, V: Codec, H: CHasher, T: Translato
Ok(())
}

/// Get the metadata associated with the last commit, or None if no commit has been made.
pub async fn get_metadata(&self) -> Result<Option<V>, Error> {
let mut last_commit = self.op_count() - self.uncommitted_ops;
if last_commit == 0 {
return Ok(None);
}
last_commit -= 1;
let section = last_commit / self.log_items_per_section;
let offset = self.locations.read(last_commit).await?.into();
let Some(Operation::CommitFloor(metadata, _)) = self.log.get(section, offset).await? else {
unreachable!("no commit operation at location of last commit {last_commit}");
};

Ok(metadata)
}

/// Sync the db to disk ensuring the current state is persisted. Batch operations will be
/// parallelized if a thread pool is provided.
pub(super) async fn sync(&mut self) -> Result<(), Error> {
Expand Down Expand Up @@ -662,7 +678,11 @@ impl<E: RStorage + Clock + Metrics, K: Array, V: Codec, H: CHasher, T: Translato
///
/// This method does not change the state of the db's snapshot, but it always changes the root
/// since it applies at least one operation.
pub(super) async fn raise_inactivity_floor(&mut self, max_steps: u64) -> Result<(), Error> {
async fn raise_inactivity_floor(
&mut self,
metadata: Option<V>,
max_steps: u64,
) -> Result<(), Error> {
for _ in 0..max_steps {
if self.inactivity_floor_loc == self.op_count() {
break;
Expand All @@ -675,7 +695,7 @@ impl<E: RStorage + Clock + Metrics, K: Array, V: Codec, H: CHasher, T: Translato
self.inactivity_floor_loc += 1;
}

self.apply_op(Operation::CommitFloor(self.inactivity_floor_loc))
self.apply_op(Operation::CommitFloor(metadata, self.inactivity_floor_loc))
.await?;

Ok(())
Expand Down Expand Up @@ -834,7 +854,7 @@ pub(super) mod test {
assert_eq!(db.op_count(), 0);

// Test calling commit on an empty db which should make it (durably) non-empty.
db.commit().await.unwrap();
db.commit(None).await.unwrap();
assert_eq!(db.op_count(), 1); // floor op added
let root = db.root(&mut hasher);
assert!(matches!(db.prune_inactive().await, Ok(())));
Expand All @@ -843,7 +863,7 @@ pub(super) mod test {

// Confirm the inactivity floor doesn't fall endlessly behind with multiple commits.
for _ in 1..100 {
db.commit().await.unwrap();
db.commit(None).await.unwrap();
assert_eq!(db.op_count() - 1, db.inactivity_floor_loc);
}

Expand Down Expand Up @@ -892,7 +912,7 @@ pub(super) mod test {
db.sync().await.unwrap();

// Advance over 3 inactive operations.
db.raise_inactivity_floor(3).await.unwrap();
db.raise_inactivity_floor(None, 3).await.unwrap();
assert_eq!(db.inactivity_floor_loc, 3);
assert_eq!(db.op_count(), 6); // 4 updates, 1 deletion, 1 commit
db.sync().await.unwrap();
Expand All @@ -917,7 +937,8 @@ pub(super) mod test {
assert_eq!(db.op_count(), 8);

// Make sure closing/reopening gets us back to the same state.
db.commit().await.unwrap();
let metadata = Some(vec![99, 100]);
db.commit(metadata.clone()).await.unwrap();
assert_eq!(db.op_count(), 9);
let root = db.root(&mut hasher);
db.close().await.unwrap();
Expand All @@ -927,9 +948,12 @@ pub(super) mod test {

// Since this db no longer has any active keys, we should be able to raise the
// inactivity floor to the tip (only the inactive commit op remains).
db.raise_inactivity_floor(100).await.unwrap();
db.raise_inactivity_floor(None, 100).await.unwrap();
assert_eq!(db.inactivity_floor_loc, db.op_count() - 1);

// Make sure we can still get the metadata.
assert_eq!(db.get_metadata().await.unwrap(), metadata);

// Re-activate the keys by updating them.
db.update(d1, v1.clone()).await.unwrap();
db.update(d2, v2.clone()).await.unwrap();
Expand All @@ -939,16 +963,17 @@ pub(super) mod test {
assert_eq!(db.snapshot.keys(), 2);

// Confirm close/reopen gets us back to the same state.
db.commit().await.unwrap();
db.commit(None).await.unwrap();
let root = db.root(&mut hasher);
db.close().await.unwrap();
let mut db = open_db(context).await;
assert_eq!(db.root(&mut hasher), root);
assert_eq!(db.snapshot.keys(), 2);
assert_eq!(db.get_metadata().await.unwrap(), None);

// Commit will raise the inactivity floor, which won't affect state but will affect the
// root.
db.commit().await.unwrap();
db.commit(None).await.unwrap();

assert!(db.root(&mut hasher) != root);

Expand Down Expand Up @@ -1007,7 +1032,7 @@ pub(super) mod test {
assert_eq!(db.snapshot.items(), 857);

// Test that commit will raise the activity floor.
db.commit().await.unwrap();
db.commit(None).await.unwrap();
assert_eq!(db.op_count(), 2336);
assert_eq!(db.oldest_retained_loc().unwrap(), 1477);
assert_eq!(db.inactivity_floor_loc, 1478);
Expand All @@ -1023,7 +1048,7 @@ pub(super) mod test {
assert_eq!(db.snapshot.items(), 857);

// Raise the inactivity floor to the point where all inactive operations can be pruned.
db.raise_inactivity_floor(3000).await.unwrap();
db.raise_inactivity_floor(None, 3000).await.unwrap();
db.prune_inactive().await.unwrap();
assert_eq!(db.inactivity_floor_loc, 4478);
// Inactivity floor should be 858 operations from tip since 858 operations are active
Expand Down Expand Up @@ -1051,7 +1076,7 @@ pub(super) mod test {
let start_pos = db.mmr.pruned_to_pos();
let start_loc = leaf_pos_to_num(start_pos).unwrap();
// Raise the inactivity floor and make sure historical inactive operations are still provable.
db.raise_inactivity_floor(100).await.unwrap();
db.raise_inactivity_floor(None, 100).await.unwrap();
db.sync().await.unwrap();
let root = db.root(&mut hasher);
assert!(start_loc < db.inactivity_floor_loc);
Expand Down Expand Up @@ -1081,7 +1106,7 @@ pub(super) mod test {
let v = vec![(i % 255) as u8; ((i % 7) + 3) as usize];
db.update(k, v).await.unwrap();
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();
let root = db.root(&mut hasher);
db.close().await.unwrap();

Expand Down Expand Up @@ -1112,14 +1137,14 @@ pub(super) mod test {
db.update(k, v.clone()).await.unwrap();
map.insert(k, v);
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();
}
let k = hash(&((ELEMENTS - 1) * 1000 + (ELEMENTS - 1)).to_be_bytes());

// Do one last delete operation which will be above the inactivity
// floor, to make sure it gets replayed on restart.
db.delete(k).await.unwrap();
db.commit().await.unwrap();
db.commit(None).await.unwrap();
assert!(db.get(&k).await.unwrap().is_none());

// Close & reopen the db, making sure the re-opened db has exactly the same state.
Expand Down Expand Up @@ -1160,7 +1185,7 @@ pub(super) mod test {
let v = vec![(i % 255) as u8; ((i % 13) + 7) as usize];
db.update(k, v.clone()).await.unwrap();
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();
let root = db.root(&mut hasher);

// Update every 3rd key
Expand All @@ -1187,7 +1212,7 @@ pub(super) mod test {
let v = vec![((i + 1) % 255) as u8; ((i % 13) + 8) as usize];
db.update(k, v.clone()).await.unwrap();
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();
let root = db.root(&mut hasher);

// Delete every 7th key
Expand All @@ -1212,7 +1237,7 @@ pub(super) mod test {
let k = hash(&i.to_be_bytes());
db.delete(k).await.unwrap();
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();

let root = db.root(&mut hasher);
assert_eq!(db.op_count(), 2787);
Expand Down Expand Up @@ -1250,7 +1275,7 @@ pub(super) mod test {
let v = vec![(i % 255) as u8; ((i % 13) + 7) as usize];
db.update(k, v).await.unwrap();
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();
let root = db.root(&mut hasher);
let expected_mmr_size = db.mmr.size();
let expected_log_size = db.log_size;
Expand Down Expand Up @@ -1279,7 +1304,7 @@ pub(super) mod test {
let v = vec![(i % 255) as u8; ((i % 13) + 8) as usize];
db.update(k, v).await.unwrap();
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();
let root = db.root(&mut hasher);
let expected_mmr_size = db.mmr.size();
let expected_log_size = db.log_size;
Expand Down Expand Up @@ -1314,7 +1339,7 @@ pub(super) mod test {
let v = vec![(i % 255) as u8; ((i % 13) + 9) as usize];
db.update(k, v).await.unwrap();
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();
let root = db.root(&mut hasher);

// Add more elements but crash after writing only the locations
Expand All @@ -1334,7 +1359,7 @@ pub(super) mod test {
let v = vec![(i % 255) as u8; ((i % 13) + 10) as usize];
db.update(k, v).await.unwrap();
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();
let root = db.root(&mut hasher);

// Add more elements but crash after writing only the oldest_retained_loc
Expand Down
4 changes: 2 additions & 2 deletions storage/src/adb/benches/variable_init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,10 @@ fn gen_random_any(cfg: Config, num_elements: u64, num_operations: u64) {
let v = vec![(rng.next_u32() % 255) as u8; ((rng.next_u32() % 24) + 20) as usize];
db.update(rand_key, v).await.unwrap();
if rng.next_u32() % COMMIT_FREQUENCY == 0 {
db.commit().await.unwrap();
db.commit(None).await.unwrap();
}
}
db.commit().await.unwrap();
db.commit(None).await.unwrap();
info!(
op_count = db.op_count(),
oldest_retained_loc = db.oldest_retained_loc().unwrap(),
Expand Down
Loading
Loading