From afbed60ff19ec1d0113faf6a15474b269b3ffe4e Mon Sep 17 00:00:00 2001
From: Frank McSherry <fmcsherry@me.com>
Date: Wed, 25 Jun 2025 08:09:56 -0400
Subject: [PATCH 1/9] Reshape ordneu to resemble trie layers

---
 .../src/trace/implementations/ord_neu.rs      | 319 ++++++++++--------
 1 file changed, 179 insertions(+), 140 deletions(-)

diff --git a/differential-dataflow/src/trace/implementations/ord_neu.rs b/differential-dataflow/src/trace/implementations/ord_neu.rs
index 4ac2acf30..8c7a3c73d 100644
--- a/differential-dataflow/src/trace/implementations/ord_neu.rs
+++ b/differential-dataflow/src/trace/implementations/ord_neu.rs
@@ -10,7 +10,10 @@
 
 use std::rc::Rc;
 
+use serde::{Deserialize, Serialize};
+
 use crate::containers::TimelyStack;
+use crate::trace::implementations::BatchContainer;
 use crate::trace::implementations::chunker::{ColumnationChunker, VecChunker};
 use crate::trace::implementations::spine_fueled::Spine;
 use crate::trace::implementations::merge_batcher::{MergeBatcher, VecMerger, ColMerger};
@@ -66,6 +69,70 @@ pub type PreferredBuilder<K, V, T, R> = RcBuilder<OrdValBuilder<Preferred<K,V,T,
 // pub type ColKeySpine<K, T, R> = Spine<Rc<OrdKeyBatch<TStack<((K,()),T,R)>>>>;
 
 
+/// A container for non-empty lists of values.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Vals<O, V> {
+    /// Offsets used to provide indexes from keys to values.
+    ///
+    /// The length of this list is one longer than `keys`, so that we can avoid bounds logic.
+    pub offs: O,
+    /// Concatenated ordered lists of values, bracketed by offsets in `offs`.
+    pub vals: V,
+}
+
+impl<O: BatchContainer, V: BatchContainer> Default for Vals<O, V> {
+    fn default() -> Self { Self::with_capacity(0, 0) }
+}
+
+impl<O: BatchContainer, V: BatchContainer> Vals<O, V> {
+    fn with_capacity(o_size: usize, v_size: usize) -> Self {
+        Self {
+            offs: <O as BatchContainer>::with_capacity(o_size),
+            vals: <V as BatchContainer>::with_capacity(v_size),
+        }
+    }
+    fn merge_capacity(this: &Self, that: &Self) -> Self {
+        Self {
+            offs: <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len()),
+            vals: <V as BatchContainer>::merge_capacity(&this.vals, &that.vals),
+        }
+    }
+}
+/// A container for non-empty lists of updates.
+///
+/// This container uses the special representiation of an empty slice to stand in for
+/// "the previous single element". An empty slice is an otherwise invalid representation.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Upds<O, T, D> {
+    /// Offsets used to provide indexes from values to updates.
+    pub offs: O,
+    /// Concatenated ordered lists of update times, bracketed by offsets in `offs`.
+    pub times: T,
+    /// Concatenated ordered lists of update diffs, bracketed by offsets in `offs`.
+    pub diffs: D,
+}
+
+impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Default for Upds<O, T, D> {
+    fn default() -> Self { Self::with_capacity(0, 0) }
+}
+impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Upds<O, T, D> {
+    fn with_capacity(o_size: usize, u_size: usize) -> Self {
+        Self {
+            offs: <O as BatchContainer>::with_capacity(o_size),
+            times: <T as BatchContainer>::with_capacity(u_size),
+            diffs: <D as BatchContainer>::with_capacity(u_size),
+        }
+    }
+    fn merge_capacity(this: &Self, that: &Self) -> Self {
+        Self {
+            offs:  <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len()),
+            times: <T as BatchContainer>::merge_capacity(&this.times, &that.times),
+            diffs: <D as BatchContainer>::merge_capacity(&this.diffs, &that.diffs),
+        }
+    }
+}
+
+
 /// Types related to forming batches with values.
 pub mod val_batch {
 
@@ -78,43 +145,35 @@ pub mod val_batch {
     use crate::trace::implementations::{BatchContainer, BuilderInput};
     use crate::IntoOwned;
 
-    use super::{Layout, Update};
+    use super::{Layout, Update, Vals, Upds};
 
     /// An immutable collection of update tuples, from a contiguous interval of logical times.
     #[derive(Debug, Serialize, Deserialize)]
+    #[serde(bound = "
+        L::KeyContainer: Serialize + for<'a> Deserialize<'a>,
+        L::ValContainer: Serialize + for<'a> Deserialize<'a>,
+        L::OffsetContainer: Serialize + for<'a> Deserialize<'a>,
+        L::TimeContainer: Serialize + for<'a> Deserialize<'a>,
+        L::DiffContainer: Serialize + for<'a> Deserialize<'a>,
+    ")]
     pub struct OrdValStorage<L: Layout> {
-        /// An ordered list of keys, corresponding to entries in `keys_offs`.
+        /// An ordered list of keys.
         pub keys: L::KeyContainer,
-        /// Offsets used to provide indexes from keys to values.
-        ///
-        /// The length of this list is one longer than `keys`, so that we can avoid bounds logic.
-        pub keys_offs: L::OffsetContainer,
-        /// Concatenated ordered lists of values, bracketed by offsets in `keys_offs`.
-        pub vals: L::ValContainer,
-        /// Offsets used to provide indexes from values to updates.
-        ///
-        /// This list has a special representation that any empty range indicates the singleton
-        /// element just before the range, as if the start were decremented by one. The empty
-        /// range is otherwise an invalid representation, and we borrow it to compactly encode
-        /// single common update values (e.g. in a snapshot, the minimal time and a diff of one).
-        ///
-        /// The length of this list is one longer than `vals`, so that we can avoid bounds logic.
-        pub vals_offs: L::OffsetContainer,
-        /// Concatenated ordered lists of update times, bracketed by offsets in `vals_offs`.
-        pub times: L::TimeContainer,
-        /// Concatenated ordered lists of update diffs, bracketed by offsets in `vals_offs`.
-        pub diffs: L::DiffContainer,
+        /// For each key in `keys`, a list of values.
+        pub vals: Vals<L::OffsetContainer, L::ValContainer>,
+        /// For each val in `vals`, a list of (time, diff) updates.
+        pub upds: Upds<L::OffsetContainer, L::TimeContainer, L::DiffContainer>,
     }
 
     impl<L: Layout> OrdValStorage<L> {
         /// Lower and upper bounds in `self.vals` corresponding to the key at `index`.
         fn values_for_key(&self, index: usize) -> (usize, usize) {
-            (self.keys_offs.index(index), self.keys_offs.index(index+1))
+            (self.vals.offs.index(index), self.vals.offs.index(index+1))
         }
         /// Lower and upper bounds in `self.updates` corresponding to the value at `index`.
         fn updates_for_value(&self, index: usize) -> (usize, usize) {
-            let mut lower = self.vals_offs.index(index);
-            let upper = self.vals_offs.index(index+1);
+            let mut lower = self.upds.offs.index(index);
+            let upper = self.upds.offs.index(index+1);
             // We use equal lower and upper to encode "singleton update; just before here".
             // It should only apply when there is a prior element, so `lower` should be greater than zero.
             if lower == upper {
@@ -186,11 +245,8 @@ pub mod val_batch {
             Self {
                 storage: OrdValStorage {
                     keys: L::KeyContainer::with_capacity(0),
-                    keys_offs: L::OffsetContainer::with_capacity(0),
-                    vals: L::ValContainer::with_capacity(0),
-                    vals_offs: L::OffsetContainer::with_capacity(0),
-                    times: L::TimeContainer::with_capacity(0),
-                    diffs: L::DiffContainer::with_capacity(0),
+                    vals: Default::default(),
+                    upds: Default::default(),
                 },
                 description: Description::new(lower, upper, Antichain::from_elem(Self::Time::minimum())),
                 updates: 0,
@@ -238,17 +294,14 @@ pub mod val_batch {
 
             let mut storage = OrdValStorage {
                 keys: L::KeyContainer::merge_capacity(&batch1.keys, &batch2.keys),
-                keys_offs: L::OffsetContainer::with_capacity(batch1.keys_offs.len() + batch2.keys_offs.len()),
-                vals: L::ValContainer::merge_capacity(&batch1.vals, &batch2.vals),
-                vals_offs: L::OffsetContainer::with_capacity(batch1.vals_offs.len() + batch2.vals_offs.len()),
-                times: L::TimeContainer::merge_capacity(&batch1.times, &batch2.times),
-                diffs: L::DiffContainer::merge_capacity(&batch1.diffs, &batch2.diffs),
+                vals: Vals::merge_capacity(&batch1.vals, &batch2.vals),
+                upds: Upds::merge_capacity(&batch1.upds, &batch2.upds),
             };
 
             // Mark explicit types because type inference fails to resolve it.
-            let keys_offs: &mut L::OffsetContainer = &mut storage.keys_offs;
+            let keys_offs: &mut L::OffsetContainer = &mut storage.vals.offs;
             keys_offs.push(0);
-            let vals_offs: &mut L::OffsetContainer = &mut storage.vals_offs;
+            let vals_offs: &mut L::OffsetContainer = &mut storage.upds.offs;
             vals_offs.push(0);
 
             OrdValMerger {
@@ -262,7 +315,7 @@ pub mod val_batch {
         }
         fn done(self) -> OrdValBatch<L> {
             OrdValBatch {
-                updates: self.result.times.len() + self.singletons,
+                updates: self.result.upds.times.len() + self.singletons,
                 storage: self.result,
                 description: self.description,
             }
@@ -270,14 +323,14 @@ pub mod val_batch {
         fn work(&mut self, source1: &OrdValBatch<L>, source2: &OrdValBatch<L>, fuel: &mut isize) {
 
             // An (incomplete) indication of the amount of work we've done so far.
-            let starting_updates = self.result.times.len() + self.singletons;
+            let starting_updates = self.result.upds.times.len() + self.singletons;
             let mut effort = 0isize;
 
             // While both mergees are still active, perform single-key merges.
             while self.key_cursor1 < source1.storage.keys.len() && self.key_cursor2 < source2.storage.keys.len() && effort < *fuel {
                 self.merge_key(&source1.storage, &source2.storage);
                 // An (incomplete) accounting of the work we've done.
-                effort = (self.result.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
             }
 
             // Merging is complete, and only copying remains. 
@@ -285,12 +338,12 @@ pub mod val_batch {
             while self.key_cursor1 < source1.storage.keys.len() && effort < *fuel {
                 self.copy_key(&source1.storage, self.key_cursor1);
                 self.key_cursor1 += 1;
-                effort = (self.result.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
             }
             while self.key_cursor2 < source2.storage.keys.len() && effort < *fuel {
                 self.copy_key(&source2.storage, self.key_cursor2);
                 self.key_cursor2 += 1;
-                effort = (self.result.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
             }
 
             *fuel -= effort;
@@ -308,21 +361,21 @@ pub mod val_batch {
         /// The caller should be certain to update the cursor, as this method does not do this.
         fn copy_key(&mut self, source: &OrdValStorage<L>, cursor: usize) {
             // Capture the initial number of values to determine if the merge was ultimately non-empty.
-            let init_vals = self.result.vals.len();
+            let init_vals = self.result.vals.vals.len();
             let (mut lower, upper) = source.values_for_key(cursor);
             while lower < upper {
                 self.stash_updates_for_val(source, lower);
                 if let Some(off) = self.consolidate_updates() {
-                    self.result.vals_offs.push(off);
-                    self.result.vals.push(source.vals.index(lower));
+                    self.result.upds.offs.push(off);
+                    self.result.vals.vals.push(source.vals.vals.index(lower));
                 }
                 lower += 1;
             }            
 
             // If we have pushed any values, copy the key as well.
-            if self.result.vals.len() > init_vals {
+            if self.result.vals.vals.len() > init_vals {
                 self.result.keys.push(source.keys.index(cursor));
-                self.result.keys_offs.push(self.result.vals.len());
+                self.result.vals.offs.push(self.result.vals.vals.len());
             }           
         }
         /// Merge the next key in each of `source1` and `source2` into `self`, updating the appropriate cursors.
@@ -342,7 +395,7 @@ pub mod val_batch {
                     let (lower2, upper2) = source2.values_for_key(self.key_cursor2);
                     if let Some(off) = self.merge_vals((source1, lower1, upper1), (source2, lower2, upper2)) {
                         self.result.keys.push(source1.keys.index(self.key_cursor1));
-                        self.result.keys_offs.push(off);
+                        self.result.vals.offs.push(off);
                     }
                     // Increment cursors in either case; the keys are merged.
                     self.key_cursor1 += 1;
@@ -364,19 +417,19 @@ pub mod val_batch {
             (source2, mut lower2, upper2): (&OrdValStorage<L>, usize, usize),
         ) -> Option<usize> {
             // Capture the initial number of values to determine if the merge was ultimately non-empty.
-            let init_vals = self.result.vals.len();
+            let init_vals = self.result.vals.vals.len();
             while lower1 < upper1 && lower2 < upper2 {
                 // We compare values, and fold in updates for the lowest values;
                 // if they are non-empty post-consolidation, we write the value.
                 // We could multi-way merge and it wouldn't be very complicated.
                 use ::std::cmp::Ordering;
-                match source1.vals.index(lower1).cmp(&source2.vals.index(lower2)) {
+                match source1.vals.vals.index(lower1).cmp(&source2.vals.vals.index(lower2)) {
                     Ordering::Less => { 
                         // Extend stash by updates, with logical compaction applied.
                         self.stash_updates_for_val(source1, lower1);
                         if let Some(off) = self.consolidate_updates() {
-                            self.result.vals_offs.push(off);
-                            self.result.vals.push(source1.vals.index(lower1));
+                            self.result.upds.offs.push(off);
+                            self.result.vals.vals.push(source1.vals.vals.index(lower1));
                         }
                         lower1 += 1;
                     },
@@ -384,8 +437,8 @@ pub mod val_batch {
                         self.stash_updates_for_val(source1, lower1);
                         self.stash_updates_for_val(source2, lower2);
                         if let Some(off) = self.consolidate_updates() {
-                            self.result.vals_offs.push(off);
-                            self.result.vals.push(source1.vals.index(lower1));
+                            self.result.upds.offs.push(off);
+                            self.result.vals.vals.push(source1.vals.vals.index(lower1));
                         }
                         lower1 += 1;
                         lower2 += 1;
@@ -394,8 +447,8 @@ pub mod val_batch {
                         // Extend stash by updates, with logical compaction applied.
                         self.stash_updates_for_val(source2, lower2);
                         if let Some(off) = self.consolidate_updates() {
-                            self.result.vals_offs.push(off);
-                            self.result.vals.push(source2.vals.index(lower2));
+                            self.result.upds.offs.push(off);
+                            self.result.vals.vals.push(source2.vals.vals.index(lower2));
                         }
                         lower2 += 1;
                     },
@@ -405,23 +458,23 @@ pub mod val_batch {
             while lower1 < upper1 {
                 self.stash_updates_for_val(source1, lower1);
                 if let Some(off) = self.consolidate_updates() {
-                    self.result.vals_offs.push(off);
-                    self.result.vals.push(source1.vals.index(lower1));
+                    self.result.upds.offs.push(off);
+                    self.result.vals.vals.push(source1.vals.vals.index(lower1));
                 }
                 lower1 += 1;
             }
             while lower2 < upper2 {
                 self.stash_updates_for_val(source2, lower2);
                 if let Some(off) = self.consolidate_updates() {
-                    self.result.vals_offs.push(off);
-                    self.result.vals.push(source2.vals.index(lower2));
+                    self.result.upds.offs.push(off);
+                    self.result.vals.vals.push(source2.vals.vals.index(lower2));
                 }
                 lower2 += 1;
             }
 
             // Values being pushed indicate non-emptiness.
-            if self.result.vals.len() > init_vals {
-                Some(self.result.vals.len())
+            if self.result.vals.vals.len() > init_vals {
+                Some(self.result.vals.vals.len())
             } else {
                 None
             }
@@ -432,8 +485,8 @@ pub mod val_batch {
             let (lower, upper) = source.updates_for_value(index);
             for i in lower .. upper {
                 // NB: Here is where we would need to look back if `lower == upper`.
-                let time = source.times.index(i);
-                let diff = source.diffs.index(i);
+                let time = source.upds.times.index(i);
+                let diff = source.upds.diffs.index(i);
                 use crate::lattice::Lattice;
                 let mut new_time: <L::Target as Update>::Time = time.into_owned();
                 new_time.advance_by(self.description.since().borrow());
@@ -448,7 +501,7 @@ pub mod val_batch {
             if !self.update_stash.is_empty() {
                 // If there is a single element, equal to a just-prior recorded update,
                 // we push nothing and report an unincremented offset to encode this case.
-                let time_diff = self.result.times.last().zip(self.result.diffs.last());
+                let time_diff = self.result.upds.times.last().zip(self.result.upds.diffs.last());
                 let last_eq = self.update_stash.last().zip(time_diff).map(|((t1, d1), (t2, d2))| {
                     let t1 = <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(t1);
                     let d1 = <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(d1);
@@ -462,11 +515,11 @@ pub mod val_batch {
                 else {
                     // Conventional; move `update_stash` into `updates`.
                     for (time, diff) in self.update_stash.drain(..) {
-                        self.result.times.push(time);
-                        self.result.diffs.push(diff);
+                        self.result.upds.times.push(time);
+                        self.result.upds.diffs.push(diff);
                     }
                 }
-                Some(self.result.times.len())
+                Some(self.result.upds.times.len())
             } else {
                 None
             }
@@ -498,12 +551,12 @@ pub mod val_batch {
         fn get_val<'a>(&self, storage: &'a Self::Storage) -> Option<Self::Val<'a>> { if self.val_valid(storage) { Some(self.val(storage)) } else { None } }
 
         fn key<'a>(&self, storage: &'a OrdValBatch<L>) -> Self::Key<'a> { storage.storage.keys.index(self.key_cursor) }
-        fn val<'a>(&self, storage: &'a OrdValBatch<L>) -> Self::Val<'a> { storage.storage.vals.index(self.val_cursor) }
+        fn val<'a>(&self, storage: &'a OrdValBatch<L>) -> Self::Val<'a> { storage.storage.vals.vals.index(self.val_cursor) }
         fn map_times<L2: FnMut(Self::TimeGat<'_>, Self::DiffGat<'_>)>(&mut self, storage: &OrdValBatch<L>, mut logic: L2) {
             let (lower, upper) = storage.storage.updates_for_value(self.val_cursor);
             for index in lower .. upper {
-                let time = storage.storage.times.index(index);
-                let diff = storage.storage.diffs.index(index);
+                let time = storage.storage.upds.times.index(index);
+                let diff = storage.storage.upds.diffs.index(index);
                 logic(time, diff);
             }
         }
@@ -531,7 +584,7 @@ pub mod val_batch {
             }
         }
         fn seek_val(&mut self, storage: &OrdValBatch<L>, val: Self::Val<'_>) {
-            self.val_cursor += storage.storage.vals.advance(self.val_cursor, storage.storage.values_for_key(self.key_cursor).1, |x| <L::ValContainer as BatchContainer>::reborrow(x).lt(&<L::ValContainer as BatchContainer>::reborrow(val)));
+            self.val_cursor += storage.storage.vals.vals.advance(self.val_cursor, storage.storage.values_for_key(self.key_cursor).1, |x| <L::ValContainer as BatchContainer>::reborrow(x).lt(&<L::ValContainer as BatchContainer>::reborrow(val)));
         }
         fn rewind_keys(&mut self, storage: &OrdValBatch<L>) {
             self.key_cursor = 0;
@@ -573,8 +626,8 @@ pub mod val_batch {
         /// to recover the singleton to push it into `updates` to join the second update.
         fn push_update(&mut self, time: <L::Target as Update>::Time, diff: <L::Target as Update>::Diff) {
             // If a just-pushed update exactly equals `(time, diff)` we can avoid pushing it.
-            if self.result.times.last().map(|t| t == <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time)) == Some(true) &&
-               self.result.diffs.last().map(|d| d == <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff)) == Some(true)
+            if self.result.upds.times.last().map(|t| t == <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time)) == Some(true) &&
+               self.result.upds.diffs.last().map(|d| d == <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff)) == Some(true)
             {
                 assert!(self.singleton.is_none());
                 self.singleton = Some((time, diff));
@@ -582,11 +635,11 @@ pub mod val_batch {
             else {
                 // If we have pushed a single element, we need to copy it out to meet this one.
                 if let Some((time, diff)) = self.singleton.take() {
-                    self.result.times.push(time);
-                    self.result.diffs.push(diff);
+                    self.result.upds.times.push(time);
+                    self.result.upds.diffs.push(diff);
                 }
-                self.result.times.push(time);
-                self.result.diffs.push(diff);
+                self.result.upds.times.push(time);
+                self.result.upds.diffs.push(diff);
             }
         }
     }
@@ -611,11 +664,8 @@ pub mod val_batch {
             Self { 
                 result: OrdValStorage {
                     keys: L::KeyContainer::with_capacity(keys),
-                    keys_offs: L::OffsetContainer::with_capacity(keys + 1),
-                    vals: L::ValContainer::with_capacity(vals),
-                    vals_offs: L::OffsetContainer::with_capacity(vals + 1),
-                    times: L::TimeContainer::with_capacity(upds),
-                    diffs: L::DiffContainer::with_capacity(upds),
+                    vals: Vals::with_capacity(keys + 1, vals),
+                    upds: Upds::with_capacity(vals + 1, upds),
                 },
                 singleton: None,
                 singletons: 0,
@@ -630,22 +680,22 @@ pub mod val_batch {
                 // Perhaps this is a continuation of an already received key.
                 if self.result.keys.last().map(|k| CI::key_eq(&key, k)).unwrap_or(false) {
                     // Perhaps this is a continuation of an already received value.
-                    if self.result.vals.last().map(|v| CI::val_eq(&val, v)).unwrap_or(false) {
+                    if self.result.vals.vals.last().map(|v| CI::val_eq(&val, v)).unwrap_or(false) {
                         self.push_update(time, diff);
                     } else {
                         // New value; complete representation of prior value.
-                        self.result.vals_offs.push(self.result.times.len());
+                        self.result.upds.offs.push(self.result.upds.times.len());
                         if self.singleton.take().is_some() { self.singletons += 1; }
                         self.push_update(time, diff);
-                        self.result.vals.push(val);
+                        self.result.vals.vals.push(val);
                     }
                 } else {
                     // New key; complete representation of prior key.
-                    self.result.vals_offs.push(self.result.times.len());
+                    self.result.upds.offs.push(self.result.upds.times.len());
                     if self.singleton.take().is_some() { self.singletons += 1; }
-                    self.result.keys_offs.push(self.result.vals.len());
+                    self.result.vals.offs.push(self.result.vals.vals.len());
                     self.push_update(time, diff);
-                    self.result.vals.push(val);
+                    self.result.vals.vals.push(val);
                     self.result.keys.push(key);
                 }
             }
@@ -654,12 +704,12 @@ pub mod val_batch {
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdValBatch<L> {
             // Record the final offsets
-            self.result.vals_offs.push(self.result.times.len());
+            self.result.upds.offs.push(self.result.upds.times.len());
             // Remove any pending singleton, and if it was set increment our count.
             if self.singleton.take().is_some() { self.singletons += 1; }
-            self.result.keys_offs.push(self.result.vals.len());
+            self.result.vals.offs.push(self.result.vals.vals.len());
             OrdValBatch {
-                updates: self.result.times.len() + self.singletons,
+                updates: self.result.upds.times.len() + self.singletons,
                 storage: self.result,
                 description,
             }
@@ -689,33 +739,28 @@ pub mod key_batch {
     use crate::trace::implementations::{BatchContainer, BuilderInput};
     use crate::IntoOwned;
 
-    use super::{Layout, Update};
+    use super::{Layout, Update, Upds};
 
     /// An immutable collection of update tuples, from a contiguous interval of logical times.
     #[derive(Debug, Serialize, Deserialize)]
+    #[serde(bound = "
+        L::KeyContainer: Serialize + for<'a> Deserialize<'a>,
+        L::OffsetContainer: Serialize + for<'a> Deserialize<'a>,
+        L::TimeContainer: Serialize + for<'a> Deserialize<'a>,
+        L::DiffContainer: Serialize + for<'a> Deserialize<'a>,
+    ")]
     pub struct OrdKeyStorage<L: Layout> {
         /// An ordered list of keys, corresponding to entries in `keys_offs`.
         pub keys: L::KeyContainer,
-        /// Offsets used to provide indexes from keys to updates.
-        ///
-        /// This list has a special representation that any empty range indicates the singleton
-        /// element just before the range, as if the start were decremented by one. The empty
-        /// range is otherwise an invalid representation, and we borrow it to compactly encode
-        /// single common update values (e.g. in a snapshot, the minimal time and a diff of one).
-        ///
-        /// The length of this list is one longer than `keys`, so that we can avoid bounds logic.
-        pub keys_offs: L::OffsetContainer,
-        /// Concatenated ordered lists of update times, bracketed by offsets in `vals_offs`.
-        pub times: L::TimeContainer,
-        /// Concatenated ordered lists of update diffs, bracketed by offsets in `vals_offs`.
-        pub diffs: L::DiffContainer,
+        /// For each key in `keys`, a list of (time, diff) updates.
+        pub upds: Upds<L::OffsetContainer, L::TimeContainer, L::DiffContainer>,
     }
 
     impl<L: Layout> OrdKeyStorage<L> {
         /// Lower and upper bounds in `self.vals` corresponding to the key at `index`.
         fn updates_for_key(&self, index: usize) -> (usize, usize) {
-            let mut lower = self.keys_offs.index(index);
-            let upper = self.keys_offs.index(index+1);
+            let mut lower = self.upds.offs.index(index);
+            let upper = self.upds.offs.index(index+1);
             // We use equal lower and upper to encode "singleton update; just before here".
             // It should only apply when there is a prior element, so `lower` should be greater than zero.
             if lower == upper {
@@ -787,9 +832,7 @@ pub mod key_batch {
             Self {
                 storage: OrdKeyStorage {
                     keys: L::KeyContainer::with_capacity(0),
-                    keys_offs: L::OffsetContainer::with_capacity(0),
-                    times: L::TimeContainer::with_capacity(0),
-                    diffs: L::DiffContainer::with_capacity(0),
+                    upds: Upds::default(),
                 },
                 description: Description::new(lower, upper, Antichain::from_elem(Self::Time::minimum())),
                 updates: 0,
@@ -837,12 +880,10 @@ pub mod key_batch {
 
             let mut storage = OrdKeyStorage {
                 keys: L::KeyContainer::merge_capacity(&batch1.keys, &batch2.keys),
-                keys_offs: L::OffsetContainer::with_capacity(batch1.keys_offs.len() + batch2.keys_offs.len()),
-                times: L::TimeContainer::merge_capacity(&batch1.times, &batch2.times),
-                diffs: L::DiffContainer::merge_capacity(&batch1.diffs, &batch2.diffs),
+                upds: Upds::merge_capacity(&batch1.upds, &batch2.upds),
             };
 
-            let keys_offs: &mut L::OffsetContainer = &mut storage.keys_offs;
+            let keys_offs: &mut L::OffsetContainer = &mut storage.upds.offs;
             keys_offs.push(0);
 
             OrdKeyMerger {
@@ -856,7 +897,7 @@ pub mod key_batch {
         }
         fn done(self) -> OrdKeyBatch<L> {
             OrdKeyBatch {
-                updates: self.result.times.len() + self.singletons,
+                updates: self.result.upds.times.len() + self.singletons,
                 storage: self.result,
                 description: self.description,
             }
@@ -864,14 +905,14 @@ pub mod key_batch {
         fn work(&mut self, source1: &OrdKeyBatch<L>, source2: &OrdKeyBatch<L>, fuel: &mut isize) {
 
             // An (incomplete) indication of the amount of work we've done so far.
-            let starting_updates = self.result.times.len() + self.singletons;
+            let starting_updates = self.result.upds.times.len() + self.singletons;
             let mut effort = 0isize;
 
             // While both mergees are still active, perform single-key merges.
             while self.key_cursor1 < source1.storage.keys.len() && self.key_cursor2 < source2.storage.keys.len() && effort < *fuel {
                 self.merge_key(&source1.storage, &source2.storage);
                 // An (incomplete) accounting of the work we've done.
-                effort = (self.result.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
             }
 
             // Merging is complete, and only copying remains.
@@ -879,12 +920,12 @@ pub mod key_batch {
             while self.key_cursor1 < source1.storage.keys.len() && effort < *fuel {
                 self.copy_key(&source1.storage, self.key_cursor1);
                 self.key_cursor1 += 1;
-                effort = (self.result.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
             }
             while self.key_cursor2 < source2.storage.keys.len() && effort < *fuel {
                 self.copy_key(&source2.storage, self.key_cursor2);
                 self.key_cursor2 += 1;
-                effort = (self.result.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
             }
 
             *fuel -= effort;
@@ -903,7 +944,7 @@ pub mod key_batch {
         fn copy_key(&mut self, source: &OrdKeyStorage<L>, cursor: usize) {
             self.stash_updates_for_key(source, cursor);
             if let Some(off) = self.consolidate_updates() {
-                self.result.keys_offs.push(off);
+                self.result.upds.offs.push(off);
                 self.result.keys.push(source.keys.index(cursor));
             }
         }
@@ -923,7 +964,7 @@ pub mod key_batch {
                     self.stash_updates_for_key(source1, self.key_cursor1);
                     self.stash_updates_for_key(source2, self.key_cursor2);
                     if let Some(off) = self.consolidate_updates() {
-                        self.result.keys_offs.push(off);
+                        self.result.upds.offs.push(off);
                         self.result.keys.push(source1.keys.index(self.key_cursor1));
                     }
                     // Increment cursors in either case; the keys are merged.
@@ -942,8 +983,8 @@ pub mod key_batch {
             let (lower, upper) = source.updates_for_key(index);
             for i in lower .. upper {
                 // NB: Here is where we would need to look back if `lower == upper`.
-                let time = source.times.index(i);
-                let diff = source.diffs.index(i);
+                let time = source.upds.times.index(i);
+                let diff = source.upds.diffs.index(i);
                 use crate::lattice::Lattice;
                 let mut new_time = time.into_owned();
                 new_time.advance_by(self.description.since().borrow());
@@ -958,7 +999,7 @@ pub mod key_batch {
             if !self.update_stash.is_empty() {
                 // If there is a single element, equal to a just-prior recorded update,
                 // we push nothing and report an unincremented offset to encode this case.
-                let time_diff = self.result.times.last().zip(self.result.diffs.last());
+                let time_diff = self.result.upds.times.last().zip(self.result.upds.diffs.last());
                 let last_eq = self.update_stash.last().zip(time_diff).map(|((t1, d1), (t2, d2))| {
                     let t1 = <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(t1);
                     let d1 = <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(d1);
@@ -972,11 +1013,11 @@ pub mod key_batch {
                 else {
                     // Conventional; move `update_stash` into `updates`.
                     for (time, diff) in self.update_stash.drain(..) {
-                        self.result.times.push(time);
-                        self.result.diffs.push(diff);
+                        self.result.upds.times.push(time);
+                        self.result.upds.diffs.push(diff);
                     }
                 }
-                Some(self.result.times.len())
+                Some(self.result.upds.times.len())
             } else {
                 None
             }
@@ -1011,8 +1052,8 @@ pub mod key_batch {
         fn map_times<L2: FnMut(Self::TimeGat<'_>, Self::DiffGat<'_>)>(&mut self, storage: &Self::Storage, mut logic: L2) {
             let (lower, upper) = storage.storage.updates_for_key(self.key_cursor);
             for index in lower .. upper {
-                let time = storage.storage.times.index(index);
-                let diff = storage.storage.diffs.index(index);
+                let time = storage.storage.upds.times.index(index);
+                let diff = storage.storage.upds.diffs.index(index);
                 logic(time, diff);
             }
         }
@@ -1079,18 +1120,18 @@ pub mod key_batch {
             // If a just-pushed update exactly equals `(time, diff)` we can avoid pushing it.
             let t1 = <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time);
             let d1 = <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff);
-            if self.result.times.last().map(|t| t == t1).unwrap_or(false) && self.result.diffs.last().map(|d| d == d1).unwrap_or(false) {
+            if self.result.upds.times.last().map(|t| t == t1).unwrap_or(false) && self.result.upds.diffs.last().map(|d| d == d1).unwrap_or(false) {
                 assert!(self.singleton.is_none());
                 self.singleton = Some((time, diff));
             }
             else {
                 // If we have pushed a single element, we need to copy it out to meet this one.
                 if let Some((time, diff)) = self.singleton.take() {
-                    self.result.times.push(time);
-                    self.result.diffs.push(diff);
+                    self.result.upds.times.push(time);
+                    self.result.upds.diffs.push(diff);
                 }
-                self.result.times.push(time);
-                self.result.diffs.push(diff);
+                self.result.upds.times.push(time);
+                self.result.upds.diffs.push(diff);
             }
         }
     }
@@ -1112,9 +1153,7 @@ pub mod key_batch {
             Self { 
                 result: OrdKeyStorage {
                     keys: L::KeyContainer::with_capacity(keys),
-                    keys_offs: L::OffsetContainer::with_capacity(keys + 1),
-                    times: L::TimeContainer::with_capacity(upds),
-                    diffs: L::DiffContainer::with_capacity(upds),
+                    upds: Upds::with_capacity(keys+1, upds),
                 },
                 singleton: None,
                 singletons: 0,
@@ -1131,7 +1170,7 @@ pub mod key_batch {
                     self.push_update(time, diff);
                 } else {
                     // New key; complete representation of prior key.
-                    self.result.keys_offs.push(self.result.times.len());
+                    self.result.upds.offs.push(self.result.upds.times.len());
                     // Remove any pending singleton, and if it was set increment our count.
                     if self.singleton.take().is_some() { self.singletons += 1; }
                     self.push_update(time, diff);
@@ -1143,11 +1182,11 @@ pub mod key_batch {
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdKeyBatch<L> {
             // Record the final offsets
-            self.result.keys_offs.push(self.result.times.len());
+            self.result.upds.offs.push(self.result.upds.times.len());
             // Remove any pending singleton, and if it was set increment our count.
             if self.singleton.take().is_some() { self.singletons += 1; }
             OrdKeyBatch {
-                updates: self.result.times.len() + self.singletons,
+                updates: self.result.upds.times.len() + self.singletons,
                 storage: self.result,
                 description,
             }

From 5c671b0d41f921b4d4cdb535bdf4eb94738dad09 Mon Sep 17 00:00:00 2001
From: Frank McSherry <fmcsherry@me.com>
Date: Wed, 25 Jun 2025 09:25:48 -0400
Subject: [PATCH 2/9] Update columnar example

---
 differential-dataflow/examples/columnar.rs | 61 ++++++++++------------
 1 file changed, 28 insertions(+), 33 deletions(-)

diff --git a/differential-dataflow/examples/columnar.rs b/differential-dataflow/examples/columnar.rs
index 52d5e141f..c413ab65b 100644
--- a/differential-dataflow/examples/columnar.rs
+++ b/differential-dataflow/examples/columnar.rs
@@ -580,7 +580,7 @@ pub mod dd_builder {
     use differential_dataflow::trace::implementations::Layout;
     use differential_dataflow::trace::implementations::Update;
     use differential_dataflow::trace::implementations::BatchContainer;
-    use differential_dataflow::trace::implementations::ord_neu::{OrdValBatch, val_batch::OrdValStorage, OrdKeyBatch};
+    use differential_dataflow::trace::implementations::ord_neu::{OrdValBatch, val_batch::OrdValStorage, OrdKeyBatch, Vals, Upds};
     use differential_dataflow::trace::implementations::ord_neu::key_batch::OrdKeyStorage;
     use crate::Column;
 
@@ -619,8 +619,8 @@ pub mod dd_builder {
         /// to recover the singleton to push it into `updates` to join the second update.
         fn push_update(&mut self, time: <L::Target as Update>::Time, diff: <L::Target as Update>::Diff) {
             // If a just-pushed update exactly equals `(time, diff)` we can avoid pushing it.
-            if self.result.times.last().map(|t| t == <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time)) == Some(true) &&
-                self.result.diffs.last().map(|d| d == <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff)) == Some(true)
+            if self.result.upds.times.last().map(|t| t == <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time)) == Some(true) &&
+                self.result.upds.diffs.last().map(|d| d == <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff)) == Some(true)
             {
                 assert!(self.singleton.is_none());
                 self.singleton = Some((time, diff));
@@ -628,11 +628,11 @@ pub mod dd_builder {
             else {
                 // If we have pushed a single element, we need to copy it out to meet this one.
                 if let Some((time, diff)) = self.singleton.take() {
-                    self.result.times.push(time);
-                    self.result.diffs.push(diff);
+                    self.result.upds.times.push(time);
+                    self.result.upds.diffs.push(diff);
                 }
-                self.result.times.push(time);
-                self.result.diffs.push(diff);
+                self.result.upds.times.push(time);
+                self.result.upds.diffs.push(diff);
             }
         }
     }
@@ -660,11 +660,8 @@ pub mod dd_builder {
             Self {
                 result: OrdValStorage {
                     keys: L::KeyContainer::with_capacity(keys),
-                    keys_offs: L::OffsetContainer::with_capacity(keys + 1),
-                    vals: L::ValContainer::with_capacity(vals),
-                    vals_offs: L::OffsetContainer::with_capacity(vals + 1),
-                    times: L::TimeContainer::with_capacity(upds),
-                    diffs: L::DiffContainer::with_capacity(upds),
+                    vals: Vals::with_capacity(keys + 1, vals),
+                    upds: Upds::with_capacity(vals + 1, upds),
                 },
                 singleton: None,
                 singletons: 0,
@@ -692,22 +689,22 @@ pub mod dd_builder {
                 // Perhaps this is a continuation of an already received key.
                 if self.result.keys.last().map(|k| <<L::KeyContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&key).eq(&k)).unwrap_or(false) {
                     // Perhaps this is a continuation of an already received value.
-                    if self.result.vals.last().map(|v| <<L::ValContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&val).eq(&v)).unwrap_or(false) {
+                    if self.result.vals.vals.last().map(|v| <<L::ValContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&val).eq(&v)).unwrap_or(false) {
                         self.push_update(time, diff);
                     } else {
                         // New value; complete representation of prior value.
-                        self.result.vals_offs.push(self.result.times.len());
+                        self.result.upds.offs.push(self.result.upds.times.len());
                         if self.singleton.take().is_some() { self.singletons += 1; }
                         self.push_update(time, diff);
-                        self.result.vals.push(&val);
+                        self.result.vals.vals.push(&val);
                     }
                 } else {
                     // New key; complete representation of prior key.
-                    self.result.vals_offs.push(self.result.times.len());
+                    self.result.upds.offs.push(self.result.upds.times.len());
                     if self.singleton.take().is_some() { self.singletons += 1; }
-                    self.result.keys_offs.push(self.result.vals.len());
+                    self.result.vals.offs.push(self.result.vals.len());
                     self.push_update(time, diff);
-                    self.result.vals.push(&val);
+                    self.result.vals.vals.push(&val);
                     self.result.keys.push(&key);
                 }
             }
@@ -716,12 +713,12 @@ pub mod dd_builder {
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdValBatch<L> {
             // Record the final offsets
-            self.result.vals_offs.push(self.result.times.len());
+            self.result.upds.offs.push(self.result.upds.times.len());
             // Remove any pending singleton, and if it was set increment our count.
             if self.singleton.take().is_some() { self.singletons += 1; }
-            self.result.keys_offs.push(self.result.vals.len());
+            self.result.vals.offs.push(self.result.vals.len());
             OrdValBatch {
-                updates: self.result.times.len() + self.singletons,
+                updates: self.result.upds.times.len() + self.singletons,
                 storage: self.result,
                 description,
             }
@@ -767,8 +764,8 @@ pub mod dd_builder {
         /// to recover the singleton to push it into `updates` to join the second update.
         fn push_update(&mut self, time: <L::Target as Update>::Time, diff: <L::Target as Update>::Diff) {
             // If a just-pushed update exactly equals `(time, diff)` we can avoid pushing it.
-            if self.result.times.last().map(|t| t == <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time)) == Some(true) &&
-                self.result.diffs.last().map(|d| d == <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff)) == Some(true)
+            if self.result.upds.times.last().map(|t| t == <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time)) == Some(true) &&
+                self.result.upds.diffs.last().map(|d| d == <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff)) == Some(true)
             {
                 assert!(self.singleton.is_none());
                 self.singleton = Some((time, diff));
@@ -776,11 +773,11 @@ pub mod dd_builder {
             else {
                 // If we have pushed a single element, we need to copy it out to meet this one.
                 if let Some((time, diff)) = self.singleton.take() {
-                    self.result.times.push(time);
-                    self.result.diffs.push(diff);
+                    self.result.upds.times.push(time);
+                    self.result.upds.diffs.push(diff);
                 }
-                self.result.times.push(time);
-                self.result.diffs.push(diff);
+                self.result.upds.times.push(time);
+                self.result.upds.diffs.push(diff);
             }
         }
     }
@@ -808,9 +805,7 @@ pub mod dd_builder {
             Self {
                 result: OrdKeyStorage {
                     keys: L::KeyContainer::with_capacity(keys),
-                    keys_offs: L::OffsetContainer::with_capacity(keys + 1),
-                    times: L::TimeContainer::with_capacity(upds),
-                    diffs: L::DiffContainer::with_capacity(upds),
+                    upds: Upds::with_capacity(keys + 1, upds),
                 },
                 singleton: None,
                 singletons: 0,
@@ -839,7 +834,7 @@ pub mod dd_builder {
                     self.push_update(time, diff);
                 } else {
                     // New key; complete representation of prior key.
-                    self.result.keys_offs.push(self.result.times.len());
+                    self.result.upds.offs.push(self.result.upds.times.len());
                     if self.singleton.take().is_some() { self.singletons += 1; }
                     self.push_update(time, diff);
                     self.result.keys.push(&key);
@@ -850,11 +845,11 @@ pub mod dd_builder {
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdKeyBatch<L> {
             // Record the final offsets
-            self.result.keys_offs.push(self.result.times.len());
+            self.result.upds.offs.push(self.result.upds.times.len());
             // Remove any pending singleton, and if it was set increment our count.
             if self.singleton.take().is_some() { self.singletons += 1; }
             OrdKeyBatch {
-                updates: self.result.times.len() + self.singletons,
+                updates: self.result.upds.times.len() + self.singletons,
                 storage: self.result,
                 description,
             }

From daf28b2e278f6fe148c6d7631e8d1a23fb0e15a4 Mon Sep 17 00:00:00 2001
From: Frank McSherry <fmcsherry@me.com>
Date: Wed, 25 Jun 2025 09:26:19 -0400
Subject: [PATCH 3/9] Extract into module; add bounds function

---
 .../src/trace/implementations/ord_neu.rs      | 203 +++++++++---------
 1 file changed, 104 insertions(+), 99 deletions(-)

diff --git a/differential-dataflow/src/trace/implementations/ord_neu.rs b/differential-dataflow/src/trace/implementations/ord_neu.rs
index 8c7a3c73d..9154e4449 100644
--- a/differential-dataflow/src/trace/implementations/ord_neu.rs
+++ b/differential-dataflow/src/trace/implementations/ord_neu.rs
@@ -10,10 +10,7 @@
 
 use std::rc::Rc;
 
-use serde::{Deserialize, Serialize};
-
 use crate::containers::TimelyStack;
-use crate::trace::implementations::BatchContainer;
 use crate::trace::implementations::chunker::{ColumnationChunker, VecChunker};
 use crate::trace::implementations::spine_fueled::Spine;
 use crate::trace::implementations::merge_batcher::{MergeBatcher, VecMerger, ColMerger};
@@ -68,71 +65,113 @@ pub type PreferredBuilder<K, V, T, R> = RcBuilder<OrdValBuilder<Preferred<K,V,T,
 // /// A trace implementation backed by columnar storage.
 // pub type ColKeySpine<K, T, R> = Spine<Rc<OrdKeyBatch<TStack<((K,()),T,R)>>>>;
 
+pub use layers::{Vals, Upds};
+/// Layers are containers of lists of some type.
+///
+/// The intent is that they "attach" to an outer layer which has as many values
+/// as the layer has lists, thereby associating a list with each outer value.
+/// A sequence of layers, each matching the number of values in its predecessor,
+/// forms a layered trie: a tree with values of some type on nodes at each depth.
+///
+/// We will form tries here by layering `[Keys, Vals, Upds]` or `[Keys, Upds]`.
+mod layers {
 
-/// A container for non-empty lists of values.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Vals<O, V> {
-    /// Offsets used to provide indexes from keys to values.
-    ///
-    /// The length of this list is one longer than `keys`, so that we can avoid bounds logic.
-    pub offs: O,
-    /// Concatenated ordered lists of values, bracketed by offsets in `offs`.
-    pub vals: V,
-}
+    use serde::{Deserialize, Serialize};
+    use crate::trace::implementations::BatchContainer;
 
-impl<O: BatchContainer, V: BatchContainer> Default for Vals<O, V> {
-    fn default() -> Self { Self::with_capacity(0, 0) }
-}
+    /// A container for non-empty lists of values.
+    #[derive(Debug, Serialize, Deserialize)]
+    pub struct Vals<O, V> {
+        /// Offsets used to provide indexes from keys to values.
+        ///
+        /// The length of this list is one longer than `keys`, so that we can avoid bounds logic.
+        pub offs: O,
+        /// Concatenated ordered lists of values, bracketed by offsets in `offs`.
+        pub vals: V,
+    }
+
+    impl<O: BatchContainer, V: BatchContainer> Default for Vals<O, V> {
+        fn default() -> Self { Self::with_capacity(0, 0) }
+    }
 
-impl<O: BatchContainer, V: BatchContainer> Vals<O, V> {
-    fn with_capacity(o_size: usize, v_size: usize) -> Self {
-        Self {
-            offs: <O as BatchContainer>::with_capacity(o_size),
-            vals: <V as BatchContainer>::with_capacity(v_size),
+    impl<O: for<'a> BatchContainer<ReadItem<'a> = usize>, V: BatchContainer> Vals<O, V> {
+        /// Lower and upper bounds in `self.vals` of the indexed list.
+        pub fn bounds(&self, index: usize) -> (usize, usize) {
+            (self.offs.index(index), self.offs.index(index+1))
         }
     }
-    fn merge_capacity(this: &Self, that: &Self) -> Self {
-        Self {
-            offs: <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len()),
-            vals: <V as BatchContainer>::merge_capacity(&this.vals, &that.vals),
+    impl<O: BatchContainer, V: BatchContainer> Vals<O, V> {
+        /// Number of lists in the container.
+        pub fn len(&self) -> usize { self.offs.len() - 1 }
+        /// Allocates with capacities for a number of lists and values.
+        pub fn with_capacity(o_size: usize, v_size: usize) -> Self {
+            Self {
+                offs: <O as BatchContainer>::with_capacity(o_size),
+                vals: <V as BatchContainer>::with_capacity(v_size),
+            }
+        }
+        /// Allocates with enough capacity to contain two inputs.
+        pub fn merge_capacity(this: &Self, that: &Self) -> Self {
+            Self {
+                offs: <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len()),
+                vals: <V as BatchContainer>::merge_capacity(&this.vals, &that.vals),
+            }
         }
     }
-}
-/// A container for non-empty lists of updates.
-///
-/// This container uses the special representiation of an empty slice to stand in for
-/// "the previous single element". An empty slice is an otherwise invalid representation.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Upds<O, T, D> {
-    /// Offsets used to provide indexes from values to updates.
-    pub offs: O,
-    /// Concatenated ordered lists of update times, bracketed by offsets in `offs`.
-    pub times: T,
-    /// Concatenated ordered lists of update diffs, bracketed by offsets in `offs`.
-    pub diffs: D,
-}
 
-impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Default for Upds<O, T, D> {
-    fn default() -> Self { Self::with_capacity(0, 0) }
-}
-impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Upds<O, T, D> {
-    fn with_capacity(o_size: usize, u_size: usize) -> Self {
-        Self {
-            offs: <O as BatchContainer>::with_capacity(o_size),
-            times: <T as BatchContainer>::with_capacity(u_size),
-            diffs: <D as BatchContainer>::with_capacity(u_size),
+    /// A container for non-empty lists of updates.
+    ///
+    /// This container uses the special representiation of an empty slice to stand in for
+    /// "the previous single element". An empty slice is an otherwise invalid representation.
+    #[derive(Debug, Serialize, Deserialize)]
+    pub struct Upds<O, T, D> {
+        /// Offsets used to provide indexes from values to updates.
+        pub offs: O,
+        /// Concatenated ordered lists of update times, bracketed by offsets in `offs`.
+        pub times: T,
+        /// Concatenated ordered lists of update diffs, bracketed by offsets in `offs`.
+        pub diffs: D,
+    }
+
+    impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Default for Upds<O, T, D> {
+        fn default() -> Self { Self::with_capacity(0, 0) }
+    }
+    impl<O: for<'a> BatchContainer<ReadItem<'a> = usize>, T: BatchContainer, D: BatchContainer> Upds<O, T, D> {
+        /// Lower and upper bounds in `self.vals` of the indexed list.
+        pub fn bounds(&self, index: usize) -> (usize, usize) {
+            let mut lower = self.offs.index(index);
+            let upper = self.offs.index(index+1);
+            // We use equal lower and upper to encode "singleton update; just before here".
+            // It should only apply when there is a prior element, so `lower` should be greater than zero.
+            if lower == upper {
+                assert!(lower > 0);
+                lower -= 1;
+            }
+            (lower, upper)
         }
     }
-    fn merge_capacity(this: &Self, that: &Self) -> Self {
-        Self {
-            offs:  <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len()),
-            times: <T as BatchContainer>::merge_capacity(&this.times, &that.times),
-            diffs: <D as BatchContainer>::merge_capacity(&this.diffs, &that.diffs),
+    impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Upds<O, T, D> {
+        /// Number of lists in the container.
+        pub fn len(&self) -> usize { self.offs.len() - 1 }
+        /// Allocates with capacities for a number of lists and values.
+        pub fn with_capacity(o_size: usize, u_size: usize) -> Self {
+            Self {
+                offs: <O as BatchContainer>::with_capacity(o_size),
+                times: <T as BatchContainer>::with_capacity(u_size),
+                diffs: <D as BatchContainer>::with_capacity(u_size),
+            }
+        }
+        /// Allocates with enough capacity to contain two inputs.
+        pub fn merge_capacity(this: &Self, that: &Self) -> Self {
+            Self {
+                offs:  <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len()),
+                times: <T as BatchContainer>::merge_capacity(&this.times, &that.times),
+                diffs: <D as BatchContainer>::merge_capacity(&this.diffs, &that.diffs),
+            }
         }
     }
 }
 
-
 /// Types related to forming batches with values.
 pub mod val_batch {
 
@@ -165,25 +204,6 @@ pub mod val_batch {
         pub upds: Upds<L::OffsetContainer, L::TimeContainer, L::DiffContainer>,
     }
 
-    impl<L: Layout> OrdValStorage<L> {
-        /// Lower and upper bounds in `self.vals` corresponding to the key at `index`.
-        fn values_for_key(&self, index: usize) -> (usize, usize) {
-            (self.vals.offs.index(index), self.vals.offs.index(index+1))
-        }
-        /// Lower and upper bounds in `self.updates` corresponding to the value at `index`.
-        fn updates_for_value(&self, index: usize) -> (usize, usize) {
-            let mut lower = self.upds.offs.index(index);
-            let upper = self.upds.offs.index(index+1);
-            // We use equal lower and upper to encode "singleton update; just before here".
-            // It should only apply when there is a prior element, so `lower` should be greater than zero.
-            if lower == upper {
-                assert!(lower > 0);
-                lower -= 1;
-            }
-            (lower, upper)
-        }
-    }
-
     /// An immutable collection of update tuples, from a contiguous interval of logical times.
     ///
     /// The `L` parameter captures how the updates should be laid out, and `C` determines which
@@ -362,7 +382,7 @@ pub mod val_batch {
         fn copy_key(&mut self, source: &OrdValStorage<L>, cursor: usize) {
             // Capture the initial number of values to determine if the merge was ultimately non-empty.
             let init_vals = self.result.vals.vals.len();
-            let (mut lower, upper) = source.values_for_key(cursor);
+            let (mut lower, upper) = source.vals.bounds(cursor);
             while lower < upper {
                 self.stash_updates_for_val(source, lower);
                 if let Some(off) = self.consolidate_updates() {
@@ -391,8 +411,8 @@ pub mod val_batch {
                 },
                 Ordering::Equal => {
                     // Keys are equal; must merge all values from both sources for this one key.
-                    let (lower1, upper1) = source1.values_for_key(self.key_cursor1);
-                    let (lower2, upper2) = source2.values_for_key(self.key_cursor2);
+                    let (lower1, upper1) = source1.vals.bounds(self.key_cursor1);
+                    let (lower2, upper2) = source2.vals.bounds(self.key_cursor2);
                     if let Some(off) = self.merge_vals((source1, lower1, upper1), (source2, lower2, upper2)) {
                         self.result.keys.push(source1.keys.index(self.key_cursor1));
                         self.result.vals.offs.push(off);
@@ -482,7 +502,7 @@ pub mod val_batch {
 
         /// Transfer updates for an indexed value in `source` into `self`, with compaction applied.
         fn stash_updates_for_val(&mut self, source: &OrdValStorage<L>, index: usize) {
-            let (lower, upper) = source.updates_for_value(index);
+            let (lower, upper) = source.upds.bounds(index);
             for i in lower .. upper {
                 // NB: Here is where we would need to look back if `lower == upper`.
                 let time = source.upds.times.index(i);
@@ -553,7 +573,7 @@ pub mod val_batch {
         fn key<'a>(&self, storage: &'a OrdValBatch<L>) -> Self::Key<'a> { storage.storage.keys.index(self.key_cursor) }
         fn val<'a>(&self, storage: &'a OrdValBatch<L>) -> Self::Val<'a> { storage.storage.vals.vals.index(self.val_cursor) }
         fn map_times<L2: FnMut(Self::TimeGat<'_>, Self::DiffGat<'_>)>(&mut self, storage: &OrdValBatch<L>, mut logic: L2) {
-            let (lower, upper) = storage.storage.updates_for_value(self.val_cursor);
+            let (lower, upper) = storage.storage.upds.bounds(self.val_cursor);
             for index in lower .. upper {
                 let time = storage.storage.upds.times.index(index);
                 let diff = storage.storage.upds.diffs.index(index);
@@ -561,7 +581,7 @@ pub mod val_batch {
             }
         }
         fn key_valid(&self, storage: &OrdValBatch<L>) -> bool { self.key_cursor < storage.storage.keys.len() }
-        fn val_valid(&self, storage: &OrdValBatch<L>) -> bool { self.val_cursor < storage.storage.values_for_key(self.key_cursor).1 }
+        fn val_valid(&self, storage: &OrdValBatch<L>) -> bool { self.val_cursor < storage.storage.vals.bounds(self.key_cursor).1 }
         fn step_key(&mut self, storage: &OrdValBatch<L>){
             self.key_cursor += 1;
             if self.key_valid(storage) {
@@ -580,11 +600,11 @@ pub mod val_batch {
         fn step_val(&mut self, storage: &OrdValBatch<L>) {
             self.val_cursor += 1; 
             if !self.val_valid(storage) {
-                self.val_cursor = storage.storage.values_for_key(self.key_cursor).1;
+                self.val_cursor = storage.storage.vals.bounds(self.key_cursor).1;
             }
         }
         fn seek_val(&mut self, storage: &OrdValBatch<L>, val: Self::Val<'_>) {
-            self.val_cursor += storage.storage.vals.vals.advance(self.val_cursor, storage.storage.values_for_key(self.key_cursor).1, |x| <L::ValContainer as BatchContainer>::reborrow(x).lt(&<L::ValContainer as BatchContainer>::reborrow(val)));
+            self.val_cursor += storage.storage.vals.vals.advance(self.val_cursor, storage.storage.vals.bounds(self.key_cursor).1, |x| <L::ValContainer as BatchContainer>::reborrow(x).lt(&<L::ValContainer as BatchContainer>::reborrow(val)));
         }
         fn rewind_keys(&mut self, storage: &OrdValBatch<L>) {
             self.key_cursor = 0;
@@ -593,7 +613,7 @@ pub mod val_batch {
             }
         }
         fn rewind_vals(&mut self, storage: &OrdValBatch<L>) {
-            self.val_cursor = storage.storage.values_for_key(self.key_cursor).0;
+            self.val_cursor = storage.storage.vals.bounds(self.key_cursor).0;
         }
     }
 
@@ -756,21 +776,6 @@ pub mod key_batch {
         pub upds: Upds<L::OffsetContainer, L::TimeContainer, L::DiffContainer>,
     }
 
-    impl<L: Layout> OrdKeyStorage<L> {
-        /// Lower and upper bounds in `self.vals` corresponding to the key at `index`.
-        fn updates_for_key(&self, index: usize) -> (usize, usize) {
-            let mut lower = self.upds.offs.index(index);
-            let upper = self.upds.offs.index(index+1);
-            // We use equal lower and upper to encode "singleton update; just before here".
-            // It should only apply when there is a prior element, so `lower` should be greater than zero.
-            if lower == upper {
-                assert!(lower > 0);
-                lower -= 1;
-            }
-            (lower, upper)
-        }
-    }
-
     /// An immutable collection of update tuples, from a contiguous interval of logical times.
     ///
     /// The `L` parameter captures how the updates should be laid out, and `C` determines which
@@ -980,7 +985,7 @@ pub mod key_batch {
 
         /// Transfer updates for an indexed value in `source` into `self`, with compaction applied.
         fn stash_updates_for_key(&mut self, source: &OrdKeyStorage<L>, index: usize) {
-            let (lower, upper) = source.updates_for_key(index);
+            let (lower, upper) = source.upds.bounds(index);
             for i in lower .. upper {
                 // NB: Here is where we would need to look back if `lower == upper`.
                 let time = source.upds.times.index(i);
@@ -1050,7 +1055,7 @@ pub mod key_batch {
         fn key<'a>(&self, storage: &'a Self::Storage) -> Self::Key<'a> { storage.storage.keys.index(self.key_cursor) }
         fn val<'a>(&self, _storage: &'a Self::Storage) -> &'a () { &() }
         fn map_times<L2: FnMut(Self::TimeGat<'_>, Self::DiffGat<'_>)>(&mut self, storage: &Self::Storage, mut logic: L2) {
-            let (lower, upper) = storage.storage.updates_for_key(self.key_cursor);
+            let (lower, upper) = storage.storage.upds.bounds(self.key_cursor);
             for index in lower .. upper {
                 let time = storage.storage.upds.times.index(index);
                 let diff = storage.storage.upds.diffs.index(index);

From 98b5de6b10ca885ae3c3f550b1d756bdecc1637c Mon Sep 17 00:00:00 2001
From: Frank McSherry <fmcsherry@me.com>
Date: Wed, 25 Jun 2025 10:21:29 -0400
Subject: [PATCH 4/9] Add indexing operators to layers

---
 .../src/trace/implementations/ord_neu.rs      | 54 +++++++++++++------
 1 file changed, 38 insertions(+), 16 deletions(-)

diff --git a/differential-dataflow/src/trace/implementations/ord_neu.rs b/differential-dataflow/src/trace/implementations/ord_neu.rs
index 9154e4449..469a4d7b9 100644
--- a/differential-dataflow/src/trace/implementations/ord_neu.rs
+++ b/differential-dataflow/src/trace/implementations/ord_neu.rs
@@ -99,10 +99,23 @@ mod layers {
         pub fn bounds(&self, index: usize) -> (usize, usize) {
             (self.offs.index(index), self.offs.index(index+1))
         }
+        /// Retrieves a value using relative indexes.
+        ///
+        /// The first index identifies a list, and the second an item within the list.
+        /// The method adds the list's lower bound to the item index, and then calls
+        /// `get_abs`. Using absolute indexes within the list's bounds can be more
+        /// efficient than using relative indexing.
+        pub fn get_rel(&self, list_idx: usize, item_idx: usize) -> V::ReadItem<'_> {
+            self.get_abs(self.bounds(list_idx).0 + item_idx)
+        }
     }
     impl<O: BatchContainer, V: BatchContainer> Vals<O, V> {
         /// Number of lists in the container.
         pub fn len(&self) -> usize { self.offs.len() - 1 }
+        /// Retrieves a value using an absolute rather than relative index.
+        pub fn get_abs(&self, index: usize) -> V::ReadItem<'_> {
+            self.vals.index(index)
+        }
         /// Allocates with capacities for a number of lists and values.
         pub fn with_capacity(o_size: usize, v_size: usize) -> Self {
             Self {
@@ -149,10 +162,23 @@ mod layers {
             }
             (lower, upper)
         }
+        /// Retrieves a value using relative indexes.
+        ///
+        /// The first index identifies a list, and the second an item within the list.
+        /// The method adds the list's lower bound to the item index, and then calls
+        /// `get_abs`. Using absolute indexes within the list's bounds can be more
+        /// efficient than using relative indexing.
+        pub fn get_rel(&self, list_idx: usize, item_idx: usize) -> (T::ReadItem<'_>, D::ReadItem<'_>) {
+            self.get_abs(self.bounds(list_idx).0 + item_idx)
+        }
     }
     impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Upds<O, T, D> {
         /// Number of lists in the container.
         pub fn len(&self) -> usize { self.offs.len() - 1 }
+        /// Retrieves a value using an absolute rather than relative index.
+        pub fn get_abs(&self, index: usize) -> (T::ReadItem<'_>, D::ReadItem<'_>) {
+            (self.times.index(index), self.diffs.index(index))
+        }
         /// Allocates with capacities for a number of lists and values.
         pub fn with_capacity(o_size: usize, u_size: usize) -> Self {
             Self {
@@ -387,7 +413,7 @@ pub mod val_batch {
                 self.stash_updates_for_val(source, lower);
                 if let Some(off) = self.consolidate_updates() {
                     self.result.upds.offs.push(off);
-                    self.result.vals.vals.push(source.vals.vals.index(lower));
+                    self.result.vals.vals.push(source.vals.get_abs(lower));
                 }
                 lower += 1;
             }            
@@ -443,13 +469,13 @@ pub mod val_batch {
                 // if they are non-empty post-consolidation, we write the value.
                 // We could multi-way merge and it wouldn't be very complicated.
                 use ::std::cmp::Ordering;
-                match source1.vals.vals.index(lower1).cmp(&source2.vals.vals.index(lower2)) {
+                match source1.vals.get_abs(lower1).cmp(&source2.vals.get_abs(lower2)) {
                     Ordering::Less => { 
                         // Extend stash by updates, with logical compaction applied.
                         self.stash_updates_for_val(source1, lower1);
                         if let Some(off) = self.consolidate_updates() {
                             self.result.upds.offs.push(off);
-                            self.result.vals.vals.push(source1.vals.vals.index(lower1));
+                            self.result.vals.vals.push(source1.vals.get_abs(lower1));
                         }
                         lower1 += 1;
                     },
@@ -458,7 +484,7 @@ pub mod val_batch {
                         self.stash_updates_for_val(source2, lower2);
                         if let Some(off) = self.consolidate_updates() {
                             self.result.upds.offs.push(off);
-                            self.result.vals.vals.push(source1.vals.vals.index(lower1));
+                            self.result.vals.vals.push(source1.vals.get_abs(lower1));
                         }
                         lower1 += 1;
                         lower2 += 1;
@@ -468,7 +494,7 @@ pub mod val_batch {
                         self.stash_updates_for_val(source2, lower2);
                         if let Some(off) = self.consolidate_updates() {
                             self.result.upds.offs.push(off);
-                            self.result.vals.vals.push(source2.vals.vals.index(lower2));
+                            self.result.vals.vals.push(source2.vals.get_abs(lower2));
                         }
                         lower2 += 1;
                     },
@@ -479,7 +505,7 @@ pub mod val_batch {
                 self.stash_updates_for_val(source1, lower1);
                 if let Some(off) = self.consolidate_updates() {
                     self.result.upds.offs.push(off);
-                    self.result.vals.vals.push(source1.vals.vals.index(lower1));
+                    self.result.vals.vals.push(source1.vals.get_abs(lower1));
                 }
                 lower1 += 1;
             }
@@ -487,7 +513,7 @@ pub mod val_batch {
                 self.stash_updates_for_val(source2, lower2);
                 if let Some(off) = self.consolidate_updates() {
                     self.result.upds.offs.push(off);
-                    self.result.vals.vals.push(source2.vals.vals.index(lower2));
+                    self.result.vals.vals.push(source2.vals.get_abs(lower2));
                 }
                 lower2 += 1;
             }
@@ -505,8 +531,7 @@ pub mod val_batch {
             let (lower, upper) = source.upds.bounds(index);
             for i in lower .. upper {
                 // NB: Here is where we would need to look back if `lower == upper`.
-                let time = source.upds.times.index(i);
-                let diff = source.upds.diffs.index(i);
+                let (time, diff) = source.upds.get_abs(i);
                 use crate::lattice::Lattice;
                 let mut new_time: <L::Target as Update>::Time = time.into_owned();
                 new_time.advance_by(self.description.since().borrow());
@@ -571,12 +596,11 @@ pub mod val_batch {
         fn get_val<'a>(&self, storage: &'a Self::Storage) -> Option<Self::Val<'a>> { if self.val_valid(storage) { Some(self.val(storage)) } else { None } }
 
         fn key<'a>(&self, storage: &'a OrdValBatch<L>) -> Self::Key<'a> { storage.storage.keys.index(self.key_cursor) }
-        fn val<'a>(&self, storage: &'a OrdValBatch<L>) -> Self::Val<'a> { storage.storage.vals.vals.index(self.val_cursor) }
+        fn val<'a>(&self, storage: &'a OrdValBatch<L>) -> Self::Val<'a> { storage.storage.vals.get_abs(self.val_cursor) }
         fn map_times<L2: FnMut(Self::TimeGat<'_>, Self::DiffGat<'_>)>(&mut self, storage: &OrdValBatch<L>, mut logic: L2) {
             let (lower, upper) = storage.storage.upds.bounds(self.val_cursor);
             for index in lower .. upper {
-                let time = storage.storage.upds.times.index(index);
-                let diff = storage.storage.upds.diffs.index(index);
+                let (time, diff) = storage.storage.upds.get_abs(index);
                 logic(time, diff);
             }
         }
@@ -988,8 +1012,7 @@ pub mod key_batch {
             let (lower, upper) = source.upds.bounds(index);
             for i in lower .. upper {
                 // NB: Here is where we would need to look back if `lower == upper`.
-                let time = source.upds.times.index(i);
-                let diff = source.upds.diffs.index(i);
+                let (time, diff) = source.upds.get_abs(i);
                 use crate::lattice::Lattice;
                 let mut new_time = time.into_owned();
                 new_time.advance_by(self.description.since().borrow());
@@ -1057,8 +1080,7 @@ pub mod key_batch {
         fn map_times<L2: FnMut(Self::TimeGat<'_>, Self::DiffGat<'_>)>(&mut self, storage: &Self::Storage, mut logic: L2) {
             let (lower, upper) = storage.storage.upds.bounds(self.key_cursor);
             for index in lower .. upper {
-                let time = storage.storage.upds.times.index(index);
-                let diff = storage.storage.upds.diffs.index(index);
+                let (time, diff) = storage.storage.upds.get_abs(index);
                 logic(time, diff);
             }
         }

From ce9c8db27aa2c393626f62411c044948d129d7f7 Mon Sep 17 00:00:00 2001
From: Frank McSherry <fmcsherry@me.com>
Date: Wed, 25 Jun 2025 16:55:39 -0400
Subject: [PATCH 5/9] Move singleton update into layer

---
 .../src/trace/implementations/ord_neu.rs      | 385 ++++++++----------
 1 file changed, 161 insertions(+), 224 deletions(-)

diff --git a/differential-dataflow/src/trace/implementations/ord_neu.rs b/differential-dataflow/src/trace/implementations/ord_neu.rs
index 469a4d7b9..53969635a 100644
--- a/differential-dataflow/src/trace/implementations/ord_neu.rs
+++ b/differential-dataflow/src/trace/implementations/ord_neu.rs
@@ -144,6 +144,11 @@ mod layers {
         pub times: T,
         /// Concatenated ordered lists of update diffs, bracketed by offsets in `offs`.
         pub diffs: D,
+        /// Tracks the total number of updates contained within.
+        ///
+        /// This may be more than `times.len()` or `diffs.len()` because we deduplicate runs
+        /// of the same single `(time, diff)` pair.
+        total: usize,
     }
 
     impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Default for Upds<O, T, D> {
@@ -185,6 +190,7 @@ mod layers {
                 offs: <O as BatchContainer>::with_capacity(o_size),
                 times: <T as BatchContainer>::with_capacity(u_size),
                 diffs: <D as BatchContainer>::with_capacity(u_size),
+                total: 0,
             }
         }
         /// Allocates with enough capacity to contain two inputs.
@@ -193,9 +199,79 @@ mod layers {
                 offs:  <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len()),
                 times: <T as BatchContainer>::merge_capacity(&this.times, &that.times),
                 diffs: <D as BatchContainer>::merge_capacity(&this.diffs, &that.diffs),
+                total: 0,
             }
         }
     }
+
+    pub struct UpdsBuilder<T: BatchContainer, D: BatchContainer> {
+        /// Local stash of updates, to use for consolidation.
+        ///
+        /// We could emulate a `ChangeBatch` here, with related compaction smarts.
+        /// A `ChangeBatch` itself needs an `i64` diff type, which we have not.
+        stash: Vec<(T::Owned, D::Owned)>,
+        /// Total number of consolidated updates.
+        ///
+        /// Tracked independently to account for duplicate compression.
+        total: usize,
+    }
+
+    impl<T: BatchContainer, D: BatchContainer> Default for UpdsBuilder<T, D> {
+        fn default() -> Self { Self { stash: Vec::default(), total: 0, } }
+    }
+
+
+    use timely::container::PushInto;
+    impl<T, D> UpdsBuilder<T, D>
+    where
+        T: BatchContainer<Owned: Ord> + PushInto<T::Owned>,
+        D: BatchContainer<Owned: crate::difference::Semigroup> + PushInto<D::Owned>,
+    {
+        /// Stages one update, but does not seal the set of updates.
+        pub fn push(&mut self, time: T::Owned, diff: D::Owned) {
+            self.stash.push((time, diff));
+        }
+
+        /// Consolidate and insert (if non-empty) the stashed updates.
+        ///
+        /// The return indicates whether the results were indeed non-empty.
+        pub fn seal<O: for<'a> BatchContainer<ReadItem<'a> = usize>>(&mut self, upds: &mut Upds<O, T, D>) -> bool {
+            use crate::consolidation;
+            consolidation::consolidate(&mut self.stash);
+            if !self.stash.is_empty() {
+                // If there is a single element, equal to a just-prior recorded update,
+                // we push nothing and report an unincremented offset to encode this case.
+                let time_diff = upds.times.last().zip(upds.diffs.last());
+                let last_eq = self.stash.last().zip(time_diff).map(|((t1, d1), (t2, d2))| {
+                    use crate::IntoOwned;
+                    let t1 = <<T as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(t1);
+                    let d1 = <<D as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(d1);
+                    t1.eq(&t2) && d1.eq(&d2)
+                });
+                if self.stash.len() == 1 && last_eq.unwrap_or(false) {
+                    // Just clear out the stash, as we won't drain it here.
+                    self.total += 1;
+                    self.stash.clear();
+                    upds.offs.push(upds.times.len());
+                }
+                else {
+                    // Conventional; move `stash` into `updates`.
+                    self.total += self.stash.len();
+                    for (time, diff) in self.stash.drain(..) {
+                        upds.times.push(time);
+                        upds.diffs.push(diff);
+                    }
+                    upds.offs.push(upds.times.len());
+                }
+                true
+            } else {
+                false
+            }
+        }
+
+        /// Completes the building and returns the total updates sealed.
+        pub fn total(&self) -> usize { self.total }
+    }
 }
 
 /// Types related to forming batches with values.
@@ -210,7 +286,7 @@ pub mod val_batch {
     use crate::trace::implementations::{BatchContainer, BuilderInput};
     use crate::IntoOwned;
 
-    use super::{Layout, Update, Vals, Upds};
+    use super::{Layout, Update, Vals, Upds, layers::UpdsBuilder};
 
     /// An immutable collection of update tuples, from a contiguous interval of logical times.
     #[derive(Debug, Serialize, Deserialize)]
@@ -310,14 +386,8 @@ pub mod val_batch {
         result: OrdValStorage<L>,
         /// description
         description: Description<<L::Target as Update>::Time>,
-
-        /// Local stash of updates, to use for consolidation.
-        ///
-        /// We could emulate a `ChangeBatch` here, with related compaction smarts.
-        /// A `ChangeBatch` itself needs an `i64` diff type, which we have not.
-        update_stash: Vec<(<L::Target as Update>::Time, <L::Target as Update>::Diff)>,
-        /// Counts the number of singleton-optimized entries, that we may correctly count the updates.
-        singletons: usize,
+        /// Staging area to consolidate owned times and diffs, before sealing.
+        staging: UpdsBuilder<L::TimeContainer, L::DiffContainer>,
     }
 
     impl<L: Layout> Merger<OrdValBatch<L>> for OrdValMerger<L>
@@ -355,13 +425,12 @@ pub mod val_batch {
                 key_cursor2: 0,
                 result: storage,
                 description,
-                update_stash: Vec::new(),
-                singletons: 0,
+                staging: UpdsBuilder::default(),
             }
         }
         fn done(self) -> OrdValBatch<L> {
             OrdValBatch {
-                updates: self.result.upds.times.len() + self.singletons,
+                updates: self.staging.total(),
                 storage: self.result,
                 description: self.description,
             }
@@ -369,14 +438,14 @@ pub mod val_batch {
         fn work(&mut self, source1: &OrdValBatch<L>, source2: &OrdValBatch<L>, fuel: &mut isize) {
 
             // An (incomplete) indication of the amount of work we've done so far.
-            let starting_updates = self.result.upds.times.len() + self.singletons;
+            let starting_updates = self.staging.total();
             let mut effort = 0isize;
 
             // While both mergees are still active, perform single-key merges.
             while self.key_cursor1 < source1.storage.keys.len() && self.key_cursor2 < source2.storage.keys.len() && effort < *fuel {
                 self.merge_key(&source1.storage, &source2.storage);
                 // An (incomplete) accounting of the work we've done.
-                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.staging.total() - starting_updates) as isize;
             }
 
             // Merging is complete, and only copying remains. 
@@ -384,12 +453,12 @@ pub mod val_batch {
             while self.key_cursor1 < source1.storage.keys.len() && effort < *fuel {
                 self.copy_key(&source1.storage, self.key_cursor1);
                 self.key_cursor1 += 1;
-                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.staging.total() - starting_updates) as isize;
             }
             while self.key_cursor2 < source2.storage.keys.len() && effort < *fuel {
                 self.copy_key(&source2.storage, self.key_cursor2);
                 self.key_cursor2 += 1;
-                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.staging.total() - starting_updates) as isize;
             }
 
             *fuel -= effort;
@@ -411,8 +480,7 @@ pub mod val_batch {
             let (mut lower, upper) = source.vals.bounds(cursor);
             while lower < upper {
                 self.stash_updates_for_val(source, lower);
-                if let Some(off) = self.consolidate_updates() {
-                    self.result.upds.offs.push(off);
+                if self.staging.seal(&mut self.result.upds) {
                     self.result.vals.vals.push(source.vals.get_abs(lower));
                 }
                 lower += 1;
@@ -473,8 +541,7 @@ pub mod val_batch {
                     Ordering::Less => { 
                         // Extend stash by updates, with logical compaction applied.
                         self.stash_updates_for_val(source1, lower1);
-                        if let Some(off) = self.consolidate_updates() {
-                            self.result.upds.offs.push(off);
+                        if self.staging.seal(&mut self.result.upds) {
                             self.result.vals.vals.push(source1.vals.get_abs(lower1));
                         }
                         lower1 += 1;
@@ -482,8 +549,7 @@ pub mod val_batch {
                     Ordering::Equal => {
                         self.stash_updates_for_val(source1, lower1);
                         self.stash_updates_for_val(source2, lower2);
-                        if let Some(off) = self.consolidate_updates() {
-                            self.result.upds.offs.push(off);
+                        if self.staging.seal(&mut self.result.upds) {
                             self.result.vals.vals.push(source1.vals.get_abs(lower1));
                         }
                         lower1 += 1;
@@ -492,8 +558,7 @@ pub mod val_batch {
                     Ordering::Greater => { 
                         // Extend stash by updates, with logical compaction applied.
                         self.stash_updates_for_val(source2, lower2);
-                        if let Some(off) = self.consolidate_updates() {
-                            self.result.upds.offs.push(off);
+                        if self.staging.seal(&mut self.result.upds) {
                             self.result.vals.vals.push(source2.vals.get_abs(lower2));
                         }
                         lower2 += 1;
@@ -503,16 +568,14 @@ pub mod val_batch {
             // Merging is complete, but we may have remaining elements to push.
             while lower1 < upper1 {
                 self.stash_updates_for_val(source1, lower1);
-                if let Some(off) = self.consolidate_updates() {
-                    self.result.upds.offs.push(off);
+                if self.staging.seal(&mut self.result.upds) {
                     self.result.vals.vals.push(source1.vals.get_abs(lower1));
                 }
                 lower1 += 1;
             }
             while lower2 < upper2 {
                 self.stash_updates_for_val(source2, lower2);
-                if let Some(off) = self.consolidate_updates() {
-                    self.result.upds.offs.push(off);
+                if self.staging.seal(&mut self.result.upds) {
                     self.result.vals.vals.push(source2.vals.get_abs(lower2));
                 }
                 lower2 += 1;
@@ -535,38 +598,8 @@ pub mod val_batch {
                 use crate::lattice::Lattice;
                 let mut new_time: <L::Target as Update>::Time = time.into_owned();
                 new_time.advance_by(self.description.since().borrow());
-                self.update_stash.push((new_time, diff.into_owned()));
-            }
-        }
-
-        /// Consolidates `self.updates_stash` and produces the offset to record, if any.
-        fn consolidate_updates(&mut self) -> Option<usize> {
-            use crate::consolidation;
-            consolidation::consolidate(&mut self.update_stash);
-            if !self.update_stash.is_empty() {
-                // If there is a single element, equal to a just-prior recorded update,
-                // we push nothing and report an unincremented offset to encode this case.
-                let time_diff = self.result.upds.times.last().zip(self.result.upds.diffs.last());
-                let last_eq = self.update_stash.last().zip(time_diff).map(|((t1, d1), (t2, d2))| {
-                    let t1 = <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(t1);
-                    let d1 = <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(d1);
-                    t1.eq(&t2) && d1.eq(&d2)
-                });
-                if self.update_stash.len() == 1 && last_eq.unwrap_or(false) {
-                    // Just clear out update_stash, as we won't drain it here.
-                    self.update_stash.clear();
-                    self.singletons += 1;
-                }
-                else {
-                    // Conventional; move `update_stash` into `updates`.
-                    for (time, diff) in self.update_stash.drain(..) {
-                        self.result.upds.times.push(time);
-                        self.result.upds.diffs.push(diff);
-                    }
-                }
-                Some(self.result.upds.times.len())
-            } else {
-                None
+                self.staging.push(new_time, diff.into_owned());
+                // self.update_stash.push((new_time, diff.into_owned()));
             }
         }
     }
@@ -647,47 +680,10 @@ pub mod val_batch {
         ///
         /// This is public to allow container implementors to set and inspect their container.
         pub result: OrdValStorage<L>,
-        singleton: Option<(<L::Target as Update>::Time, <L::Target as Update>::Diff)>,
-        /// Counts the number of singleton optimizations we performed.
-        ///
-        /// This number allows us to correctly gauge the total number of updates reflected in a batch,
-        /// even though `updates.len()` may be much shorter than this amount.
-        singletons: usize,
+        staging: UpdsBuilder<L::TimeContainer, L::DiffContainer>,
         _marker: PhantomData<CI>,
     }
 
-    impl<L: Layout, CI> OrdValBuilder<L, CI> {
-        /// Pushes a single update, which may set `self.singleton` rather than push.
-        ///
-        /// This operation is meant to be equivalent to `self.results.updates.push((time, diff))`.
-        /// However, for "clever" reasons it does not do this. Instead, it looks for opportunities
-        /// to encode a singleton update with an "absert" update: repeating the most recent offset.
-        /// This otherwise invalid state encodes "look back one element".
-        ///
-        /// When `self.singleton` is `Some`, it means that we have seen one update and it matched the
-        /// previously pushed update exactly. In that case, we do not push the update into `updates`.
-        /// The update tuple is retained in `self.singleton` in case we see another update and need
-        /// to recover the singleton to push it into `updates` to join the second update.
-        fn push_update(&mut self, time: <L::Target as Update>::Time, diff: <L::Target as Update>::Diff) {
-            // If a just-pushed update exactly equals `(time, diff)` we can avoid pushing it.
-            if self.result.upds.times.last().map(|t| t == <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time)) == Some(true) &&
-               self.result.upds.diffs.last().map(|d| d == <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff)) == Some(true)
-            {
-                assert!(self.singleton.is_none());
-                self.singleton = Some((time, diff));
-            }
-            else {
-                // If we have pushed a single element, we need to copy it out to meet this one.
-                if let Some((time, diff)) = self.singleton.take() {
-                    self.result.upds.times.push(time);
-                    self.result.upds.diffs.push(diff);
-                }
-                self.result.upds.times.push(time);
-                self.result.upds.diffs.push(diff);
-            }
-        }
-    }
-
     impl<L, CI> Builder for OrdValBuilder<L, CI>
     where
         L: for<'a> Layout<
@@ -704,15 +700,22 @@ pub mod val_batch {
         type Output = OrdValBatch<L>;
 
         fn with_capacity(keys: usize, vals: usize, upds: usize) -> Self {
-            // We don't introduce zero offsets as they will be introduced by the first `push` call.
+
+            let mut result = OrdValStorage {
+                keys: L::KeyContainer::with_capacity(keys),
+                vals: Vals::with_capacity(keys + 1, vals),
+                upds: Upds::with_capacity(vals + 1, upds),
+            };
+
+            // Mark explicit types because type inference fails to resolve it.
+            let keys_offs: &mut L::OffsetContainer = &mut result.vals.offs;
+            keys_offs.push(0);
+            let vals_offs: &mut L::OffsetContainer = &mut result.upds.offs;
+            vals_offs.push(0);
+
             Self { 
-                result: OrdValStorage {
-                    keys: L::KeyContainer::with_capacity(keys),
-                    vals: Vals::with_capacity(keys + 1, vals),
-                    upds: Upds::with_capacity(vals + 1, upds),
-                },
-                singleton: None,
-                singletons: 0,
+                result,
+                staging: UpdsBuilder::default(),
                 _marker: PhantomData,
             }
         }
@@ -721,24 +724,30 @@ pub mod val_batch {
         fn push(&mut self, chunk: &mut Self::Input) {
             for item in chunk.drain() {
                 let (key, val, time, diff) = CI::into_parts(item);
+
+                // Pre-load the first update.
+                if self.result.keys.is_empty() {
+                    self.result.vals.vals.push(val);
+                    self.result.keys.push(key);
+                    self.staging.push(time, diff);
+                }
                 // Perhaps this is a continuation of an already received key.
-                if self.result.keys.last().map(|k| CI::key_eq(&key, k)).unwrap_or(false) {
+                else if self.result.keys.last().map(|k| CI::key_eq(&key, k)).unwrap_or(false) {
                     // Perhaps this is a continuation of an already received value.
                     if self.result.vals.vals.last().map(|v| CI::val_eq(&val, v)).unwrap_or(false) {
-                        self.push_update(time, diff);
+                        // self.push_update(time, diff);
+                        self.staging.push(time, diff);
                     } else {
                         // New value; complete representation of prior value.
-                        self.result.upds.offs.push(self.result.upds.times.len());
-                        if self.singleton.take().is_some() { self.singletons += 1; }
-                        self.push_update(time, diff);
+                        self.staging.seal(&mut self.result.upds);
+                        self.staging.push(time, diff);
                         self.result.vals.vals.push(val);
                     }
                 } else {
                     // New key; complete representation of prior key.
-                    self.result.upds.offs.push(self.result.upds.times.len());
-                    if self.singleton.take().is_some() { self.singletons += 1; }
+                    self.staging.seal(&mut self.result.upds);
+                    self.staging.push(time, diff);
                     self.result.vals.offs.push(self.result.vals.vals.len());
-                    self.push_update(time, diff);
                     self.result.vals.vals.push(val);
                     self.result.keys.push(key);
                 }
@@ -748,12 +757,13 @@ pub mod val_batch {
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdValBatch<L> {
             // Record the final offsets
-            self.result.upds.offs.push(self.result.upds.times.len());
-            // Remove any pending singleton, and if it was set increment our count.
-            if self.singleton.take().is_some() { self.singletons += 1; }
+            self.staging.seal(&mut self.result.upds);
+            // self.result.upds.offs.push(self.result.upds.times.len());
+            // // Remove any pending singleton, and if it was set increment our count.
+            // if self.singleton.take().is_some() { self.singletons += 1; }
             self.result.vals.offs.push(self.result.vals.vals.len());
             OrdValBatch {
-                updates: self.result.upds.times.len() + self.singletons,
+                updates: self.staging.total(),
                 storage: self.result,
                 description,
             }
@@ -783,7 +793,7 @@ pub mod key_batch {
     use crate::trace::implementations::{BatchContainer, BuilderInput};
     use crate::IntoOwned;
 
-    use super::{Layout, Update, Upds};
+    use super::{Layout, Update, Upds, layers::UpdsBuilder};
 
     /// An immutable collection of update tuples, from a contiguous interval of logical times.
     #[derive(Debug, Serialize, Deserialize)]
@@ -881,12 +891,7 @@ pub mod key_batch {
         description: Description<<L::Target as Update>::Time>,
 
         /// Local stash of updates, to use for consolidation.
-        ///
-        /// We could emulate a `ChangeBatch` here, with related compaction smarts.
-        /// A `ChangeBatch` itself needs an `i64` diff type, which we have not.
-        update_stash: Vec<(<L::Target as Update>::Time, <L::Target as Update>::Diff)>,
-        /// Counts the number of singleton-optimized entries, that we may correctly count the updates.
-        singletons: usize,
+        staging: UpdsBuilder<L::TimeContainer, L::DiffContainer>,
     }
 
     impl<L: Layout> Merger<OrdKeyBatch<L>> for OrdKeyMerger<L>
@@ -920,13 +925,12 @@ pub mod key_batch {
                 key_cursor2: 0,
                 result: storage,
                 description,
-                update_stash: Vec::new(),
-                singletons: 0,
+                staging: UpdsBuilder::default(),
             }
         }
         fn done(self) -> OrdKeyBatch<L> {
             OrdKeyBatch {
-                updates: self.result.upds.times.len() + self.singletons,
+                updates: self.staging.total(),
                 storage: self.result,
                 description: self.description,
             }
@@ -934,14 +938,14 @@ pub mod key_batch {
         fn work(&mut self, source1: &OrdKeyBatch<L>, source2: &OrdKeyBatch<L>, fuel: &mut isize) {
 
             // An (incomplete) indication of the amount of work we've done so far.
-            let starting_updates = self.result.upds.times.len() + self.singletons;
+            let starting_updates = self.staging.total();
             let mut effort = 0isize;
 
             // While both mergees are still active, perform single-key merges.
             while self.key_cursor1 < source1.storage.keys.len() && self.key_cursor2 < source2.storage.keys.len() && effort < *fuel {
                 self.merge_key(&source1.storage, &source2.storage);
                 // An (incomplete) accounting of the work we've done.
-                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.staging.total() - starting_updates) as isize;
             }
 
             // Merging is complete, and only copying remains.
@@ -949,12 +953,12 @@ pub mod key_batch {
             while self.key_cursor1 < source1.storage.keys.len() && effort < *fuel {
                 self.copy_key(&source1.storage, self.key_cursor1);
                 self.key_cursor1 += 1;
-                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.staging.total() - starting_updates) as isize;
             }
             while self.key_cursor2 < source2.storage.keys.len() && effort < *fuel {
                 self.copy_key(&source2.storage, self.key_cursor2);
                 self.key_cursor2 += 1;
-                effort = (self.result.upds.times.len() + self.singletons - starting_updates) as isize;
+                effort = (self.staging.total() - starting_updates) as isize;
             }
 
             *fuel -= effort;
@@ -972,8 +976,7 @@ pub mod key_batch {
         /// The caller should be certain to update the cursor, as this method does not do this.
         fn copy_key(&mut self, source: &OrdKeyStorage<L>, cursor: usize) {
             self.stash_updates_for_key(source, cursor);
-            if let Some(off) = self.consolidate_updates() {
-                self.result.upds.offs.push(off);
+            if self.staging.seal(&mut self.result.upds) {
                 self.result.keys.push(source.keys.index(cursor));
             }
         }
@@ -992,8 +995,7 @@ pub mod key_batch {
                     // Keys are equal; must merge all updates from both sources for this one key.
                     self.stash_updates_for_key(source1, self.key_cursor1);
                     self.stash_updates_for_key(source2, self.key_cursor2);
-                    if let Some(off) = self.consolidate_updates() {
-                        self.result.upds.offs.push(off);
+                    if self.staging.seal(&mut self.result.upds) {
                         self.result.keys.push(source1.keys.index(self.key_cursor1));
                     }
                     // Increment cursors in either case; the keys are merged.
@@ -1016,38 +1018,7 @@ pub mod key_batch {
                 use crate::lattice::Lattice;
                 let mut new_time = time.into_owned();
                 new_time.advance_by(self.description.since().borrow());
-                self.update_stash.push((new_time, diff.into_owned()));
-            }
-        }
-
-        /// Consolidates `self.updates_stash` and produces the offset to record, if any.
-        fn consolidate_updates(&mut self) -> Option<usize> {
-            use crate::consolidation;
-            consolidation::consolidate(&mut self.update_stash);
-            if !self.update_stash.is_empty() {
-                // If there is a single element, equal to a just-prior recorded update,
-                // we push nothing and report an unincremented offset to encode this case.
-                let time_diff = self.result.upds.times.last().zip(self.result.upds.diffs.last());
-                let last_eq = self.update_stash.last().zip(time_diff).map(|((t1, d1), (t2, d2))| {
-                    let t1 = <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(t1);
-                    let d1 = <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(d1);
-                    t1.eq(&t2) && d1.eq(&d2)
-                });
-                if self.update_stash.len() == 1 && last_eq.unwrap_or(false) {
-                    // Just clear out update_stash, as we won't drain it here.
-                    self.update_stash.clear();
-                    self.singletons += 1;
-                }
-                else {
-                    // Conventional; move `update_stash` into `updates`.
-                    for (time, diff) in self.update_stash.drain(..) {
-                        self.result.upds.times.push(time);
-                        self.result.upds.diffs.push(diff);
-                    }
-                }
-                Some(self.result.upds.times.len())
-            } else {
-                None
+                self.staging.push(new_time, diff.into_owned());
             }
         }
     }
@@ -1122,47 +1093,10 @@ pub mod key_batch {
         ///
         /// This is public to allow container implementors to set and inspect their container.
         pub result: OrdKeyStorage<L>,
-        singleton: Option<(<L::Target as Update>::Time, <L::Target as Update>::Diff)>,
-        /// Counts the number of singleton optimizations we performed.
-        ///
-        /// This number allows us to correctly gauge the total number of updates reflected in a batch,
-        /// even though `updates.len()` may be much shorter than this amount.
-        singletons: usize,
+        staging: UpdsBuilder<L::TimeContainer, L::DiffContainer>,
         _marker: PhantomData<CI>,
     }
 
-    impl<L: Layout, CI> OrdKeyBuilder<L, CI> {
-        /// Pushes a single update, which may set `self.singleton` rather than push.
-        ///
-        /// This operation is meant to be equivalent to `self.results.updates.push((time, diff))`.
-        /// However, for "clever" reasons it does not do this. Instead, it looks for opportunities
-        /// to encode a singleton update with an "absert" update: repeating the most recent offset.
-        /// This otherwise invalid state encodes "look back one element".
-        ///
-        /// When `self.singleton` is `Some`, it means that we have seen one update and it matched the
-        /// previously pushed update exactly. In that case, we do not push the update into `updates`.
-        /// The update tuple is retained in `self.singleton` in case we see another update and need
-        /// to recover the singleton to push it into `updates` to join the second update.
-        fn push_update(&mut self, time: <L::Target as Update>::Time, diff: <L::Target as Update>::Diff) {
-            // If a just-pushed update exactly equals `(time, diff)` we can avoid pushing it.
-            let t1 = <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time);
-            let d1 = <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff);
-            if self.result.upds.times.last().map(|t| t == t1).unwrap_or(false) && self.result.upds.diffs.last().map(|d| d == d1).unwrap_or(false) {
-                assert!(self.singleton.is_none());
-                self.singleton = Some((time, diff));
-            }
-            else {
-                // If we have pushed a single element, we need to copy it out to meet this one.
-                if let Some((time, diff)) = self.singleton.take() {
-                    self.result.upds.times.push(time);
-                    self.result.upds.diffs.push(diff);
-                }
-                self.result.upds.times.push(time);
-                self.result.upds.diffs.push(diff);
-            }
-        }
-    }
-
     impl<L: Layout, CI> Builder for OrdKeyBuilder<L, CI>
     where
         L: for<'a> Layout<KeyContainer: PushInto<CI::Key<'a>>>,
@@ -1176,14 +1110,18 @@ pub mod key_batch {
         type Output = OrdKeyBatch<L>;
 
         fn with_capacity(keys: usize, _vals: usize, upds: usize) -> Self {
-            // We don't introduce zero offsets as they will be introduced by the first `push` call.
+
+            let mut result = OrdKeyStorage {
+                keys: L::KeyContainer::with_capacity(keys),
+                upds: Upds::with_capacity(keys+1, upds),
+            };
+
+            let keys_offs: &mut L::OffsetContainer = &mut result.upds.offs;
+            keys_offs.push(0);
+
             Self { 
-                result: OrdKeyStorage {
-                    keys: L::KeyContainer::with_capacity(keys),
-                    upds: Upds::with_capacity(keys+1, upds),
-                },
-                singleton: None,
-                singletons: 0,
+                result,
+                staging: UpdsBuilder::default(),
                 _marker: PhantomData,
             }
         }
@@ -1192,15 +1130,16 @@ pub mod key_batch {
         fn push(&mut self, chunk: &mut Self::Input) {
             for item in chunk.drain() {
                 let (key, _val, time, diff) = CI::into_parts(item);
+                if self.result.keys.is_empty() {
+                    self.result.keys.push(key);
+                    self.staging.push(time, diff);
+                }
                 // Perhaps this is a continuation of an already received key.
-                if self.result.keys.last().map(|k| CI::key_eq(&key, k)).unwrap_or(false) {
-                    self.push_update(time, diff);
+                else if self.result.keys.last().map(|k| CI::key_eq(&key, k)).unwrap_or(false) {
+                    self.staging.push(time, diff);
                 } else {
-                    // New key; complete representation of prior key.
-                    self.result.upds.offs.push(self.result.upds.times.len());
-                    // Remove any pending singleton, and if it was set increment our count.
-                    if self.singleton.take().is_some() { self.singletons += 1; }
-                    self.push_update(time, diff);
+                    self.staging.seal(&mut self.result.upds);
+                    self.staging.push(time, diff);
                     self.result.keys.push(key);
                 }
             }
@@ -1209,11 +1148,9 @@ pub mod key_batch {
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdKeyBatch<L> {
             // Record the final offsets
-            self.result.upds.offs.push(self.result.upds.times.len());
-            // Remove any pending singleton, and if it was set increment our count.
-            if self.singleton.take().is_some() { self.singletons += 1; }
+            self.staging.seal(&mut self.result.upds);
             OrdKeyBatch {
-                updates: self.result.upds.times.len() + self.singletons,
+                updates: self.staging.total(),
                 storage: self.result,
                 description,
             }

From bdc34d998090a7207b0f063c51b276cf55138871 Mon Sep 17 00:00:00 2001
From: Frank McSherry <fmcsherry@me.com>
Date: Wed, 25 Jun 2025 17:07:53 -0400
Subject: [PATCH 6/9] Move zero push into containers

---
 .../src/trace/implementations/ord_neu.rs      | 128 ++++++++----------
 1 file changed, 54 insertions(+), 74 deletions(-)

diff --git a/differential-dataflow/src/trace/implementations/ord_neu.rs b/differential-dataflow/src/trace/implementations/ord_neu.rs
index 53969635a..94bbd0764 100644
--- a/differential-dataflow/src/trace/implementations/ord_neu.rs
+++ b/differential-dataflow/src/trace/implementations/ord_neu.rs
@@ -90,7 +90,7 @@ mod layers {
         pub vals: V,
     }
 
-    impl<O: BatchContainer, V: BatchContainer> Default for Vals<O, V> {
+    impl<O: for<'a> BatchContainer<ReadItem<'a> = usize>, V: BatchContainer> Default for Vals<O, V> {
         fn default() -> Self { Self::with_capacity(0, 0) }
     }
 
@@ -108,8 +108,7 @@ mod layers {
         pub fn get_rel(&self, list_idx: usize, item_idx: usize) -> V::ReadItem<'_> {
             self.get_abs(self.bounds(list_idx).0 + item_idx)
         }
-    }
-    impl<O: BatchContainer, V: BatchContainer> Vals<O, V> {
+
         /// Number of lists in the container.
         pub fn len(&self) -> usize { self.offs.len() - 1 }
         /// Retrieves a value using an absolute rather than relative index.
@@ -118,15 +117,19 @@ mod layers {
         }
         /// Allocates with capacities for a number of lists and values.
         pub fn with_capacity(o_size: usize, v_size: usize) -> Self {
+            let mut offs = <O as BatchContainer>::with_capacity(o_size);
+            offs.push(0);
             Self {
-                offs: <O as BatchContainer>::with_capacity(o_size),
+                offs,
                 vals: <V as BatchContainer>::with_capacity(v_size),
             }
         }
         /// Allocates with enough capacity to contain two inputs.
         pub fn merge_capacity(this: &Self, that: &Self) -> Self {
+            let mut offs = <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len());
+            offs.push(0);
             Self {
-                offs: <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len()),
+                offs,
                 vals: <V as BatchContainer>::merge_capacity(&this.vals, &that.vals),
             }
         }
@@ -151,7 +154,7 @@ mod layers {
         total: usize,
     }
 
-    impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Default for Upds<O, T, D> {
+    impl<O: for<'a> BatchContainer<ReadItem<'a> = usize>, T: BatchContainer, D: BatchContainer> Default for Upds<O, T, D> {
         fn default() -> Self { Self::with_capacity(0, 0) }
     }
     impl<O: for<'a> BatchContainer<ReadItem<'a> = usize>, T: BatchContainer, D: BatchContainer> Upds<O, T, D> {
@@ -176,8 +179,7 @@ mod layers {
         pub fn get_rel(&self, list_idx: usize, item_idx: usize) -> (T::ReadItem<'_>, D::ReadItem<'_>) {
             self.get_abs(self.bounds(list_idx).0 + item_idx)
         }
-    }
-    impl<O: BatchContainer, T: BatchContainer, D: BatchContainer> Upds<O, T, D> {
+
         /// Number of lists in the container.
         pub fn len(&self) -> usize { self.offs.len() - 1 }
         /// Retrieves a value using an absolute rather than relative index.
@@ -186,8 +188,10 @@ mod layers {
         }
         /// Allocates with capacities for a number of lists and values.
         pub fn with_capacity(o_size: usize, u_size: usize) -> Self {
+            let mut offs = <O as BatchContainer>::with_capacity(o_size);
+            offs.push(0);
             Self {
-                offs: <O as BatchContainer>::with_capacity(o_size),
+                offs,
                 times: <T as BatchContainer>::with_capacity(u_size),
                 diffs: <D as BatchContainer>::with_capacity(u_size),
                 total: 0,
@@ -195,8 +199,10 @@ mod layers {
         }
         /// Allocates with enough capacity to contain two inputs.
         pub fn merge_capacity(this: &Self, that: &Self) -> Self {
+            let mut offs = <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len());
+            offs.push(0);
             Self {
-                offs:  <O as BatchContainer>::with_capacity(this.offs.len() + that.offs.len()),
+                offs,
                 times: <T as BatchContainer>::merge_capacity(&this.times, &that.times),
                 diffs: <D as BatchContainer>::merge_capacity(&this.diffs, &that.diffs),
                 total: 0,
@@ -326,7 +332,7 @@ pub mod val_batch {
         /// The number of updates reflected in the batch.
         ///
         /// We track this separately from `storage` because due to the singleton optimization,
-        /// we may have many more updates than `storage.updates.len()`. It should equal that 
+        /// we may have many more updates than `storage.updates.len()`. It should equal that
         /// length, plus the number of singleton optimizations employed.
         pub updates: usize,
     }
@@ -340,14 +346,14 @@ pub mod val_batch {
         type DiffGat<'a> = <L::DiffContainer as BatchContainer>::ReadItem<'a>;
 
         type Cursor = OrdValCursor<L>;
-        fn cursor(&self) -> Self::Cursor { 
+        fn cursor(&self) -> Self::Cursor {
             OrdValCursor {
                 key_cursor: 0,
                 val_cursor: 0,
                 phantom: PhantomData,
             }
         }
-        fn len(&self) -> usize { 
+        fn len(&self) -> usize {
             // Normally this would be `self.updates.len()`, but we have a clever compact encoding.
             // Perhaps we should count such exceptions to the side, to provide a correct accounting.
             self.updates
@@ -408,22 +414,14 @@ pub mod val_batch {
             let batch1 = &batch1.storage;
             let batch2 = &batch2.storage;
 
-            let mut storage = OrdValStorage {
-                keys: L::KeyContainer::merge_capacity(&batch1.keys, &batch2.keys),
-                vals: Vals::merge_capacity(&batch1.vals, &batch2.vals),
-                upds: Upds::merge_capacity(&batch1.upds, &batch2.upds),
-            };
-
-            // Mark explicit types because type inference fails to resolve it.
-            let keys_offs: &mut L::OffsetContainer = &mut storage.vals.offs;
-            keys_offs.push(0);
-            let vals_offs: &mut L::OffsetContainer = &mut storage.upds.offs;
-            vals_offs.push(0);
-
             OrdValMerger {
                 key_cursor1: 0,
                 key_cursor2: 0,
-                result: storage,
+                result: OrdValStorage {
+                    keys: L::KeyContainer::merge_capacity(&batch1.keys, &batch2.keys),
+                    vals: Vals::merge_capacity(&batch1.vals, &batch2.vals),
+                    upds: Upds::merge_capacity(&batch1.upds, &batch2.upds),
+                },
                 description,
                 staging: UpdsBuilder::default(),
             }
@@ -448,7 +446,7 @@ pub mod val_batch {
                 effort = (self.staging.total() - starting_updates) as isize;
             }
 
-            // Merging is complete, and only copying remains. 
+            // Merging is complete, and only copying remains.
             // Key-by-key copying allows effort interruption, and compaction.
             while self.key_cursor1 < source1.storage.keys.len() && effort < *fuel {
                 self.copy_key(&source1.storage, self.key_cursor1);
@@ -484,13 +482,13 @@ pub mod val_batch {
                     self.result.vals.vals.push(source.vals.get_abs(lower));
                 }
                 lower += 1;
-            }            
+            }
 
             // If we have pushed any values, copy the key as well.
             if self.result.vals.vals.len() > init_vals {
                 self.result.keys.push(source.keys.index(cursor));
                 self.result.vals.offs.push(self.result.vals.vals.len());
-            }           
+            }
         }
         /// Merge the next key in each of `source1` and `source2` into `self`, updating the appropriate cursors.
         ///
@@ -499,7 +497,7 @@ pub mod val_batch {
         fn merge_key(&mut self, source1: &OrdValStorage<L>, source2: &OrdValStorage<L>) {
             use ::std::cmp::Ordering;
             match source1.keys.index(self.key_cursor1).cmp(&source2.keys.index(self.key_cursor2)) {
-                Ordering::Less => { 
+                Ordering::Less => {
                     self.copy_key(source1, self.key_cursor1);
                     self.key_cursor1 += 1;
                 },
@@ -526,8 +524,8 @@ pub mod val_batch {
         /// If the compacted result contains values with non-empty updates, the function returns
         /// an offset that should be recorded to indicate the upper extent of the result values.
         fn merge_vals(
-            &mut self, 
-            (source1, mut lower1, upper1): (&OrdValStorage<L>, usize, usize), 
+            &mut self,
+            (source1, mut lower1, upper1): (&OrdValStorage<L>, usize, usize),
             (source2, mut lower2, upper2): (&OrdValStorage<L>, usize, usize),
         ) -> Option<usize> {
             // Capture the initial number of values to determine if the merge was ultimately non-empty.
@@ -538,7 +536,7 @@ pub mod val_batch {
                 // We could multi-way merge and it wouldn't be very complicated.
                 use ::std::cmp::Ordering;
                 match source1.vals.get_abs(lower1).cmp(&source2.vals.get_abs(lower2)) {
-                    Ordering::Less => { 
+                    Ordering::Less => {
                         // Extend stash by updates, with logical compaction applied.
                         self.stash_updates_for_val(source1, lower1);
                         if self.staging.seal(&mut self.result.upds) {
@@ -555,7 +553,7 @@ pub mod val_batch {
                         lower1 += 1;
                         lower2 += 1;
                     },
-                    Ordering::Greater => { 
+                    Ordering::Greater => {
                         // Extend stash by updates, with logical compaction applied.
                         self.stash_updates_for_val(source2, lower2);
                         if self.staging.seal(&mut self.result.upds) {
@@ -655,7 +653,7 @@ pub mod val_batch {
             }
         }
         fn step_val(&mut self, storage: &OrdValBatch<L>) {
-            self.val_cursor += 1; 
+            self.val_cursor += 1;
             if !self.val_valid(storage) {
                 self.val_cursor = storage.storage.vals.bounds(self.key_cursor).1;
             }
@@ -701,20 +699,12 @@ pub mod val_batch {
 
         fn with_capacity(keys: usize, vals: usize, upds: usize) -> Self {
 
-            let mut result = OrdValStorage {
-                keys: L::KeyContainer::with_capacity(keys),
-                vals: Vals::with_capacity(keys + 1, vals),
-                upds: Upds::with_capacity(vals + 1, upds),
-            };
-
-            // Mark explicit types because type inference fails to resolve it.
-            let keys_offs: &mut L::OffsetContainer = &mut result.vals.offs;
-            keys_offs.push(0);
-            let vals_offs: &mut L::OffsetContainer = &mut result.upds.offs;
-            vals_offs.push(0);
-
-            Self { 
-                result,
+            Self {
+                result: OrdValStorage {
+                    keys: L::KeyContainer::with_capacity(keys),
+                    vals: Vals::with_capacity(keys + 1, vals),
+                    upds: Upds::with_capacity(vals + 1, upds),
+                },
                 staging: UpdsBuilder::default(),
                 _marker: PhantomData,
             }
@@ -775,7 +765,7 @@ pub mod val_batch {
             for mut chunk in chain.drain(..) {
                 builder.push(&mut chunk);
             }
-    
+
             builder.done(description)
         }
     }
@@ -835,7 +825,7 @@ pub mod key_batch {
     }
 
     impl<L: Layout> BatchReader for OrdKeyBatch<L> {
-        
+
         type Key<'a> = <L::KeyContainer as BatchContainer>::ReadItem<'a>;
         type Val<'a> = &'a ();
         type Time = <L::Target as Update>::Time;
@@ -912,18 +902,13 @@ pub mod key_batch {
             let batch1 = &batch1.storage;
             let batch2 = &batch2.storage;
 
-            let mut storage = OrdKeyStorage {
-                keys: L::KeyContainer::merge_capacity(&batch1.keys, &batch2.keys),
-                upds: Upds::merge_capacity(&batch1.upds, &batch2.upds),
-            };
-
-            let keys_offs: &mut L::OffsetContainer = &mut storage.upds.offs;
-            keys_offs.push(0);
-
             OrdKeyMerger {
                 key_cursor1: 0,
                 key_cursor2: 0,
-                result: storage,
+                result: OrdKeyStorage {
+                    keys: L::KeyContainer::merge_capacity(&batch1.keys, &batch2.keys),
+                    upds: Upds::merge_capacity(&batch1.upds, &batch2.upds),
+                },
                 description,
                 staging: UpdsBuilder::default(),
             }
@@ -971,8 +956,8 @@ pub mod key_batch {
         ///
         /// The method extracts the key in `source` at `cursor`, and merges it in to `self`.
         /// If the result does not wholly cancel, they key will be present in `self` with the
-        /// compacted values and updates. 
-        /// 
+        /// compacted values and updates.
+        ///
         /// The caller should be certain to update the cursor, as this method does not do this.
         fn copy_key(&mut self, source: &OrdKeyStorage<L>, cursor: usize) {
             self.stash_updates_for_key(source, cursor);
@@ -987,7 +972,7 @@ pub mod key_batch {
         fn merge_key(&mut self, source1: &OrdKeyStorage<L>, source2: &OrdKeyStorage<L>) {
             use ::std::cmp::Ordering;
             match source1.keys.index(self.key_cursor1).cmp(&source2.keys.index(self.key_cursor2)) {
-                Ordering::Less => { 
+                Ordering::Less => {
                     self.copy_key(source1, self.key_cursor1);
                     self.key_cursor1 += 1;
                 },
@@ -1111,16 +1096,11 @@ pub mod key_batch {
 
         fn with_capacity(keys: usize, _vals: usize, upds: usize) -> Self {
 
-            let mut result = OrdKeyStorage {
-                keys: L::KeyContainer::with_capacity(keys),
-                upds: Upds::with_capacity(keys+1, upds),
-            };
-
-            let keys_offs: &mut L::OffsetContainer = &mut result.upds.offs;
-            keys_offs.push(0);
-
-            Self { 
-                result,
+            Self {
+                result: OrdKeyStorage {
+                    keys: L::KeyContainer::with_capacity(keys),
+                    upds: Upds::with_capacity(keys+1, upds),
+                },
                 staging: UpdsBuilder::default(),
                 _marker: PhantomData,
             }
@@ -1162,7 +1142,7 @@ pub mod key_batch {
             for mut chunk in chain.drain(..) {
                 builder.push(&mut chunk);
             }
-    
+
             builder.done(description)
         }
     }

From 84a5405f750ba4d700ea6058648c7f32396f0532 Mon Sep 17 00:00:00 2001
From: Frank McSherry <fmcsherry@me.com>
Date: Wed, 25 Jun 2025 19:04:25 -0400
Subject: [PATCH 7/9] Tidy comments; make members private

---
 .../src/trace/implementations/ord_neu.rs       | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/differential-dataflow/src/trace/implementations/ord_neu.rs b/differential-dataflow/src/trace/implementations/ord_neu.rs
index 94bbd0764..b3175dbd7 100644
--- a/differential-dataflow/src/trace/implementations/ord_neu.rs
+++ b/differential-dataflow/src/trace/implementations/ord_neu.rs
@@ -74,7 +74,7 @@ pub use layers::{Vals, Upds};
 /// forms a layered trie: a tree with values of some type on nodes at each depth.
 ///
 /// We will form tries here by layering `[Keys, Vals, Upds]` or `[Keys, Upds]`.
-mod layers {
+pub mod layers {
 
     use serde::{Deserialize, Serialize};
     use crate::trace::implementations::BatchContainer;
@@ -142,11 +142,11 @@ mod layers {
     #[derive(Debug, Serialize, Deserialize)]
     pub struct Upds<O, T, D> {
         /// Offsets used to provide indexes from values to updates.
-        pub offs: O,
+        offs: O,
         /// Concatenated ordered lists of update times, bracketed by offsets in `offs`.
-        pub times: T,
+        times: T,
         /// Concatenated ordered lists of update diffs, bracketed by offsets in `offs`.
-        pub diffs: D,
+        diffs: D,
         /// Tracks the total number of updates contained within.
         ///
         /// This may be more than `times.len()` or `diffs.len()` because we deduplicate runs
@@ -210,6 +210,7 @@ mod layers {
         }
     }
 
+    /// Helper type for constructing `Upds` containers.
     pub struct UpdsBuilder<T: BatchContainer, D: BatchContainer> {
         /// Local stash of updates, to use for consolidation.
         ///
@@ -597,7 +598,6 @@ pub mod val_batch {
                 let mut new_time: <L::Target as Update>::Time = time.into_owned();
                 new_time.advance_by(self.description.since().borrow());
                 self.staging.push(new_time, diff.into_owned());
-                // self.update_stash.push((new_time, diff.into_owned()));
             }
         }
     }
@@ -698,7 +698,6 @@ pub mod val_batch {
         type Output = OrdValBatch<L>;
 
         fn with_capacity(keys: usize, vals: usize, upds: usize) -> Self {
-
             Self {
                 result: OrdValStorage {
                     keys: L::KeyContainer::with_capacity(keys),
@@ -725,7 +724,6 @@ pub mod val_batch {
                 else if self.result.keys.last().map(|k| CI::key_eq(&key, k)).unwrap_or(false) {
                     // Perhaps this is a continuation of an already received value.
                     if self.result.vals.vals.last().map(|v| CI::val_eq(&val, v)).unwrap_or(false) {
-                        // self.push_update(time, diff);
                         self.staging.push(time, diff);
                     } else {
                         // New value; complete representation of prior value.
@@ -746,11 +744,7 @@ pub mod val_batch {
 
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdValBatch<L> {
-            // Record the final offsets
             self.staging.seal(&mut self.result.upds);
-            // self.result.upds.offs.push(self.result.upds.times.len());
-            // // Remove any pending singleton, and if it was set increment our count.
-            // if self.singleton.take().is_some() { self.singletons += 1; }
             self.result.vals.offs.push(self.result.vals.vals.len());
             OrdValBatch {
                 updates: self.staging.total(),
@@ -1095,7 +1089,6 @@ pub mod key_batch {
         type Output = OrdKeyBatch<L>;
 
         fn with_capacity(keys: usize, _vals: usize, upds: usize) -> Self {
-
             Self {
                 result: OrdKeyStorage {
                     keys: L::KeyContainer::with_capacity(keys),
@@ -1127,7 +1120,6 @@ pub mod key_batch {
 
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdKeyBatch<L> {
-            // Record the final offsets
             self.staging.seal(&mut self.result.upds);
             OrdKeyBatch {
                 updates: self.staging.total(),

From 4960543696e5ce003d05145a04d19d664667589b Mon Sep 17 00:00:00 2001
From: Frank McSherry <fmcsherry@me.com>
Date: Wed, 25 Jun 2025 19:04:36 -0400
Subject: [PATCH 8/9] Update columnar example

---
 differential-dataflow/examples/columnar.rs | 136 +++++----------------
 1 file changed, 30 insertions(+), 106 deletions(-)

diff --git a/differential-dataflow/examples/columnar.rs b/differential-dataflow/examples/columnar.rs
index c413ab65b..cf58ee3c2 100644
--- a/differential-dataflow/examples/columnar.rs
+++ b/differential-dataflow/examples/columnar.rs
@@ -580,7 +580,7 @@ pub mod dd_builder {
     use differential_dataflow::trace::implementations::Layout;
     use differential_dataflow::trace::implementations::Update;
     use differential_dataflow::trace::implementations::BatchContainer;
-    use differential_dataflow::trace::implementations::ord_neu::{OrdValBatch, val_batch::OrdValStorage, OrdKeyBatch, Vals, Upds};
+    use differential_dataflow::trace::implementations::ord_neu::{OrdValBatch, val_batch::OrdValStorage, OrdKeyBatch, Vals, Upds, layers::UpdsBuilder};
     use differential_dataflow::trace::implementations::ord_neu::key_batch::OrdKeyStorage;
     use crate::Column;
 
@@ -597,44 +597,7 @@ pub mod dd_builder {
         ///
         /// This is public to allow container implementors to set and inspect their container.
         pub result: OrdValStorage<L>,
-        singleton: Option<(<L::Target as Update>::Time, <L::Target as Update>::Diff)>,
-        /// Counts the number of singleton optimizations we performed.
-        ///
-        /// This number allows us to correctly gauge the total number of updates reflected in a batch,
-        /// even though `updates.len()` may be much shorter than this amount.
-        singletons: usize,
-    }
-
-    impl<L: Layout> OrdValBuilder<L> {
-        /// Pushes a single update, which may set `self.singleton` rather than push.
-        ///
-        /// This operation is meant to be equivalent to `self.results.updates.push((time, diff))`.
-        /// However, for "clever" reasons it does not do this. Instead, it looks for opportunities
-        /// to encode a singleton update with an "absert" update: repeating the most recent offset.
-        /// This otherwise invalid state encodes "look back one element".
-        ///
-        /// When `self.singleton` is `Some`, it means that we have seen one update and it matched the
-        /// previously pushed update exactly. In that case, we do not push the update into `updates`.
-        /// The update tuple is retained in `self.singleton` in case we see another update and need
-        /// to recover the singleton to push it into `updates` to join the second update.
-        fn push_update(&mut self, time: <L::Target as Update>::Time, diff: <L::Target as Update>::Diff) {
-            // If a just-pushed update exactly equals `(time, diff)` we can avoid pushing it.
-            if self.result.upds.times.last().map(|t| t == <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time)) == Some(true) &&
-                self.result.upds.diffs.last().map(|d| d == <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff)) == Some(true)
-            {
-                assert!(self.singleton.is_none());
-                self.singleton = Some((time, diff));
-            }
-            else {
-                // If we have pushed a single element, we need to copy it out to meet this one.
-                if let Some((time, diff)) = self.singleton.take() {
-                    self.result.upds.times.push(time);
-                    self.result.upds.diffs.push(diff);
-                }
-                self.result.upds.times.push(time);
-                self.result.upds.diffs.push(diff);
-            }
-        }
+        staging: UpdsBuilder<L::TimeContainer, L::DiffContainer>,
     }
 
     // The layout `L` determines the key, val, time, and diff types.
@@ -656,15 +619,13 @@ pub mod dd_builder {
         type Output = OrdValBatch<L>;
 
         fn with_capacity(keys: usize, vals: usize, upds: usize) -> Self {
-            // We don't introduce zero offsets as they will be introduced by the first `push` call.
             Self {
                 result: OrdValStorage {
                     keys: L::KeyContainer::with_capacity(keys),
                     vals: Vals::with_capacity(keys + 1, vals),
                     upds: Upds::with_capacity(vals + 1, upds),
                 },
-                singleton: None,
-                singletons: 0,
+                staging: UpdsBuilder::default(),
             }
         }
 
@@ -686,24 +647,28 @@ pub mod dd_builder {
                 let time = <<L::TimeContainer as BatchContainer>::Owned as Columnar>::into_owned(time);
                 let diff = <<L::DiffContainer as BatchContainer>::Owned as Columnar>::into_owned(diff);
 
+                // Pre-load the first update.
+                if self.result.keys.is_empty() {
+                    self.result.vals.vals.push(&val);
+                    self.result.keys.push(&key);
+                    self.staging.push(time, diff);
+                }
                 // Perhaps this is a continuation of an already received key.
-                if self.result.keys.last().map(|k| <<L::KeyContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&key).eq(&k)).unwrap_or(false) {
+                else if self.result.keys.last().map(|k| <<L::KeyContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&key).eq(&k)).unwrap_or(false) {
                     // Perhaps this is a continuation of an already received value.
                     if self.result.vals.vals.last().map(|v| <<L::ValContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&val).eq(&v)).unwrap_or(false) {
-                        self.push_update(time, diff);
+                        self.staging.push(time, diff);
                     } else {
                         // New value; complete representation of prior value.
-                        self.result.upds.offs.push(self.result.upds.times.len());
-                        if self.singleton.take().is_some() { self.singletons += 1; }
-                        self.push_update(time, diff);
+                        self.staging.seal(&mut self.result.upds);
+                        self.staging.push(time, diff);
                         self.result.vals.vals.push(&val);
                     }
                 } else {
                     // New key; complete representation of prior key.
-                    self.result.upds.offs.push(self.result.upds.times.len());
-                    if self.singleton.take().is_some() { self.singletons += 1; }
+                    self.staging.seal(&mut self.result.upds);
+                    self.staging.push(time, diff);
                     self.result.vals.offs.push(self.result.vals.len());
-                    self.push_update(time, diff);
                     self.result.vals.vals.push(&val);
                     self.result.keys.push(&key);
                 }
@@ -712,13 +677,10 @@ pub mod dd_builder {
 
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdValBatch<L> {
-            // Record the final offsets
-            self.result.upds.offs.push(self.result.upds.times.len());
-            // Remove any pending singleton, and if it was set increment our count.
-            if self.singleton.take().is_some() { self.singletons += 1; }
+            self.staging.seal(&mut self.result.upds);
             self.result.vals.offs.push(self.result.vals.len());
             OrdValBatch {
-                updates: self.result.upds.times.len() + self.singletons,
+                updates: self.staging.total(),
                 storage: self.result,
                 description,
             }
@@ -742,44 +704,7 @@ pub mod dd_builder {
         ///
         /// This is public to allow container implementors to set and inspect their container.
         pub result: OrdKeyStorage<L>,
-        singleton: Option<(<L::Target as Update>::Time, <L::Target as Update>::Diff)>,
-        /// Counts the number of singleton optimizations we performed.
-        ///
-        /// This number allows us to correctly gauge the total number of updates reflected in a batch,
-        /// even though `updates.len()` may be much shorter than this amount.
-        singletons: usize,
-    }
-
-    impl<L: Layout> OrdKeyBuilder<L> {
-        /// Pushes a single update, which may set `self.singleton` rather than push.
-        ///
-        /// This operation is meant to be equivalent to `self.results.updates.push((time, diff))`.
-        /// However, for "clever" reasons it does not do this. Instead, it looks for opportunities
-        /// to encode a singleton update with an "absert" update: repeating the most recent offset.
-        /// This otherwise invalid state encodes "look back one element".
-        ///
-        /// When `self.singleton` is `Some`, it means that we have seen one update and it matched the
-        /// previously pushed update exactly. In that case, we do not push the update into `updates`.
-        /// The update tuple is retained in `self.singleton` in case we see another update and need
-        /// to recover the singleton to push it into `updates` to join the second update.
-        fn push_update(&mut self, time: <L::Target as Update>::Time, diff: <L::Target as Update>::Diff) {
-            // If a just-pushed update exactly equals `(time, diff)` we can avoid pushing it.
-            if self.result.upds.times.last().map(|t| t == <<L::TimeContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&time)) == Some(true) &&
-                self.result.upds.diffs.last().map(|d| d == <<L::DiffContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&diff)) == Some(true)
-            {
-                assert!(self.singleton.is_none());
-                self.singleton = Some((time, diff));
-            }
-            else {
-                // If we have pushed a single element, we need to copy it out to meet this one.
-                if let Some((time, diff)) = self.singleton.take() {
-                    self.result.upds.times.push(time);
-                    self.result.upds.diffs.push(diff);
-                }
-                self.result.upds.times.push(time);
-                self.result.upds.diffs.push(diff);
-            }
-        }
+        staging: UpdsBuilder<L::TimeContainer, L::DiffContainer>,
     }
 
     // The layout `L` determines the key, val, time, and diff types.
@@ -801,14 +726,12 @@ pub mod dd_builder {
         type Output = OrdKeyBatch<L>;
 
         fn with_capacity(keys: usize, _vals: usize, upds: usize) -> Self {
-            // We don't introduce zero offsets as they will be introduced by the first `push` call.
             Self {
                 result: OrdKeyStorage {
                     keys: L::KeyContainer::with_capacity(keys),
                     upds: Upds::with_capacity(keys + 1, upds),
                 },
-                singleton: None,
-                singletons: 0,
+                staging: UpdsBuilder::default(),
             }
         }
 
@@ -829,14 +752,18 @@ pub mod dd_builder {
                 let time = <<L::TimeContainer as BatchContainer>::Owned as Columnar>::into_owned(time);
                 let diff = <<L::DiffContainer as BatchContainer>::Owned as Columnar>::into_owned(diff);
 
+                // Pre-load the first update.
+                if self.result.keys.is_empty() {
+                    self.result.keys.push(&key);
+                    self.staging.push(time, diff);
+                }
                 // Perhaps this is a continuation of an already received key.
-                if self.result.keys.last().map(|k| <<L::KeyContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&key).eq(&k)).unwrap_or(false) {
-                    self.push_update(time, diff);
+                else if self.result.keys.last().map(|k| <<L::KeyContainer as BatchContainer>::ReadItem<'_> as IntoOwned>::borrow_as(&key).eq(&k)).unwrap_or(false) {
+                    self.staging.push(time, diff);
                 } else {
                     // New key; complete representation of prior key.
-                    self.result.upds.offs.push(self.result.upds.times.len());
-                    if self.singleton.take().is_some() { self.singletons += 1; }
-                    self.push_update(time, diff);
+                    self.staging.seal(&mut self.result.upds);
+                    self.staging.push(time, diff);
                     self.result.keys.push(&key);
                 }
             }
@@ -844,12 +771,9 @@ pub mod dd_builder {
 
         #[inline(never)]
         fn done(mut self, description: Description<Self::Time>) -> OrdKeyBatch<L> {
-            // Record the final offsets
-            self.result.upds.offs.push(self.result.upds.times.len());
-            // Remove any pending singleton, and if it was set increment our count.
-            if self.singleton.take().is_some() { self.singletons += 1; }
+            self.staging.seal(&mut self.result.upds);
             OrdKeyBatch {
-                updates: self.result.upds.times.len() + self.singletons,
+                updates: self.staging.total(),
                 storage: self.result,
                 description,
             }

From 4e3cc24de7524bc5494445d7708f4858cd72da7c Mon Sep 17 00:00:00 2001
From: Frank McSherry <fmcsherry@me.com>
Date: Fri, 11 Jul 2025 10:41:00 -0400
Subject: [PATCH 9/9] Respond to Copilot feedback

---
 differential-dataflow/src/trace/implementations/ord_neu.rs | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/differential-dataflow/src/trace/implementations/ord_neu.rs b/differential-dataflow/src/trace/implementations/ord_neu.rs
index b3175dbd7..cb0d12bf2 100644
--- a/differential-dataflow/src/trace/implementations/ord_neu.rs
+++ b/differential-dataflow/src/trace/implementations/ord_neu.rs
@@ -147,11 +147,6 @@ pub mod layers {
         times: T,
         /// Concatenated ordered lists of update diffs, bracketed by offsets in `offs`.
         diffs: D,
-        /// Tracks the total number of updates contained within.
-        ///
-        /// This may be more than `times.len()` or `diffs.len()` because we deduplicate runs
-        /// of the same single `(time, diff)` pair.
-        total: usize,
     }
 
     impl<O: for<'a> BatchContainer<ReadItem<'a> = usize>, T: BatchContainer, D: BatchContainer> Default for Upds<O, T, D> {
@@ -194,7 +189,6 @@ pub mod layers {
                 offs,
                 times: <T as BatchContainer>::with_capacity(u_size),
                 diffs: <D as BatchContainer>::with_capacity(u_size),
-                total: 0,
             }
         }
         /// Allocates with enough capacity to contain two inputs.
@@ -205,7 +199,6 @@ pub mod layers {
                 offs,
                 times: <T as BatchContainer>::merge_capacity(&this.times, &that.times),
                 diffs: <D as BatchContainer>::merge_capacity(&this.diffs, &that.diffs),
-                total: 0,
             }
         }
     }