Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 668c097

Browse files
joseph-isaacsjoseph-isaacs
authored
chore[btrblocks]: move validity in deltas & enabled delta encoding (vortex-data#4905)
Think we need a fused RLE+Delta kernel? We believe this Array was unused since it not never wired into the default compressor. If you find a problem reading a `DeltaArray` please reach out. --------- Signed-off-by: Joe Isaacs <[email protected]>
1 parent d711afb commit 668c097

12 files changed

Lines changed: 142 additions & 146 deletions

File tree

‎bench-vortex/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ vortex = { workspace = true, features = [
8282
"files",
8383
"tokio",
8484
"zstd",
85+
"unstable_encodings",
8586
] }
8687
vortex-datafusion = { workspace = true }
8788
vortex-duckdb = { workspace = true }

‎encodings/fastlanes/src/delta/compress.rs‎

Lines changed: 19 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
use arrayref::{array_mut_ref, array_ref};
55
use fastlanes::{Delta, FastLanes, Transpose};
66
use num_traits::{WrappingAdd, WrappingSub};
7-
use vortex_array::ToCanonical;
87
use vortex_array::arrays::PrimitiveArray;
98
use vortex_array::validity::Validity;
109
use vortex_array::vtable::ValidityHelper;
10+
use vortex_array::{Array, ToCanonical};
1111
use vortex_buffer::{Buffer, BufferMut};
12-
use vortex_dtype::{NativePType, Nullability, match_each_unsigned_integer_ptype};
12+
use vortex_dtype::{NativePType, match_each_unsigned_integer_ptype};
1313
use vortex_error::VortexResult;
1414

1515
use crate::DeltaArray;
@@ -22,16 +22,10 @@ pub fn delta_compress(array: &PrimitiveArray) -> VortexResult<(PrimitiveArray, P
2222
let (bases, deltas) = match_each_unsigned_integer_ptype!(array.ptype(), |T| {
2323
const LANES: usize = T::LANES;
2424
let (bases, deltas) = compress_primitive::<T, LANES>(array.as_slice::<T>());
25-
let (base_validity, delta_validity) =
26-
if array.validity().nullability() != Nullability::NonNullable {
27-
(Validity::AllValid, Validity::AllValid)
28-
} else {
29-
(Validity::NonNullable, Validity::NonNullable)
30-
};
3125
(
3226
// To preserve nullability, we include Validity
33-
PrimitiveArray::new(bases, base_validity),
34-
PrimitiveArray::new(deltas, delta_validity),
27+
PrimitiveArray::new(bases, array.dtype().nullability().into()),
28+
PrimitiveArray::new(deltas, array.validity().clone()),
3529
)
3630
});
3731

@@ -103,7 +97,7 @@ pub fn delta_decompress(array: &DeltaArray) -> PrimitiveArray {
10397

10498
PrimitiveArray::new(
10599
decompress_primitive::<T, LANES>(bases.as_slice(), deltas.as_slice()),
106-
array.validity().clone(),
100+
Validity::from_mask(array.deltas().validity_mask(), array.dtype().nullability()),
107101
)
108102
});
109103

@@ -168,26 +162,30 @@ mod test {
168162

169163
#[test]
170164
fn test_compress() {
171-
do_roundtrip_test((0u32..10_000).collect::<Vec<_>>());
165+
do_roundtrip_test::<u32>((0u32..10_000).collect());
166+
}
167+
168+
#[test]
169+
fn test_compress_nullable() {
170+
do_roundtrip_test::<u32>(PrimitiveArray::from_option_iter(
171+
(0u32..10_000).map(|i| (i % 2 == 0).then_some(i)),
172+
));
172173
}
173174

174175
#[test]
175176
fn test_compress_overflow() {
176-
do_roundtrip_test(
177-
(0..10_000)
178-
.map(|i| (i % (u8::MAX as i32)) as u8)
179-
.collect::<Vec<_>>(),
180-
);
177+
do_roundtrip_test::<u8>((0..10_000).map(|i| (i % (u8::MAX as i32)) as u8).collect());
181178
}
182179

183-
fn do_roundtrip_test<T: NativePType>(input: Vec<T>) {
184-
let delta = DeltaArray::try_from_vec(input.clone()).unwrap();
180+
fn do_roundtrip_test<T: NativePType>(input: PrimitiveArray) {
181+
let delta = DeltaArray::try_from_primitive_array(&input).unwrap();
185182
assert_eq!(delta.len(), input.len());
186183
let decompressed = delta_decompress(&delta);
187184
let decompressed_slice = decompressed.as_slice::<T>();
188185
assert_eq!(decompressed_slice.len(), input.len());
189-
for (actual, expected) in decompressed_slice.iter().zip(input) {
190-
assert_eq!(actual, &expected);
186+
for (actual, expected) in decompressed_slice.iter().zip(input.as_slice()) {
187+
assert_eq!(actual, expected);
191188
}
189+
assert_eq!(decompressed.validity(), input.validity());
192190
}
193191
}

‎encodings/fastlanes/src/delta/compute/cast.rs‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use vortex_array::compute::{CastKernel, CastKernelAdapter, cast};
5-
use vortex_array::vtable::ValidityHelper;
65
use vortex_array::{ArrayRef, IntoArray, register_kernel};
76
use vortex_dtype::DType;
7+
use vortex_dtype::Nullability::NonNullable;
88
use vortex_error::VortexResult;
99

1010
use crate::delta::{DeltaArray, DeltaVTable};
@@ -20,20 +20,12 @@ impl CastKernel for DeltaVTable {
2020
}
2121

2222
// Cast both bases and deltas to the target type
23-
let casted_bases = cast(array.bases(), dtype)?;
23+
let casted_bases = cast(array.bases(), &dtype.with_nullability(NonNullable))?;
2424
let casted_deltas = cast(array.deltas(), dtype)?;
2525

2626
// Create a new DeltaArray with the casted components
2727
Ok(Some(
28-
DeltaArray::try_from_delta_compress_parts(
29-
casted_bases,
30-
casted_deltas,
31-
array
32-
.validity()
33-
.clone()
34-
.cast_nullability(dtype.nullability(), array.len())?,
35-
)?
36-
.into_array(),
28+
DeltaArray::try_from_delta_compress_parts(casted_bases, casted_deltas)?.into_array(),
3729
))
3830
}
3931
}

‎encodings/fastlanes/src/delta/mod.rs‎

Lines changed: 30 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ use vortex_array::arrays::PrimitiveArray;
99
use vortex_array::stats::{ArrayStats, StatsSetRef};
1010
use vortex_array::validity::Validity;
1111
use vortex_array::vtable::{
12-
ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityHelper,
13-
ValidityVTableFromValidityHelper,
12+
ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityChildSliceHelper,
13+
ValidityVTableFromChildSliceHelper,
1414
};
1515
use vortex_array::{Array, ArrayRef, Canonical, EncodingId, EncodingRef, IntoArray, vtable};
1616
use vortex_buffer::Buffer;
@@ -31,7 +31,7 @@ impl VTable for DeltaVTable {
3131
type ArrayVTable = Self;
3232
type CanonicalVTable = Self;
3333
type OperationsVTable = Self;
34-
type ValidityVTable = ValidityVTableFromValidityHelper;
34+
type ValidityVTable = ValidityVTableFromChildSliceHelper;
3535
type VisitorVTable = Self;
3636
type ComputeVTable = NotSupported;
3737
type EncodeVTable = NotSupported;
@@ -47,20 +47,6 @@ impl VTable for DeltaVTable {
4747
}
4848
}
4949

50-
#[derive(Clone, Debug)]
51-
pub struct DeltaArray {
52-
offset: usize,
53-
len: usize,
54-
dtype: DType,
55-
bases: ArrayRef,
56-
deltas: ArrayRef,
57-
validity: Validity,
58-
stats_set: ArrayStats,
59-
}
60-
61-
#[derive(Clone, Debug)]
62-
pub struct DeltaEncoding;
63-
6450
/// A FastLanes-style delta-encoded array of primitive values.
6551
///
6652
/// A [`DeltaArray`] comprises a sequence of _chunks_ each representing 1,024 delta-encoded values,
@@ -93,6 +79,21 @@ pub struct DeltaEncoding;
9379
///
9480
/// If the chunk physically has fewer than 1,024 values, then it is stored as a traditional,
9581
/// non-SIMD-amenable, delta-encoded vector.
82+
///
83+
/// Note the validity is stored in the deltas array.
84+
#[derive(Clone, Debug)]
85+
pub struct DeltaArray {
86+
offset: usize,
87+
len: usize,
88+
dtype: DType,
89+
bases: ArrayRef,
90+
deltas: ArrayRef,
91+
stats_set: ArrayStats,
92+
}
93+
94+
#[derive(Clone, Debug)]
95+
pub struct DeltaEncoding;
96+
9697
impl DeltaArray {
9798
// TODO(ngates): remove constructing from vec
9899
pub fn try_from_vec<T: NativePType>(vec: Vec<T>) -> VortexResult<Self> {
@@ -105,26 +106,19 @@ impl DeltaArray {
105106
pub fn try_from_primitive_array(array: &PrimitiveArray) -> VortexResult<Self> {
106107
let (bases, deltas) = delta_compress(array)?;
107108

108-
Self::try_from_delta_compress_parts(
109-
bases.into_array(),
110-
deltas.into_array(),
111-
Validity::NonNullable,
112-
)
109+
Self::try_from_delta_compress_parts(bases.into_array(), deltas.into_array())
113110
}
114111

115-
pub fn try_from_delta_compress_parts(
116-
bases: ArrayRef,
117-
deltas: ArrayRef,
118-
validity: Validity,
119-
) -> VortexResult<Self> {
112+
/// Create a [`DeltaArray`] from the given `bases` and `deltas` arrays.
113+
/// Note the `deltas` might be nullable
114+
pub fn try_from_delta_compress_parts(bases: ArrayRef, deltas: ArrayRef) -> VortexResult<Self> {
120115
let logical_len = deltas.len();
121-
Self::try_new(bases, deltas, validity, 0, logical_len)
116+
Self::try_new(bases, deltas, 0, logical_len)
122117
}
123118

124119
pub fn try_new(
125120
bases: ArrayRef,
126121
deltas: ArrayRef,
127-
validity: Validity,
128122
offset: usize,
129123
logical_len: usize,
130124
) -> VortexResult<Self> {
@@ -139,7 +133,7 @@ impl DeltaArray {
139133
deltas.len()
140134
)
141135
}
142-
if bases.dtype() != deltas.dtype() {
136+
if !bases.dtype().eq_ignore_nullability(deltas.dtype()) {
143137
vortex_bail!(
144138
"DeltaArray: bases and deltas must have the same dtype, got {:?} and {:?}",
145139
bases.dtype(),
@@ -157,16 +151,6 @@ impl DeltaArray {
157151
vortex_bail!("DeltaArray: ptype must be an integer, got {}", ptype);
158152
}
159153

160-
if let Some(vlen) = validity.maybe_len()
161-
&& vlen != logical_len
162-
{
163-
vortex_bail!(
164-
"DeltaArray: validity length ({}) must match logical_len ({})",
165-
vlen,
166-
logical_len
167-
);
168-
}
169-
170154
let lanes = lane_count(ptype);
171155

172156
if (deltas.len() % 1024 == 0) != (bases.len() % lanes == 0) {
@@ -179,23 +163,21 @@ impl DeltaArray {
179163
}
180164

181165
// SAFETY: validation done above
182-
Ok(unsafe { Self::new_unchecked(bases, deltas, validity, offset, logical_len) })
166+
Ok(unsafe { Self::new_unchecked(bases, deltas, offset, logical_len) })
183167
}
184168

185169
pub(crate) unsafe fn new_unchecked(
186170
bases: ArrayRef,
187171
deltas: ArrayRef,
188-
validity: Validity,
189172
offset: usize,
190173
logical_len: usize,
191174
) -> Self {
192175
Self {
193176
offset,
194177
len: logical_len,
195-
dtype: bases.dtype().clone(),
178+
dtype: bases.dtype().with_nullability(deltas.dtype().nullability()),
196179
bases,
197180
deltas,
198-
validity,
199181
stats_set: Default::default(),
200182
}
201183
}
@@ -236,9 +218,10 @@ pub(crate) fn lane_count(ptype: PType) -> usize {
236218
match_each_unsigned_integer_ptype!(ptype, |T| { T::LANES })
237219
}
238220

239-
impl ValidityHelper for DeltaArray {
240-
fn validity(&self) -> &Validity {
241-
&self.validity
221+
impl ValidityChildSliceHelper for DeltaArray {
222+
fn unsliced_child_and_slice(&self) -> (&ArrayRef, usize, usize) {
223+
let (start, len) = (self.offset(), self.len());
224+
(self.deltas(), start, start + len)
242225
}
243226
}
244227

‎encodings/fastlanes/src/delta/ops.rs‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
use std::cmp::min;
55
use std::ops::Range;
66

7-
use vortex_array::vtable::{OperationsVTable, ValidityHelper};
7+
use vortex_array::vtable::OperationsVTable;
88
use vortex_array::{Array, ArrayRef, IntoArray, ToCanonical};
99
use vortex_scalar::Scalar;
1010

@@ -20,7 +20,6 @@ impl OperationsVTable<DeltaVTable> for DeltaVTable {
2020

2121
let bases = array.bases();
2222
let deltas = array.deltas();
23-
let validity = array.validity();
2423
let lanes = array.lanes();
2524

2625
let new_bases = bases.slice(
@@ -31,18 +30,10 @@ impl OperationsVTable<DeltaVTable> for DeltaVTable {
3130
min(start_chunk * 1024, array.deltas_len())..min(stop_chunk * 1024, array.deltas_len()),
3231
);
3332

34-
let new_validity = validity.slice(range.clone());
35-
3633
// SAFETY: slicing valid bases/deltas preserves correctness
3734
unsafe {
38-
DeltaArray::new_unchecked(
39-
new_bases,
40-
new_deltas,
41-
new_validity,
42-
physical_start % 1024,
43-
range.len(),
44-
)
45-
.into_array()
35+
DeltaArray::new_unchecked(new_bases, new_deltas, physical_start % 1024, range.len())
36+
.into_array()
4637
}
4738
}
4839

‎encodings/fastlanes/src/delta/serde.rs‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,13 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use vortex_array::serde::ArrayChildren;
5-
use vortex_array::validity::Validity;
6-
use vortex_array::vtable::{SerdeVTable, ValidityHelper, VisitorVTable};
5+
use vortex_array::vtable::{SerdeVTable, VisitorVTable};
76
use vortex_array::{
87
Array, ArrayBufferVisitor, ArrayChildVisitor, DeserializeMetadata, ProstMetadata,
98
};
109
use vortex_buffer::ByteBuffer;
1110
use vortex_dtype::{DType, PType, match_each_unsigned_integer_ptype};
12-
use vortex_error::{VortexResult, vortex_bail, vortex_err};
11+
use vortex_error::{VortexResult, vortex_err};
1312

1413
use super::DeltaEncoding;
1514
use crate::{DeltaArray, DeltaVTable};
@@ -41,18 +40,7 @@ impl SerdeVTable<DeltaVTable> for DeltaVTable {
4140
_buffers: &[ByteBuffer],
4241
children: &dyn ArrayChildren,
4342
) -> VortexResult<DeltaArray> {
44-
let validity = if children.len() == 2 {
45-
Validity::from(dtype.nullability())
46-
} else if children.len() == 3 {
47-
let validity = children.get(2, &Validity::DTYPE, len)?;
48-
Validity::Array(validity)
49-
} else {
50-
vortex_bail!(
51-
"DeltaArray: expected 2 or 3 children, got {}",
52-
children.len()
53-
);
54-
};
55-
43+
assert_eq!(children.len(), 2);
5644
let ptype = PType::try_from(dtype)?;
5745
let lanes =
5846
match_each_unsigned_integer_ptype!(ptype, |T| { <T as fastlanes::FastLanes>::LANES });
@@ -67,7 +55,7 @@ impl SerdeVTable<DeltaVTable> for DeltaVTable {
6755
let bases = children.get(0, dtype, bases_len)?;
6856
let deltas = children.get(1, dtype, deltas_len)?;
6957

70-
DeltaArray::try_new(bases, deltas, validity, metadata.offset as usize, len)
58+
DeltaArray::try_new(bases, deltas, metadata.offset as usize, len)
7159
}
7260
}
7361

@@ -77,7 +65,6 @@ impl VisitorVTable<DeltaVTable> for DeltaVTable {
7765
fn visit_children(array: &DeltaArray, visitor: &mut dyn ArrayChildVisitor) {
7866
visitor.visit_child("bases", array.bases());
7967
visitor.visit_child("deltas", array.deltas());
80-
visitor.visit_validity(array.validity(), array.len());
8168
}
8269
}
8370

0 commit comments

Comments
 (0)