arrow2/array/
mod.rs

1//! Contains the [`Array`] and [`MutableArray`] trait objects declaring arrays,
2//! as well as concrete arrays (such as [`Utf8Array`] and [`MutableUtf8Array`]).
3//!
4//! Fixed-length containers with optional values
5//! that are layed in memory according to the Arrow specification.
6//! Each array type has its own `struct`. The following are the main array types:
7//! * [`PrimitiveArray`] and [`MutablePrimitiveArray`], an array of values with a fixed length such as integers, floats, etc.
8//! * [`BooleanArray`] and [`MutableBooleanArray`], an array of boolean values (stored as a bitmap)
9//! * [`Utf8Array`] and [`MutableUtf8Array`], an array of variable length utf8 values
10//! * [`BinaryArray`] and [`MutableBinaryArray`], an array of opaque variable length values
11//! * [`ListArray`] and [`MutableListArray`], an array of arrays (e.g. `[[1, 2], None, [], [None]]`)
12//! * [`StructArray`] and [`MutableStructArray`], an array of arrays identified by a string (e.g. `{"a": [1, 2], "b": [true, false]}`)
13//! All immutable arrays implement the trait object [`Array`] and that can be downcasted
14//! to a concrete struct based on [`PhysicalType`](crate::datatypes::PhysicalType) available from [`Array::data_type`].
15//! All immutable arrays are backed by [`Buffer`](crate::buffer::Buffer) and thus cloning and slicing them is `O(1)`.
16//!
17//! Most arrays contain a [`MutableArray`] counterpart that is neither clonable nor slicable, but
18//! can be operated in-place.
19use std::any::Any;
20use std::sync::Arc;
21
22use crate::error::Result;
23use crate::{
24    bitmap::{Bitmap, MutableBitmap},
25    datatypes::DataType,
26};
27
28pub(self) mod physical_binary;
29
30/// A trait representing an immutable Arrow array. Arrow arrays are trait objects
31/// that are infallibly downcasted to concrete types according to the [`Array::data_type`].
32pub trait Array: Send + Sync + dyn_clone::DynClone + 'static {
33    /// Converts itself to a reference of [`Any`], which enables downcasting to concrete types.
34    fn as_any(&self) -> &dyn Any;
35
36    /// Converts itself to a mutable reference of [`Any`], which enables mutable downcasting to concrete types.
37    fn as_any_mut(&mut self) -> &mut dyn Any;
38
39    /// The length of the [`Array`]. Every array has a length corresponding to the number of
40    /// elements (slots).
41    fn len(&self) -> usize;
42
43    /// whether the array is empty
44    fn is_empty(&self) -> bool {
45        self.len() == 0
46    }
47
48    /// The [`DataType`] of the [`Array`]. In combination with [`Array::as_any`], this can be
49    /// used to downcast trait objects (`dyn Array`) to concrete arrays.
50    fn data_type(&self) -> &DataType;
51
52    /// The validity of the [`Array`]: every array has an optional [`Bitmap`] that, when available
53    /// specifies whether the array slot is valid or not (null).
54    /// When the validity is [`None`], all slots are valid.
55    fn validity(&self) -> Option<&Bitmap>;
56
57    /// The number of null slots on this [`Array`].
58    /// # Implementation
59    /// This is `O(1)` since the number of null elements is pre-computed.
60    #[inline]
61    fn null_count(&self) -> usize {
62        if self.data_type() == &DataType::Null {
63            return self.len();
64        };
65        self.validity()
66            .as_ref()
67            .map(|x| x.unset_bits())
68            .unwrap_or(0)
69    }
70
71    /// Returns whether slot `i` is null.
72    /// # Panic
73    /// Panics iff `i >= self.len()`.
74    #[inline]
75    fn is_null(&self, i: usize) -> bool {
76        assert!(i < self.len());
77        unsafe { self.is_null_unchecked(i) }
78    }
79
80    /// Returns whether slot `i` is null.
81    /// # Safety
82    /// The caller must ensure `i < self.len()`
83    #[inline]
84    unsafe fn is_null_unchecked(&self, i: usize) -> bool {
85        self.validity()
86            .as_ref()
87            .map(|x| !x.get_bit_unchecked(i))
88            .unwrap_or(false)
89    }
90
91    /// Returns whether slot `i` is valid.
92    /// # Panic
93    /// Panics iff `i >= self.len()`.
94    #[inline]
95    fn is_valid(&self, i: usize) -> bool {
96        !self.is_null(i)
97    }
98
99    /// Slices this [`Array`].
100    /// # Implementation
101    /// This operation is `O(1)` over `len`.
102    /// # Panic
103    /// This function panics iff `offset + length > self.len()`.
104    fn slice(&mut self, offset: usize, length: usize);
105
106    /// Slices the [`Array`].
107    /// # Implementation
108    /// This operation is `O(1)`.
109    /// # Safety
110    /// The caller must ensure that `offset + length <= self.len()`
111    unsafe fn slice_unchecked(&mut self, offset: usize, length: usize);
112
113    /// Returns a slice of this [`Array`].
114    /// # Implementation
115    /// This operation is `O(1)` over `len`.
116    /// # Panic
117    /// This function panics iff `offset + length > self.len()`.
118    #[must_use]
119    fn sliced(&self, offset: usize, length: usize) -> Box<dyn Array> {
120        let mut new = self.to_boxed();
121        new.slice(offset, length);
122        new
123    }
124
125    /// Returns a slice of this [`Array`].
126    /// # Implementation
127    /// This operation is `O(1)` over `len`, as it amounts to increase two ref counts
128    /// and moving the struct to the heap.
129    /// # Safety
130    /// The caller must ensure that `offset + length <= self.len()`
131    #[must_use]
132    unsafe fn sliced_unchecked(&self, offset: usize, length: usize) -> Box<dyn Array> {
133        let mut new = self.to_boxed();
134        new.slice_unchecked(offset, length);
135        new
136    }
137
138    /// Clones this [`Array`] with a new new assigned bitmap.
139    /// # Panic
140    /// This function panics iff `validity.len() != self.len()`.
141    fn with_validity(&self, validity: Option<Bitmap>) -> Box<dyn Array>;
142
143    /// Clone a `&dyn Array` to an owned `Box<dyn Array>`.
144    fn to_boxed(&self) -> Box<dyn Array>;
145}
146
147dyn_clone::clone_trait_object!(Array);
148
149/// A trait describing an array with a backing store that can be preallocated to
150/// a given size.
151pub(crate) trait Container {
152    /// Create this array with a given capacity.
153    fn with_capacity(capacity: usize) -> Self
154    where
155        Self: Sized;
156}
157
158/// A trait describing a mutable array; i.e. an array whose values can be changed.
159/// Mutable arrays cannot be cloned but can be mutated in place,
160/// thereby making them useful to perform numeric operations without allocations.
161/// As in [`Array`], concrete arrays (such as [`MutablePrimitiveArray`]) implement how they are mutated.
162pub trait MutableArray: std::fmt::Debug + Send + Sync {
163    /// The [`DataType`] of the array.
164    fn data_type(&self) -> &DataType;
165
166    /// The length of the array.
167    fn len(&self) -> usize;
168
169    /// Whether the array is empty.
170    fn is_empty(&self) -> bool {
171        self.len() == 0
172    }
173
174    /// The optional validity of the array.
175    fn validity(&self) -> Option<&MutableBitmap>;
176
177    /// Convert itself to an (immutable) [`Array`].
178    fn as_box(&mut self) -> Box<dyn Array>;
179
180    /// Convert itself to an (immutable) atomically reference counted [`Array`].
181    // This provided implementation has an extra allocation as it first
182    // boxes `self`, then converts the box into an `Arc`. Implementors may wish
183    // to avoid an allocation by skipping the box completely.
184    fn as_arc(&mut self) -> std::sync::Arc<dyn Array> {
185        self.as_box().into()
186    }
187
188    /// Convert to `Any`, to enable dynamic casting.
189    fn as_any(&self) -> &dyn Any;
190
191    /// Convert to mutable `Any`, to enable dynamic casting.
192    fn as_mut_any(&mut self) -> &mut dyn Any;
193
194    /// Adds a new null element to the array.
195    fn push_null(&mut self);
196
197    /// Whether `index` is valid / set.
198    /// # Panic
199    /// Panics if `index >= self.len()`.
200    #[inline]
201    fn is_valid(&self, index: usize) -> bool {
202        self.validity()
203            .as_ref()
204            .map(|x| x.get(index))
205            .unwrap_or(true)
206    }
207
208    /// Reserves additional slots to its capacity.
209    fn reserve(&mut self, additional: usize);
210
211    /// Shrink the array to fit its length.
212    fn shrink_to_fit(&mut self);
213}
214
215impl MutableArray for Box<dyn MutableArray> {
216    fn len(&self) -> usize {
217        self.as_ref().len()
218    }
219
220    fn validity(&self) -> Option<&MutableBitmap> {
221        self.as_ref().validity()
222    }
223
224    fn as_box(&mut self) -> Box<dyn Array> {
225        self.as_mut().as_box()
226    }
227
228    fn as_arc(&mut self) -> Arc<dyn Array> {
229        self.as_mut().as_arc()
230    }
231
232    fn data_type(&self) -> &DataType {
233        self.as_ref().data_type()
234    }
235
236    fn as_any(&self) -> &dyn std::any::Any {
237        self.as_ref().as_any()
238    }
239
240    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
241        self.as_mut().as_mut_any()
242    }
243
244    #[inline]
245    fn push_null(&mut self) {
246        self.as_mut().push_null()
247    }
248
249    fn shrink_to_fit(&mut self) {
250        self.as_mut().shrink_to_fit();
251    }
252
253    fn reserve(&mut self, additional: usize) {
254        self.as_mut().reserve(additional);
255    }
256}
257
258macro_rules! general_dyn {
259    ($array:expr, $ty:ty, $f:expr) => {{
260        let array = $array.as_any().downcast_ref::<$ty>().unwrap();
261        ($f)(array)
262    }};
263}
264
265macro_rules! fmt_dyn {
266    ($array:expr, $ty:ty, $f:expr) => {{
267        let mut f = |x: &$ty| x.fmt($f);
268        general_dyn!($array, $ty, f)
269    }};
270}
271
272macro_rules! match_integer_type {(
273    $key_type:expr, | $_:tt $T:ident | $($body:tt)*
274) => ({
275    macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )}
276    use crate::datatypes::IntegerType::*;
277    match $key_type {
278        Int8 => __with_ty__! { i8 },
279        Int16 => __with_ty__! { i16 },
280        Int32 => __with_ty__! { i32 },
281        Int64 => __with_ty__! { i64 },
282        UInt8 => __with_ty__! { u8 },
283        UInt16 => __with_ty__! { u16 },
284        UInt32 => __with_ty__! { u32 },
285        UInt64 => __with_ty__! { u64 },
286    }
287})}
288
289macro_rules! with_match_primitive_type {(
290    $key_type:expr, | $_:tt $T:ident | $($body:tt)*
291) => ({
292    macro_rules! __with_ty__ {( $_ $T:ident ) => ( $($body)* )}
293    use crate::datatypes::PrimitiveType::*;
294    use crate::types::{days_ms, months_days_ns, f16, i256};
295    match $key_type {
296        Int8 => __with_ty__! { i8 },
297        Int16 => __with_ty__! { i16 },
298        Int32 => __with_ty__! { i32 },
299        Int64 => __with_ty__! { i64 },
300        Int128 => __with_ty__! { i128 },
301        Int256 => __with_ty__! { i256 },
302        DaysMs => __with_ty__! { days_ms },
303        MonthDayNano => __with_ty__! { months_days_ns },
304        UInt8 => __with_ty__! { u8 },
305        UInt16 => __with_ty__! { u16 },
306        UInt32 => __with_ty__! { u32 },
307        UInt64 => __with_ty__! { u64 },
308        Float16 => __with_ty__! { f16 },
309        Float32 => __with_ty__! { f32 },
310        Float64 => __with_ty__! { f64 },
311    }
312})}
313
314impl std::fmt::Debug for dyn Array + '_ {
315    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
316        use crate::datatypes::PhysicalType::*;
317        match self.data_type().to_physical_type() {
318            Null => fmt_dyn!(self, NullArray, f),
319            Boolean => fmt_dyn!(self, BooleanArray, f),
320            Primitive(primitive) => with_match_primitive_type!(primitive, |$T| {
321                fmt_dyn!(self, PrimitiveArray<$T>, f)
322            }),
323            Binary => fmt_dyn!(self, BinaryArray<i32>, f),
324            LargeBinary => fmt_dyn!(self, BinaryArray<i64>, f),
325            FixedSizeBinary => fmt_dyn!(self, FixedSizeBinaryArray, f),
326            Utf8 => fmt_dyn!(self, Utf8Array::<i32>, f),
327            LargeUtf8 => fmt_dyn!(self, Utf8Array::<i64>, f),
328            List => fmt_dyn!(self, ListArray::<i32>, f),
329            LargeList => fmt_dyn!(self, ListArray::<i64>, f),
330            FixedSizeList => fmt_dyn!(self, FixedSizeListArray, f),
331            Struct => fmt_dyn!(self, StructArray, f),
332            Union => fmt_dyn!(self, UnionArray, f),
333            Dictionary(key_type) => {
334                match_integer_type!(key_type, |$T| {
335                    fmt_dyn!(self, DictionaryArray::<$T>, f)
336                })
337            }
338            Map => fmt_dyn!(self, MapArray, f),
339        }
340    }
341}
342
343/// Creates a new [`Array`] with a [`Array::len`] of 0.
344pub fn new_empty_array(data_type: DataType) -> Box<dyn Array> {
345    use crate::datatypes::PhysicalType::*;
346    match data_type.to_physical_type() {
347        Null => Box::new(NullArray::new_empty(data_type)),
348        Boolean => Box::new(BooleanArray::new_empty(data_type)),
349        Primitive(primitive) => with_match_primitive_type!(primitive, |$T| {
350            Box::new(PrimitiveArray::<$T>::new_empty(data_type))
351        }),
352        Binary => Box::new(BinaryArray::<i32>::new_empty(data_type)),
353        LargeBinary => Box::new(BinaryArray::<i64>::new_empty(data_type)),
354        FixedSizeBinary => Box::new(FixedSizeBinaryArray::new_empty(data_type)),
355        Utf8 => Box::new(Utf8Array::<i32>::new_empty(data_type)),
356        LargeUtf8 => Box::new(Utf8Array::<i64>::new_empty(data_type)),
357        List => Box::new(ListArray::<i32>::new_empty(data_type)),
358        LargeList => Box::new(ListArray::<i64>::new_empty(data_type)),
359        FixedSizeList => Box::new(FixedSizeListArray::new_empty(data_type)),
360        Struct => Box::new(StructArray::new_empty(data_type)),
361        Union => Box::new(UnionArray::new_empty(data_type)),
362        Map => Box::new(MapArray::new_empty(data_type)),
363        Dictionary(key_type) => {
364            match_integer_type!(key_type, |$T| {
365                Box::new(DictionaryArray::<$T>::new_empty(data_type))
366            })
367        }
368    }
369}
370
371/// Creates a new [`Array`] of [`DataType`] `data_type` and `length`.
372/// The array is guaranteed to have [`Array::null_count`] equal to [`Array::len`]
373/// for all types except Union, which does not have a validity.
374pub fn new_null_array(data_type: DataType, length: usize) -> Box<dyn Array> {
375    use crate::datatypes::PhysicalType::*;
376    match data_type.to_physical_type() {
377        Null => Box::new(NullArray::new_null(data_type, length)),
378        Boolean => Box::new(BooleanArray::new_null(data_type, length)),
379        Primitive(primitive) => with_match_primitive_type!(primitive, |$T| {
380            Box::new(PrimitiveArray::<$T>::new_null(data_type, length))
381        }),
382        Binary => Box::new(BinaryArray::<i32>::new_null(data_type, length)),
383        LargeBinary => Box::new(BinaryArray::<i64>::new_null(data_type, length)),
384        FixedSizeBinary => Box::new(FixedSizeBinaryArray::new_null(data_type, length)),
385        Utf8 => Box::new(Utf8Array::<i32>::new_null(data_type, length)),
386        LargeUtf8 => Box::new(Utf8Array::<i64>::new_null(data_type, length)),
387        List => Box::new(ListArray::<i32>::new_null(data_type, length)),
388        LargeList => Box::new(ListArray::<i64>::new_null(data_type, length)),
389        FixedSizeList => Box::new(FixedSizeListArray::new_null(data_type, length)),
390        Struct => Box::new(StructArray::new_null(data_type, length)),
391        Union => Box::new(UnionArray::new_null(data_type, length)),
392        Map => Box::new(MapArray::new_null(data_type, length)),
393        Dictionary(key_type) => {
394            match_integer_type!(key_type, |$T| {
395                Box::new(DictionaryArray::<$T>::new_null(data_type, length))
396            })
397        }
398    }
399}
400
401/// Trait providing bi-directional conversion between arrow2 [`Array`] and arrow-rs [`ArrayData`]
402///
403/// [`ArrayData`]: arrow_data::ArrayData
404#[cfg(feature = "arrow")]
405pub trait Arrow2Arrow: Array {
406    /// Convert this [`Array`] into [`ArrayData`]
407    fn to_data(&self) -> arrow_data::ArrayData;
408
409    /// Create this [`Array`] from [`ArrayData`]
410    fn from_data(data: &arrow_data::ArrayData) -> Self;
411}
412
413#[cfg(feature = "arrow")]
414macro_rules! to_data_dyn {
415    ($array:expr, $ty:ty) => {{
416        let f = |x: &$ty| x.to_data();
417        general_dyn!($array, $ty, f)
418    }};
419}
420
421#[cfg(feature = "arrow")]
422impl From<Box<dyn Array>> for arrow_array::ArrayRef {
423    fn from(value: Box<dyn Array>) -> Self {
424        value.as_ref().into()
425    }
426}
427
428#[cfg(feature = "arrow")]
429impl From<&dyn Array> for arrow_array::ArrayRef {
430    fn from(value: &dyn Array) -> Self {
431        arrow_array::make_array(to_data(value))
432    }
433}
434
435#[cfg(feature = "arrow")]
436impl From<arrow_array::ArrayRef> for Box<dyn Array> {
437    fn from(value: arrow_array::ArrayRef) -> Self {
438        value.as_ref().into()
439    }
440}
441
442#[cfg(feature = "arrow")]
443impl From<&dyn arrow_array::Array> for Box<dyn Array> {
444    fn from(value: &dyn arrow_array::Array) -> Self {
445        from_data(&value.to_data())
446    }
447}
448
449/// Convert an arrow2 [`Array`] to [`arrow_data::ArrayData`]
450#[cfg(feature = "arrow")]
451pub fn to_data(array: &dyn Array) -> arrow_data::ArrayData {
452    use crate::datatypes::PhysicalType::*;
453    match array.data_type().to_physical_type() {
454        Null => to_data_dyn!(array, NullArray),
455        Boolean => to_data_dyn!(array, BooleanArray),
456        Primitive(primitive) => with_match_primitive_type!(primitive, |$T| {
457            to_data_dyn!(array, PrimitiveArray<$T>)
458        }),
459        Binary => to_data_dyn!(array, BinaryArray<i32>),
460        LargeBinary => to_data_dyn!(array, BinaryArray<i64>),
461        FixedSizeBinary => to_data_dyn!(array, FixedSizeBinaryArray),
462        Utf8 => to_data_dyn!(array, Utf8Array::<i32>),
463        LargeUtf8 => to_data_dyn!(array, Utf8Array::<i64>),
464        List => to_data_dyn!(array, ListArray::<i32>),
465        LargeList => to_data_dyn!(array, ListArray::<i64>),
466        FixedSizeList => to_data_dyn!(array, FixedSizeListArray),
467        Struct => to_data_dyn!(array, StructArray),
468        Union => to_data_dyn!(array, UnionArray),
469        Dictionary(key_type) => {
470            match_integer_type!(key_type, |$T| {
471                to_data_dyn!(array, DictionaryArray::<$T>)
472            })
473        }
474        Map => to_data_dyn!(array, MapArray),
475    }
476}
477
478/// Convert an [`arrow_data::ArrayData`] to arrow2 [`Array`]
479#[cfg(feature = "arrow")]
480pub fn from_data(data: &arrow_data::ArrayData) -> Box<dyn Array> {
481    use crate::datatypes::PhysicalType::*;
482    let data_type: DataType = data.data_type().clone().into();
483    match data_type.to_physical_type() {
484        Null => Box::new(NullArray::from_data(data)),
485        Boolean => Box::new(BooleanArray::from_data(data)),
486        Primitive(primitive) => with_match_primitive_type!(primitive, |$T| {
487            Box::new(PrimitiveArray::<$T>::from_data(data))
488        }),
489        Binary => Box::new(BinaryArray::<i32>::from_data(data)),
490        LargeBinary => Box::new(BinaryArray::<i64>::from_data(data)),
491        FixedSizeBinary => Box::new(FixedSizeBinaryArray::from_data(data)),
492        Utf8 => Box::new(Utf8Array::<i32>::from_data(data)),
493        LargeUtf8 => Box::new(Utf8Array::<i64>::from_data(data)),
494        List => Box::new(ListArray::<i32>::from_data(data)),
495        LargeList => Box::new(ListArray::<i64>::from_data(data)),
496        FixedSizeList => Box::new(FixedSizeListArray::from_data(data)),
497        Struct => Box::new(StructArray::from_data(data)),
498        Union => Box::new(UnionArray::from_data(data)),
499        Dictionary(key_type) => {
500            match_integer_type!(key_type, |$T| {
501                Box::new(DictionaryArray::<$T>::from_data(data))
502            })
503        }
504        Map => Box::new(MapArray::from_data(data)),
505    }
506}
507
508macro_rules! clone_dyn {
509    ($array:expr, $ty:ty) => {{
510        let f = |x: &$ty| Box::new(x.clone());
511        general_dyn!($array, $ty, f)
512    }};
513}
514
515// macro implementing `sliced` and `sliced_unchecked`
516macro_rules! impl_sliced {
517    () => {
518        /// Returns this array sliced.
519        /// # Implementation
520        /// This function is `O(1)`.
521        /// # Panics
522        /// iff `offset + length > self.len()`.
523        #[inline]
524        #[must_use]
525        pub fn sliced(self, offset: usize, length: usize) -> Self {
526            assert!(
527                offset + length <= self.len(),
528                "the offset of the new Buffer cannot exceed the existing length"
529            );
530            unsafe { self.sliced_unchecked(offset, length) }
531        }
532
533        /// Returns this array sliced.
534        /// # Implementation
535        /// This function is `O(1)`.
536        /// # Safety
537        /// The caller must ensure that `offset + length <= self.len()`.
538        #[inline]
539        #[must_use]
540        pub unsafe fn sliced_unchecked(mut self, offset: usize, length: usize) -> Self {
541            self.slice_unchecked(offset, length);
542            self
543        }
544    };
545}
546
547// macro implementing `with_validity` and `set_validity`
548macro_rules! impl_mut_validity {
549    () => {
550        /// Returns this array with a new validity.
551        /// # Panic
552        /// Panics iff `validity.len() != self.len()`.
553        #[must_use]
554        #[inline]
555        pub fn with_validity(mut self, validity: Option<Bitmap>) -> Self {
556            self.set_validity(validity);
557            self
558        }
559
560        /// Sets the validity of this array.
561        /// # Panics
562        /// This function panics iff `values.len() != self.len()`.
563        #[inline]
564        pub fn set_validity(&mut self, validity: Option<Bitmap>) {
565            if matches!(&validity, Some(bitmap) if bitmap.len() != self.len()) {
566                panic!("validity must be equal to the array's length")
567            }
568            self.validity = validity;
569        }
570    }
571}
572
573// macro implementing `with_validity`, `set_validity` and `apply_validity` for mutable arrays
574macro_rules! impl_mutable_array_mut_validity {
575    () => {
576        /// Returns this array with a new validity.
577        /// # Panic
578        /// Panics iff `validity.len() != self.len()`.
579        #[must_use]
580        #[inline]
581        pub fn with_validity(mut self, validity: Option<MutableBitmap>) -> Self {
582            self.set_validity(validity);
583            self
584        }
585
586        /// Sets the validity of this array.
587        /// # Panics
588        /// This function panics iff `values.len() != self.len()`.
589        #[inline]
590        pub fn set_validity(&mut self, validity: Option<MutableBitmap>) {
591            if matches!(&validity, Some(bitmap) if bitmap.len() != self.len()) {
592                panic!("validity must be equal to the array's length")
593            }
594            self.validity = validity;
595        }
596
597        /// Applies a function `f` to the validity of this array.
598        ///
599        /// This is an API to leverage clone-on-write
600        /// # Panics
601        /// This function panics if the function `f` modifies the length of the [`Bitmap`].
602        #[inline]
603        pub fn apply_validity<F: FnOnce(MutableBitmap) -> MutableBitmap>(&mut self, f: F) {
604            if let Some(validity) = std::mem::take(&mut self.validity) {
605                self.set_validity(Some(f(validity)))
606            }
607        }
608
609    }
610}
611
612// macro implementing `boxed` and `arced`
613macro_rules! impl_into_array {
614    () => {
615        /// Boxes this array into a [`Box<dyn Array>`].
616        pub fn boxed(self) -> Box<dyn Array> {
617            Box::new(self)
618        }
619
620        /// Arcs this array into a [`std::sync::Arc<dyn Array>`].
621        pub fn arced(self) -> std::sync::Arc<dyn Array> {
622            std::sync::Arc::new(self)
623        }
624    };
625}
626
627// macro implementing common methods of trait `Array`
628macro_rules! impl_common_array {
629    () => {
630        #[inline]
631        fn as_any(&self) -> &dyn std::any::Any {
632            self
633        }
634
635        #[inline]
636        fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
637            self
638        }
639
640        #[inline]
641        fn len(&self) -> usize {
642            self.len()
643        }
644
645        #[inline]
646        fn data_type(&self) -> &DataType {
647            &self.data_type
648        }
649
650        #[inline]
651        fn slice(&mut self, offset: usize, length: usize) {
652            self.slice(offset, length);
653        }
654
655        #[inline]
656        unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) {
657            self.slice_unchecked(offset, length);
658        }
659
660        #[inline]
661        fn to_boxed(&self) -> Box<dyn Array> {
662            Box::new(self.clone())
663        }
664    };
665}
666
667/// Clones a dynamic [`Array`].
668/// # Implementation
669/// This operation is `O(1)` over `len`, as it amounts to increase two ref counts
670/// and moving the concrete struct under a `Box`.
671pub fn clone(array: &dyn Array) -> Box<dyn Array> {
672    use crate::datatypes::PhysicalType::*;
673    match array.data_type().to_physical_type() {
674        Null => clone_dyn!(array, NullArray),
675        Boolean => clone_dyn!(array, BooleanArray),
676        Primitive(primitive) => with_match_primitive_type!(primitive, |$T| {
677            clone_dyn!(array, PrimitiveArray<$T>)
678        }),
679        Binary => clone_dyn!(array, BinaryArray<i32>),
680        LargeBinary => clone_dyn!(array, BinaryArray<i64>),
681        FixedSizeBinary => clone_dyn!(array, FixedSizeBinaryArray),
682        Utf8 => clone_dyn!(array, Utf8Array::<i32>),
683        LargeUtf8 => clone_dyn!(array, Utf8Array::<i64>),
684        List => clone_dyn!(array, ListArray::<i32>),
685        LargeList => clone_dyn!(array, ListArray::<i64>),
686        FixedSizeList => clone_dyn!(array, FixedSizeListArray),
687        Struct => clone_dyn!(array, StructArray),
688        Union => clone_dyn!(array, UnionArray),
689        Map => clone_dyn!(array, MapArray),
690        Dictionary(key_type) => {
691            match_integer_type!(key_type, |$T| {
692                clone_dyn!(array, DictionaryArray::<$T>)
693            })
694        }
695    }
696}
697
698// see https://users.rust-lang.org/t/generic-for-dyn-a-or-box-dyn-a-or-arc-dyn-a/69430/3
699// for details
700impl<'a> AsRef<(dyn Array + 'a)> for dyn Array {
701    fn as_ref(&self) -> &(dyn Array + 'a) {
702        self
703    }
704}
705
706mod binary;
707mod boolean;
708mod dictionary;
709mod fixed_size_binary;
710mod fixed_size_list;
711mod list;
712mod map;
713mod null;
714mod primitive;
715mod specification;
716mod struct_;
717mod union;
718mod utf8;
719
720mod equal;
721mod ffi;
722mod fmt;
723#[doc(hidden)]
724pub mod indexable;
725mod iterator;
726
727pub mod growable;
728pub mod ord;
729
730pub(crate) use iterator::ArrayAccessor;
731pub use iterator::ArrayValuesIter;
732
733pub use equal::equal;
734pub use fmt::{get_display, get_value_display};
735
736pub use binary::{BinaryArray, BinaryValueIter, MutableBinaryArray, MutableBinaryValuesArray};
737pub use boolean::{BooleanArray, MutableBooleanArray};
738pub use dictionary::{DictionaryArray, DictionaryKey, MutableDictionaryArray};
739pub use fixed_size_binary::{FixedSizeBinaryArray, MutableFixedSizeBinaryArray};
740pub use fixed_size_list::{FixedSizeListArray, MutableFixedSizeListArray};
741pub use list::{ListArray, ListValuesIter, MutableListArray};
742pub use map::MapArray;
743pub use null::{MutableNullArray, NullArray};
744pub use primitive::*;
745pub use struct_::{MutableStructArray, StructArray};
746pub use union::UnionArray;
747pub use utf8::{MutableUtf8Array, MutableUtf8ValuesArray, Utf8Array, Utf8ValuesIter};
748
749pub(crate) use self::ffi::offset_buffers_children_dictionary;
750pub(crate) use self::ffi::FromFfi;
751pub(crate) use self::ffi::ToFfi;
752
753/// A trait describing the ability of a struct to create itself from a iterator.
754/// This is similar to [`Extend`], but accepted the creation to error.
755pub trait TryExtend<A> {
756    /// Fallible version of [`Extend::extend`].
757    fn try_extend<I: IntoIterator<Item = A>>(&mut self, iter: I) -> Result<()>;
758}
759
760/// A trait describing the ability of a struct to receive new items.
761pub trait TryPush<A> {
762    /// Tries to push a new element.
763    fn try_push(&mut self, item: A) -> Result<()>;
764}
765
766/// A trait describing the ability of a struct to receive new items.
767pub trait PushUnchecked<A> {
768    /// Push a new element that holds the invariants of the struct.
769    /// # Safety
770    /// The items must uphold the invariants of the struct
771    /// Read the specific implementation of the trait to understand what these are.
772    unsafe fn push_unchecked(&mut self, item: A);
773}
774
775/// A trait describing the ability of a struct to extend from a reference of itself.
776/// Specialization of [`TryExtend`].
777pub trait TryExtendFromSelf {
778    /// Tries to extend itself with elements from `other`, failing only on overflow.
779    fn try_extend_from_self(&mut self, other: &Self) -> Result<()>;
780}
781
782/// Trait that [`BinaryArray`] and [`Utf8Array`] implement for the purposes of DRY.
783/// # Safety
784/// The implementer must ensure that
785/// 1. `offsets.len() > 0`
786/// 2. `offsets[i] >= offsets[i-1] for all i`
787/// 3. `offsets[i] < values.len() for all i`
788pub unsafe trait GenericBinaryArray<O: crate::offset::Offset>: Array {
789    /// The values of the array
790    fn values(&self) -> &[u8];
791    /// The offsets of the array
792    fn offsets(&self) -> &[O];
793}
arrow2/array/mod.rs

arrow2/array/
mod.rs
