]>> From for BooleanArray {
diff --git a/crates/polars-arrow/src/array/boolean/mod.rs b/crates/polars-arrow/src/array/boolean/mod.rs
index eb3401ee8d83..0737dcfece55 100644
--- a/crates/polars-arrow/src/array/boolean/mod.rs
+++ b/crates/polars-arrow/src/array/boolean/mod.rs
@@ -136,6 +136,7 @@ impl BooleanArray {
}
/// Returns the element at index `i` as bool
+ ///
/// # Safety
/// Caller must be sure that `i < self.len()`
#[inline]
@@ -173,6 +174,7 @@ impl BooleanArray {
/// Slices this [`BooleanArray`].
/// # Implementation
/// This operation is `O(1)` as it amounts to increase two ref counts.
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`.
#[inline]
@@ -279,6 +281,7 @@ impl BooleanArray {
/// Creates a new [`BooleanArray`] from an [`TrustedLen`] of `bool`.
/// Use this over [`BooleanArray::from_trusted_len_iter`] when the iterator is trusted len
/// but this crate does not mark it as such.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -298,6 +301,7 @@ impl BooleanArray {
/// Creates a [`BooleanArray`] from an iterator of trusted length.
/// Use this over [`BooleanArray::from_trusted_len_iter`] when the iterator is trusted len
/// but this crate does not mark it as such.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -321,6 +325,7 @@ impl BooleanArray {
}
/// Creates a [`BooleanArray`] from an falible iterator of trusted length.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
diff --git a/crates/polars-arrow/src/array/boolean/mutable.rs b/crates/polars-arrow/src/array/boolean/mutable.rs
index 50b50574a7a2..fd3b4a1989d7 100644
--- a/crates/polars-arrow/src/array/boolean/mutable.rs
+++ b/crates/polars-arrow/src/array/boolean/mutable.rs
@@ -1,4 +1,3 @@
-use std::iter::FromIterator;
use std::sync::Arc;
use polars_error::{polars_bail, PolarsResult};
@@ -136,12 +135,13 @@ impl MutableBooleanArray {
where
I: TrustedLen- ,
{
- // Safety: `I` is `TrustedLen`
+ // SAFETY: `I` is `TrustedLen`
unsafe { self.extend_trusted_len_values_unchecked(iterator) }
}
/// Extends the [`MutableBooleanArray`] from an iterator of values of trusted len.
/// This differs from `extend_trusted_len_unchecked`, which accepts in iterator of optional values.
+ ///
/// # Safety
/// The iterator must be trusted len.
#[inline]
@@ -167,11 +167,12 @@ impl MutableBooleanArray {
P: std::borrow::Borrow,
I: TrustedLen
- >,
{
- // Safety: `I` is `TrustedLen`
+ // SAFETY: `I` is `TrustedLen`
unsafe { self.extend_trusted_len_unchecked(iterator) }
}
/// Extends the [`MutableBooleanArray`] from an iterator of trusted len.
+ ///
/// # Safety
/// The iterator must be trusted len.
#[inline]
@@ -255,6 +256,7 @@ impl MutableBooleanArray {
/// Creates a new [`MutableBooleanArray`] from an [`TrustedLen`] of `bool`.
/// Use this over [`BooleanArray::from_trusted_len_iter`] when the iterator is trusted len
/// but this crate does not mark it as such.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -276,6 +278,7 @@ impl MutableBooleanArray {
/// Creates a [`BooleanArray`] from an iterator of trusted length.
/// Use this over [`BooleanArray::from_trusted_len_iter`] when the iterator is trusted len
/// but this crate does not mark it as such.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -297,11 +300,12 @@ impl MutableBooleanArray {
P: std::borrow::Borrow,
I: TrustedLen
- >,
{
- // Safety: `I` is `TrustedLen`
+ // SAFETY: `I` is `TrustedLen`
unsafe { Self::from_trusted_len_iter_unchecked(iterator) }
}
/// Creates a [`BooleanArray`] from an falible iterator of trusted length.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -331,7 +335,7 @@ impl MutableBooleanArray {
P: std::borrow::Borrow,
I: TrustedLen
- , E>>,
{
- // Safety: `I` is `TrustedLen`
+ // SAFETY: `I` is `TrustedLen`
unsafe { Self::try_from_trusted_len_iter_unchecked(iterator) }
}
@@ -555,7 +559,7 @@ impl TryExtendFromSelf for MutableBooleanArray {
extend_validity(self.len(), &mut self.validity, &other.validity);
let slice = other.values.as_slice();
- // safety: invariant offset + length <= slice.len()
+ // SAFETY: invariant offset + length <= slice.len()
unsafe {
self.values
.extend_from_slice_unchecked(slice, 0, other.values.len());
diff --git a/crates/polars-arrow/src/array/dictionary/data.rs b/crates/polars-arrow/src/array/dictionary/data.rs
index 29eb1da057b3..e7159e4bfff2 100644
--- a/crates/polars-arrow/src/array/dictionary/data.rs
+++ b/crates/polars-arrow/src/array/dictionary/data.rs
@@ -13,7 +13,7 @@ impl Arrow2Arrow for DictionaryArray {
.data_type(self.data_type.clone().into())
.child_data(vec![to_data(self.values.as_ref())]);
- // Safety: Dictionary is valid
+ // SAFETY: Dictionary is valid
unsafe { builder.build_unchecked() }
}
@@ -35,7 +35,7 @@ impl Arrow2Arrow for DictionaryArray {
.len(data.len())
.nulls(data.nulls().cloned());
- // Safety: Dictionary is valid
+ // SAFETY: Dictionary is valid
let key_data = unsafe { key_builder.build_unchecked() };
let keys = PrimitiveArray::from_data(&key_data);
let values = from_data(&data.child_data()[0]);
diff --git a/crates/polars-arrow/src/array/dictionary/mod.rs b/crates/polars-arrow/src/array/dictionary/mod.rs
index df59504c0b98..6947c9e071c7 100644
--- a/crates/polars-arrow/src/array/dictionary/mod.rs
+++ b/crates/polars-arrow/src/array/dictionary/mod.rs
@@ -37,6 +37,7 @@ pub unsafe trait DictionaryKey: NativeType + TryInto + TryFrom + H
const KEY_TYPE: IntegerType;
/// Represents this key as a `usize`.
+ ///
/// # Safety
/// The caller _must_ have checked that the value can be casted to `usize`.
#[inline]
@@ -146,7 +147,7 @@ impl DictionaryArray {
if keys.null_count() != keys.len() {
if K::always_fits_usize() {
- // safety: we just checked that conversion to `usize` always
+ // SAFETY: we just checked that conversion to `usize` always
// succeeds
unsafe { check_indexes_unchecked(keys.values(), values.len()) }?;
} else {
@@ -178,6 +179,7 @@ impl DictionaryArray {
/// * the `data_type`'s logical type is not a `DictionaryArray`
/// * the `data_type`'s keys is not compatible with `keys`
/// * the `data_type`'s values's data_type is not equal with `values.data_type()`
+ ///
/// # Safety
/// The caller must ensure that every keys's values is represented in `usize` and is `< values.len()`
pub unsafe fn try_new_unchecked(
@@ -292,6 +294,7 @@ impl DictionaryArray {
}
/// Slices this [`DictionaryArray`].
+ ///
/// # Safety
/// Safe iff `offset + length <= self.len()`.
pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) {
@@ -340,14 +343,14 @@ impl DictionaryArray {
/// Returns an iterator of the keys' values of the [`DictionaryArray`] as `usize`
#[inline]
pub fn keys_values_iter(&self) -> impl TrustedLen
- + Clone + '_ {
- // safety - invariant of the struct
+ // SAFETY: invariant of the struct
self.keys.values_iter().map(|x| unsafe { x.as_usize() })
}
/// Returns an iterator of the keys' of the [`DictionaryArray`] as `usize`
#[inline]
pub fn keys_iter(&self) -> impl TrustedLen
- > + Clone + '_ {
- // safety - invariant of the struct
+ // SAFETY: invariant of the struct
self.keys.iter().map(|x| x.map(|x| unsafe { x.as_usize() }))
}
@@ -356,7 +359,7 @@ impl DictionaryArray {
/// This function panics iff `index >= self.len()`
#[inline]
pub fn key_value(&self, index: usize) -> usize {
- // safety - invariant of the struct
+ // SAFETY: invariant of the struct
unsafe { self.keys.values()[index].as_usize() }
}
@@ -374,7 +377,7 @@ impl DictionaryArray {
/// This function panics iff `index >= self.len()`
#[inline]
pub fn value(&self, index: usize) -> Box {
- // safety - invariant of this struct
+ // SAFETY: invariant of this struct
let index = unsafe { self.keys.value(index).as_usize() };
new_scalar(self.values.as_ref(), index)
}
diff --git a/crates/polars-arrow/src/array/dictionary/mutable.rs b/crates/polars-arrow/src/array/dictionary/mutable.rs
index 9c6e783dd28e..d55ba6484443 100644
--- a/crates/polars-arrow/src/array/dictionary/mutable.rs
+++ b/crates/polars-arrow/src/array/dictionary/mutable.rs
@@ -21,7 +21,7 @@ pub struct MutableDictionaryArray {
impl From> for DictionaryArray {
fn from(other: MutableDictionaryArray) -> Self {
- // Safety - the invariant of this struct ensures that this is up-held
+ // SAFETY: the invariant of this struct ensures that this is up-held
unsafe {
DictionaryArray::::try_new_unchecked(
other.data_type,
diff --git a/crates/polars-arrow/src/array/dictionary/value_map.rs b/crates/polars-arrow/src/array/dictionary/value_map.rs
index 2be9a7ca1047..5b6bdb9528ba 100644
--- a/crates/polars-arrow/src/array/dictionary/value_map.rs
+++ b/crates/polars-arrow/src/array/dictionary/value_map.rs
@@ -81,15 +81,16 @@ impl ValueMap {
);
for index in 0..values.len() {
let key = K::try_from(index).map_err(|_| polars_err!(ComputeError: "overflow"))?;
- // safety: we only iterate within bounds
+ // SAFETY: we only iterate within bounds
let value = unsafe { values.value_unchecked_at(index) };
let hash = ahash_hash(value.borrow());
- match map.raw_entry_mut().from_hash(hash, |item| {
- // safety: invariant of the struct, it's always in bounds since we maintain it
+ let entry = map.raw_entry_mut().from_hash(hash, |item| {
+ // SAFETY: invariant of the struct, it's always in bounds since we maintain it
let stored_value = unsafe { values.value_unchecked_at(item.key.as_usize()) };
stored_value.borrow() == value.borrow()
- }) {
+ });
+ match entry {
RawEntryMut::Occupied(_) => {
polars_bail!(InvalidOperation: "duplicate value in dictionary values array")
},
@@ -133,26 +134,25 @@ impl ValueMap {
M::Type: Eq + Hash,
{
let hash = ahash_hash(value.as_indexed());
- Ok(
- match self.map.raw_entry_mut().from_hash(hash, |item| {
- // safety: we've already checked (the inverse) when we pushed it, so it should be ok?
- let index = unsafe { item.key.as_usize() };
- // safety: invariant of the struct, it's always in bounds since we maintain it
- let stored_value = unsafe { self.values.value_unchecked_at(index) };
- stored_value.borrow() == value.as_indexed()
- }) {
- RawEntryMut::Occupied(entry) => entry.key().key,
- RawEntryMut::Vacant(entry) => {
- let index = self.values.len();
- let key =
- K::try_from(index).map_err(|_| polars_err!(ComputeError: "overflow"))?;
- entry.insert_hashed_nocheck(hash, Hashed { hash, key }, ()); // NB: don't use .insert() here!
- push(&mut self.values, value)?;
- debug_assert_eq!(self.values.len(), index + 1);
- key
- },
+ let entry = self.map.raw_entry_mut().from_hash(hash, |item| {
+ // SAFETY: we've already checked (the inverse) when we pushed it, so it should be ok?
+ let index = unsafe { item.key.as_usize() };
+ // SAFETY: invariant of the struct, it's always in bounds since we maintain it
+ let stored_value = unsafe { self.values.value_unchecked_at(index) };
+ stored_value.borrow() == value.as_indexed()
+ });
+ let out = match entry {
+ RawEntryMut::Occupied(entry) => entry.key().key,
+ RawEntryMut::Vacant(entry) => {
+ let index = self.values.len();
+ let key = K::try_from(index).map_err(|_| polars_err!(ComputeError: "overflow"))?;
+ entry.insert_hashed_nocheck(hash, Hashed { hash, key }, ()); // NB: don't use .insert() here!
+ push(&mut self.values, value)?;
+ debug_assert_eq!(self.values.len(), index + 1);
+ key
},
- )
+ };
+ Ok(out)
}
pub fn shrink_to_fit(&mut self) {
diff --git a/crates/polars-arrow/src/array/ffi.rs b/crates/polars-arrow/src/array/ffi.rs
index e1dd62488b70..9806eac25e97 100644
--- a/crates/polars-arrow/src/array/ffi.rs
+++ b/crates/polars-arrow/src/array/ffi.rs
@@ -26,6 +26,7 @@ pub(crate) unsafe trait ToFfi {
/// [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html) (FFI).
pub(crate) trait FromFfi: Sized {
/// Convert itself from FFI.
+ ///
/// # Safety
/// This function is intrinsically `unsafe` as it requires the FFI to be made according
/// to the [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html)
diff --git a/crates/polars-arrow/src/array/fixed_size_binary/data.rs b/crates/polars-arrow/src/array/fixed_size_binary/data.rs
index a0662d7d6555..f99822eb0fbb 100644
--- a/crates/polars-arrow/src/array/fixed_size_binary/data.rs
+++ b/crates/polars-arrow/src/array/fixed_size_binary/data.rs
@@ -13,7 +13,7 @@ impl Arrow2Arrow for FixedSizeBinaryArray {
.buffers(vec![self.values.clone().into()])
.nulls(self.validity.as_ref().map(|b| b.clone().into()));
- // Safety: Array is valid
+ // SAFETY: Array is valid
unsafe { builder.build_unchecked() }
}
diff --git a/crates/polars-arrow/src/array/fixed_size_binary/mod.rs b/crates/polars-arrow/src/array/fixed_size_binary/mod.rs
index 70e421e746ad..e439aac214aa 100644
--- a/crates/polars-arrow/src/array/fixed_size_binary/mod.rs
+++ b/crates/polars-arrow/src/array/fixed_size_binary/mod.rs
@@ -105,6 +105,7 @@ impl FixedSizeBinaryArray {
/// Slices this [`FixedSizeBinaryArray`].
/// # Implementation
/// This operation is `O(1)`.
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`.
pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) {
@@ -151,6 +152,7 @@ impl FixedSizeBinaryArray {
}
/// Returns the element at index `i` as &str
+ ///
/// # Safety
/// Assumes that the `i < self.len`.
#[inline]
@@ -264,20 +266,3 @@ impl FixedSizeBinaryArray {
MutableFixedSizeBinaryArray::from(slice).into()
}
}
-
-pub trait FixedSizeBinaryValues {
- fn values(&self) -> &[u8];
- fn size(&self) -> usize;
-}
-
-impl FixedSizeBinaryValues for FixedSizeBinaryArray {
- #[inline]
- fn values(&self) -> &[u8] {
- &self.values
- }
-
- #[inline]
- fn size(&self) -> usize {
- self.size
- }
-}
diff --git a/crates/polars-arrow/src/array/fixed_size_binary/mutable.rs b/crates/polars-arrow/src/array/fixed_size_binary/mutable.rs
index aba3904bef4e..8f81ce86f6d8 100644
--- a/crates/polars-arrow/src/array/fixed_size_binary/mutable.rs
+++ b/crates/polars-arrow/src/array/fixed_size_binary/mutable.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
use polars_error::{polars_bail, PolarsResult};
-use super::{FixedSizeBinaryArray, FixedSizeBinaryValues};
+use super::FixedSizeBinaryArray;
use crate::array::physical_binary::extend_validity;
use crate::array::{Array, MutableArray, TryExtendFromSelf};
use crate::bitmap::MutableBitmap;
@@ -200,6 +200,7 @@ impl MutableFixedSizeBinaryArray {
}
/// Returns the element at index `i` as `&[u8]`
+ ///
/// # Safety
/// Assumes that the `i < self.len`.
#[inline]
@@ -289,18 +290,6 @@ impl MutableArray for MutableFixedSizeBinaryArray {
}
}
-impl FixedSizeBinaryValues for MutableFixedSizeBinaryArray {
- #[inline]
- fn values(&self) -> &[u8] {
- &self.values
- }
-
- #[inline]
- fn size(&self) -> usize {
- self.size
- }
-}
-
impl PartialEq for MutableFixedSizeBinaryArray {
fn eq(&self, other: &Self) -> bool {
self.iter().eq(other.iter())
diff --git a/crates/polars-arrow/src/array/fixed_size_list/data.rs b/crates/polars-arrow/src/array/fixed_size_list/data.rs
index 5a0b902e9b51..f98fa452c6ea 100644
--- a/crates/polars-arrow/src/array/fixed_size_list/data.rs
+++ b/crates/polars-arrow/src/array/fixed_size_list/data.rs
@@ -12,7 +12,7 @@ impl Arrow2Arrow for FixedSizeListArray {
.nulls(self.validity.as_ref().map(|b| b.clone().into()))
.child_data(vec![to_data(self.values.as_ref())]);
- // Safety: Array is valid
+ // SAFETY: Array is valid
unsafe { builder.build_unchecked() }
}
diff --git a/crates/polars-arrow/src/array/fixed_size_list/mod.rs b/crates/polars-arrow/src/array/fixed_size_list/mod.rs
index 612e134a5a5d..2cefb2e8ddaf 100644
--- a/crates/polars-arrow/src/array/fixed_size_list/mod.rs
+++ b/crates/polars-arrow/src/array/fixed_size_list/mod.rs
@@ -111,6 +111,7 @@ impl FixedSizeListArray {
/// Slices this [`FixedSizeListArray`].
/// # Implementation
/// This operation is `O(1)`.
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`.
pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) {
@@ -156,6 +157,7 @@ impl FixedSizeListArray {
}
/// Returns the `Vec` at position `i`.
+ ///
/// # Safety
/// Caller must ensure that `i < self.len()`
#[inline]
diff --git a/crates/polars-arrow/src/array/growable/binview.rs b/crates/polars-arrow/src/array/growable/binview.rs
index 76cf75e8bfa7..d167a09a8fe8 100644
--- a/crates/polars-arrow/src/array/growable/binview.rs
+++ b/crates/polars-arrow/src/array/growable/binview.rs
@@ -1,5 +1,10 @@
+use std::hash::{Hash, Hasher};
use std::sync::Arc;
+use polars_utils::aliases::PlIndexSet;
+use polars_utils::slice::GetSaferUnchecked;
+use polars_utils::unwrap::UnwrapUncheckedRelease;
+
use super::Growable;
use crate::array::binview::{BinaryViewArrayGeneric, View, ViewType};
use crate::array::growable::utils::{extend_validity, prepare_validity};
@@ -8,14 +13,35 @@ use crate::bitmap::MutableBitmap;
use crate::buffer::Buffer;
use crate::datatypes::ArrowDataType;
+struct BufferKey<'a> {
+ inner: &'a Buffer,
+}
+
+impl Hash for BufferKey<'_> {
+ fn hash(&self, state: &mut H) {
+ state.write_u64(self.inner.as_ptr() as u64)
+ }
+}
+
+impl PartialEq for BufferKey<'_> {
+ #[inline]
+ fn eq(&self, other: &Self) -> bool {
+ self.inner.as_ptr() == other.inner.as_ptr()
+ }
+}
+
+impl Eq for BufferKey<'_> {}
+
/// Concrete [`Growable`] for the [`BinaryArray`].
pub struct GrowableBinaryViewArray<'a, T: ViewType + ?Sized> {
arrays: Vec<&'a BinaryViewArrayGeneric>,
data_type: ArrowDataType,
validity: Option,
views: Vec,
- buffers: Vec>,
- buffers_idx_offsets: Vec,
+ // We need to use a set/hashmap to deduplicate
+ // A growable can be called with many chunks from self.
+ // See: #14201
+ buffers: PlIndexSet>,
total_bytes_len: usize,
total_buffer_len: usize,
}
@@ -37,21 +63,16 @@ impl<'a, T: ViewType + ?Sized> GrowableBinaryViewArray<'a, T> {
use_validity = true;
};
- let mut cum_sum = 0;
- let cum_offset = arrays
- .iter()
- .map(|binview| {
- let out = cum_sum;
- cum_sum += binview.data_buffers().len() as u32;
- out
- })
- .collect::>();
-
let buffers = arrays
.iter()
- .flat_map(|array| array.data_buffers().as_ref())
- .cloned()
- .collect::>();
+ .flat_map(|array| {
+ array
+ .data_buffers()
+ .as_ref()
+ .iter()
+ .map(|buf| BufferKey { inner: buf })
+ })
+ .collect::>();
let total_buffer_len = arrays
.iter()
.map(|arr| arr.data_buffers().len())
@@ -63,7 +84,6 @@ impl<'a, T: ViewType + ?Sized> GrowableBinaryViewArray<'a, T> {
validity: prepare_validity(use_validity, capacity),
views: Vec::with_capacity(capacity),
buffers,
- buffers_idx_offsets: cum_offset,
total_bytes_len: 0,
total_buffer_len,
}
@@ -77,7 +97,12 @@ impl<'a, T: ViewType + ?Sized> GrowableBinaryViewArray<'a, T> {
BinaryViewArrayGeneric::::new_unchecked(
self.data_type.clone(),
views.into(),
- Arc::from(buffers),
+ Arc::from(
+ buffers
+ .into_iter()
+ .map(|buf| buf.inner.clone())
+ .collect::>(),
+ ),
validity.map(|v| v.into()),
self.total_bytes_len,
self.total_buffer_len,
@@ -90,6 +115,7 @@ impl<'a, T: ViewType + ?Sized> GrowableBinaryViewArray<'a, T> {
/// doesn't check bounds
pub unsafe fn extend_unchecked(&mut self, index: usize, start: usize, len: usize) {
let array = *self.arrays.get_unchecked(index);
+ let local_buffers = array.data_buffers();
extend_validity(&mut self.validity, array, start, len);
@@ -102,8 +128,11 @@ impl<'a, T: ViewType + ?Sized> GrowableBinaryViewArray<'a, T> {
self.total_bytes_len += len;
if len > 12 {
- let buffer_idx = *self.buffers_idx_offsets.get_unchecked(index);
- view.buffer_idx += buffer_idx;
+ let buffer = local_buffers.get_unchecked_release(view.buffer_idx as usize);
+ let key = BufferKey { inner: buffer };
+ let idx = self.buffers.get_full(&key).unwrap_unchecked_release().0;
+
+ view.buffer_idx = idx as u32;
}
view
}));
@@ -111,6 +140,7 @@ impl<'a, T: ViewType + ?Sized> GrowableBinaryViewArray<'a, T> {
#[inline]
/// Ignores the buffers and doesn't update the view. This is only correct in a filter.
+ ///
/// # Safety
/// doesn't check bounds
pub unsafe fn extend_unchecked_no_buffers(&mut self, index: usize, start: usize, len: usize) {
@@ -163,7 +193,12 @@ impl<'a, T: ViewType + ?Sized> From> for BinaryVi
BinaryViewArrayGeneric::::new_unchecked(
val.data_type,
val.views.into(),
- Arc::from(val.buffers),
+ Arc::from(
+ val.buffers
+ .into_iter()
+ .map(|buf| buf.inner.clone())
+ .collect::>(),
+ ),
val.validity.map(|v| v.into()),
val.total_bytes_len,
val.total_buffer_len,
diff --git a/crates/polars-arrow/src/array/growable/boolean.rs b/crates/polars-arrow/src/array/growable/boolean.rs
index e293d0051ca8..ea18791a804d 100644
--- a/crates/polars-arrow/src/array/growable/boolean.rs
+++ b/crates/polars-arrow/src/array/growable/boolean.rs
@@ -57,7 +57,7 @@ impl<'a> Growable<'a> for GrowableBoolean<'a> {
let values = array.values();
let (slice, offset, _) = values.as_slice();
- // safety: invariant offset + length <= slice.len()
+ // SAFETY: invariant offset + length <= slice.len()
unsafe {
self.values
.extend_from_slice_unchecked(slice, start + offset, len);
diff --git a/crates/polars-arrow/src/array/growable/dictionary.rs b/crates/polars-arrow/src/array/growable/dictionary.rs
index 3c08b1cd65d9..dd2dbc01fde4 100644
--- a/crates/polars-arrow/src/array/growable/dictionary.rs
+++ b/crates/polars-arrow/src/array/growable/dictionary.rs
@@ -82,7 +82,7 @@ impl<'a, T: DictionaryKey> GrowableDictionary<'a, T> {
validity.map(|v| v.into()),
);
- // Safety - the invariant of this struct ensures that this is up-held
+ // SAFETY: the invariant of this struct ensures that this is up-held
unsafe {
DictionaryArray::::try_new_unchecked(
self.data_type.clone(),
diff --git a/crates/polars-arrow/src/array/growable/mod.rs b/crates/polars-arrow/src/array/growable/mod.rs
index aea9cdd8789e..ca8fc87a5a86 100644
--- a/crates/polars-arrow/src/array/growable/mod.rs
+++ b/crates/polars-arrow/src/array/growable/mod.rs
@@ -1,8 +1,6 @@
//! Contains the trait [`Growable`] and corresponding concreate implementations, one per concrete array,
//! that offer the ability to create a new [`Array`] out of slices of existing [`Array`]s.
-use std::sync::Arc;
-
use crate::array::*;
use crate::datatypes::*;
@@ -37,11 +35,13 @@ mod utils;
pub trait Growable<'a> {
/// Extends this [`Growable`] with elements from the bounded [`Array`] at index `index` from
/// a slice starting at `start` and length `len`.
+ ///
/// # Safety
/// Doesn't do any bound checks
unsafe fn extend(&mut self, index: usize, start: usize, len: usize);
/// Extends this [`Growable`] with null elements, disregarding the bound arrays
+ ///
/// # Safety
/// Doesn't do any bound checks
fn extend_validity(&mut self, additional: usize);
diff --git a/crates/polars-arrow/src/array/growable/utils.rs b/crates/polars-arrow/src/array/growable/utils.rs
index 7357f661b199..7cb4b667a5c1 100644
--- a/crates/polars-arrow/src/array/growable/utils.rs
+++ b/crates/polars-arrow/src/array/growable/utils.rs
@@ -38,7 +38,7 @@ pub(super) fn extend_validity(
Some(validity) => {
debug_assert!(start + len <= validity.len());
let (slice, offset, _) = validity.as_slice();
- // safety: invariant offset + length <= slice.len()
+ // SAFETY: invariant offset + length <= slice.len()
unsafe {
mutable_validity.extend_from_slice_unchecked(slice, start + offset, len);
}
diff --git a/crates/polars-arrow/src/array/indexable.rs b/crates/polars-arrow/src/array/indexable.rs
index d3f466722aa6..b4f455ab00c4 100644
--- a/crates/polars-arrow/src/array/indexable.rs
+++ b/crates/polars-arrow/src/array/indexable.rs
@@ -22,6 +22,7 @@ pub trait Indexable {
fn value_at(&self, index: usize) -> Self::Value<'_>;
/// Returns the element at index `i`.
+ ///
/// # Safety
/// Assumes that the `i < self.len`.
#[inline]
diff --git a/crates/polars-arrow/src/array/list/data.rs b/crates/polars-arrow/src/array/list/data.rs
index 6f3424c96ce6..212778a05abb 100644
--- a/crates/polars-arrow/src/array/list/data.rs
+++ b/crates/polars-arrow/src/array/list/data.rs
@@ -14,7 +14,7 @@ impl Arrow2Arrow for ListArray {
.nulls(self.validity.as_ref().map(|b| b.clone().into()))
.child_data(vec![to_data(self.values.as_ref())]);
- // Safety: Array is valid
+ // SAFETY: Array is valid
unsafe { builder.build_unchecked() }
}
diff --git a/crates/polars-arrow/src/array/list/mod.rs b/crates/polars-arrow/src/array/list/mod.rs
index d53747e7f228..6c2934fa1061 100644
--- a/crates/polars-arrow/src/array/list/mod.rs
+++ b/crates/polars-arrow/src/array/list/mod.rs
@@ -114,6 +114,7 @@ impl ListArray {
}
/// Slices this [`ListArray`].
+ ///
/// # Safety
/// The caller must ensure that `offset + length < self.len()`.
pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) {
@@ -144,20 +145,21 @@ impl ListArray {
#[inline]
pub fn value(&self, i: usize) -> Box {
assert!(i < self.len());
- // Safety: invariant of this function
+ // SAFETY: invariant of this function
unsafe { self.value_unchecked(i) }
}
/// Returns the element at index `i` as &str
+ ///
/// # Safety
/// Assumes that the `i < self.len`.
#[inline]
pub unsafe fn value_unchecked(&self, i: usize) -> Box {
- // safety: the invariant of the function
+ // SAFETY: the invariant of the function
let (start, end) = self.offsets.start_end_unchecked(i);
let length = end - start;
- // safety: the invariant of the struct
+ // SAFETY: the invariant of the struct
self.values.sliced_unchecked(start, length)
}
diff --git a/crates/polars-arrow/src/array/map/data.rs b/crates/polars-arrow/src/array/map/data.rs
index cb8862a4df3d..8eb586e05f4c 100644
--- a/crates/polars-arrow/src/array/map/data.rs
+++ b/crates/polars-arrow/src/array/map/data.rs
@@ -14,7 +14,7 @@ impl Arrow2Arrow for MapArray {
.nulls(self.validity.as_ref().map(|b| b.clone().into()))
.child_data(vec![to_data(self.field.as_ref())]);
- // Safety: Array is valid
+ // SAFETY: Array is valid
unsafe { builder.build_unchecked() }
}
diff --git a/crates/polars-arrow/src/array/map/iterator.rs b/crates/polars-arrow/src/array/map/iterator.rs
index f424e91b8043..558405ddc8de 100644
--- a/crates/polars-arrow/src/array/map/iterator.rs
+++ b/crates/polars-arrow/src/array/map/iterator.rs
@@ -32,7 +32,7 @@ impl<'a> Iterator for MapValuesIter<'a> {
}
let old = self.index;
self.index += 1;
- // Safety:
+ // SAFETY:
// self.end is maximized by the length of the array
Some(unsafe { self.array.value_unchecked(old) })
}
@@ -52,7 +52,7 @@ impl<'a> DoubleEndedIterator for MapValuesIter<'a> {
None
} else {
self.end -= 1;
- // Safety:
+ // SAFETY:
// self.end is maximized by the length of the array
Some(unsafe { self.array.value_unchecked(self.end) })
}
diff --git a/crates/polars-arrow/src/array/map/mod.rs b/crates/polars-arrow/src/array/map/mod.rs
index d057192ef612..c6ebfc353a06 100644
--- a/crates/polars-arrow/src/array/map/mod.rs
+++ b/crates/polars-arrow/src/array/map/mod.rs
@@ -111,6 +111,7 @@ impl MapArray {
}
/// Returns a slice of this [`MapArray`].
+ ///
/// # Safety
/// The caller must ensure that `offset + length < self.len()`.
#[inline]
@@ -168,6 +169,7 @@ impl MapArray {
}
/// Returns the element at index `i`.
+ ///
/// # Safety
/// Assumes that the `i < self.len`.
#[inline]
diff --git a/crates/polars-arrow/src/array/mod.rs b/crates/polars-arrow/src/array/mod.rs
index 24845b33bc89..b2a109f5af39 100644
--- a/crates/polars-arrow/src/array/mod.rs
+++ b/crates/polars-arrow/src/array/mod.rs
@@ -75,6 +75,7 @@ pub trait Array: Send + Sync + dyn_clone::DynClone + 'static {
}
/// Returns whether slot `i` is null.
+ ///
/// # Safety
/// The caller must ensure `i < self.len()`
#[inline]
@@ -103,6 +104,7 @@ pub trait Array: Send + Sync + dyn_clone::DynClone + 'static {
/// Slices the [`Array`].
/// # Implementation
/// This operation is `O(1)`.
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`
unsafe fn slice_unchecked(&mut self, offset: usize, length: usize);
@@ -123,6 +125,7 @@ pub trait Array: Send + Sync + dyn_clone::DynClone + 'static {
/// # Implementation
/// This operation is `O(1)` over `len`, as it amounts to increase two ref counts
/// and moving the struct to the heap.
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`
#[must_use]
@@ -143,15 +146,6 @@ pub trait Array: Send + Sync + dyn_clone::DynClone + 'static {
dyn_clone::clone_trait_object!(Array);
-/// A trait describing an array with a backing store that can be preallocated to
-/// a given size.
-pub(crate) trait Container {
- /// Create this array with a given capacity.
- fn with_capacity(capacity: usize) -> Self
- where
- Self: Sized;
-}
-
/// A trait describing a mutable array; i.e. an array whose values can be changed.
/// Mutable arrays cannot be cloned but can be mutated in place,
/// thereby making them useful to perform numeric operations without allocations.
@@ -496,6 +490,7 @@ macro_rules! impl_sliced {
/// Returns this array sliced.
/// # Implementation
/// This function is `O(1)`.
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`.
#[inline]
@@ -530,6 +525,12 @@ macro_rules! impl_mut_validity {
}
self.validity = validity;
}
+
+ /// Takes the validity of this array, leaving it without a validity mask.
+ #[inline]
+ pub fn take_validity(&mut self) -> Option {
+ self.validity.take()
+ }
}
}
@@ -740,6 +741,7 @@ pub trait TryPush {
/// A trait describing the ability of a struct to receive new items.
pub trait PushUnchecked {
/// Push a new element that holds the invariants of the struct.
+ ///
/// # Safety
/// The items must uphold the invariants of the struct
/// Read the specific implementation of the trait to understand what these are.
diff --git a/crates/polars-arrow/src/array/null.rs b/crates/polars-arrow/src/array/null.rs
index 82269e3c2066..753700abba16 100644
--- a/crates/polars-arrow/src/array/null.rs
+++ b/crates/polars-arrow/src/array/null.rs
@@ -62,6 +62,7 @@ impl NullArray {
}
/// Returns a slice of the [`NullArray`].
+ ///
/// # Safety
/// The caller must ensure that `offset + length < self.len()`.
pub unsafe fn slice_unchecked(&mut self, _offset: usize, length: usize) {
@@ -187,7 +188,7 @@ mod arrow {
pub fn to_data(&self) -> ArrayData {
let builder = ArrayDataBuilder::new(arrow_schema::DataType::Null).len(self.len());
- // Safety: safe by construction
+ // SAFETY: safe by construction
unsafe { builder.build_unchecked() }
}
diff --git a/crates/polars-arrow/src/array/physical_binary.rs b/crates/polars-arrow/src/array/physical_binary.rs
index 36e4ecf52d35..d92085d11d25 100644
--- a/crates/polars-arrow/src/array/physical_binary.rs
+++ b/crates/polars-arrow/src/array/physical_binary.rs
@@ -219,7 +219,7 @@ pub(crate) fn extend_validity(
if let Some(other) = other {
if let Some(validity) = validity {
let slice = other.as_slice();
- // safety: invariant offset + length <= slice.len()
+ // SAFETY: invariant offset + length <= slice.len()
unsafe { validity.extend_from_slice_unchecked(slice, 0, other.len()) }
} else {
let mut new_validity = MutableBitmap::from_len_set(length);
diff --git a/crates/polars-arrow/src/array/primitive/data.rs b/crates/polars-arrow/src/array/primitive/data.rs
index d4879f796812..1a32b230f54f 100644
--- a/crates/polars-arrow/src/array/primitive/data.rs
+++ b/crates/polars-arrow/src/array/primitive/data.rs
@@ -14,7 +14,7 @@ impl Arrow2Arrow for PrimitiveArray {
.buffers(vec![self.values.clone().into()])
.nulls(self.validity.as_ref().map(|b| b.clone().into()));
- // Safety: Array is valid
+ // SAFETY: Array is valid
unsafe { builder.build_unchecked() }
}
diff --git a/crates/polars-arrow/src/array/primitive/from_natural.rs b/crates/polars-arrow/src/array/primitive/from_natural.rs
index 0530c748af7e..a70259a8eeff 100644
--- a/crates/polars-arrow/src/array/primitive/from_natural.rs
+++ b/crates/polars-arrow/src/array/primitive/from_natural.rs
@@ -1,5 +1,3 @@
-use std::iter::FromIterator;
-
use super::{MutablePrimitiveArray, PrimitiveArray};
use crate::types::NativeType;
diff --git a/crates/polars-arrow/src/array/primitive/mod.rs b/crates/polars-arrow/src/array/primitive/mod.rs
index c85cbc8420b3..8f70d233baeb 100644
--- a/crates/polars-arrow/src/array/primitive/mod.rs
+++ b/crates/polars-arrow/src/array/primitive/mod.rs
@@ -208,6 +208,7 @@ impl PrimitiveArray {
/// Returns the value at index `i`.
/// The value on null slots is undetermined (it can be anything).
+ ///
/// # Safety
/// Caller must be sure that `i < self.len()`
#[inline]
@@ -243,6 +244,7 @@ impl PrimitiveArray {
/// Slices this [`PrimitiveArray`] by an offset and length.
/// # Implementation
/// This operation is `O(1)`.
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`.
#[inline]
@@ -341,7 +343,7 @@ impl PrimitiveArray {
/// This function returns a [`MutablePrimitiveArray`] (via [`std::sync::Arc::get_mut`]) iff both values
/// and validity have not been cloned / are unique references to their underlying vectors.
///
- /// This function is primarily used to re-use memory regions.
+ /// This function is primarily used to reuse memory regions.
#[must_use]
pub fn into_mut(self) -> Either> {
use Either::*;
@@ -420,6 +422,7 @@ impl PrimitiveArray {
}
/// Creates a new [`PrimitiveArray`] from an iterator over values
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -433,6 +436,7 @@ impl PrimitiveArray {
}
/// Creates a [`PrimitiveArray`] from an iterator of optional values.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -448,6 +452,37 @@ impl PrimitiveArray {
pub fn new(data_type: ArrowDataType, values: Buffer, validity: Option) -> Self {
Self::try_new(data_type, values, validity).unwrap()
}
+
+ /// Transmute this PrimitiveArray into another PrimitiveArray.
+ ///
+ /// T and U must have the same size and alignment.
+ pub fn transmute(self) -> PrimitiveArray {
+ let PrimitiveArray {
+ values, validity, ..
+ } = self;
+
+ // SAFETY: this is fine, we checked size and alignment, and NativeType
+ // is always Pod.
+ assert_eq!(std::mem::size_of::(), std::mem::size_of::());
+ assert_eq!(std::mem::align_of::(), std::mem::align_of::());
+ let new_values = unsafe { std::mem::transmute::, Buffer>(values) };
+ PrimitiveArray::new(U::PRIMITIVE.into(), new_values, validity)
+ }
+
+ /// Fills this entire array with the given value, leaving the validity mask intact.
+ ///
+ /// Reuses the memory of the PrimitiveArray if possible.
+ pub fn fill_with(mut self, value: T) -> Self {
+ if let Some(values) = self.get_mut_values() {
+ for x in values.iter_mut() {
+ *x = value;
+ }
+ self
+ } else {
+ let values = vec![value; self.len()];
+ Self::new(T::PRIMITIVE.into(), values.into(), self.validity)
+ }
+ }
}
impl Array for PrimitiveArray {
diff --git a/crates/polars-arrow/src/array/primitive/mutable.rs b/crates/polars-arrow/src/array/primitive/mutable.rs
index 3c7a8489b77e..eab3498d8ed9 100644
--- a/crates/polars-arrow/src/array/primitive/mutable.rs
+++ b/crates/polars-arrow/src/array/primitive/mutable.rs
@@ -1,8 +1,6 @@
-use std::iter::FromIterator;
use std::sync::Arc;
use polars_error::PolarsResult;
-use polars_utils::total_ord::TotalOrdWrap;
use super::{check, PrimitiveArray};
use crate::array::physical_binary::extend_validity;
@@ -195,6 +193,7 @@ impl MutablePrimitiveArray {
}
/// Extends the [`MutablePrimitiveArray`] from an iterator of trusted len.
+ ///
/// # Safety
/// The iterator must be trusted len.
#[inline]
@@ -224,6 +223,7 @@ impl MutablePrimitiveArray {
/// Extends the [`MutablePrimitiveArray`] from an iterator of values of trusted len.
/// This differs from `extend_trusted_len_unchecked` which accepts in iterator of optional values.
+ ///
/// # Safety
/// The iterator must be trusted len.
#[inline]
@@ -312,7 +312,7 @@ impl MutablePrimitiveArray {
/// Panics iff index is larger than `self.len()`.
pub fn set(&mut self, index: usize, value: Option) {
assert!(index < self.len());
- // Safety:
+ // SAFETY:
// we just checked bounds
unsafe { self.set_unchecked(index, value) }
}
@@ -320,6 +320,7 @@ impl MutablePrimitiveArray {
/// Sets position `index` to `value`.
/// Note that if it is the first time a null appears in this array,
/// this initializes the validity bitmap (`O(N)`).
+ ///
/// # Safety
/// Caller must ensure `index < self.len()`
pub unsafe fn set_unchecked(&mut self, index: usize, value: Option) {
@@ -364,14 +365,6 @@ impl Extend> for MutablePrimitiveArray {
}
}
-impl Extend>> for MutablePrimitiveArray {
- fn extend>>>(&mut self, iter: I) {
- let iter = iter.into_iter();
- self.reserve(iter.size_hint().0);
- iter.for_each(|x| self.push(x.map(|x| x.0)))
- }
-}
-
impl TryExtend> for MutablePrimitiveArray {
/// This is infallible and is implemented for consistency with all other types
fn try_extend>>(&mut self, iter: I) -> PolarsResult<()> {
@@ -448,6 +441,7 @@ impl MutablePrimitiveArray {
}
/// Creates a [`MutablePrimitiveArray`] from an iterator of trusted length.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. `size_hint().1` correctly reports its length.
@@ -477,6 +471,7 @@ impl MutablePrimitiveArray {
}
/// Creates a [`MutablePrimitiveArray`] from an fallible iterator of trusted length.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -525,6 +520,7 @@ impl MutablePrimitiveArray {
}
/// Creates a new [`MutablePrimitiveArray`] from an iterator over values
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
diff --git a/crates/polars-arrow/src/array/specification.rs b/crates/polars-arrow/src/array/specification.rs
index 72fb80ae2202..e5759144bf44 100644
--- a/crates/polars-arrow/src/array/specification.rs
+++ b/crates/polars-arrow/src/array/specification.rs
@@ -93,7 +93,7 @@ pub fn try_check_utf8(offsets: &[O], values: &[u8]) -> PolarsResult<(
for start in starts {
let start = start.to_usize();
- // Safety: `try_check_offsets_bounds` just checked for bounds
+ // SAFETY: `try_check_offsets_bounds` just checked for bounds
let b = *unsafe { values.get_unchecked(start) };
// A valid code-point iff it does not start with 0b10xxxxxx
diff --git a/crates/polars-arrow/src/array/static_array.rs b/crates/polars-arrow/src/array/static_array.rs
index 4ae0f44212ba..ac8fbc4cec32 100644
--- a/crates/polars-arrow/src/array/static_array.rs
+++ b/crates/polars-arrow/src/array/static_array.rs
@@ -25,7 +25,7 @@ pub trait StaticArray:
type ZeroableValueT<'a>: Zeroable + From>
where
Self: 'a;
- type ValueIterT<'a>: Iterator
- > + TrustedLen
+ type ValueIterT<'a>: DoubleEndedIterator
- > + TrustedLen + Send + Sync
where
Self: 'a;
diff --git a/crates/polars-arrow/src/array/static_array_collect.rs b/crates/polars-arrow/src/array/static_array_collect.rs
index 27f86fec1f5b..9413b0a16778 100644
--- a/crates/polars-arrow/src/array/static_array_collect.rs
+++ b/crates/polars-arrow/src/array/static_array_collect.rs
@@ -552,7 +552,7 @@ impl ArrayFromIter> for BinaryViewArray {
// fn try_arr_from_iter_trusted(iter: I) -> Result
}
-/// We use this to re-use the binary collect implementation for strings.
+/// We use this to reuse the binary collect implementation for strings.
/// # Safety
/// The array must be valid UTF-8.
unsafe fn into_utf8array(arr: BinaryArray) -> Utf8Array {
@@ -807,12 +807,14 @@ impl ArrayFromIter> for BooleanArray {
// as Rust considers that AsRef for Option<&dyn Array> could be implemented.
trait AsArray {
fn as_array(&self) -> &dyn Array;
+ #[cfg(feature = "dtype-array")]
fn into_boxed_array(self) -> Box; // Prevents unnecessary re-boxing.
}
impl AsArray for Box {
fn as_array(&self) -> &dyn Array {
self.as_ref()
}
+ #[cfg(feature = "dtype-array")]
fn into_boxed_array(self) -> Box {
self
}
@@ -821,6 +823,7 @@ impl<'a> AsArray for &'a dyn Array {
fn as_array(&self) -> &'a dyn Array {
*self
}
+ #[cfg(feature = "dtype-array")]
fn into_boxed_array(self) -> Box {
self.to_boxed()
}
diff --git a/crates/polars-arrow/src/array/struct_/data.rs b/crates/polars-arrow/src/array/struct_/data.rs
index b96dc4ffe28b..4dfcb0010a73 100644
--- a/crates/polars-arrow/src/array/struct_/data.rs
+++ b/crates/polars-arrow/src/array/struct_/data.rs
@@ -12,7 +12,7 @@ impl Arrow2Arrow for StructArray {
.nulls(self.validity.as_ref().map(|b| b.clone().into()))
.child_data(self.values.iter().map(|x| to_data(x.as_ref())).collect());
- // Safety: Array is valid
+ // SAFETY: Array is valid
unsafe { builder.build_unchecked() }
}
diff --git a/crates/polars-arrow/src/array/struct_/iterator.rs b/crates/polars-arrow/src/array/struct_/iterator.rs
index cb8e6aafbb09..4e89af3a6a7f 100644
--- a/crates/polars-arrow/src/array/struct_/iterator.rs
+++ b/crates/polars-arrow/src/array/struct_/iterator.rs
@@ -31,7 +31,7 @@ impl<'a> Iterator for StructValueIter<'a> {
let old = self.index;
self.index += 1;
- // Safety:
+ // SAFETY:
// self.end is maximized by the length of the array
Some(
self.array
@@ -58,7 +58,7 @@ impl<'a> DoubleEndedIterator for StructValueIter<'a> {
} else {
self.end -= 1;
- // Safety:
+ // SAFETY:
// self.end is maximized by the length of the array
Some(
self.array
diff --git a/crates/polars-arrow/src/array/struct_/mod.rs b/crates/polars-arrow/src/array/struct_/mod.rs
index 6f796ac18ac4..21d3247bbc85 100644
--- a/crates/polars-arrow/src/array/struct_/mod.rs
+++ b/crates/polars-arrow/src/array/struct_/mod.rs
@@ -178,6 +178,7 @@ impl StructArray {
/// Slices this [`StructArray`].
/// # Implementation
/// This operation is `O(F)` where `F` is the number of fields.
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`.
pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) {
diff --git a/crates/polars-arrow/src/array/union/data.rs b/crates/polars-arrow/src/array/union/data.rs
index 8bbac00b6c6e..4303ab7b4356 100644
--- a/crates/polars-arrow/src/array/union/data.rs
+++ b/crates/polars-arrow/src/array/union/data.rs
@@ -25,7 +25,7 @@ impl Arrow2Arrow for UnionArray {
),
};
- // Safety: Array is valid
+ // SAFETY: Array is valid
unsafe { builder.build_unchecked() }
}
diff --git a/crates/polars-arrow/src/array/union/mod.rs b/crates/polars-arrow/src/array/union/mod.rs
index 703a5f8d86e3..86d7cbed7397 100644
--- a/crates/polars-arrow/src/array/union/mod.rs
+++ b/crates/polars-arrow/src/array/union/mod.rs
@@ -130,7 +130,7 @@ impl UnionArray {
Some(hash)
} else {
- // Safety: every type in types is smaller than number of fields
+ // SAFETY: every type in types is smaller than number of fields
let mut is_valid = true;
for &type_ in types.iter() {
if type_ < 0 || type_ >= number_of_fields {
@@ -240,6 +240,7 @@ impl UnionArray {
/// Returns a slice of this [`UnionArray`].
/// # Implementation
/// This operation is `O(F)` where `F` is the number of fields.
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`.
#[inline]
@@ -296,20 +297,21 @@ impl UnionArray {
/// Returns the index and slot of the field to select from `self.fields`.
/// The first value is guaranteed to be `< self.fields().len()`
+ ///
/// # Safety
/// This function is safe iff `index < self.len`.
#[inline]
pub unsafe fn index_unchecked(&self, index: usize) -> (usize, usize) {
debug_assert!(index < self.len());
- // Safety: assumption of the function
+ // SAFETY: assumption of the function
let type_ = unsafe { *self.types.get_unchecked(index) };
- // Safety: assumption of the struct
+ // SAFETY: assumption of the struct
let type_ = self
.map
.as_ref()
.map(|map| unsafe { *map.get_unchecked(type_ as usize) })
.unwrap_or(type_ as usize);
- // Safety: assumption of the function
+ // SAFETY: assumption of the function
let index = self.field_slot_unchecked(index);
(type_, index)
}
@@ -323,12 +325,13 @@ impl UnionArray {
}
/// Returns the slot `index` as a [`Scalar`].
+ ///
/// # Safety
/// This function is safe iff `i < self.len`.
pub unsafe fn value_unchecked(&self, index: usize) -> Box {
debug_assert!(index < self.len());
let (type_, index) = self.index_unchecked(index);
- // Safety: assumption of the struct
+ // SAFETY: assumption of the struct
debug_assert!(type_ < self.fields.len());
let field = self.fields.get_unchecked(type_).as_ref();
new_scalar(field, index)
diff --git a/crates/polars-arrow/src/array/utf8/data.rs b/crates/polars-arrow/src/array/utf8/data.rs
index 16674c969372..577a43677c05 100644
--- a/crates/polars-arrow/src/array/utf8/data.rs
+++ b/crates/polars-arrow/src/array/utf8/data.rs
@@ -15,7 +15,7 @@ impl Arrow2Arrow for Utf8Array {
])
.nulls(self.validity.as_ref().map(|b| b.clone().into()));
- // Safety: Array is valid
+ // SAFETY: Array is valid
unsafe { builder.build_unchecked() }
}
@@ -28,7 +28,7 @@ impl Arrow2Arrow for Utf8Array {
let buffers = data.buffers();
- // Safety: ArrayData is valid
+ // SAFETY: ArrayData is valid
let mut offsets = unsafe { OffsetsBuffer::new_unchecked(buffers[0].clone().into()) };
offsets.slice(data.offset(), data.len() + 1);
diff --git a/crates/polars-arrow/src/array/utf8/from.rs b/crates/polars-arrow/src/array/utf8/from.rs
index c1dcaf09b10d..6f90bac99495 100644
--- a/crates/polars-arrow/src/array/utf8/from.rs
+++ b/crates/polars-arrow/src/array/utf8/from.rs
@@ -1,5 +1,3 @@
-use std::iter::FromIterator;
-
use super::{MutableUtf8Array, Utf8Array};
use crate::offset::Offset;
diff --git a/crates/polars-arrow/src/array/utf8/mod.rs b/crates/polars-arrow/src/array/utf8/mod.rs
index c25234c3cb2a..218e71323abf 100644
--- a/crates/polars-arrow/src/array/utf8/mod.rs
+++ b/crates/polars-arrow/src/array/utf8/mod.rs
@@ -156,6 +156,7 @@ impl Utf8Array {
}
/// Returns the value of the element at index `i`, ignoring the array's validity.
+ ///
/// # Safety
/// This function is safe iff `i < self.len`.
#[inline]
@@ -223,6 +224,7 @@ impl Utf8Array {
/// Slices this [`Utf8Array`].
/// # Implementation
/// This function is `O(1)`
+ ///
/// # Safety
/// The caller must ensure that `offset + length <= self.len()`.
pub unsafe fn slice_unchecked(&mut self, offset: usize, length: usize) {
@@ -256,7 +258,7 @@ impl Utf8Array {
use Either::*;
if let Some(bitmap) = self.validity {
match bitmap.into_mut() {
- // Safety: invariants are preserved
+ // SAFETY: invariants are preserved
Left(bitmap) => Left(unsafe {
Utf8Array::new_unchecked(
self.data_type,
@@ -267,7 +269,7 @@ impl Utf8Array {
}),
Right(mutable_bitmap) => match (self.values.into_mut(), self.offsets.into_mut()) {
(Left(values), Left(offsets)) => {
- // Safety: invariants are preserved
+ // SAFETY: invariants are preserved
Left(unsafe {
Utf8Array::new_unchecked(
self.data_type,
@@ -278,7 +280,7 @@ impl Utf8Array {
})
},
(Left(values), Right(offsets)) => {
- // Safety: invariants are preserved
+ // SAFETY: invariants are preserved
Left(unsafe {
Utf8Array::new_unchecked(
self.data_type,
@@ -289,7 +291,7 @@ impl Utf8Array {
})
},
(Right(values), Left(offsets)) => {
- // Safety: invariants are preserved
+ // SAFETY: invariants are preserved
Left(unsafe {
Utf8Array::new_unchecked(
self.data_type,
@@ -362,6 +364,7 @@ impl Utf8Array {
/// * The last offset is not equal to the values' length.
/// * the validity's length is not equal to `offsets.len()`.
/// * The `data_type`'s [`crate::datatypes::PhysicalType`] is not equal to either `Utf8` or `LargeUtf8`.
+ ///
/// # Safety
/// This function is unsound iff:
/// * The `values` between two consecutive `offsets` are not valid utf8
@@ -430,6 +433,7 @@ impl Utf8Array {
}
/// Creates a [`Utf8Array`] from an iterator of trusted length.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -453,6 +457,7 @@ impl Utf8Array {
}
/// Creates a [`Utf8Array`] from an falible iterator of trusted length.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
diff --git a/crates/polars-arrow/src/array/utf8/mutable.rs b/crates/polars-arrow/src/array/utf8/mutable.rs
index a67d2e00b4c2..ef9a5e8527b7 100644
--- a/crates/polars-arrow/src/array/utf8/mutable.rs
+++ b/crates/polars-arrow/src/array/utf8/mutable.rs
@@ -1,4 +1,3 @@
-use std::iter::FromIterator;
use std::sync::Arc;
use polars_error::{polars_bail, PolarsResult};
@@ -75,6 +74,7 @@ impl MutableUtf8Array {
}
/// Create a [`MutableUtf8Array`] out of low-end APIs.
+ ///
/// # Safety
/// The caller must ensure that every value between offsets is a valid utf8.
/// # Panics
@@ -145,14 +145,13 @@ impl MutableUtf8Array {
}
/// Returns the value of the element at index `i`, ignoring the array's validity.
- /// # Safety
- /// This function is safe iff `i < self.len`.
#[inline]
pub fn value(&self, i: usize) -> &str {
self.values.value(i)
}
/// Returns the value of the element at index `i`, ignoring the array's validity.
+ ///
/// # Safety
/// This function is safe iff `i < self.len`.
#[inline]
@@ -329,6 +328,7 @@ impl MutableUtf8Array {
/// Extends the [`MutableUtf8Array`] from an iterator of values of trusted len.
/// This differs from `extended_trusted_len_unchecked` which accepts iterator of optional
/// values.
+ ///
/// # Safety
/// The iterator must be trusted len.
#[inline]
@@ -357,6 +357,7 @@ impl MutableUtf8Array {
}
/// Extends [`MutableUtf8Array`] from an iterator of trusted len.
+ ///
/// # Safety
/// The iterator must be trusted len.
#[inline]
@@ -376,6 +377,7 @@ impl MutableUtf8Array {
}
/// Creates a [`MutableUtf8Array`] from an iterator of trusted length.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -404,6 +406,7 @@ impl MutableUtf8Array {
}
/// Creates a [`MutableUtf8Array`] from an iterator of trusted length of `&str`.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
@@ -440,6 +443,7 @@ impl MutableUtf8Array {
}
/// Creates a [`MutableUtf8Array`] from an falible iterator of trusted length.
+ ///
/// # Safety
/// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
/// I.e. that `size_hint().1` correctly reports its length.
diff --git a/crates/polars-arrow/src/array/utf8/mutable_values.rs b/crates/polars-arrow/src/array/utf8/mutable_values.rs
index 92fd10661785..ce3c2f71f20c 100644
--- a/crates/polars-arrow/src/array/utf8/mutable_values.rs
+++ b/crates/polars-arrow/src/array/utf8/mutable_values.rs
@@ -1,4 +1,3 @@
-use std::iter::FromIterator;
use std::sync::Arc;
use polars_error::{polars_bail, PolarsResult};
@@ -23,7 +22,7 @@ pub struct MutableUtf8ValuesArray {
impl From> for Utf8Array {
fn from(other: MutableUtf8ValuesArray) -> Self {
- // Safety:
+ // SAFETY:
// `MutableUtf8ValuesArray` has the same invariants as `Utf8Array` and thus
// `Utf8Array` can be safely created from `MutableUtf8ValuesArray` without checks.
unsafe {
@@ -39,7 +38,7 @@ impl From> for Utf8Array {
impl From> for MutableUtf8Array {
fn from(other: MutableUtf8ValuesArray) -> Self {
- // Safety:
+ // SAFETY:
// `MutableUtf8ValuesArray` has the same invariants as `MutableUtf8Array`
unsafe {
MutableUtf8Array::::new_unchecked(other.data_type, other.offsets, other.values, None)
@@ -95,6 +94,7 @@ impl MutableUtf8ValuesArray {
/// This function does not panic iff:
/// * The last offset is equal to the values' length.
/// * The `data_type`'s [`crate::datatypes::PhysicalType`] is equal to either `Utf8` or `LargeUtf8`.
+ ///
/// # Safety
/// This function is safe iff:
/// * the offsets are monotonically increasing
@@ -187,7 +187,7 @@ impl MutableUtf8ValuesArray {
self.offsets.pop()?;
let start = self.offsets.last().to_usize();
let value = self.values.split_off(start);
- // Safety: utf8 is validated on initialization
+ // SAFETY: utf8 is validated on initialization
Some(unsafe { String::from_utf8_unchecked(value) })
}
@@ -201,6 +201,7 @@ impl MutableUtf8ValuesArray {
}
/// Returns the value of the element at index `i`.
+ ///
/// # Safety
/// This function is safe iff `i < self.len`.
#[inline]
@@ -309,6 +310,7 @@ impl MutableUtf8ValuesArray {
}
/// Extends [`MutableUtf8ValuesArray`] from an iterator of trusted len.
+ ///
/// # Safety
/// The iterator must be trusted len.
#[inline]
@@ -333,6 +335,7 @@ impl
+ 
+ 
@@ -40,12 +40,12 @@
-
R
|
- User Guide
+ User guide
|
Discord
-## Polars: Blazingly fast DataFrames in Rust, Python, Node.js, R and SQL
+## Polars: Blazingly fast DataFrames in Rust, Python, Node.js, R, and SQL
Polars is a DataFrame interface on top of an OLAP Query Engine implemented in Rust using
[Apache Arrow Columnar Format](https://arrow.apache.org/docs/format/Columnar.html) as the memory model.
@@ -55,10 +55,10 @@ Polars is a DataFrame interface on top of an OLAP Query Engine implemented in Ru
- SIMD
- Query optimization
- Powerful expression API
-- Hybrid Streaming (larger than RAM datasets)
+- Hybrid Streaming (larger-than-RAM datasets)
- Rust | Python | NodeJS | R | ...
-To learn more, read the [User Guide](https://docs.pola.rs/).
+To learn more, read the [user guide](https://docs.pola.rs/).
## Python
@@ -103,9 +103,9 @@ shape: (5, 8)
```python
>>> df = pl.scan_ipc("file.arrow")
->>> # create a sql context, registering the frame as a table
+>>> # create a SQL context, registering the frame as a table
>>> sql = pl.SQLContext(my_table=df)
->>> # create a sql query to execute
+>>> # create a SQL query to execute
>>> query = """
... SELECT sum(v1) as sum_v1, min(v2) as min_v2 FROM my_table
... WHERE id1 = 'id016'
@@ -136,7 +136,7 @@ shape: (5, 8)
SQL commands can also be run directly from your terminal using the Polars CLI:
```bash
-# run an inline sql query
+# run an inline SQL query
> polars -c "SELECT sum(v1) as sum_v1, min(v2) as min_v2 FROM read_ipc('file.arrow') WHERE id1 = 'id016' LIMIT 10"
# run interactively
@@ -156,7 +156,7 @@ Refer to the [Polars CLI repository](https://github.com/pola-rs/polars-cli) for
Polars is very fast. In fact, it is one of the best performing solutions available.
See the results in [DuckDB's db-benchmark](https://duckdblabs.github.io/db-benchmark/).
-In the [TPCH benchmarks](https://www.pola.rs/benchmarks.html) Polars is orders of magnitudes faster than pandas, dask, modin and vaex
+In the [TPC-H benchmarks](https://www.pola.rs/benchmarks.html) Polars is orders of magnitude faster than pandas, dask, modin and vaex
on full queries (including IO).
### Lightweight
@@ -167,18 +167,18 @@ Polars is also very lightweight. It comes with zero required dependencies, and t
- numpy: 104ms
- pandas: 520ms
-### Handles larger than RAM data
+### Handles larger-than-RAM data
-If you have data that does not fit into memory, polars lazy is able to process your query (or parts of your query) in a
-streaming fashion, this drastically reduces memory requirements so you might be able to process your 250GB dataset on your
-laptop. Collect with `collect(streaming=True)` to run the query streaming. (This might be a little slower, but
-it is still very fast!)
+If you have data that does not fit into memory, Polars' query engine is able to process your query (or parts of your query) in a streaming fashion.
+This drastically reduces memory requirements, so you might be able to process your 250GB dataset on your laptop.
+Collect with `collect(streaming=True)` to run the query streaming.
+(This might be a little slower, but it is still very fast!)
## Setup
### Python
-Install the latest polars version with:
+Install the latest Polars version with:
```sh
pip install polars
@@ -210,10 +210,10 @@ pip install 'polars[numpy,pandas,pyarrow]'
| openpyxl | Support for reading from Excel files with native types |
| deltalake | Support for reading and writing Delta Lake Tables |
| pyiceberg | Support for reading from Apache Iceberg tables |
-| plot | Support for plot functions on Dataframes |
-| timezone | Timezone support, only needed if are on Python<3.9 or you are on Windows |
+| plot | Support for plot functions on DataFrames |
+| timezone | Timezone support, only needed if you are on Python<3.9 or Windows |
-Releases happen quite often (weekly / every few days) at the moment, so updating polars regularly to get the latest bugfixes / features might not be a bad idea.
+Releases happen quite often (weekly / every few days) at the moment, so updating Polars regularly to get the latest bugfixes / features might not be a bad idea.
### Rust
@@ -224,7 +224,7 @@ point to the `main` branch of this repo.
polars = { git = "https://github.com/pola-rs/polars", rev = "
](https://www.jetbrains.com)
+[