Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions kernel/src/engine/arrow_conversion/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -336,21 +336,37 @@
))
}
}

Check warning on line 339 in kernel/src/engine/arrow_conversion/mod.rs

View workflow job for this annotation

GitHub Actions / format

Diff in /home/runner/work/delta-kernel-rs-v2/delta-kernel-rs-v2/kernel/src/engine/arrow_conversion/mod.rs

Check warning on line 339 in kernel/src/engine/arrow_conversion/mod.rs

View workflow job for this annotation

GitHub Actions / format

Diff in /home/runner/work/delta-kernel-rs-v2/delta-kernel-rs-v2/kernel/src/engine/arrow_conversion/mod.rs
impl TryFromKernel<&DataType> for ArrowDataType {
fn try_from_kernel(t: &DataType) -> Result<Self, ArrowError> {

// Scoped changes: Just for deltafusion compaction
// set env var DELTA_KERNEL_PREFER_LARGE_VARLEN to true to prefer large varlen types
// for deltafusion compaction, by default we prefer default varlen types.

let mut arrow_string_type = ArrowDataType::Utf8;
let mut arrow_binary_type = ArrowDataType::Binary;

let prefer_large_varlen = std::env::var("DELTA_KERNEL_PREFER_LARGE_VARLEN")
.map(|v| v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if prefer_large_varlen {
arrow_string_type = ArrowDataType::LargeUtf8;
arrow_binary_type = ArrowDataType::LargeBinary;
}
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

match t {
DataType::Primitive(p) => {
match p {
PrimitiveType::String => Ok(ArrowDataType::Utf8),
PrimitiveType::String => Ok(arrow_string_type),
PrimitiveType::Long => Ok(ArrowDataType::Int64), // undocumented type
PrimitiveType::Integer => Ok(ArrowDataType::Int32),
PrimitiveType::Short => Ok(ArrowDataType::Int16),
PrimitiveType::Byte => Ok(ArrowDataType::Int8),
PrimitiveType::Float => Ok(ArrowDataType::Float32),
PrimitiveType::Double => Ok(ArrowDataType::Float64),
PrimitiveType::Boolean => Ok(ArrowDataType::Boolean),
PrimitiveType::Binary => Ok(ArrowDataType::Binary),
PrimitiveType::Binary => Ok(arrow_binary_type),
PrimitiveType::Decimal(dtype) => Ok(ArrowDataType::Decimal128(
dtype.precision(),
dtype.scale() as i8, // 0..=38
Expand Down
69 changes: 42 additions & 27 deletions kernel/src/engine/arrow_expression/evaluate_expression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use crate::arrow::error::ArrowError;
use crate::arrow::json::writer::{make_encoder, EncoderOptions};
use crate::arrow::json::StructMode;
use crate::engine::arrow_conversion::{TryFromKernel, TryIntoArrow};
use crate::engine::arrow_data::as_string_accessor;
use crate::engine::arrow_expression::opaque::{
ArrowOpaqueExpressionOpAdaptor, ArrowOpaquePredicateOpAdaptor,
};
Expand Down Expand Up @@ -799,15 +800,9 @@ fn evaluate_map_to_struct(
.downcast_ref::<MapArray>()
.ok_or_else(|| Error::generic("MapToStruct requires a MapArray as input"))?;

let map_keys = map_array
.keys()
.as_any()
.downcast_ref::<StringArray>()
let map_keys = as_string_accessor(map_array.keys().as_ref())
.ok_or_else(|| Error::generic("MapToStruct requires maps with string keys"))?;
let map_values = map_array
.values()
.as_any()
.downcast_ref::<StringArray>()
let map_values = as_string_accessor(map_array.values().as_ref())
.ok_or_else(|| Error::generic("MapToStruct requires maps with string values"))?;

let num_rows = map_array.len();
Expand Down Expand Up @@ -2060,11 +2055,7 @@ mod tests {
let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap();
let structs = result.as_any().downcast_ref::<StructArray>().unwrap();

let regions = structs
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let regions = as_string_accessor(structs.column(0).as_ref()).unwrap();
let ids = structs
.column(1)
.as_any()
Expand All @@ -2087,7 +2078,7 @@ mod tests {
assert!(dates.is_null(1));

// Row 2: null map → all null
assert!(regions.is_null(2));
assert!(!regions.is_valid(2));
assert!(ids.is_null(2));
assert!(dates.is_null(2));
}
Expand All @@ -2101,14 +2092,10 @@ mod tests {
let expr = Expr::map_to_struct(column_expr!("pv"));
let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap();
let structs = result.as_any().downcast_ref::<StructArray>().unwrap();
let col = structs
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
assert!(col.is_null(0));
assert!(col.is_null(1));
assert!(col.is_null(2));
let col = as_string_accessor(structs.column(0).as_ref()).unwrap();
assert!(!col.is_valid(0));
assert!(!col.is_valid(1));
assert!(!col.is_valid(2));
}

#[test]
Expand Down Expand Up @@ -2157,11 +2144,7 @@ mod tests {
let expr = Expr::map_to_struct(column_expr!("pv"));
let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap();
let structs = result.as_any().downcast_ref::<StructArray>().unwrap();
let col = structs
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let col = as_string_accessor(structs.column(0).as_ref()).unwrap();
// Rightmost entry wins
assert_eq!(col.value(0), "last");
}
Expand Down Expand Up @@ -2272,6 +2255,38 @@ mod tests {
assert!(result.is_err());
}

#[test]
fn test_map_to_struct_large_utf8_keys_and_values() {
use crate::arrow::array::LargeStringBuilder;

let mut builder =
MapBuilder::new(None, LargeStringBuilder::new(), LargeStringBuilder::new());
builder.keys().append_value("region");
builder.values().append_value("us");
builder.append(true).unwrap();
builder.keys().append_value("region");
builder.values().append_value("eu");
builder.append(true).unwrap();
let map_array = builder.finish();

let schema = ArrowSchema::new(vec![ArrowField::new(
"pv",
map_array.data_type().clone(),
true,
)]);
let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(map_array)]).unwrap();

let output_schema =
StructType::new_unchecked(vec![StructField::nullable("region", DataType::STRING)]);
let result_type = DataType::Struct(Box::new(output_schema));
let expr = Expr::map_to_struct(column_expr!("pv"));
let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap();
let structs = result.as_any().downcast_ref::<StructArray>().unwrap();
let regions = as_string_accessor(structs.column(0).as_ref()).unwrap();
assert_eq!(regions.value(0), "us");
assert_eq!(regions.value(1), "eu");
}

/// Helper to build a batch with Int32 column `a` and a Boolean column `is_valid`.
fn create_batch_with_bool_col(
a_vals: Vec<Option<i32>>,
Expand Down
56 changes: 45 additions & 11 deletions kernel/src/engine/arrow_expression/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,24 @@ impl Scalar {
}};
}

// Use append_value in a loop for builders without batch append (String, Binary)
// TODO: Remove after https://github.com/apache/arrow-rs/pull/9426 gets in
macro_rules! append_val_as {
($t:ty, $val:expr) => {{
let builder = builder_as!($t);
for _ in 0..num_rows {
builder.append_value($val);
// Utf8/Binary may be standard or Large depending on DELTA_KERNEL_PREFER_LARGE_VARLEN.
// `make_builder` creates the matching builder; try both concrete types.
// TODO: Use append_value_n after https://github.com/apache/arrow-rs/pull/9426 gets in
macro_rules! append_val_as_varlen {
($std:ty, $large:ty, $val:expr) => {{
if let Some(builder) = builder.as_any_mut().downcast_mut::<$std>() {
for _ in 0..num_rows {
builder.append_value($val);
}
} else if let Some(builder) = builder.as_any_mut().downcast_mut::<$large>() {
for _ in 0..num_rows {
builder.append_value($val);
}
} else {
return Err(Error::invalid_expression(format!(
"Invalid builder for {}",
self.data_type()
)));
}
}};
}
Expand All @@ -93,7 +104,9 @@ impl Scalar {
Byte(val) => append_val_n_as!(array::Int8Builder, *val),
Float(val) => append_val_n_as!(array::Float32Builder, *val),
Double(val) => append_val_n_as!(array::Float64Builder, *val),
String(val) => append_val_as!(array::StringBuilder, val),
String(val) => {
append_val_as_varlen!(array::StringBuilder, array::LargeStringBuilder, val)
}
Boolean(val) => builder_as!(array::BooleanBuilder).append_n(num_rows, *val),
Timestamp(val) | TimestampNtz(val) => {
// timezone was already set at builder construction time
Expand All @@ -104,7 +117,9 @@ impl Scalar {
append_val_n_as!(array::TimestampNanosecondBuilder, *val)
}
Date(val) => append_val_n_as!(array::Date32Builder, *val),
Binary(val) => append_val_as!(array::BinaryBuilder, val),
Binary(val) => {
append_val_as_varlen!(array::BinaryBuilder, array::LargeBinaryBuilder, val)
}
// precision and scale were already set at builder construction time
Decimal(val) => append_val_n_as!(array::Decimal128Builder, val.bits()),
Struct(data) => {
Expand Down Expand Up @@ -169,22 +184,41 @@ impl Scalar {
}};
}

// Utf8/Binary may be standard or Large depending on DELTA_KERNEL_PREFER_LARGE_VARLEN.
macro_rules! append_nulls_as_varlen {
($std:ty, $large:ty) => {{
if let Some(builder) = builder.as_any_mut().downcast_mut::<$std>() {
builder.append_nulls(num_rows);
} else if let Some(builder) = builder.as_any_mut().downcast_mut::<$large>() {
builder.append_nulls(num_rows);
} else {
return Err(Error::invalid_expression(format!(
"Invalid builder for {data_type}"
)));
}
}};
}

match *data_type {
DataType::INTEGER => append_nulls_as!(array::Int32Builder),
DataType::LONG => append_nulls_as!(array::Int64Builder),
DataType::SHORT => append_nulls_as!(array::Int16Builder),
DataType::BYTE => append_nulls_as!(array::Int8Builder),
DataType::FLOAT => append_nulls_as!(array::Float32Builder),
DataType::DOUBLE => append_nulls_as!(array::Float64Builder),
DataType::STRING => append_nulls_as!(array::StringBuilder),
DataType::STRING => {
append_nulls_as_varlen!(array::StringBuilder, array::LargeStringBuilder)
}
DataType::BOOLEAN => append_nulls_as!(array::BooleanBuilder),
DataType::TIMESTAMP | DataType::TIMESTAMP_NTZ => {
append_nulls_as!(array::TimestampMicrosecondBuilder)
}
#[cfg(feature = "nanosecond-timestamps")]
DataType::TIMESTAMP_NANOS => append_nulls_as!(array::TimestampNanosecondBuilder),
DataType::DATE => append_nulls_as!(array::Date32Builder),
DataType::BINARY => append_nulls_as!(array::BinaryBuilder),
DataType::BINARY => {
append_nulls_as_varlen!(array::BinaryBuilder, array::LargeBinaryBuilder)
}
DataType::Primitive(PrimitiveType::Decimal(_)) => {
append_nulls_as!(array::Decimal128Builder)
}
Expand Down
42 changes: 42 additions & 0 deletions kernel/src/engine/arrow_expression/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1489,3 +1489,45 @@ fn test_create_many_nested_struct() {
.unwrap();
assert_create_many(&[row1, row2], schema, expected);
}

// === LargeUtf8 / LargeBinary scalar builder support ===
//
// When DELTA_KERNEL_PREFER_LARGE_VARLEN is set, try_from_kernel maps String/Binary to
// LargeUtf8/LargeBinary and make_builder creates Large* builders. append_to / append_null
// must accept those builders (not only the standard Utf8/Binary ones).

#[test]
fn test_append_string_and_null_to_large_string_builder() {
let mut builder = array::make_builder(&DataType::LargeUtf8, 3);
Scalar::String("hello".into())
.append_to(builder.as_mut(), 2)
.unwrap();
Scalar::append_null(builder.as_mut(), &KernelDataType::STRING, 1).unwrap();
let arr = builder.finish();
assert_eq!(arr.data_type(), &DataType::LargeUtf8);
let arr = arr
.as_any()
.downcast_ref::<GenericStringArray<i64>>()
.unwrap();
assert_eq!(arr.value(0), "hello");
assert_eq!(arr.value(1), "hello");
assert!(arr.is_null(2));
}

#[test]
fn test_append_binary_and_null_to_large_binary_builder() {
let mut builder = array::make_builder(&DataType::LargeBinary, 3);
Scalar::Binary(b"abc".to_vec())
.append_to(builder.as_mut(), 2)
.unwrap();
Scalar::append_null(builder.as_mut(), &KernelDataType::BINARY, 1).unwrap();
let arr = builder.finish();
assert_eq!(arr.data_type(), &DataType::LargeBinary);
let arr = arr
.as_any()
.downcast_ref::<crate::arrow::array::LargeBinaryArray>()
.unwrap();
assert_eq!(arr.value(0), b"abc");
assert_eq!(arr.value(1), b"abc");
assert!(arr.is_null(2));
}
Loading