From ec8600e25177df137407aaac01b6af7341e9b36f Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Fri, 13 Sep 2024 16:03:30 +0200 Subject: [PATCH 1/3] Support reading tables with type widening in default parquet reader --- kernel/src/engine/arrow_utils.rs | 106 ++++++++++++++++++ kernel/src/features/mod.rs | 14 +++ ...-9f5b-f0fccaceb415-c000.snappy.parquet.crc | Bin 0 -> 40 bytes ...-984c-4376799a50d9-c000.snappy.parquet.crc | Bin 0 -> 40 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 32 bytes .../_delta_log/.00000000000000000001.json.crc | Bin 0 -> 24 bytes .../_delta_log/.00000000000000000002.json.crc | Bin 0 -> 44 bytes .../_delta_log/00000000000000000000.json | 4 + .../_delta_log/00000000000000000001.json | 3 + .../_delta_log/00000000000000000002.json | 3 + ...416b-9f5b-f0fccaceb415-c000.snappy.parquet | Bin 0 -> 4001 bytes ...42b1-984c-4376799a50d9-c000.snappy.parquet | Bin 0 -> 3694 bytes kernel/tests/read.rs | 44 ++++++++ 13 files changed, 174 insertions(+) create mode 100644 kernel/tests/data/type-widening/.part-00000-61accb66-b740-416b-9f5b-f0fccaceb415-c000.snappy.parquet.crc create mode 100644 kernel/tests/data/type-widening/.part-00000-f6dbc649-d5bc-42b1-984c-4376799a50d9-c000.snappy.parquet.crc create mode 100644 kernel/tests/data/type-widening/_delta_log/.00000000000000000000.json.crc create mode 100644 kernel/tests/data/type-widening/_delta_log/.00000000000000000001.json.crc create mode 100644 kernel/tests/data/type-widening/_delta_log/.00000000000000000002.json.crc create mode 100644 kernel/tests/data/type-widening/_delta_log/00000000000000000000.json create mode 100644 kernel/tests/data/type-widening/_delta_log/00000000000000000001.json create mode 100644 kernel/tests/data/type-widening/_delta_log/00000000000000000002.json create mode 100644 kernel/tests/data/type-widening/part-00000-61accb66-b740-416b-9f5b-f0fccaceb415-c000.snappy.parquet create mode 100644 kernel/tests/data/type-widening/part-00000-f6dbc649-d5bc-42b1-984c-4376799a50d9-c000.snappy.parquet diff --git a/kernel/src/engine/arrow_utils.rs b/kernel/src/engine/arrow_utils.rs index 9b2de32c1..cfcf2a991 100644 --- a/kernel/src/engine/arrow_utils.rs +++ b/kernel/src/engine/arrow_utils.rs @@ -73,12 +73,23 @@ fn check_cast_compat( target_type: ArrowDataType, source_type: &ArrowDataType, ) -> DeltaResult { + use ArrowDataType::*; + match (source_type, &target_type) { (source_type, target_type) if source_type == target_type => Ok(DataTypeCompat::Identical), (&ArrowDataType::Timestamp(_, _), &ArrowDataType::Timestamp(_, _)) => { // timestamps are able to be cast between each other Ok(DataTypeCompat::NeedsCast(target_type)) } + // Allow up-casting to a larger type if it's safe and can't cause overflow or loss of precision. + (Int8, Int16 | Int32 | Int64 | Float64) => Ok(DataTypeCompat::NeedsCast(target_type)), + (Int16, Int32 | Int64 | Float64) => Ok(DataTypeCompat::NeedsCast(target_type)), + (Int32, Int64 | Float64) => Ok(DataTypeCompat::NeedsCast(target_type)), + (Float32, Float64) => Ok(DataTypeCompat::NeedsCast(target_type)), + (_, Decimal128(p, s) | Decimal256(p, s)) if can_upcast_to_decimal(source_type, *p, *s) => { + Ok(DataTypeCompat::NeedsCast(target_type)) + } + (Date32, Timestamp(_, None)) => Ok(DataTypeCompat::NeedsCast(target_type)), _ => Err(make_arrow_error(format!( "Incorrect datatype. Expected {}, got {}", target_type, source_type @@ -86,6 +97,31 @@ fn check_cast_compat( } } +// Returns whether the given source type can be safely cast to a decimal with the given precision and scale without +// loss of information. +pub(crate) fn can_upcast_to_decimal( + source_type: &ArrowDataType, + target_precision: u8, + target_scale: i8, +) -> bool { + use ArrowDataType::*; + + let (source_precision, source_scale) = match source_type { + Decimal128(p, s) => (*p, *s), + Decimal256(p, s) => (*p, *s), + // Allow converting integers to a decimal that can hold all possible values. + Int8 => (3u8, 0i8), + Int16 => (5u8, 0i8), + Int32 => (10u8, 0i8), + Int64 => (20u8, 0i8), + _ => return false, + }; + + target_precision >= source_precision + && target_scale >= source_scale + && target_precision - source_precision >= (target_scale - source_scale) as u8 +} + /// Ensure a kernel data type matches an arrow data type. This only ensures that the actual "type" /// is the same, but does so recursively into structs, and ensures lists and maps have the correct /// associated types as well. This returns an `Ok(DataTypeCompat)` if the types are compatible, and @@ -1427,4 +1463,74 @@ mod tests { assert_eq!(mask_indices, expect_mask); assert_eq!(reorder_indices, expect_reorder); } + + #[test] + fn accepts_safe_decimal_casts() { + use super::can_upcast_to_decimal; + use ArrowDataType::*; + + assert!(can_upcast_to_decimal(&Decimal128(1, 0), 2u8, 0i8)); + assert!(can_upcast_to_decimal(&Decimal128(1, 0), 2u8, 1i8)); + assert!(can_upcast_to_decimal(&Decimal128(5, -2), 6u8, -2i8)); + assert!(can_upcast_to_decimal(&Decimal128(5, -2), 6u8, -1i8)); + assert!(can_upcast_to_decimal(&Decimal128(5, 1), 6u8, 1i8)); + assert!(can_upcast_to_decimal(&Decimal128(5, 1), 6u8, 2i8)); + assert!(can_upcast_to_decimal( + &Decimal128(10, 5), + arrow_schema::DECIMAL128_MAX_PRECISION, + arrow_schema::DECIMAL128_MAX_SCALE - 5 + )); + assert!(can_upcast_to_decimal(&Decimal256(17, 5), 30u8, 5i8)); + assert!(can_upcast_to_decimal(&Decimal256(17, 5), 30u8, 7i8)); + assert!(can_upcast_to_decimal(&Decimal256(17, 5), 30u8, 7i8)); + assert!(can_upcast_to_decimal(&Decimal256(17, -5), 30u8, -5i8)); + assert!(can_upcast_to_decimal(&Decimal256(17, -5), 30u8, -3i8)); + assert!(can_upcast_to_decimal( + &Decimal256(10, 5), + arrow_schema::DECIMAL256_MAX_PRECISION, + arrow_schema::DECIMAL256_MAX_SCALE - 5 + )); + + assert!(can_upcast_to_decimal(&Int8, 3u8, 0i8)); + assert!(can_upcast_to_decimal(&Int8, 4u8, 0i8)); + assert!(can_upcast_to_decimal(&Int8, 4u8, 1i8)); + assert!(can_upcast_to_decimal(&Int8, 7u8, 2i8)); + + assert!(can_upcast_to_decimal(&Int16, 5u8, 0i8)); + assert!(can_upcast_to_decimal(&Int16, 6u8, 0i8)); + assert!(can_upcast_to_decimal(&Int16, 6u8, 1i8)); + assert!(can_upcast_to_decimal(&Int16, 9u8, 2i8)); + + assert!(can_upcast_to_decimal(&Int32, 10u8, 0i8)); + assert!(can_upcast_to_decimal(&Int32, 11u8, 0i8)); + assert!(can_upcast_to_decimal(&Int32, 11u8, 1i8)); + assert!(can_upcast_to_decimal(&Int32, 14u8, 2i8)); + + assert!(can_upcast_to_decimal(&Int64, 20u8, 0i8)); + assert!(can_upcast_to_decimal(&Int64, 21u8, 0i8)); + assert!(can_upcast_to_decimal(&Int64, 21u8, 1i8)); + assert!(can_upcast_to_decimal(&Int64, 24u8, 2i8)); + } + + #[test] + fn rejects_unsafe_decimal_casts() { + use super::can_upcast_to_decimal; + use ArrowDataType::*; + + assert!(!can_upcast_to_decimal(&Decimal128(2, 0), 2u8, 1i8)); + assert!(!can_upcast_to_decimal(&Decimal128(2, 0), 2u8, -1i8)); + assert!(!can_upcast_to_decimal(&Decimal128(5, 2), 6u8, 4i8)); + assert!(!can_upcast_to_decimal(&Decimal256(2, 0), 2u8, 1i8)); + assert!(!can_upcast_to_decimal(&Decimal256(2, 0), 2u8, -1i8)); + assert!(!can_upcast_to_decimal(&Decimal256(5, 2), 6u8, 4i8)); + + assert!(!can_upcast_to_decimal(&Int8, 2u8, 0i8)); + assert!(!can_upcast_to_decimal(&Int8, 3u8, 1i8)); + assert!(!can_upcast_to_decimal(&Int16, 4u8, 0i8)); + assert!(!can_upcast_to_decimal(&Int16, 5u8, 1i8)); + assert!(!can_upcast_to_decimal(&Int32, 9u8, 0i8)); + assert!(!can_upcast_to_decimal(&Int32, 10u8, 1i8)); + assert!(!can_upcast_to_decimal(&Int64, 19u8, 0i8)); + assert!(!can_upcast_to_decimal(&Int64, 20u8, 1i8)); + } } diff --git a/kernel/src/features/mod.rs b/kernel/src/features/mod.rs index 8ac28a3ce..cef49d906 100644 --- a/kernel/src/features/mod.rs +++ b/kernel/src/features/mod.rs @@ -31,6 +31,11 @@ pub enum ReaderFeatures { #[strum(serialize = "timestampNtz")] #[serde(rename = "timestampNtz")] TimestampWithoutTimezone, + // Allow columns to change type + TypeWidening, + #[strum(serialize = "typeWidening-preview")] + #[serde(rename = "typeWidening-preview")] + TypeWideningPreview, /// version 2 of checkpointing V2Checkpoint, } @@ -74,6 +79,11 @@ pub enum WriterFeatures { #[strum(serialize = "timestampNtz")] #[serde(rename = "timestampNtz")] TimestampWithoutTimezone, + // Allow columns to change type + TypeWidening, + #[strum(serialize = "typeWidening-preview")] + #[serde(rename = "typeWidening-preview")] + TypeWideningPreview, /// domain specific metadata DomainMetadata, /// version 2 of checkpointing @@ -94,6 +104,8 @@ mod tests { (ReaderFeatures::ColumnMapping, "columnMapping"), (ReaderFeatures::DeletionVectors, "deletionVectors"), (ReaderFeatures::TimestampWithoutTimezone, "timestampNtz"), + (ReaderFeatures::TypeWidening, "typeWidening"), + (ReaderFeatures::TypeWideningPreview, "typeWidening-preview"), (ReaderFeatures::V2Checkpoint, "v2Checkpoint"), ]; @@ -126,6 +138,8 @@ mod tests { (WriterFeatures::DeletionVectors, "deletionVectors"), (WriterFeatures::RowTracking, "rowTracking"), (WriterFeatures::TimestampWithoutTimezone, "timestampNtz"), + (WriterFeatures::TypeWidening, "typeWidening"), + (WriterFeatures::TypeWideningPreview, "typeWidening-preview"), (WriterFeatures::DomainMetadata, "domainMetadata"), (WriterFeatures::V2Checkpoint, "v2Checkpoint"), (WriterFeatures::IcebergCompatV1, "icebergCompatV1"), diff --git a/kernel/tests/data/type-widening/.part-00000-61accb66-b740-416b-9f5b-f0fccaceb415-c000.snappy.parquet.crc b/kernel/tests/data/type-widening/.part-00000-61accb66-b740-416b-9f5b-f0fccaceb415-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..1e4c85e727490da15d59d5ae8e415b9b442b31e8 GIT binary patch literal 40 wcmYc;N@ieSU}CuDubOdseu-F+wow149S=nmi(DLH-#9TX3z65E)|E9G023|_qyPW_ literal 0 HcmV?d00001 diff --git a/kernel/tests/data/type-widening/.part-00000-f6dbc649-d5bc-42b1-984c-4376799a50d9-c000.snappy.parquet.crc b/kernel/tests/data/type-widening/.part-00000-f6dbc649-d5bc-42b1-984c-4376799a50d9-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..ce45ba244ba99b6b9a39a11b523c5d61699f9f0a GIT binary patch literal 40 wcmYc;N@ieSU}C7+AsKY_m~_}lqY#(5)608)6gOIL6nSy)oahU-*Y#Wr05cd7MF0Q* literal 0 HcmV?d00001 diff --git a/kernel/tests/data/type-widening/_delta_log/.00000000000000000000.json.crc b/kernel/tests/data/type-widening/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..caa5f5500584b3b4149f2287f9c9bd383f953cc8 GIT binary patch literal 32 ocmYc;N@ieSU}8vkvbRA|dQ$mU%0H;3+?EnA( literal 0 HcmV?d00001 diff --git a/kernel/tests/data/type-widening/_delta_log/.00000000000000000001.json.crc b/kernel/tests/data/type-widening/_delta_log/.00000000000000000001.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..ca2f40f469d1fb8e1f55ba44a7dc1b0cbea09fe8 GIT binary patch literal 24 gcmYc;N@ieSU}Ct@%cR+KKD=#t+|92XVh`s50AIWbX#fBK literal 0 HcmV?d00001 diff --git a/kernel/tests/data/type-widening/_delta_log/.00000000000000000002.json.crc b/kernel/tests/data/type-widening/_delta_log/.00000000000000000002.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..b27eccb1118ed65fb981acd5f6b1713cc8e95c4c GIT binary patch literal 44 zcmYc;N@ieSU}CV_TDWpToq=v6gGQ|FeSNuKKTI0wVI9zqda^rqN@28*6n?4i_N#Fh1XGxOfOkXVw4mNN6r z_rCZ0ec$`u`@WayXJ61U!ZAF6AAEK4b1;Sl_zYkPp~#qk5bErB0E(EbfA_~~pn=JiUpj~hS^w=1HVzxDe=1fzAb6frUmmRg*wFcN>e-W} zU*E(Y-1;{MYmE|c<>N1>(KVhRXXCSe4SD5Gx$7BiLfF(!K zm^{Oz%?>6W`(j04+O5aCzP+ zy^=1^-m>xrK98_CC@9^TJGPb16$^9jx0Ef|-e(Ag!#8t9!%myUg-p&un5cT38N4Q3 zzEv#M8ifc9wTa+9EGXwqYc`uVa%uW58+i-7F>=;m2#a_q@e+BGBjHYXj2hDST*)$Q ztHkDnoH_6$a|Gh8>zd00paPZv6rR+e*YlLK2s;*GVnNoU$8nq+mz{C4;GSDyvRE_- zhvb4|a)Cnhnc!vG&gQMMZRF?E1^e~E{_CrKu>@EkBn#14v>KOUSd69uNP?Zka)1R{ zkgv%%;?kslh4PVh2PAK zr6RN$$`6F)&|x8Un=Tl4J{ZSZXPX>Hgkw8IBB=sL#7oL42hQQVE>DB~Bv)rS5t8JW zT29ZWY55HBJ_z9{6yFhj-qX)5quvq!1@%XSdb@{EKLl!l?nAx+I80OWRj`E+4hFYB1QnE`TNd#C;N2|BHvcPhdB~1a<{(3-s9YrN#jz z2Y)31n}BvAUm|CwUdxmGL`d$3$tKtb61oX$suf%NePHqX87Devr+tpo^Gqj%X}OoI z{TAG8XgCeZ9*b%#2vN~i=}RQ4dM(d&_biF37ow6s4a!1Xk`IIZ&{7kMQY_7B#=HUD zOmiBW_NrAXXNv`;E2*WlQKSslS(I<*H0Xi17c5&V6<^cxmTf?>Hk6~_qH5onx6}!> zY?l^hZ8f3Z%vw3KtWLbRs1~48!4NCcFzQ*8UI1T(gRA zN#8XWkcQ}~7SQ=xRS!0n&|AQ&x!2T#U2F-Ay8_ii>3U08{AH;gY_v77`iqqhS##e4 z^{FX-c?l`kyh`v?zBJ&KjsacMlbSv-lv&8;%mFhqrjIA}k;`T(b16Ay&899VN6hg{ hBeSMq=@)ftY-HRtGsbYV8bhd=fAn?_Leuc${BN0QTc!X2 literal 0 HcmV?d00001 diff --git a/kernel/tests/data/type-widening/part-00000-f6dbc649-d5bc-42b1-984c-4376799a50d9-c000.snappy.parquet b/kernel/tests/data/type-widening/part-00000-f6dbc649-d5bc-42b1-984c-4376799a50d9-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9a6d79ad03ecbdba81c002cbd6d17451e365faf6 GIT binary patch literal 3694 zcmb`KU1%It6o79go83vWtyXh~ov_NXn2^9a>^6x_LXj$^NGVJ4O;ncI%rsq`-A#8V zN)34sfAJv?k_Q_EBPfVO9}0#(i1ei;zA46+CHSI*SbRvKprlwl=ia$LNtQOLLb!A9 znKR!x_ndpr*_?mw#R4U?kEZC2_rJN6B3U;>KTHZB)7xWo>C|Bhf}Bu2lZ{`_7k-v{6ZLl{C%JZ{OPf zTHn#pQNE*zJm@I3?{9r|SKpOTd$(Vz{eSUUX?^-xM;o_0l|QSuf4pXBD#D)_cSl6} zkGnsanwCgcqDY8+%&=22OkDJ!hHVKk#sI`DXPOuj1Y(vaO^itaG0Tl6#zcUa~AsqXUS&@Tn`+VsGa=T;s!k1=g{VODA(lM`yfGI=4WyJ5+z&z1w9 zf7;7F<&r%gCz}`R4WT`d0q~;|+=F1ji2=Ib^QS6RHz2pc+3OqrLa=< zo1t5sDb>O^2a@LkaMv>Iw7oHim&_iE6VukBO%2Pip0~~nnkV^1D2d*0QGl9-PEYs@s!L>WMWO?BQn)WXMTdrkGgUDE1hW?i#)y031>FWT1C)5sfwGL zfh}3`lvKBQ6>VPcrq7bW9nI$buZc9fgJwn1R5LWYU#gp3L$mjo*){CfA}RV^lL1_d zK>)FxJm{4Fiq!9VFn|l#;v?$rTPm>&U4T``8}p)8y0do>dy8S$HLQNWC3GF3SpE5k zP_@#bA07qh$Gzz5Qd}i_SBa2uUW!AUMdMI2jPrh}Zea3khrRt4o$DMJZS2(O~IIsJSW~E+B zb&WV<&T*0&-lx&L6gbdfF3tL()2P4URQ=F}+UTZ!NX%u!%QJp z+||ebGo5FF_ihDys5I<_JWL+;G7g;x{fCFz4@wUa@qNEk6bXODJc#t_JV~96uK3f{ zMkcBWnSxUoaSEBC@@ysWGG2MKFg{YqPk3YH6C Result<(), Box> { )?; Ok(()) } + +#[test] +fn type_widening_basic() -> Result<(), Box> { + let expected = vec![ + "+---------------------+---------------------+--------------------+----------------+----------------+----------------+----------------------------+", + "| byte_long | int_long | float_double | byte_double | short_double | int_double | date_timestamp_ntz |", + "+---------------------+---------------------+--------------------+----------------+----------------+----------------+----------------------------+", + "| 1 | 2 | 3.4000000953674316 | 5.0 | 6.0 | 7.0 | 2024-09-09T00:00:00 |", + "| 9223372036854775807 | 9223372036854775807 | 1.234567890123 | 1.234567890123 | 1.234567890123 | 1.234567890123 | 2024-09-09T12:34:56.123456 |", + "+---------------------+---------------------+--------------------+----------------+----------------+----------------+----------------------------+", + ]; + let select_cols: Option<&[&str]> = Some(&[ + "byte_long", + "int_long", + "float_double", + "byte_double", + "short_double", + "int_double", + "date_timestamp_ntz", + ]); + + read_table_data_str("./tests/data/type-widening/", select_cols, None, expected) +} + +#[test] +fn type_widening_decimal() -> Result<(), Box> { + let expected = vec![ + "+----------------------------+-------------------------------+--------------+---------------+--------------+----------------------+", + "| decimal_decimal_same_scale | decimal_decimal_greater_scale | byte_decimal | short_decimal | int_decimal | long_decimal |", + "+----------------------------+-------------------------------+--------------+---------------+--------------+----------------------+", + "| 123.45 | 67.89000 | 1.0 | 2.0 | 3.0 | 4.0 |", + "| 12345678901234.56 | 12345678901.23456 | 123.4 | 12345.6 | 1234567890.1 | 123456789012345678.9 |", + "+----------------------------+-------------------------------+--------------+---------------+--------------+----------------------+", + ]; + let select_cols: Option<&[&str]> = Some(&[ + "decimal_decimal_same_scale", + "decimal_decimal_greater_scale", + "byte_decimal", + "short_decimal", + "int_decimal", + "long_decimal", + ]); + read_table_data_str("./tests/data/type-widening/", select_cols, None, expected) +} From 1423cc583aa350a97fba5028bfc9d3cb6e443bf9 Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Tue, 17 Sep 2024 14:48:31 +0200 Subject: [PATCH 2/3] Remove Decimal256, not supported by Delta spec --- kernel/src/engine/arrow_utils.rs | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/kernel/src/engine/arrow_utils.rs b/kernel/src/engine/arrow_utils.rs index cfcf2a991..20499db9e 100644 --- a/kernel/src/engine/arrow_utils.rs +++ b/kernel/src/engine/arrow_utils.rs @@ -86,7 +86,7 @@ fn check_cast_compat( (Int16, Int32 | Int64 | Float64) => Ok(DataTypeCompat::NeedsCast(target_type)), (Int32, Int64 | Float64) => Ok(DataTypeCompat::NeedsCast(target_type)), (Float32, Float64) => Ok(DataTypeCompat::NeedsCast(target_type)), - (_, Decimal128(p, s) | Decimal256(p, s)) if can_upcast_to_decimal(source_type, *p, *s) => { + (_, Decimal128(p, s)) if can_upcast_to_decimal(source_type, *p, *s) => { Ok(DataTypeCompat::NeedsCast(target_type)) } (Date32, Timestamp(_, None)) => Ok(DataTypeCompat::NeedsCast(target_type)), @@ -108,7 +108,6 @@ pub(crate) fn can_upcast_to_decimal( let (source_precision, source_scale) = match source_type { Decimal128(p, s) => (*p, *s), - Decimal256(p, s) => (*p, *s), // Allow converting integers to a decimal that can hold all possible values. Int8 => (3u8, 0i8), Int16 => (5u8, 0i8), @@ -1480,16 +1479,6 @@ mod tests { arrow_schema::DECIMAL128_MAX_PRECISION, arrow_schema::DECIMAL128_MAX_SCALE - 5 )); - assert!(can_upcast_to_decimal(&Decimal256(17, 5), 30u8, 5i8)); - assert!(can_upcast_to_decimal(&Decimal256(17, 5), 30u8, 7i8)); - assert!(can_upcast_to_decimal(&Decimal256(17, 5), 30u8, 7i8)); - assert!(can_upcast_to_decimal(&Decimal256(17, -5), 30u8, -5i8)); - assert!(can_upcast_to_decimal(&Decimal256(17, -5), 30u8, -3i8)); - assert!(can_upcast_to_decimal( - &Decimal256(10, 5), - arrow_schema::DECIMAL256_MAX_PRECISION, - arrow_schema::DECIMAL256_MAX_SCALE - 5 - )); assert!(can_upcast_to_decimal(&Int8, 3u8, 0i8)); assert!(can_upcast_to_decimal(&Int8, 4u8, 0i8)); @@ -1520,9 +1509,6 @@ mod tests { assert!(!can_upcast_to_decimal(&Decimal128(2, 0), 2u8, 1i8)); assert!(!can_upcast_to_decimal(&Decimal128(2, 0), 2u8, -1i8)); assert!(!can_upcast_to_decimal(&Decimal128(5, 2), 6u8, 4i8)); - assert!(!can_upcast_to_decimal(&Decimal256(2, 0), 2u8, 1i8)); - assert!(!can_upcast_to_decimal(&Decimal256(2, 0), 2u8, -1i8)); - assert!(!can_upcast_to_decimal(&Decimal256(5, 2), 6u8, 4i8)); assert!(!can_upcast_to_decimal(&Int8, 2u8, 0i8)); assert!(!can_upcast_to_decimal(&Int8, 3u8, 1i8)); From 9230d4bd2ad15620bcdf2a43442417a34c1e6d29 Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Thu, 19 Sep 2024 09:07:50 +0200 Subject: [PATCH 3/3] Remove pub modifier on can_upcast_to_decimal --- kernel/src/engine/arrow_utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/src/engine/arrow_utils.rs b/kernel/src/engine/arrow_utils.rs index 20499db9e..6d1a66d23 100644 --- a/kernel/src/engine/arrow_utils.rs +++ b/kernel/src/engine/arrow_utils.rs @@ -99,7 +99,7 @@ fn check_cast_compat( // Returns whether the given source type can be safely cast to a decimal with the given precision and scale without // loss of information. -pub(crate) fn can_upcast_to_decimal( +fn can_upcast_to_decimal( source_type: &ArrowDataType, target_precision: u8, target_scale: i8,