Skip to content
This repository was archived by the owner on Oct 9, 2023. It is now read-only.

Commit 48c37de

Browse files
authored
Structured Dataset with generic format should be castable to Flyte Schema (#536)
* Structured Dataset with generic format should be castable to Schema Signed-off-by: Kevin Su <[email protected]> * Structured Dataset with generic format should be castable to Schema Signed-off-by: Kevin Su <[email protected]> * update Signed-off-by: Kevin Su <[email protected]> --------- Signed-off-by: Kevin Su <[email protected]>
1 parent 328a41c commit 48c37de

File tree

2 files changed

+21
-2
lines changed

2 files changed

+21
-2
lines changed

pkg/compiler/validators/typing.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ func (t schemaTypeChecker) CastsFrom(upstreamType *flyte.LiteralType) bool {
115115
}
116116

117117
// Flyte Schema can only be serialized to parquet
118-
if !strings.EqualFold(structuredDatasetType.Format, "parquet") {
118+
if len(structuredDatasetType.Format) != 0 && !strings.EqualFold(structuredDatasetType.Format, "parquet") {
119119
return false
120120
}
121121

@@ -147,7 +147,8 @@ func (t structuredDatasetChecker) CastsFrom(upstreamType *flyte.LiteralType) boo
147147
}
148148
if schemaType != nil {
149149
// Flyte Schema can only be serialized to parquet
150-
if !strings.EqualFold(t.literalType.GetStructuredDatasetType().Format, "parquet") {
150+
format := t.literalType.GetStructuredDatasetType().Format
151+
if len(format) != 0 && !strings.EqualFold(format, "parquet") {
151152
return false
152153
}
153154
return structuredDatasetCastFromSchema(schemaType, t.literalType.GetStructuredDatasetType())

pkg/compiler/validators/typing_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -579,6 +579,14 @@ func TestSchemaCasting(t *testing.T) {
579579
},
580580
},
581581
}
582+
genericStructuredDataset := &core.LiteralType{
583+
Type: &core.LiteralType_StructuredDatasetType{
584+
StructuredDatasetType: &core.StructuredDatasetType{
585+
Columns: []*core.StructuredDatasetType_DatasetColumn{},
586+
Format: "",
587+
},
588+
},
589+
}
582590
subsetIntegerSchema := &core.LiteralType{
583591
Type: &core.LiteralType_Schema{
584592
Schema: &core.SchemaType{
@@ -657,6 +665,16 @@ func TestSchemaCasting(t *testing.T) {
657665
assert.True(t, castable, "Schema(a=Integer, b=Float) should be castable to Schema(a=Integer)")
658666
})
659667

668+
t.Run("GenericToSubsetTypedSchema", func(t *testing.T) {
669+
castable := AreTypesCastable(genericStructuredDataset, subsetIntegerSchema)
670+
assert.True(t, castable, "StructuredDataset() with generic format should be castable to Schema(a=Integer)")
671+
})
672+
673+
t.Run("SubsetTypedSchemaToGeneric", func(t *testing.T) {
674+
castable := AreTypesCastable(subsetIntegerSchema, genericStructuredDataset)
675+
assert.True(t, castable, "Schema(a=Integer) should be castable to StructuredDataset() with generic format")
676+
})
677+
660678
t.Run("SupersetStructuredToSubsetTypedSchema", func(t *testing.T) {
661679
castable := AreTypesCastable(supersetStructuredDataset, subsetIntegerSchema)
662680
assert.True(t, castable, "StructuredDataset(a=Integer, b=Float) should be castable to Schema(a=Integer)")

0 commit comments

Comments
 (0)