Skip to content

Commit

Permalink
support nested map chunk encoding
Browse files Browse the repository at this point in the history
  • Loading branch information
@chaokunyang
chaokunyang committed Jan 27, 2025
1 parent 4e2a1b8 commit 5fcb56a
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 36 deletions.
99 changes: 63 additions & 36 deletions java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,9 @@ public InvokeHint(boolean genNewMethod, Expression... cutPoints) {
}

public InvokeHint add(Expression cutPoint) {
cutPoints.add(cutPoint);
if (cutPoint != null) {
cutPoints.add(cutPoint);
}
return this;
}

Expand Down Expand Up @@ -367,7 +369,8 @@ protected Expression writeRefOrNull(Expression buffer, Expression object) {

protected Expression serializeForNotNull(
Expression inputObject, Expression buffer, TypeRef<?> typeRef) {
return serializeForNotNull(inputObject, buffer, typeRef, null, false);
boolean genNewMethod = useCollectionSerialization(typeRef) || useMapSerialization(typeRef);
return serializeForNotNull(inputObject, buffer, typeRef, null, genNewMethod);
}

/**
Expand All @@ -379,6 +382,12 @@ private Expression serializeForNotNull(
return serializeForNotNull(inputObject, buffer, typeRef, null, generateNewMethod);
}

private Expression serializeForNotNull(
Expression inputObject, Expression buffer, TypeRef<?> typeRef, Expression serializer) {
boolean genNewMethod = useCollectionSerialization(typeRef) || useMapSerialization(typeRef);
return serializeForNotNull(inputObject, buffer, typeRef, serializer, genNewMethod);
}

private Expression serializeForNotNull(
Expression inputObject,
Expression buffer,
Expand Down Expand Up @@ -681,10 +690,13 @@ protected Expression readClassInfo(Class<?> cls, Expression buffer, boolean inli
}

protected TypeRef<?> getSerializerType(TypeRef<?> objType) {
Class<?> rawType = getRawType(objType);
if (classResolver.isCollection(rawType)) {
return getSerializerType(objType.getRawType());
}

protected TypeRef<?> getSerializerType(Class<?> objType) {
if (classResolver.isCollection(objType)) {
return COLLECTION_SERIALIZER_TYPE;
} else if (classResolver.isMap(rawType)) {
} else if (classResolver.isMap(objType)) {
return MAP_SERIALIZER_TYPE;
}
return SERIALIZER_TYPE;
Expand Down Expand Up @@ -1282,7 +1294,7 @@ protected Expression writeChunk(
neq(inlineInvoke(key, "getClass", CLASS_TYPE), keyTypeExpr),
neq(inlineInvoke(value, "getClass", CLASS_TYPE), valueTypeExpr));
}
Expression writeKey = new Invoke(keySerializer, "write", buffer, key);
Expression writeKey = serializeForNotNull(key, buffer, keyType, keySerializer);
if (trackingKeyRef) {
writeKey =
new If(
Expand All @@ -1297,7 +1309,8 @@ protected Expression writeChunk(
key))),
writeKey);
}
Expression writeValue = new Invoke(valueSerializer, "write", buffer, value);
Expression writeValue =
serializeForNotNull(value, buffer, valueType, valueSerializer);
if (trackingValueRef) {
writeValue =
new If(
Expand Down Expand Up @@ -1533,6 +1546,7 @@ protected Expression deserializeForCollection(
false);
if (invokeHint != null && invokeHint.genNewMethod) {
invokeHint.add(buffer);
invokeHint.add(serializer);
return invokeGenerated(
ctx,
invokeHint.cutPoints,
Expand Down Expand Up @@ -1745,8 +1759,11 @@ protected Expression deserializeForMap(
Expression newMap = new Invoke(serializer, "newMap", MAP_TYPE, buffer);
Expression size = new Invoke(serializer, "getAndClearNumElements", "size", PRIMITIVE_INT_TYPE);
Expression chunkHeader =
new Invoke(buffer, "readUnsignedByte", "chunkHeader", PRIMITIVE_INT_TYPE);
expressions.add(newMap, size, new If(eq(size, ofInt(0)), new Return()), chunkHeader);
new If(
eq(size, ofInt(0)),
ofInt(0),
inlineInvoke(buffer, "readUnsignedByte", PRIMITIVE_INT_TYPE));
expressions.add(newMap, size, chunkHeader);
boolean keyMonomorphic = isMonomorphic(keyType);
boolean valueMonomorphic = isMonomorphic(valueType);
boolean inline = keyMonomorphic && valueMonomorphic;
Expand Down Expand Up @@ -1795,6 +1812,7 @@ chunkHeader, cast(bitand(sizeAndHeader2, ofInt(0xff)), PRIMITIVE_INT_TYPE)),
new If(supportHook, map, new Invoke(serializer, "read", OBJECT_TYPE, buffer), false);
if (invokeHint != null && invokeHint.genNewMethod) {
invokeHint.add(buffer);
invokeHint.add(serializer);
return invokeGenerated(
ctx,
invokeHint.cutPoints,
Expand All @@ -1821,45 +1839,29 @@ private Expression readChunk(
visitFury(fury -> fury.getClassResolver().needToWriteRef(keyTypeRawType));
boolean trackingValueRef =
visitFury(fury -> fury.getClassResolver().needToWriteRef(valueTypeRawType));
ListExpression expressions = new ListExpression();
ListExpression expressions = new ListExpression(buffer);
Expression trackKeyRef = neq(bitand(chunkHeader, ofInt(TRACKING_KEY_REF)), ofInt(0));
Expression trackValueRef = neq(bitand(chunkHeader, ofInt(TRACKING_VALUE_REF)), ofInt(0));
Expression keyIsDeclaredType = neq(bitand(chunkHeader, ofInt(KEY_DECL_TYPE)), ofInt(0));
Expression valueIsDeclaredType = neq(bitand(chunkHeader, ofInt(VALUE_DECL_TYPE)), ofInt(0));
Expression chunkSize = new Invoke(buffer, "readUnsignedByte", "chunkSize", PRIMITIVE_INT_TYPE);
expressions.add(chunkSize);

Expression keySerializer = getOrCreateSerializer(keyType.getRawType());
Expression valueSerializer = getOrCreateSerializer(valueType.getRawType());
Expression keySerializer, valueSerializer;
if (!keyMonomorphic && !valueMonomorphic) {
keySerializer =
new If(
keyIsDeclaredType,
keySerializer,
inlineInvoke(readClassInfo(keyTypeRawType, buffer), "getSerializer", SERIALIZER_TYPE),
false);
keySerializer = readOrGetSerializerForDeclaredType(buffer, keyTypeRawType, keyIsDeclaredType);
valueSerializer =
new If(
valueIsDeclaredType,
valueSerializer,
inlineInvoke(
readClassInfo(valueTypeRawType, buffer), "getSerializer", SERIALIZER_TYPE),
false);
readOrGetSerializerForDeclaredType(buffer, valueTypeRawType, valueIsDeclaredType);
} else if (!keyMonomorphic) {
keySerializer =
new If(
keyIsDeclaredType,
keySerializer,
inlineInvoke(readClassInfo(keyTypeRawType, buffer), "getSerializer", SERIALIZER_TYPE),
false);
keySerializer = readOrGetSerializerForDeclaredType(buffer, keyTypeRawType, keyIsDeclaredType);
valueSerializer = getOrCreateSerializer(valueTypeRawType);
} else if (!valueMonomorphic) {
keySerializer = getOrCreateSerializer(keyTypeRawType);
valueSerializer =
new If(
valueIsDeclaredType,
valueSerializer,
inlineInvoke(
readClassInfo(valueTypeRawType, buffer), "getSerializer", SERIALIZER_TYPE),
false);
readOrGetSerializerForDeclaredType(buffer, valueTypeRawType, valueIsDeclaredType);
} else {
keySerializer = getOrCreateSerializer(keyTypeRawType);
valueSerializer = getOrCreateSerializer(valueTypeRawType);
}
Expression keySerializerExpr = uninline(keySerializer);
Expression valueSerializerExpr = uninline(valueSerializer);
Expand Down Expand Up @@ -1947,6 +1949,31 @@ private Expression readChunk(
}
}

private Expression readOrGetSerializerForDeclaredType(
Expression buffer, Class<?> type, Expression isDeclaredType) {
if (isMonomorphic(type)) {
return getOrCreateSerializer(type);
}
TypeRef<?> serializerType = getSerializerType(type);
if (ReflectionUtils.isAbstract(type) || type.isInterface()) {
Invoke serializer =
new Invoke(readClassInfo(type, buffer), "getSerializer", SERIALIZER_TYPE, false);
if (!serializerType.equals(SERIALIZER_TYPE)) {
return new Cast(serializer.inline(), serializerType, "serializer");
} else {
return serializer;
}
} else {
return new If(
isDeclaredType,
getOrCreateSerializer(type),
new Cast(
inlineInvoke(readClassInfo(type, buffer), "getSerializer", SERIALIZER_TYPE),
serializerType),
false);
}
}

@Override
protected Expression beanClassExpr() {
if (GraalvmSupport.isGraalBuildtime()) {
Expand Down
26 changes: 26 additions & 0 deletions java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -811,4 +811,30 @@ public void testMapFieldStructCodegen3(boolean referenceTrackingConfig) {
struct1.map3 = ofHashMap(1, beanB, 2, beanB, 3, BeanB.createBeanB(2));
serDeCheck(fury, struct1);
}

@Data
public static class NestedMapFieldStruct1 {
public Map<Object, Map<String, String>> map1;
public Map<String, Map<String, Integer>> map2;
public Map<Integer, Map<String, BeanB>> map3;
public Map<Object, Map<Object, Map<String, BeanB>>> map4;
}

@Test(dataProvider = "referenceTrackingConfig")
public void testNestedMapFieldStructCodegen(boolean referenceTrackingConfig) {
Fury fury =
Fury.builder()
.withRefTracking(referenceTrackingConfig)
.withCodegen(true)
.requireClassRegistration(false)
.build();
NestedMapFieldStruct1 struct1 = new NestedMapFieldStruct1();
BeanB beanB = BeanB.createBeanB(2);
struct1.map1 = ofHashMap(1, ofHashMap("k1", "v1", "k2", "v2"));
struct1.map2 = ofHashMap("k1", ofHashMap("k1", 1, "k2", 2));
struct1.map3 = ofHashMap(1, ofHashMap("k1", beanB, "k2", beanB, "k3", BeanB.createBeanB(1)));
struct1.map4 = ofHashMap(2, ofHashMap(true,
ofHashMap("k1", beanB, "k2", beanB, "k3", BeanB.createBeanB(1))));
serDeCheck(fury, struct1);
}
}

0 comments on commit 5fcb56a

Please sign in to comment.