From 2907327cb452e5eaf35d2bc8158c51d5d6fb409b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 28 Jan 2025 00:33:44 +0800 Subject: [PATCH] remove old map serialization protocol --- .../collection/AbstractMapSerializer.java | 456 +----------------- .../serializer/collection/MapSerializer.java | 6 +- .../collection/MapSerializersTest.java | 5 +- 3 files changed, 5 insertions(+), 462 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 436e922e20..8461e9772d 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -129,11 +129,7 @@ public void setValueSerializer(Serializer valueSerializer) { @Override public void write(MemoryBuffer buffer, T value) { Map map = onMapWrite(buffer, value); - if (useChunkSerialize) { - chunkWriteElements(fury, buffer, map); - } else { - writeElements(fury, buffer, map); - } + chunkWriteElements(fury, buffer, map); } @Override @@ -142,41 +138,6 @@ public void xwrite(MemoryBuffer buffer, T value) { xwriteElements(fury, buffer, map); } - protected final void writeElements(Fury fury, MemoryBuffer buffer, Map map) { - Serializer keySerializer = this.keySerializer; - Serializer valueSerializer = this.valueSerializer; - // clear the elemSerializer to avoid conflict if the nested - // serialization has collection field. - // TODO use generics for compatible serializer. - this.keySerializer = null; - this.valueSerializer = null; - if (keySerializer != null && valueSerializer != null) { - javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); - } else if (keySerializer != null) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - fury.writeRef(buffer, entry.getKey(), keySerializer); - Object value = entry.getValue(); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, value, valueClassInfoWriteCache); - } - } else if (valueSerializer != null) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, key, keyClassInfoWriteCache); - fury.writeRef(buffer, entry.getValue(), valueSerializer); - } - } else { - genericJavaWrite(fury, buffer, map); - } - } - protected final void chunkWriteElements(Fury fury, MemoryBuffer buffer, Map map) { Serializer keySerializer = this.keySerializer; Serializer valueSerializer = this.valueSerializer; @@ -490,193 +451,6 @@ private Entry writeJavaChunkGeneric( return entry; } - private void javaWriteWithKVSerializers( - Fury fury, - MemoryBuffer buffer, - Map map, - Serializer keySerializer, - Serializer valueSerializer) { - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - fury.writeRef(buffer, key, keySerializer); - fury.writeRef(buffer, value, valueSerializer); - } - } - - private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null) { - generalJavaWrite(fury, buffer, map); - } else { - - // type parameters count for `Map field` will be 0; - // type parameters count for `SubMap field` which SubMap is - // `SubMap implements Map` will be 1; - if (genericType.getTypeParametersCount() < 2) { - genericType = getKVGenericType(genericType); - } - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - if (keyGenericType == objType && valueGenericType == objType) { - generalJavaWrite(fury, buffer, map); - return; - } - // Can't avoid push generics repeatedly in loop by stack depth, because push two - // generic type changed generics stack top, which is depth index, update stack top - // and depth will have some cost too. - // Stack depth to avoid push generics repeatedly in loop. - // Note push two generic type changed generics stack top, which is depth index, - // stack top should be updated when using for serialization k/v. - // int depth = fury.getDepth(); - // // depth + 1 to leave a slot for value generics, otherwise value generics will - // // be overwritten by nested key generics. - // fury.setDepth(depth + 1); - // generics.pushGenericType(keyGenericType); - // fury.setDepth(depth); - // generics.pushGenericType(valueGenericType); - boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); - boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); - if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else if (keyGenericTypeFinal) { - javaKeyTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else if (valueGenericTypeFinal) { - javaValueTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else { - javaKVTypesNonFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } - } - } - - private void javaKVTypesFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.writeRef(buffer, entry.getKey(), keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - fury.writeRef(buffer, entry.getValue(), valueSerializer); - generics.popGenericType(); - } - } - - private void javaKeyTypeFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.writeRef(buffer, entry.getKey(), keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingValueRef, - buffer, - entry.getValue(), - valueClassInfoWriteCache); - generics.popGenericType(); - } - } - - private void javaValueTypeFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingKeyRef, - buffer, - entry.getKey(), - keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - fury.writeRef(buffer, entry.getValue(), valueSerializer); - generics.popGenericType(); - } - } - - private void javaKVTypesNonFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingKeyRef, - buffer, - entry.getKey(), - keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingValueRef, - buffer, - entry.getValue(), - valueClassInfoWriteCache); - generics.popGenericType(); - } - } - - private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); - } - } - public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value) { Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); @@ -1013,170 +787,6 @@ private long readJavaChunkGeneric( return size > 0 ? (size << 8) | buffer.readUnsignedByte() : 0; } - @SuppressWarnings("unchecked") - protected final void readElements(MemoryBuffer buffer, int size, Map map) { - Serializer keySerializer = this.keySerializer; - Serializer valueSerializer = this.valueSerializer; - // clear the elemSerializer to avoid conflict if the nested - // serialization has collection field. - // TODO use generics for compatible serializer. - this.keySerializer = null; - this.valueSerializer = null; - if (keySerializer != null && valueSerializer != null) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer, keySerializer); - Object value = fury.readRef(buffer, valueSerializer); - map.put(key, value); - } - } else if (keySerializer != null) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer, keySerializer); - map.put(key, fury.readRef(buffer, keyClassInfoReadCache)); - } - } else if (valueSerializer != null) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer); - Object value = fury.readRef(buffer, valueSerializer); - map.put(key, value); - } - } else { - genericJavaRead(fury, buffer, map, size); - } - } - - private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null) { - generalJavaRead(fury, buffer, map, size); - } else { - if (genericType.getTypeParametersCount() < 2) { - genericType = getKVGenericType(genericType); - } - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - if (keyGenericType == objType && valueGenericType == objType) { - generalJavaRead(fury, buffer, map, size); - return; - } - boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); - boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); - if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else if (keyGenericTypeFinal) { - javaKeyTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else if (valueGenericTypeFinal) { - javaValueTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else { - javaKVTypesNonFinalRead( - fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } - generics.popGenericType(); - } - } - - private void javaKVTypesFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = fury.readRef(buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = fury.readRef(buffer, valueSerializer); - generics.popGenericType(); - map.put(key, value); - } - } - - private void javaKeyTypeFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - RefResolver refResolver = fury.getRefResolver(); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = fury.readRef(buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = - readJavaRefOptimized( - fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); - generics.popGenericType(); - map.put(key, value); - } - } - - private void javaValueTypeFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - RefResolver refResolver = fury.getRefResolver(); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = - readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = fury.readRef(buffer, valueSerializer); - generics.popGenericType(); - map.put(key, value); - } - } - - private void javaKVTypesNonFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); - boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = - readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = - readJavaRefOptimized( - fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); - generics.popGenericType(); - map.put(key, value); - } - } - - private void generalJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer, keyClassInfoReadCache); - Object value = fury.readRef(buffer, valueClassInfoReadCache); - map.put(key, value); - } - } - @SuppressWarnings("unchecked") public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int size) { Generics generics = fury.getGenerics(); @@ -1249,10 +859,6 @@ public final boolean supportCodegenHook() { return supportCodegenHook; } - public boolean isUseChunkSerialize() { - return useChunkSerialize; - } - public void setUseChunkSerialize(boolean useChunkSerialize) { this.useChunkSerialize = useChunkSerialize; } @@ -1270,41 +876,6 @@ public void setUseChunkSerialize(boolean useChunkSerialize) { */ public abstract Map onMapWrite(MemoryBuffer buffer, T value); - /** Check null first to avoid ref tracking for some types with ref tracking disabled. */ - private void writeJavaRefOptimized( - Fury fury, - ClassResolver classResolver, - RefResolver refResolver, - MemoryBuffer buffer, - Object obj, - ClassInfoHolder classInfoHolder) { - if (!refResolver.writeNullFlag(buffer, obj)) { - fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } - - private void writeJavaRefOptimized( - Fury fury, - ClassResolver classResolver, - RefResolver refResolver, - boolean trackingRef, - MemoryBuffer buffer, - Object obj, - ClassInfoHolder classInfoHolder) { - if (trackingRef) { - if (!refResolver.writeNullFlag(buffer, obj)) { - fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } else { - if (obj == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - fury.writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } - } - @Override public abstract T read(MemoryBuffer buffer); @@ -1373,29 +944,4 @@ public void setNumElements(int numElements) { public abstract T onMapCopy(Map map); public abstract T onMapRead(Map map); - - private Object readJavaRefOptimized( - Fury fury, - RefResolver refResolver, - boolean trackingRef, - MemoryBuffer buffer, - ClassInfoHolder classInfoHolder) { - if (trackingRef) { - int nextReadRefId = refResolver.tryPreserveRefId(buffer); - if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { - Object obj = fury.readNonRef(buffer, classInfoHolder); - refResolver.setReadObject(nextReadRefId, obj); - return obj; - } else { - return refResolver.getReadObject(); - } - } else { - byte headFlag = buffer.readByte(); - if (headFlag == Fury.NULL_FLAG) { - return null; - } else { - return fury.readNonRef(buffer, classInfoHolder); - } - } - } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java index ca40de4c0f..9ab384dccb 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java @@ -65,11 +65,7 @@ public T onMapCopy(Map map) { @Override public T read(MemoryBuffer buffer) { Map map = newMap(buffer); - if (useChunkSerialize) { - chunkReadElements(buffer, getAndClearNumElements(), map); - } else { - readElements(buffer, getAndClearNumElements(), map); - } + chunkReadElements(buffer, getAndClearNumElements(), map); return onMapRead(map); } diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java index f422b8174e..0aad9f47f6 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java @@ -833,8 +833,9 @@ public void testNestedMapFieldStructCodegen(boolean referenceTrackingConfig) { struct1.map1 = ofHashMap(1, ofHashMap("k1", "v1", "k2", "v2")); struct1.map2 = ofHashMap("k1", ofHashMap("k1", 1, "k2", 2)); struct1.map3 = ofHashMap(1, ofHashMap("k1", beanB, "k2", beanB, "k3", BeanB.createBeanB(1))); - struct1.map4 = ofHashMap(2, ofHashMap(true, - ofHashMap("k1", beanB, "k2", beanB, "k3", BeanB.createBeanB(1)))); + struct1.map4 = + ofHashMap( + 2, ofHashMap(true, ofHashMap("k1", beanB, "k2", beanB, "k3", BeanB.createBeanB(1)))); serDeCheck(fury, struct1); } }