Skip to content

Speed up Deserialization #5237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft

Speed up Deserialization #5237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
64db0b5
serializing uuids
antonydellavecchia Aug 22, 2025
852d403
Merge remote-tracking branch 'origin/master'
antonydellavecchia Aug 22, 2025
29e87bd
removing more tryparse UUID
antonydellavecchia Aug 22, 2025
761a919
minor updates
antonydellavecchia Aug 22, 2025
2818379
small improvements
antonydellavecchia Aug 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions src/Serialization/basic_types.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,16 @@
################################################################################
# UUID
@register_serialization_type UUID "UUID"
save_object(s::SerializerState, x::UUID) = save_data_basic(s, x)
load_object(s::DeserializerState, ::Type{UUID}) = load_ref(s, UUID(s.obj))

################################################################################

function save_object(s::SerializerState, x::T) where T <: Union{BasicTypeUnion, VersionNumber}
save_data_basic(s, x)
end


################################################################################
# Bool
@register_serialization_type Bool
Expand All @@ -24,11 +33,8 @@ end

load_object(s::DeserializerState, T::Type{ZZRingElem}, ::ZZRing) = load_object(s, T)

function load_object(s::DeserializerState, ::Type{ZZRingElem})
load_node(s) do str
return ZZRingElem(str)
end
end
load_object(s::DeserializerState, ::Type{ZZRingElem}) = ZZRingElem(s.obj)


################################################################################
# QQFieldElem
Expand Down
43 changes: 20 additions & 23 deletions src/Serialization/containers.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,16 +111,13 @@ function save_object(s::SerializerState, x::AbstractVector{S}) where S
end

function load_object(s::DeserializerState, T::Type{<: Vector{params}}) where params
l = length(s.obj)
load_node(s) do v
if serialize_with_id(params)
loaded_v::Vector{params} = load_array_node(s) do _
load_ref(s)
end
loaded_v = params[load_object(s, UUID, k) for k in 1:l]
else
isempty(s.obj) && return params[]
loaded_v = load_array_node(s) do obj
load_object(s, params)
end
loaded_v = params[load_object(s, params, k) for k in 1:l]
end
return loaded_v
end
Expand All @@ -129,21 +126,20 @@ end

function load_object(s::DeserializerState, ::Type{Vector{T}}, params::S) where {T, S}
isempty(s.obj) && return T[]
v = load_array_node(s) do _
if serialize_with_id(T)
load_ref(s)
else
load_object(s, T, params)
end
l = length(s.obj)
if serialize_with_id(T)
return T[load_object(s, UUID, k)::T for k in 1:l]
else
return T[load_object(s, T, params, k) for k in 1:l]
end
return v
end

function load_object(s::DeserializerState, T::Type{Vector{U}}, ::Nothing) where U
isempty(s.obj) && return U[]
return load_array_node(s) do _
load_object(s, U)
end
l = length(s.obj)
return U[
load_object(s, U, k) for k in 1:l
]
end

################################################################################
Expand Down Expand Up @@ -252,10 +248,11 @@ end

function load_type_params(s::DeserializerState, T::Type{Tuple})
subtype, params = load_node(s, :params) do _
tuple_params = load_array_node(s) do _
U = decode_type(s)
load_type_params(s, U)
end
l = length(s.obj)
U = decode_type(s)
tuple_params = [
load_type_params(s, U, k) for k in 1:l
]
return Tuple([x[1] for x in tuple_params]), Tuple(x[2] for x in tuple_params)
end
return T{subtype...}, params
Expand All @@ -278,7 +275,7 @@ function load_object(s::DeserializerState, T::Type{<:Tuple}, params::Tuple)
entries = load_array_node(s) do (i, entry)
S = fieldtype(T, i)
if serialize_with_id(S)
return load_ref(s)
return load_object(s, UUID)
else
return load_object(s, S, params[i])
end
Expand Down Expand Up @@ -326,7 +323,7 @@ function save_type_params(s::SerializerState, tp::TypeParams{<:NamedTuple})
end

function load_type_params(s::DeserializerState, T::Type{NamedTuple})
subtype, params = load_node(s, :params) do obj
subtype, params = load_node(s, :params) do _
tuple_params = load_array_node(s, :tuple_params) do _
U = decode_type(s)
load_type_params(s, U)
Expand Down Expand Up @@ -395,7 +392,7 @@ function save_type_params(
end

function load_type_params(s::DeserializerState, T::Type{Dict})
subtype, params = load_node(s, :params) do obj
subtype, params = load_node(s, :params) do _
if haskey(s, :value_params)
S, key_params = load_node(s, :key_params) do params
params isa String && return decode_type(s), nothing
Expand Down
56 changes: 22 additions & 34 deletions src/Serialization/main.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,20 +122,7 @@ function decode_type(s::String)
end

function decode_type(s::DeserializerState)
if s.obj isa String
if !isnothing(tryparse(UUID, s.obj))
id = s.obj
obj = s.obj
if isnothing(s.refs)
return typeof(global_serializer_state.id_to_obj[UUID(id)])
end
s.obj = s.refs[Symbol(id)]
T = decode_type(s)
s.obj = obj
return T
end
return decode_type(s.obj)
end
s.obj isa String && return decode_type(s.obj)

if type_key in keys(s.obj)
return load_node(s, type_key) do _
Expand Down Expand Up @@ -245,7 +232,9 @@ function save_typed_object(s::SerializerState, x::T, key::Symbol) where T
if serialize_with_id(x)
# key should already be set before function call
ref = save_as_ref(s, x)
save_object(s, ref)
save_data_dict(s) do
save_typed_object(s, ref)
end
else
save_data_dict(s) do
save_typed_object(s, x)
Expand Down Expand Up @@ -329,7 +318,9 @@ function save_type_params(s::SerializerState,
save_data_array(s, Symbol(param.first)) do
for entry in param.second
if serialize_with_id(entry)
save_object(s, save_as_ref(s, entry))
save_data_dict(s) do
save_typed_object(s, save_as_ref(s, entry))
end
else
save_data_dict(s) do
save_typed_object(s, entry)
Expand Down Expand Up @@ -357,19 +348,13 @@ end
function load_type_array_params(s::DeserializerState)
load_array_node(s) do obj
T = decode_type(s)
if obj isa String
!isnothing(tryparse(UUID, s.obj)) && return load_ref(s)
return T
end
obj isa String && return T
return load_type_params(s, T)[2]
end
end

function load_type_params(s::DeserializerState, T::Type)
if s.obj isa String
if !isnothing(tryparse(UUID, s.obj))
return T, load_ref(s)
end
return T, nothing
end
if haskey(s, :params)
Expand All @@ -391,11 +376,8 @@ function load_type_params(s::DeserializerState, T::Type)
if obj isa JSON3.Array || obj isa Vector
return load_type_array_params(s)
end

U = decode_type(s)
if obj isa String && isnothing(tryparse(UUID, obj))
return U
end
obj isa String && return U
return load_type_params(s, U)[2]
end
end
Expand Down Expand Up @@ -428,7 +410,6 @@ function load_typed_object(s::DeserializerState; override_params::Any = nothing)
T, _ = load_type_params(s, T, type_key)
params = override_params
else
s.obj isa String && !isnothing(tryparse(UUID, s.obj)) && return load_ref(s)
T, params = load_type_params(s, T, type_key)
end
obj = load_node(s, :data) do _
Expand All @@ -446,8 +427,15 @@ end

function load_object(s::DeserializerState, T::Type, params::S,
key::Union{Symbol, Int}) where S
load_node(s, T, key) do _
load_object(s, T, params)::T
end::T
end

function load_object(s::DeserializerState, ::Type{UUID}, params::S,
key::Union{Symbol, Int}) where S
load_node(s, key) do _
load_object(s, T, params)
load_object(s, UUID, params)
end
end

Expand All @@ -469,7 +457,7 @@ function load_attrs(s::DeserializerState, obj::T) where T
!with_attrs(s) && return

haskey(s, :attrs) && load_node(s, :attrs) do d
for attr in keys(d)
for attr in keys(s)
set_attribute!(obj, attr, load_typed_object(s, attr))
end
end
Expand Down Expand Up @@ -755,9 +743,9 @@ function load(io::IO; params::Any = nothing, type::Any = nothing,
serializer=JSONSerializer(), with_attrs::Bool=true)
s = deserializer_open(io, serializer, with_attrs)
if haskey(s.obj, :id)
id = s.obj[:id]
if haskey(global_serializer_state.id_to_obj, UUID(id))
return global_serializer_state.id_to_obj[UUID(id)]
id = UUID(s.obj[:id])
if haskey(global_serializer_state.id_to_obj, id)
return global_serializer_state.id_to_obj[id]
end
end

Expand Down Expand Up @@ -819,7 +807,7 @@ function load(io::IO; params::Any = nothing, type::Any = nothing,
loaded = load_typed_object(s; override_params=params)
end

if :id in keys(s.obj)
if haskey(s, :id)
load_node(s, :id) do id
global_serializer_state.obj_to_id[loaded] = UUID(id)
global_serializer_state.id_to_obj[UUID(id)] = loaded
Expand Down
Loading
Loading