Improve read/write performance

benchmark/benchmarks.jl

@@ -42,6 +42,12 @@ SUITE["read"]["big"]["sample17/128analogchannels.c2d"]["full"] =
     @benchmarkable readc3d(fn) seconds=10 setup=(fn=artifact"sample17/128analogchannels.c3d")
 SUITE["read"]["big"]["sample19/sample19.c3d"]["full"] =
     @benchmarkable readc3d(fn) seconds=10 setup=(fn=artifact"sample19/sample19.c3d")
+SUITE["read"]["big"]["sample33/bigparlove.c3d"]["full"] =
+    @benchmarkable readc3d(fn) seconds=10 setup=(fn=artifact"sample33/bigparlove.c3d")
+SUITE["read"]["big"]["sample12/c24089 13.c3d"]["full"] =
+    @benchmarkable readc3d(fn) seconds=10 setup=(fn=artifact"sample12/c24089 13.c3d")
+SUITE["read"]["big"]["sample31/large01.c3d"]["full"] =
+    @benchmarkable readc3d(fn) seconds=10 setup=(fn=artifact"sample31/large01.c3d")
 
 SUITE["read"]["pc-real"]["sample01"]["params"] =
     @benchmarkable readc3d(fn; paramsonly=true) seconds=10 setup=(fn=artifact"sample01/Eb015pr.c3d")
@@ -75,6 +81,12 @@ SUITE["read"]["big"]["sample17/128analogchannels.c3d"]["params"] =
     @benchmarkable readc3d(fn; paramsonly=true) seconds=10 setup=(fn=artifact"sample17/128analogchannels.c3d")
 SUITE["read"]["big"]["sample19/sample19.c3d"]["params"] =
     @benchmarkable readc3d(fn; paramsonly=true) seconds=10 setup=(fn=artifact"sample19/sample19.c3d")
+SUITE["read"]["big"]["sample33/bigparlove.c3d"]["params"] =
+    @benchmarkable readc3d(fn; paramsonly=true) seconds=10 setup=(fn=artifact"sample33/bigparlove.c3d")
+SUITE["read"]["big"]["sample12/c24089 13.c3d"]["params"] =
+    @benchmarkable readc3d(fn; paramsonly=true) seconds=10 setup=(fn=artifact"sample12/c24089 13.c3d")
+SUITE["read"]["big"]["sample31/large01.c3d"]["params"] =
+    @benchmarkable readc3d(fn; paramsonly=true) seconds=10 setup=(fn=artifact"sample31/large01.c3d")
 
 SUITE["show"] = BenchmarkGroup()
 

src/C3D.jl

@@ -42,13 +42,6 @@ function C3DFile(name::String, header::Header{END}, groups::LittleDict{Symbol,Gr
     cameras = OrderedDict{String,Vector{UInt8}}()
     fanalog = OrderedDict{String,Vector{Float32}}()
 
-    numpts = groups[:POINT][Int, :USED]
-
-    ptlabel_keys = get_multipled_parameter_names(groups, :POINT, :LABELS)
-    pt_labels = Iterators.flatten(
-        groups[:POINT][Vector{String}, label] for label in ptlabel_keys
-        )
-
     if strip_prefixes
         if haskey(groups, :SUBJECTS) && groups[:SUBJECTS][Int, :USES_PREFIXES] == 1
             rgx = Regex(
@@ -60,16 +53,27 @@ function C3DFile(name::String, header::Header{END}, groups::LittleDict{Symbol,Gr
         end
     end
 
-    nolabel_count = 1
+    numpts = groups[:POINT][Int, :USED]
     if !iszero(numpts)
+        if haskey(groups[:POINT], :LABELS2)
+            ptlabel_keys = get_multipled_parameter_names(groups, :POINT, :LABELS)
+            pt_labels = Iterators.flatten(
+                groups[:POINT][Vector{String}, label] for label in ptlabel_keys
+                )
+        else
+            pt_labels = groups[:POINT][Vector{String}, :LABELS]
+        end
+
         sizehint!(fpoint, numpts)
         sizehint!(fresiduals, numpts)
         sizehint!(cameras, numpts)
 
         invalidpoints = Vector{Bool}(undef, size(point, 1))
         calculatedpoints = Vector{Bool}(undef, size(point, 1))
         goodpoints = Vector{Bool}(undef, size(point, 1))
+        abs_scale = abs(groups[:POINT][Float32, :SCALE])
 
+        nolabel_count = 1
         for (idx, ptname) in enumerate(pt_labels)
             idx > numpts && break # can't slice Iterators.flatten
             og_ptname = ptname
@@ -95,7 +99,7 @@ function C3DFile(name::String, header::Header{END}, groups::LittleDict{Symbol,Gr
                 while ptname*"_$cnt" ∈ keys(fpoint); cnt+=1 end
                 ptname *= "_$cnt"
                 dedupped_ptname = ptname
-                @warn "Duplicate marker label detected (\"$(og_ptname)\"). Duplicate renamed to \"$dedupped_ptname\"." _id=dedupped_ptname
+                @warn "Duplicate marker label detected (\"$(og_ptname)\"). Duplicate renamed to \"$dedupped_ptname\"." _id=hash(name*dedupped_ptname) maxlog=1
             end
             ptname ∉ keys(fpoint) || throw(DuplicateMarkerError(
                 "Markers labels must be unique but found duplicate marker label \"$ptname\""*
@@ -112,36 +116,39 @@ function C3DFile(name::String, header::Header{END}, groups::LittleDict{Symbol,Gr
             invalidpoints .= ((@view(residuals[:,idx]) .% UInt16) .& 0x8000) .!== 0x0000
             calculatedpoints .= iszero.(@view(residuals[:,idx]) .& 0xff)
             goodpoints .= .~(invalidpoints .| calculatedpoints)
-            calcresiduals!(fresiduals[ptname], goodpoints, abs(groups[:POINT][Float32, :SCALE]))
+            calcresiduals!(fresiduals[ptname], goodpoints, abs_scale)
 
             if missingpoints
+                pt_residual = fresiduals[ptname]
+                pt_point = fpoint[ptname]
                 for i in eachindex(fresiduals[ptname])
                     if calculatedpoints[i] & ~invalidpoints[i]
-                        fresiduals[ptname][i] = 0.0f0
+                        pt_residual[i] = 0.0f0
                     end
 
                     if invalidpoints[i]
-                        fpoint[ptname][i, :] .= missing
-                        fresiduals[ptname][i] = missing
+                        pt_point[i, :] .= (missing, missing, missing)
+                        pt_residual[i] = missing
                     end
                 end
             end
         end
     end
 
-    anlabel_keys = collect(filter(keys(groups[:ANALOG])) do k
-        contains(string(k), r"^LABELS\d*")
-    end)
-    sort!(anlabel_keys; by=_naturalsortby)
-    an_labels = Iterators.flatten(
-        groups[:ANALOG][Vector{String}, label] for label in anlabel_keys
-        )
-
     numanalogs = groups[:ANALOG][Int, :USED]
-
-    nolabel_count = 1
     if !iszero(numanalogs)
+        if haskey(groups[:ANALOG], :LABELS2)
+            anlabel_keys = get_multipled_parameter_names(groups, :ANALOG, :LABELS)
+            an_labels = Iterators.flatten(
+                groups[:ANALOG][Vector{String}, label] for label in anlabel_keys
+                )
+        else
+            an_labels = groups[:ANALOG][Vector{String}, :LABELS]
+        end
+
         sizehint!(fanalog, numanalogs)
+
+        nolabel_count = 1
         for (idx, name) in enumerate(an_labels)
             idx > numanalogs && break # can't slice Iterators.flatten
             og_name = name
@@ -156,7 +163,7 @@ function C3DFile(name::String, header::Header{END}, groups::LittleDict{Symbol,Gr
                 while name*"_$cnt" ∈ keys(fanalog); cnt+=1 end
                 name *= "_$cnt"
                 dedupped_name = name
-                @warn "Duplicate analog signal label detected (\"$(og_name)\"). Duplicate renamed to \"$dedupped_name\"." _id=dedupped_name
+                @warn "Duplicate analog signal label detected (\"$(og_name)\"). Duplicate renamed to \"$dedupped_name\"." _id=hash(name*dedupped_name) maxlog=1
             end
 
             fanalog[name] = analog[:, idx]

src/parameters.jl

@@ -226,7 +226,7 @@ function readparam(io::IO, ::Type{END}) where {END<:AbstractEndian}
     @assert gid != 0
     _name = read(io, abs(nl))
     @assert any(!iscntrl∘Char, _name)
-    name = Symbol(replace(strip(transcode(String, view(_name, :))), r"[^a-zA-Z0-9_]" => '_'))
+    name = Symbol(replace(transcode(String, rstrip_vectorstring(!isspace∘Char, view(_name, :))), r"[^a-zA-Z0-9_]" => '_'))
 
     # @debug "Parameter $name at $pos has unofficially supported characters.
         # Unexpected results may occur" maxlog=occursin(r"[^a-zA-Z0-9_ ]", transcode(String, copy(_name)))
@@ -256,7 +256,7 @@ function readparam(io::IO, ::Type{END}) where {END<:AbstractEndian}
     end
 
     dl = read(io, UInt8)::UInt8
-    desc = copy(rstrip_cntrl_null_space(read(io, dl)))
+    desc = copy(rstrip_vectorstring(read(io, dl)))
 
     pointer = pos + np + abs(nl) + 2
     # @debug "wrong pointer in $name" position(io) pointer maxlog=(position(io) != pointer)
@@ -295,14 +295,9 @@ function _readarrayparameter(io::IO, END::Type{<:AbstractEndian{T}}, dims) where
     return read!(io, a, END)
 end
 
-# Taken from Base.iscntrl and Base.isspace, but for bytes instead of chars
-_iscntrl(c::Char) = iscntrl(c)
-_isspace(c::Char) = isspace(c)
-_iscntrl(c::Union{Int8,UInt8}) = c <= 0x1f || 0x7f <= c <= 0x9f
-_isspace(c::Union{Int8,UInt8}) = c == 0x20 || 0x09 <= c <= 0x0d || c == 0x85 || 0xa0 <= c
-
-function rstrip_cntrl_null_space(s)
-    l = findlast(c -> !_iscntrl(c) && !_isspace(c), s)
+rstrip_vectorstring(s) = rstrip_vectorstring((c -> !isspace(c) && !iscntrl(c))∘Char, s)
+function rstrip_vectorstring(f, s)
+    l = findlast(f, s)
     if isnothing(l)
         return @view s[end:end-1]
     else
@@ -321,13 +316,15 @@ function _readarrayparameter(io::IO, ::Type{<:AbstractEndian{String}}, dims)::Ar
     if length(dims) > 1
         _, rdims... = dims
         data = Array{String}(undef, rdims)::Array{String}
+        temp = Vector{UInt16}(undef, dims[1])
         for ijk::CartesianIndex in CartesianIndices(data)
-            data[ijk] = string(rstrip(x -> iscntrl(x) || isspace(x),
-                transcode(String, transcode(UInt16, @view tdata[:, ijk]))))::String
+            temp .= tdata[:, ijk]
+            data[ijk] = transcode(String,
+                rstrip_vectorstring(convert(Vector{UInt16}, temp)))
             # @debug "" @view(tdata[:, ijk]), data[ijk]
         end
     else
-        data = [ rstrip(x -> iscntrl(x) || isspace(x), transcode(String, transcode(UInt16, _tdata))) ]
+        data = [ transcode(String, rstrip_vectorstring(convert(Vector{UInt16}, _tdata))) ]
     end
     return data
 end

src/read.jl

@@ -86,17 +86,20 @@ function readdata(
         # Analog Samples Per Frame => ASPF
         analog = zeros(Float32, numchannels, aspf*numframes)
 
+        nbanalog = numchannels*aspf
         analogtmp = Matrix{F}(undef, (numchannels,aspf))
     else
         analog = Array{Float32,2}(undef, 0,0)
     end
 
+    pos = position(io) # calculate position to avoid repeatedly acquire/release of file lock
     @inbounds for i in 1:numframes
         if hasmarkers
-            if _iosize - position(io) ≥ sizeof(pointtmp)
+            if _iosize - pos ≥ sizeof(pointtmp)
                 read!(io, pointtmp, END)
                 point[:,i] .= convert.(Float32, pointview)
                 residuals[:,i] .= convert.(Int32, resview) .% Int16
+                pos += nb*sizeof(F)
             else
                 # Make marker data for missing frames be treated as missing
                 residuals[:,i] .= -ones(Int16, nummarkers)
@@ -106,9 +109,10 @@ function readdata(
             end
         end
         if haschannels
-            if _iosize - position(io) ≥ sizeof(analogtmp)
+            if _iosize - pos ≥ sizeof(analogtmp)
                 read!(io, analogtmp, END)
                 analog[:,((i-1)*aspf+1):(i*aspf)] .= convert.(Float32, analogtmp)
+                pos += nbanalog*sizeof(F)
             else
                 @debug "End-of-file reached before expected; frame$(length(i:numframes) > 1 ? "s" : "") $(i:numframes) \
                     are missing"
@@ -129,8 +133,8 @@ function readdata(
             SCALE = groups[:ANALOG][Float32, :GEN_SCALE] * groups[:ANALOG][Float32, :SCALE]
             analog .= (analog .- ANALOG_OFFSET) .* SCALE
         else
-            off_labels = get_multipled_parameter_names(groups, :ANALOG, :OFFSET)
-            if length(off_labels) > 1
+            if haskey(groups[:ANALOG], :OFFSET2)
+                off_labels = get_multipled_parameter_names(groups, :ANALOG, :OFFSET)
                 VECANALOG_OFFSET = convert(Vector{Float32}, reduce(vcat,
                     groups[:ANALOG][Vector{Int}, offset]
                     for offset in off_labels))[1:numchannels]
@@ -139,8 +143,8 @@ function readdata(
                     groups[:ANALOG][Vector{Int}, :OFFSET][1:numchannels])
             end
 
-            scale_labels = get_multipled_parameter_names(groups, :ANALOG, :SCALE)
-            if length(scale_labels) > 1
+            if haskey(groups[:ANALOG], :SCALE2)
+                scale_labels = get_multipled_parameter_names(groups, :ANALOG, :SCALE)
                 VECSCALE = convert(Vector{Float32}, reduce(vcat,
                     groups[:ANALOG][Vector{Int}, scale]
                     for scale in scale_labels))[1:numchannels]
@@ -153,7 +157,7 @@ function readdata(
         end
     end
 
-    return (permutedims(point), permutedims(residuals), permutedims(analog))
+    return (point', permutedims(residuals), analog')
 end
 
 """