diff --git a/src/bit_vector.jl b/src/bit_vector.jl index f7c55b3..94c33d7 100644 --- a/src/bit_vector.jl +++ b/src/bit_vector.jl @@ -18,6 +18,22 @@ end struct BlobBitVector <: AbstractArray{Bool, 1} data::Blob{UInt64} length::Int64 + + function BlobBitVector(data::Blob{UInt64}, length::Int64) + @assert length >= 0 + @boundscheck begin + if div(length, self_size(UInt64), RoundUp) > available_size(data) + throw(InvalidBlobError( + BlobBitVector, + getfield(data, :base), + getfield(data, :offset), + getfield(data, :limit), + div(length + 7, 8)), + ) + end + end + new(data, length) + end end Base.@propagate_inbounds function get_address(blob::BlobBitVector, i::Int)::BlobBit diff --git a/src/blob.jl b/src/blob.jl index 7715f71..6690b02 100644 --- a/src/blob.jl +++ b/src/blob.jl @@ -1,5 +1,58 @@ +struct InvalidBlobError <: Exception + type::Type + base::Ptr{Nothing} + offset::Int64 + limit::Int64 + length::Int64 +end + +function Base.showerror(io::IO, e::InvalidBlobError) + print(io, "InvalidBlobError: $(e.type) needs $(e.length) * $(self_size(e.type)) bytes. \ + Got length($(e.offset):$(e.limit)) == $(e.limit - e.offset) bytes") +end + """ -A pointer to a `T` stored inside a Blob. + Blob{T} + +A pointer to a memory array that stores a `T`. + +The fields are stored compact in memory without alignment, i.e each basic field f takes up +`sizeof(fieldtype(T, :f))` bytes. Blobs inside `T` take just the offset, i.e. 8 bytes. +This is different from Julia memory layout. + +You can just store struct of only primitive types or structs of primitive out of the box, +for example: + +```julia +struct Foo + x::Int64 + y::Float64 +end + +blob = Blobs.malloc(Foo) +blob[] = Foo(42, 3.14) + +In order to store variable size data structures (`BlobVector`, `BlobBitVector`, +`BlobString` or your own implementation) or a `Blob``, you need to implement `child_size` +and `init` for your type. + +Example: +```julia + struct FooString + s::BlobString + i::Int64 + end + + function Blobs.child_size(::FooString, string_length::Int64) + return child_size(BlobString, string_length) + end + + function Blobs.init(blob::Blob{FooString}, free::Blob{Nothing}, string_length::Int64) + free = Blobs.init(blob.s, free, string_length) + blob.i[] = 0 + return free + end +``` """ struct Blob{T} base::Ptr{Nothing} @@ -8,54 +61,99 @@ struct Blob{T} function Blob{T}(base::Ptr{Nothing}, offset::Int64, limit::Int64) where {T} @assert isbitstype(T) - new(base, offset, limit) + @boundscheck _bounds_check(base, offset, limit, self_size(T), T) + new{T}(base, offset, limit) + end +end + +@noinline function _bounds_check( + base::Ptr{Nothing}, + offset::Int64, + limit::Int64, + self_size_T::Int64, + @nospecialize(T::DataType)) + if offset < 0 || offset + self_size_T > limit + throw(InvalidBlobError(Blob{T}, base, offset, limit, 1)) + end + if limit > 0 && base == Ptr{Nothing}(0) + throw(AssertionError("Null pointer reference Blob{$(T)}")) end end +""" + Blob{T}(ref::Base.RefValue{T}) where T + +Create a `Blob{T}` from an Julia allocated object. +**Danger**: This only works if memory layout of Julia struct is the same as of the Blob. +""" function Blob(ref::Base.RefValue{T}) where T - Blob{T}(pointer_from_objref(ref), 0, sizeof(T)) + @assert self_size(T) == sizeof(T) "$(T) cannot of aligned fields or Blobs" + @inbounds Blob{T}(pointer_from_objref(ref), 0, self_size(T)) end +""" + Blob{T}(base::Ptr{T}, offset::Int64 = 0, limit::Int64 = sizeof(T)) where T + +Create a `Blob{T}` from a pointer. +""" +Base.@propagate_inbounds \ function Blob(base::Ptr{T}, offset::Int64 = 0, limit::Int64 = sizeof(T)) where {T} Blob{T}(Ptr{Nothing}(base), offset, limit) end -function Blob{T}(blob::Blob) where T +""" + Blob{T}(blob::Blob) + + Make a copy and potentially change the type of a `Blob`. +""" +Base.@propagate_inbounds function Blob{T}(blob::Blob) where T Blob{T}(getfield(blob, :base), getfield(blob, :offset), getfield(blob, :limit)) end +""" + available_size(blob::Blob{T}) where T + +The size of memory this `Blob` and it's children own. `blob.limit - blob.offset`. +""" +available_size(blob::Blob{T}) where T = getfield(blob, :limit) - getfield(blob, :offset) + function assert_same_allocation(blob1::Blob, blob2::Blob) @assert getfield(blob1, :base) == getfield(blob2, :base) "These blobs do not share the same allocation: $blob1 - $blob2" end +"""" + pointer(blob::Blob{T}) where T + +Get a pointer to the data in the `blob`. Note that you cannot `unsafe_load` +from this pointer, since the data is not aligned. +""" function Base.pointer(blob::Blob{T}) where T convert(Ptr{T}, getfield(blob, :base) + getfield(blob, :offset)) end -function Base.:+(blob::Blob{T}, offset::Integer) where T +"""" + Base:+(::Blob, ::Integer) + +Increase the offset of a `Blob` by `offset`. +""" +Base.@propagate_inbounds function Base.:+(blob::Blob{T}, offset::Integer) where T Blob{T}(getfield(blob, :base), getfield(blob, :offset) + offset, getfield(blob, :limit)) end +""" + Base:-(::Blob, ::Blob) + +Get the offset difference of two blobs in the same allocation. +""" function Base.:-(blob1::Blob, blob2::Blob) assert_same_allocation(blob1, blob2) getfield(blob1, :offset) - getfield(blob2, :offset) end -function boundscheck(blob::Blob{T}) where T - begin - if (getfield(blob, :offset) < 0) || (getfield(blob, :offset) + self_size(T) > getfield(blob, :limit)) - throw(BoundsError(blob)) - end - @assert (getfield(blob, :base) != Ptr{Nothing}(0)) "Null pointer dereference in $(typeof(blob))" - end -end - -Base.@propagate_inbounds function Base.getindex(blob::Blob{T}) where T - @boundscheck boundscheck(blob) +function Base.getindex(blob::Blob{T}) where T unsafe_load(blob) end -# TODO(jamii) do we need to align data? """ self_size(::Type{T}, args...) where {T} @@ -88,10 +186,10 @@ end @generated function Base.getindex(blob::Blob{T}, ::Type{Val{field}}) where {T, field} i = findfirst(isequal(field), fieldnames(T)) - @assert i != nothing "$T has no field $field" + @assert i !== nothing "$T has no field $field" quote $(Expr(:meta, :inline)) - Blob{$(fieldtype(T, i))}(blob + $(blob_offset(T, i))) + @inbounds Blob{$(fieldtype(T, i))}(blob) + $(blob_offset(T, i)) end end @@ -99,11 +197,10 @@ end @boundscheck if i < 1 || i > fieldcount(T) throw(BoundsError(blob, i)) end - return Blob{fieldtype(T, i)}(blob + Blobs.blob_offset(T, i)) + return @inbounds Blob{fieldtype(T, i)}(blob) + Blobs.blob_offset(T, i) end Base.@propagate_inbounds function Base.setindex!(blob::Blob{T}, value::T) where T - @boundscheck boundscheck(blob) unsafe_store!(blob, value) end diff --git a/src/layout.jl b/src/layout.jl index 9c19c80..5f79733 100644 --- a/src/layout.jl +++ b/src/layout.jl @@ -15,7 +15,7 @@ Initialize `blob`. Assumes that `blob` is at least `self_size(T) + child_size(T, args...)` bytes long. """ function init(blob::Blob{T}, args...) where T - init(blob, Blob{Nothing}(blob + self_size(T)), args...) + init(blob, Blob{Nothing}(blob) + self_size(T), args...) end """ @@ -78,7 +78,9 @@ Allocate an uninitialized `Blob{T}`. """ function malloc(::Type{T}, args...)::Blob{T} where T size = self_size(T) + child_size(T, args...) - Blob{T}(Libc.malloc(size), 0, size) + base = Libc.malloc(size) + base == Ptr{Nothing}(0) && throw(OutOfMemoryError()) + return @inbounds Blob{T}(base, 0, size) end """ @@ -88,7 +90,9 @@ Allocate a zero-initialized `Blob{T}`. """ function calloc(::Type{T}, args...)::Blob{T} where T size = self_size(T) + child_size(T, args...) - Blob{T}(Libc.calloc(1, size), 0, size) + base = Libc.calloc(1, size) + base == Ptr{Nothing}(0) && throw(OutOfMemoryError()) + return @inbounds Blob{T}(base, 0, size) end """ @@ -97,11 +101,10 @@ end Allocate and initialize a new `Blob{T}`. """ function malloc_and_init(::Type{T}, args...)::Blob{T} where T - size = self_size(T) + child_size(T, args...) - blob = Blob{T}(Libc.malloc(size), 0, size) - used = init(blob, args...) - @assert used - blob == size - blob + blob = malloc(T, args...) + used = @inbounds init(blob, args...) + @assert used - blob == available_size(blob) + return blob end """ diff --git a/src/string.jl b/src/string.jl index 0a29aa8..2f45bec 100644 --- a/src/string.jl +++ b/src/string.jl @@ -2,6 +2,19 @@ struct BlobString <: AbstractString data::Blob{UInt8} len::Int64 # in bytes + + function BlobString(data::Blob{UInt8}, len::Int64) + @assert len >= 0 + @boundscheck begin + if len * self_size(UInt8) > available_size(data) + throw(InvalidBlobError( + BlobString, getfield(data, :base), getfield(data, :offset), + getfield(data, :limit), len), + ) + end + end + new(data, len) + end end Base.pointer(blob::BlobString) = pointer(blob, 1) @@ -187,4 +200,3 @@ end ## overload methods for efficiency ## Base.isvalid(s::BlobString, i::Int) = checkbounds(Bool, s, i) && thisind(s, i) == i - diff --git a/src/vector.jl b/src/vector.jl index f0ee062..afc5c8f 100644 --- a/src/vector.jl +++ b/src/vector.jl @@ -2,6 +2,19 @@ struct BlobVector{T} <: AbstractArray{T, 1} data::Blob{T} length::Int64 + + function BlobVector{T}(data::Blob{T}, length::Int64) where T + @assert length >= 0 + @boundscheck begin + if length * self_size(T) > available_size(data) + throw(InvalidBlobError( + BlobVector{T}, getfield(data, :base), getfield(data, :offset), + getfield(data, :limit), length), + ) + end + end + new{T}(data, length) + end end function Base.pointer(bv::BlobVector{T}, i::Integer=1) where {T} diff --git a/test/Blobs-tests.jl b/test/Blobs-tests.jl index ac0076d..68a3240 100644 --- a/test/Blobs-tests.jl +++ b/test/Blobs-tests.jl @@ -2,7 +2,7 @@ module TestBlobs -using Blobs +using Blobs: Blobs, Blob, BlobBitVector, BlobString, BlobVector, InvalidBlobError using Test struct Foo @@ -12,15 +12,6 @@ end # Blob -blob = Blob{Int64}(Libc.malloc(16), 0, 8) -@test_nowarn blob[] -@test_throws BoundsError (blob+1)[] -if Base.JLOptions().check_bounds == 0 - # @inbounds only kicks in if compiled - f1(blob) = @inbounds (blob+1)[] - f1(blob) -end - foo = Blobs.malloc_and_init(Foo) foo.x[] = 1 @test foo.x[] == 1 @@ -31,9 +22,10 @@ foo.y[] = 2.5 @test foo == foo @test pointer(foo.y) == pointer(foo) + sizeof(Int64) -foo2_ref = Ref(Foo(42, 3.14)) -foo2 = Blob(foo2_ref) -@test foo2[] == Foo(42, 3.14) +println("foo $(sizeof(Foo)) $(Blobs.self_size(Foo))") + +# Cannot create a blob directly from Ref{Foo} because of different alignment +foo2 = Blobs.malloc_and_init(Foo) foo3_arr = [Foo(1, -1), Foo(2, -2)] foo31 = Blob(pointer(foo3_arr), 0, 2sizeof(Foo)) @@ -58,7 +50,8 @@ foo.x[] = 1 @test Blobs.self_size(BlobVector{Int64}) == 16 data = Blob{Int64}(Libc.malloc(sizeof(Int64) * 4), 0, sizeof(Int64) * 3) -bv = BlobVector{Int64}(data, 4) +@test_throws InvalidBlobError BlobVector{Int64}(data, 4) +bv = BlobVector{Int64}(data, 3) @test_nowarn bv[3] @test_throws BoundsError bv[4] if Base.JLOptions().check_bounds == 0 @@ -121,7 +114,8 @@ copy!(bv3, 1, bv3, 2, 4) @test Blobs.child_size(BlobBitVector, 64*3 + 1) == 8*4 data = Blob{UInt64}(Libc.malloc(sizeof(UInt64)*4), 0, sizeof(UInt64)*3) -bv = BlobBitVector(data, 64*4) +@test_throws InvalidBlobError BlobBitVector(data, 64*4) +bv = BlobBitVector(data, 64*3) @test_nowarn bv[64*3] @test_throws BoundsError bv[64*3 + 1] if Base.JLOptions().check_bounds == 0 @@ -173,8 +167,8 @@ bv2[] = true data = Blob{UInt8}(Libc.malloc(8), 0, 8) @test_nowarn BlobString(data, 8)[8] -# pretty much any access to a unicode string touches beginning and end -@test_throws BoundsError BlobString(data, 16)[8] +@test_throws BoundsError BlobString(data, 8)[9] +@test_throws InvalidBlobError BlobString(data, 16) # @inbounds doesn't work for strings - too much work to propagate # test strings and unicode @@ -334,5 +328,53 @@ bt[] = (Toto{1}((0x0,), 8),) @test_throws ErrorException Blobs.malloc_and_init(String) end - end # testitem + +@testitem "references no-alignment" begin + using Blobs + + struct S + x::Int64 + y::Float64 + end + @assert sizeof(S) == Blobs.self_size(S) + + s = S(1, 2.5) + bs = Blob(Ref(s)) + @test bs.x[] == 1 + @test bs.y[] == 2.5 + @test s == bs[] +end + +@testitem "references unaligned" begin + using Blobs + + struct S + x::Int16 # Aligned to word size + y::Float64 + end + @assert sizeof(S) > Blobs.self_size(S) + + s = S(1, 2.5) + @test_throws AssertionError Blob(Ref(s)) + + su = Blobs.malloc(S) + try + su[] = s + @test su.x[] == 1 + @test su.y[] == 2.5 + finally + Blobs.free(su) + end +end + +@testitem "invalid Blob" begin + using Blobs: Blob, InvalidBlobError + + @test_throws InvalidBlobError Blob{Int64}(Ptr{Nothing}(1), 0, 7) + @test_throws InvalidBlobError Blob{Int64}(Ptr{Nothing}(1), 4, 9) + + # @inbounds won't work directly from @test @inbounds ... + inbounds_blob() = @inbounds Blob{Int64}(Ptr{Nothing}(1), 4, 9) + @test inbounds_blob() !== nothing +end