diff --git a/.gitignore b/.gitignore index 53cfc24de2..baf782d2ea 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ MODULE.bazel MODULE.bazel.lock .DS_Store **/.DS_Store +csharp/**/bin +csharp/**/obj diff --git a/csharp/.csharpierrc.yaml b/csharp/.csharpierrc.yaml new file mode 100644 index 0000000000..9141bb0619 --- /dev/null +++ b/csharp/.csharpierrc.yaml @@ -0,0 +1 @@ +printWidth: 120 diff --git a/csharp/Fury.Testing/Fakes/SinglePrimitiveFieldObject.cs b/csharp/Fury.Testing/Fakes/SinglePrimitiveFieldObject.cs new file mode 100644 index 0000000000..22b07a0bde --- /dev/null +++ b/csharp/Fury.Testing/Fakes/SinglePrimitiveFieldObject.cs @@ -0,0 +1,36 @@ +using Fury.Serializer; + +namespace Fury.Testing.Fakes; + +public sealed class SinglePrimitiveFieldObject +{ + public int Value { get; set; } + + public sealed class Serializer : AbstractSerializer + { + public override void Write(SerializationContext context, in SinglePrimitiveFieldObject value) + { + context.Writer.Write(value.Value); + } + } + + public sealed class Deserializer : AbstractDeserializer + { + public override ValueTask> CreateInstanceAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + return new ValueTask>(new SinglePrimitiveFieldObject()); + } + + public override async ValueTask ReadAndFillAsync( + DeserializationContext context, + Box instance, + CancellationToken cancellationToken = default + ) + { + instance.Value!.Value = await context.Reader.ReadAsync(cancellationToken); + } + } +} diff --git a/csharp/Fury.Testing/Fury.Testing.csproj b/csharp/Fury.Testing/Fury.Testing.csproj new file mode 100644 index 0000000000..934c882342 --- /dev/null +++ b/csharp/Fury.Testing/Fury.Testing.csproj @@ -0,0 +1,29 @@ + + + + + net8.0 + enable + enable + 12 + false + true + + + + + + + + + + + + + + + + + + + diff --git a/csharp/Fury.Testing/MetaStringTest.cs b/csharp/Fury.Testing/MetaStringTest.cs new file mode 100644 index 0000000000..2582b38110 --- /dev/null +++ b/csharp/Fury.Testing/MetaStringTest.cs @@ -0,0 +1,252 @@ +using System.Text; +using Bogus; +using Fury.Meta; + +namespace Fury.Testing; + +public sealed class MetaStringTest +{ + public static readonly IEnumerable Lengths = Enumerable.Range(0, 9).Select(i => new object[] { i }); + + private static readonly string LowerSpecialChars = Enumerable + .Range(0, 1 << AbstractLowerSpecialEncoding.BitsPerChar) + .Select(i => (AbstractLowerSpecialEncoding.TryDecodeByte((byte)i, out var c), c)) + .Where(t => t.Item1) + .Aggregate(new StringBuilder(), (builder, t) => builder.Append(t.c)) + .ToString(); + + private static readonly string AllToLowerSpecialChars = Enumerable + .Range(0, 1 << AbstractLowerSpecialEncoding.BitsPerChar) + .Select(i => (AbstractLowerSpecialEncoding.TryDecodeByte((byte)i, out var c), c)) + .Where(t => t.Item1 && t.c != AllToLowerSpecialEncoding.UpperCaseFlag) + .Aggregate(new StringBuilder(), (builder, t) => builder.Append(t.c).Append(char.ToUpperInvariant(t.c))) + .ToString(); + + private static readonly string TypeNameLowerUpperDigitSpecialChars = Enumerable + .Range(0, 1 << LowerUpperDigitSpecialEncoding.BitsPerChar) + .Select(i => (Encodings.TypeNameEncoding.LowerUpperDigit.TryDecodeByte((byte)i, out var c), c)) + .Where(t => t.Item1 && char.IsLetterOrDigit(t.c)) + .Aggregate(new StringBuilder(), (builder, t) => builder.Append(t.c)) + .ToString(); + + private static readonly string NamespaceLowerUpperDigitSpecialChars = Enumerable + .Range(0, 1 << LowerUpperDigitSpecialEncoding.BitsPerChar) + .Select(i => (Encodings.NamespaceEncoding.LowerUpperDigit.TryDecodeByte((byte)i, out var c), c)) + .Where(t => t.Item1 && char.IsLetterOrDigit(t.c)) + .Aggregate(new StringBuilder(), (builder, t) => builder.Append(t.c)) + .ToString(); + + [Theory] + [MemberData(nameof(Lengths))] + public void LowerSpecialEncoding_InputString_ShouldReturnTheSame(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, LowerSpecialChars); + + var bufferLength = LowerSpecialEncoding.Instance.GetByteCount(stubString); + Span buffer = stackalloc byte[bufferLength]; + LowerSpecialEncoding.Instance.GetBytes(stubString, buffer); + var output = LowerSpecialEncoding.Instance.GetString(buffer); + + Assert.Equal(stubString, output); + } + + [Theory] + [MemberData(nameof(Lengths))] + public void LowerSpecialEncoding_InputSeparatedBytes_ShouldReturnConcatenatedString(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, LowerSpecialChars); + + var bufferLength = LowerSpecialEncoding.Instance.GetByteCount(stubString); + Span bytes = stackalloc byte[bufferLength]; + Span chars = stackalloc char[stubString.Length]; + LowerSpecialEncoding.Instance.GetBytes(stubString, bytes); + var decoder = LowerSpecialEncoding.Instance.GetDecoder(); + var emptyChars = chars; + for (var i = 0; i < bytes.Length; i++) + { + var slicedBytes = bytes.Slice(i, 1); + decoder.Convert(slicedBytes, emptyChars, i == bytes.Length - 1, out _, out var charsUsed, out _); + emptyChars = emptyChars.Slice(charsUsed); + } + + var output = chars.ToString(); + + Assert.Equal(stubString, output); + } + + [Theory] + [MemberData(nameof(Lengths))] + public void FirstToLowerSpecialEncoding_InputString_ShouldReturnTheSame(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, LowerSpecialChars); + if (stubString.Length > 0 && char.IsLower(stubString[0])) + { + Span stubSpan = stackalloc char[stubString.Length]; + stubString.AsSpan().CopyTo(stubSpan); + stubSpan[0] = char.ToUpperInvariant(stubSpan[0]); + stubString = stubSpan.ToString(); + } + + var bufferLength = FirstToLowerSpecialEncoding.Instance.GetByteCount(stubString); + Span buffer = stackalloc byte[bufferLength]; + FirstToLowerSpecialEncoding.Instance.GetBytes(stubString, buffer); + var output = FirstToLowerSpecialEncoding.Instance.GetString(buffer); + + Assert.Equal(stubString, output); + } + + [Theory] + [MemberData(nameof(Lengths))] + public void FirstToLowerSpecialEncoding_InputSeparatedBytes_ShouldReturnConcatenatedString(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, LowerSpecialChars); + if (stubString.Length > 0 && char.IsLower(stubString[0])) + { + Span stubSpan = stackalloc char[stubString.Length]; + stubString.AsSpan().CopyTo(stubSpan); + stubSpan[0] = char.ToUpperInvariant(stubSpan[0]); + stubString = stubSpan.ToString(); + } + + var bufferLength = FirstToLowerSpecialEncoding.Instance.GetByteCount(stubString); + Span bytes = stackalloc byte[bufferLength]; + Span chars = stackalloc char[stubString.Length]; + FirstToLowerSpecialEncoding.Instance.GetBytes(stubString, bytes); + var decoder = FirstToLowerSpecialEncoding.Instance.GetDecoder(); + var emptyChars = chars; + for (var i = 0; i < bytes.Length; i++) + { + var slicedBytes = bytes.Slice(i, 1); + decoder.Convert(slicedBytes, emptyChars, i == bytes.Length - 1, out _, out var charsUsed, out _); + emptyChars = emptyChars.Slice(charsUsed); + } + + var output = chars.ToString(); + + Assert.Equal(stubString, output); + } + + [Theory] + [MemberData(nameof(Lengths))] + public void AllToLowerSpecialEncoding_InputString_ShouldReturnTheSame(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, AllToLowerSpecialChars); + + var bufferLength = AllToLowerSpecialEncoding.Instance.GetByteCount(stubString); + Span buffer = stackalloc byte[bufferLength]; + AllToLowerSpecialEncoding.Instance.GetBytes(stubString, buffer); + var output = AllToLowerSpecialEncoding.Instance.GetString(buffer); + + Assert.Equal(stubString, output); + } + + [Theory] + [MemberData(nameof(Lengths))] + public void AllToLowerSpecialEncoding_InputSeparatedBytes_ShouldReturnConcatenatedString(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, AllToLowerSpecialChars); + + var bufferLength = AllToLowerSpecialEncoding.Instance.GetByteCount(stubString); + Span bytes = stackalloc byte[bufferLength]; + Span chars = stackalloc char[stubString.Length]; + AllToLowerSpecialEncoding.Instance.GetBytes(stubString, bytes); + var decoder = AllToLowerSpecialEncoding.Instance.GetDecoder(); + var emptyChars = chars; + for (var i = 0; i < bytes.Length; i++) + { + var slicedBytes = bytes.Slice(i, 1); + decoder.Convert(slicedBytes, emptyChars, i == bytes.Length - 1, out _, out var charsUsed, out _); + emptyChars = emptyChars.Slice(charsUsed); + } + + var output = chars.ToString(); + + Assert.Equal(stubString, output); + } + + [Theory] + [MemberData(nameof(Lengths))] + public void TypeNameLowerUpperDigitSpecialEncoding_InputString_ShouldReturnTheSame(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, TypeNameLowerUpperDigitSpecialChars); + + var bufferLength = Encodings.TypeNameEncoding.LowerUpperDigit.GetByteCount(stubString); + Span buffer = stackalloc byte[bufferLength]; + Encodings.TypeNameEncoding.LowerUpperDigit.GetBytes(stubString, buffer); + var output = Encodings.TypeNameEncoding.LowerUpperDigit.GetString(buffer); + + Assert.Equal(stubString, output); + } + + [Theory] + [MemberData(nameof(Lengths))] + public void TypeNameLowerUpperDigitSpecialEncoding_InputSeparatedBytes_ShouldReturnConcatenatedString(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, TypeNameLowerUpperDigitSpecialChars); + + var bufferLength = Encodings.TypeNameEncoding.LowerUpperDigit.GetByteCount(stubString); + Span bytes = stackalloc byte[bufferLength]; + Span chars = stackalloc char[stubString.Length]; + Encodings.TypeNameEncoding.LowerUpperDigit.GetBytes(stubString, bytes); + var decoder = Encodings.TypeNameEncoding.LowerUpperDigit.GetDecoder(); + var emptyChars = chars; + for (var i = 0; i < bytes.Length; i++) + { + var slicedBytes = bytes.Slice(i, 1); + decoder.Convert(slicedBytes, emptyChars, i == bytes.Length - 1, out _, out var charsUsed, out _); + emptyChars = emptyChars.Slice(charsUsed); + } + + var output = chars.ToString(); + + Assert.Equal(stubString, output); + } + + [Theory] + [MemberData(nameof(Lengths))] + public void NamespaceLowerUpperDigitSpecialEncoding_InputString_ShouldReturnTheSame(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, NamespaceLowerUpperDigitSpecialChars); + + var bufferLength = Encodings.NamespaceEncoding.LowerUpperDigit.GetByteCount(stubString); + Span buffer = stackalloc byte[bufferLength]; + Encodings.NamespaceEncoding.LowerUpperDigit.GetBytes(stubString, buffer); + var output = Encodings.NamespaceEncoding.LowerUpperDigit.GetString(buffer); + + Assert.Equal(stubString, output); + } + + [Theory] + [MemberData(nameof(Lengths))] + public void NamespaceLowerUpperDigitSpecialEncoding_InputSeparatedBytes_ShouldReturnConcatenatedString(int length) + { + var faker = new Faker(); + var stubString = faker.Random.String2(length, NamespaceLowerUpperDigitSpecialChars); + + var bufferLength = Encodings.NamespaceEncoding.LowerUpperDigit.GetByteCount(stubString); + Span bytes = stackalloc byte[bufferLength]; + Span chars = stackalloc char[stubString.Length]; + Encodings.NamespaceEncoding.LowerUpperDigit.GetBytes(stubString, bytes); + var decoder = Encodings.NamespaceEncoding.LowerUpperDigit.GetDecoder(); + var emptyChars = chars; + for (var i = 0; i < bytes.Length; i++) + { + var slicedBytes = bytes.Slice(i, 1); + decoder.Convert(slicedBytes, emptyChars, i == bytes.Length - 1, out _, out var charsUsed, out _); + emptyChars = emptyChars.Slice(charsUsed); + } + + var output = chars.ToString(); + + Assert.Equal(stubString, output); + } +} diff --git a/csharp/Fury.slnx b/csharp/Fury.slnx new file mode 100644 index 0000000000..9379444e0a --- /dev/null +++ b/csharp/Fury.slnx @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/csharp/Fury/Backports/BitOperations.cs b/csharp/Fury/Backports/BitOperations.cs new file mode 100644 index 0000000000..89577ba955 --- /dev/null +++ b/csharp/Fury/Backports/BitOperations.cs @@ -0,0 +1,21 @@ +#if !NET8_0_OR_GREATER +using System.Runtime.CompilerServices; + +// ReSharper disable once CheckNamespace +namespace System.Numerics; + +internal static class BitOperations +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ulong RotateLeft(ulong value, int offset) => (value << offset) | (value >> (64 - offset)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint RotateLeft(uint value, int offset) => (value << offset) | (value >> (32 - offset)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ulong RotateRight(ulong value, int offset) => (value >> offset) | (value << (64 - offset)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint RotateRight(uint value, int offset) => (value >> offset) | (value << (32 - offset)); +} +#endif diff --git a/csharp/Fury/Backports/NotReturnAttributes.cs b/csharp/Fury/Backports/NotReturnAttributes.cs new file mode 100644 index 0000000000..8b12c14c81 --- /dev/null +++ b/csharp/Fury/Backports/NotReturnAttributes.cs @@ -0,0 +1,7 @@ +#if !NET8_0_OR_GREATER +// ReSharper disable once CheckNamespace +namespace System.Diagnostics.CodeAnalysis; + +[AttributeUsage(AttributeTargets.Method, Inherited = false)] +internal class DoesNotReturnAttribute : Attribute; +#endif diff --git a/csharp/Fury/Backports/NullableAttributes.cs b/csharp/Fury/Backports/NullableAttributes.cs new file mode 100644 index 0000000000..34cd531b8f --- /dev/null +++ b/csharp/Fury/Backports/NullableAttributes.cs @@ -0,0 +1,50 @@ +#if !NET8_0_OR_GREATER +// ReSharper disable once CheckNamespace +namespace System.Diagnostics.CodeAnalysis; + +/// Specifies that when a method returns , the parameter will not be even if the corresponding type allows it. +[AttributeUsage(AttributeTargets.Parameter)] +internal sealed class NotNullWhenAttribute : Attribute +{ + /// Initializes the attribute with the specified return value condition. + /// The return value condition. If the method returns this value, the associated parameter will not be . + public NotNullWhenAttribute(bool returnValue) => this.ReturnValue = returnValue; + + /// Gets the return value condition. + /// The return value condition. If the method returns this value, the associated parameter will not be . + public bool ReturnValue { get; } +} + +/// Specifies that the method will not return if the associated parameter is passed the specified value. +[AttributeUsage(AttributeTargets.Parameter)] +internal sealed class DoesNotReturnIfAttribute : Attribute +{ + /// Initializes a new instance of the class with the specified parameter value. + /// The condition parameter value. Code after the method is considered unreachable by diagnostics if the argument to the associated parameter matches this value. + public DoesNotReturnIfAttribute(bool parameterValue) => this.ParameterValue = parameterValue; + + /// Gets the condition parameter value. + /// The condition parameter value. Code after the method is considered unreachable by diagnostics if the argument to the associated parameter matches this value. + public bool ParameterValue { get; } +} + +/// Specifies that the method or property will ensure that the listed field and property members have not-null values. +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, Inherited = false, AllowMultiple = true)] +internal sealed class MemberNotNullAttribute : Attribute +{ + /// Initializes the attribute with a field or property member. + /// + /// The field or property member that is promised to be not-null. + /// + public MemberNotNullAttribute(string member) => Members = new[] { member }; + + /// Initializes the attribute with the list of field and property members. + /// + /// The list of field and property members that are promised to be not-null. + /// + public MemberNotNullAttribute(params string[] members) => Members = members; + + /// Gets field or property member names. + public string[] Members { get; } +} +#endif diff --git a/csharp/Fury/Backports/UInt128.cs b/csharp/Fury/Backports/UInt128.cs new file mode 100644 index 0000000000..6006593398 --- /dev/null +++ b/csharp/Fury/Backports/UInt128.cs @@ -0,0 +1,6 @@ +#if !NET8_0_OR_GREATER +// ReSharper disable once CheckNamespace +namespace System; + +public record struct UInt128(ulong Upper, ulong Lower); +#endif diff --git a/csharp/Fury/BatchReader.Read.cs b/csharp/Fury/BatchReader.Read.cs new file mode 100644 index 0000000000..cd6908ec9e --- /dev/null +++ b/csharp/Fury/BatchReader.Read.cs @@ -0,0 +1,345 @@ +using System; +using System.Buffers; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Fury; + +public sealed partial class BatchReader +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe TValue ReadFixedSized(ReadOnlySequence buffer, int size) + where TValue : unmanaged + { + TValue result = default; + buffer.Slice(0, size).CopyTo(new Span(&result, size)); + return result; + } + + public async ValueTask ReadAsync(CancellationToken cancellationToken = default) + where T : unmanaged + { + var requiredSize = Unsafe.SizeOf(); + var result = await ReadAtLeastAsync(requiredSize, cancellationToken); + var buffer = result.Buffer; + if (buffer.Length < requiredSize) + { + ThrowHelper.ThrowBadDeserializationInputException_InsufficientData(); + } + + var value = ReadFixedSized(buffer, requiredSize); + AdvanceTo(requiredSize); + return value; + } + + public async ValueTask ReadAsAsync(int size, CancellationToken cancellationToken = default) + where T : unmanaged + { + var result = await ReadAtLeastAsync(size, cancellationToken); + var buffer = result.Buffer; + if (buffer.Length < size) + { + ThrowHelper.ThrowBadDeserializationInputException_InsufficientData(); + } + + var value = ReadFixedSized(buffer, size); + AdvanceTo(size); + return value; + } + + public async ValueTask ReadMemoryAsync( + Memory destination, + CancellationToken cancellationToken = default + ) + where TElement : unmanaged + { + var requiredSize = destination.Length; + var result = await ReadAtLeastAsync(requiredSize, cancellationToken); + var buffer = result.Buffer; + if (result.IsCompleted && buffer.Length < requiredSize) + { + ThrowHelper.ThrowBadDeserializationInputException_InsufficientData(); + } + + buffer.Slice(0, requiredSize).CopyTo(MemoryMarshal.AsBytes(destination.Span)); + AdvanceTo(requiredSize); + } + + public async ValueTask ReadStringAsync( + int byteCount, + Encoding encoding, + CancellationToken cancellationToken = default + ) + { + var result = await ReadAtLeastAsync(byteCount, cancellationToken); + var buffer = result.Buffer; + if (result.IsCompleted && buffer.Length < byteCount) + { + ThrowHelper.ThrowBadDeserializationInputException_InsufficientData(); + } + + var value = DoReadString(byteCount, buffer, encoding); + AdvanceTo(byteCount); + return value; + } + + private static unsafe string DoReadString(int byteCount, ReadOnlySequence byteSequence, Encoding encoding) + { + const int maxStackBufferSize = StaticConfigs.StackAllocLimit / sizeof(char); + var decoder = encoding.GetDecoder(); + int writtenChars; + string result; + if (byteCount < maxStackBufferSize) + { + // Fast path + Span stringBuffer = stackalloc char[byteCount]; + writtenChars = ReadStringCommon(decoder, byteSequence, stringBuffer); + result = stringBuffer.Slice(0, writtenChars).ToString(); + } + else + { + var rentedBuffer = ArrayPool.Shared.Rent(byteCount); + writtenChars = ReadStringCommon(decoder, byteSequence, rentedBuffer); + result = new string(rentedBuffer, 0, writtenChars); + ArrayPool.Shared.Return(rentedBuffer); + } + + return result; + } + + private static unsafe int ReadStringCommon( + Decoder decoder, + ReadOnlySequence byteSequence, + Span unwrittenBuffer + ) + { + var writtenChars = 0; + foreach (var byteMemory in byteSequence) + { + int charsUsed; + var byteSpan = byteMemory.Span; + fixed (char* pUnWrittenBuffer = unwrittenBuffer) + fixed (byte* pBytes = byteMemory.Span) + { + decoder.Convert( + pBytes, + byteSpan.Length, + pUnWrittenBuffer, + unwrittenBuffer.Length, + false, + out _, + out charsUsed, + out _ + ); + } + + unwrittenBuffer = unwrittenBuffer.Slice(charsUsed); + writtenChars += charsUsed; + } + + return writtenChars; + } + + public async ValueTask Read7BitEncodedIntAsync(CancellationToken cancellationToken = default) + { + var result = await Read7BitEncodedUintAsync(cancellationToken); + return (int)BitOperations.RotateRight(result, 1); + } + + public async ValueTask Read7BitEncodedUintAsync(CancellationToken cancellationToken = default) + { + var result = await ReadAtLeastAsync(MaxBytesOfVarInt32WithoutOverflow + 1, cancellationToken); + var buffer = result.Buffer; + + // Fast path + var value = DoRead7BitEncodedUintFast(buffer.First.Span, out var consumed); + if (consumed == 0) + { + // Slow path + value = DoRead7BitEncodedUintSlow(buffer, out consumed); + } + + AdvanceTo(consumed); + + return value; + } + + private const int MaxBytesOfVarInt32WithoutOverflow = 4; + + private static uint DoRead7BitEncodedUintFast(ReadOnlySpan buffer, out int consumed) + { + if (buffer.Length <= MaxBytesOfVarInt32WithoutOverflow) + { + consumed = 0; + return 0; + } + uint result = 0; + consumed = 0; + uint readByte; + for (var i = 0; i < MaxBytesOfVarInt32WithoutOverflow; i++) + { + readByte = buffer[i]; + result |= (readByte & 0x7F) << (i * 7); + if ((readByte & 0x80) == 0) + { + consumed = i + 1; + return result; + } + } + + readByte = buffer[MaxBytesOfVarInt32WithoutOverflow]; + if (readByte > 0b_1111u) + { + ThrowHelper.ThrowBadDeserializationInputException_VarInt32Overflow(); + } + + result |= readByte << (MaxBytesOfVarInt32WithoutOverflow * 7); + consumed = MaxBytesOfVarInt32WithoutOverflow + 1; + return result; + } + + private static uint DoRead7BitEncodedUintSlow(ReadOnlySequence buffer, out int consumed) + { + uint result = 0; + var consumedBytes = 0; + foreach (var memory in buffer) + { + var span = memory.Span; + foreach (uint readByte in span) + { + if (consumedBytes < MaxBytesOfVarInt32WithoutOverflow) + { + result |= (readByte & 0x7F) << (7 * consumedBytes); + ++consumedBytes; + if ((readByte & 0x80) == 0) + { + consumed = consumedBytes; + return result; + } + } + else + { + if (readByte > 0b_1111u) + { + ThrowHelper.ThrowBadDeserializationInputException_VarInt32Overflow(); + } + result |= readByte << (7 * MaxBytesOfVarInt32WithoutOverflow); + consumed = consumedBytes + 1; + return result; + } + } + } + consumed = 0; + return result; + } + + public async ValueTask Read7BitEncodedLongAsync(CancellationToken cancellationToken = default) + { + var result = await Read7BitEncodedUlongAsync(cancellationToken); + return (long)BitOperations.RotateRight(result, 1); + } + + public async ValueTask Read7BitEncodedUlongAsync(CancellationToken cancellationToken = default) + { + var result = await ReadAtLeastAsync(MaxBytesOfVarInt64WithoutOverflow + 1, cancellationToken); + var buffer = result.Buffer; + + // Fast path + var value = DoRead7BitEncodedUlongFast(buffer.First.Span, out var consumed); + if (consumed == 0) + { + // Slow path + value = DoRead7BitEncodedUlongSlow(buffer, out consumed); + } + + AdvanceTo(consumed); + + return value; + } + + private const int MaxBytesOfVarInt64WithoutOverflow = 8; + + private static ulong DoRead7BitEncodedUlongFast(ReadOnlySpan buffer, out int consumed) + { + if (buffer.Length <= MaxBytesOfVarInt64WithoutOverflow) + { + consumed = 0; + return 0; + } + ulong result = 0; + consumed = 0; + ulong readByte; + for (var i = 0; i < MaxBytesOfVarInt64WithoutOverflow; i++) + { + readByte = buffer[i]; + result |= (readByte & 0x7F) << (i * 7); + if ((readByte & 0x80) == 0) + { + consumed = i + 1; + return result; + } + } + + readByte = buffer[MaxBytesOfVarInt64WithoutOverflow]; + result |= readByte << (MaxBytesOfVarInt64WithoutOverflow * 7); + return result; + } + + private static ulong DoRead7BitEncodedUlongSlow(ReadOnlySequence buffer, out int consumed) + { + ulong result = 0; + var consumedBytes = 0; + foreach (var memory in buffer) + { + var span = memory.Span; + foreach (ulong readByte in span) + { + if (consumedBytes < MaxBytesOfVarInt64WithoutOverflow) + { + result |= (readByte & 0x7F) << (7 * consumedBytes); + ++consumedBytes; + if ((readByte & 0x80) == 0) + { + consumed = consumedBytes; + return result; + } + } + else + { + result |= readByte << (7 * MaxBytesOfVarInt64WithoutOverflow); + consumed = consumedBytes + 1; + return result; + } + } + } + consumed = 0; + return result; + } + + public async ValueTask ReadCountAsync(CancellationToken cancellationToken) + { + return (int)await Read7BitEncodedUintAsync(cancellationToken); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal async ValueTask ReadReferenceFlagAsync(CancellationToken cancellationToken = default) + { + return (ReferenceFlag)await ReadAsync(cancellationToken); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal async ValueTask ReadTypeIdAsync(CancellationToken cancellationToken = default) + { + return new TypeId((int)await Read7BitEncodedUintAsync(cancellationToken)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal async ValueTask ReadRefIdAsync(CancellationToken cancellationToken = default) + { + return new RefId((int)await Read7BitEncodedUintAsync(cancellationToken)); + } +} diff --git a/csharp/Fury/BatchReader.cs b/csharp/Fury/BatchReader.cs new file mode 100644 index 0000000000..83e1f0ebce --- /dev/null +++ b/csharp/Fury/BatchReader.cs @@ -0,0 +1,40 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Threading; +using System.Threading.Tasks; + +namespace Fury; + +public sealed partial class BatchReader(PipeReader reader) +{ + private ReadOnlySequence _cachedBuffer; + private bool _isCanceled; + private bool _isCompleted; + + public async ValueTask ReadAtLeastAsync(int minimumSize, CancellationToken cancellationToken = default) + { + if (_cachedBuffer.Length < minimumSize) + { + reader.AdvanceTo(_cachedBuffer.Start); + var result = await reader.ReadAtLeastAsync(minimumSize, cancellationToken); + _cachedBuffer = result.Buffer; + _isCanceled = result.IsCanceled; + _isCompleted = result.IsCompleted; + } + + return new ReadResult(_cachedBuffer, _isCanceled, _isCompleted); + } + + public void AdvanceTo(int consumed) + { + _cachedBuffer = _cachedBuffer.Slice(consumed); + } + + public void Complete() + { + reader.AdvanceTo(_cachedBuffer.Start); + _cachedBuffer = default; + _isCompleted = true; + reader.Complete(); + } +} diff --git a/csharp/Fury/BatchWriter.cs b/csharp/Fury/BatchWriter.cs new file mode 100644 index 0000000000..9b5dfd30a1 --- /dev/null +++ b/csharp/Fury/BatchWriter.cs @@ -0,0 +1,43 @@ +using System; +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Fury; + +// This is used to reduce the virtual call overhead of the PipeWriter + +[StructLayout(LayoutKind.Auto)] +public ref partial struct BatchWriter(PipeWriter writer) +{ + private Span _cachedBuffer = Span.Empty; + private int _consumed = 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Advance(int count) + { + _consumed += count; + _cachedBuffer = _cachedBuffer.Slice(count); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Span GetSpan(int sizeHint = 0) + { + if (_cachedBuffer.Length < sizeHint) + { + writer.Advance(_consumed); + _consumed = 0; + _cachedBuffer = writer.GetSpan(sizeHint); + } + + return _cachedBuffer; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Flush() + { + writer.Advance(_consumed); + _consumed = 0; + _cachedBuffer = Span.Empty; + } +} diff --git a/csharp/Fury/BatchWriter.write.cs b/csharp/Fury/BatchWriter.write.cs new file mode 100644 index 0000000000..26d1aa3a30 --- /dev/null +++ b/csharp/Fury/BatchWriter.write.cs @@ -0,0 +1,254 @@ +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +namespace Fury; + +public ref partial struct BatchWriter +{ + public void Write(T value) + where T : unmanaged + { + var size = Unsafe.SizeOf(); + var buffer = GetSpan(size); +#if NET8_0_OR_GREATER + MemoryMarshal.Write(buffer, in value); +#else + MemoryMarshal.Write(buffer, ref value); +#endif + Advance(size); + } + + public void Write(Span values) + { + var buffer = GetSpan(values.Length); + values.CopyTo(buffer); + Advance(values.Length); + } + + public void Write(Span values) + where TElement : unmanaged + { + Write(MemoryMarshal.AsBytes(values)); + } + + public unsafe void Write(ReadOnlySpan value, Encoding encoding, int byteCountHint) + { + var buffer = GetSpan(byteCountHint); + int actualByteCount; + + fixed (char* pChars = value) + fixed (byte* pBytes = buffer) + { + actualByteCount = encoding.GetBytes(pChars, value.Length, pBytes, buffer.Length); + } + + Advance(actualByteCount); + } + + public unsafe void Write(ReadOnlySpan value, Encoding encoding) + { + const int fastPathBufferSize = 128; + + var possibleMaxByteCount = encoding.GetMaxByteCount(value.Length); + int bufferLength; + if (possibleMaxByteCount <= fastPathBufferSize) + { + bufferLength = possibleMaxByteCount; + } + else + { + fixed (char* pChars = value) + { + bufferLength = encoding.GetByteCount(pChars, value.Length); + } + } + + Write(value, encoding, bufferLength); + } + + public void Write7BitEncodedInt(int value) + { + var zigzag = BitOperations.RotateLeft((uint)value, 1); + Write7BitEncodedUint(zigzag); + } + + public void Write7BitEncodedUint(uint value) + { + switch (value) + { + case < 1u << 7: + Write((byte)value); + return; + case < 1u << 14: + { + var buffer = GetSpan(2); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)(value >>> 7); + Advance(2); + break; + } + case < 1u << 21: + { + var buffer = GetSpan(3); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer[2] = (byte)(value >>> 14); + Advance(3); + break; + } + case < 1u << 28: + { + var buffer = GetSpan(4); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer[2] = (byte)((value >>> 14) | ~0x7Fu); + buffer[3] = (byte)(value >>> 21); + Advance(4); + break; + } + default: + var buffer2 = GetSpan(5); + buffer2[0] = (byte)(value | ~0x7Fu); + buffer2[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer2[2] = (byte)((value >>> 14) | ~0x7Fu); + buffer2[3] = (byte)((value >>> 21) | ~0x7Fu); + buffer2[4] = (byte)(value >>> 28); + Advance(5); + break; + } + } + + public void Write7BitEncodedLong(long value) + { + var zigzag = BitOperations.RotateLeft((ulong)value, 1); + Write7BitEncodedUlong(zigzag); + } + + public void Write7BitEncodedUlong(ulong value) + { + switch (value) + { + case < 1ul << 7: + Write((byte)value); + return; + case < 1ul << 14: + { + var buffer = GetSpan(2); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)(value >>> 7); + Advance(2); + break; + } + case < 1ul << 21: + { + var buffer = GetSpan(3); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer[2] = (byte)(value >>> 14); + Advance(3); + break; + } + case < 1ul << 28: + { + var buffer = GetSpan(4); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer[2] = (byte)((value >>> 14) | ~0x7Fu); + buffer[3] = (byte)(value >>> 21); + Advance(4); + break; + } + case < 1ul << 35: + { + var buffer = GetSpan(5); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer[2] = (byte)((value >>> 14) | ~0x7Fu); + buffer[3] = (byte)((value >>> 21) | ~0x7Fu); + buffer[4] = (byte)(value >>> 28); + Advance(5); + break; + } + case < 1ul << 42: + { + var buffer = GetSpan(6); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer[2] = (byte)((value >>> 14) | ~0x7Fu); + buffer[3] = (byte)((value >>> 21) | ~0x7Fu); + buffer[4] = (byte)((value >>> 28) | ~0x7Fu); + buffer[5] = (byte)(value >>> 35); + Advance(6); + break; + } + case < 1ul << 49: + { + var buffer = GetSpan(7); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer[2] = (byte)((value >>> 14) | ~0x7Fu); + buffer[3] = (byte)((value >>> 21) | ~0x7Fu); + buffer[4] = (byte)((value >>> 28) | ~0x7Fu); + buffer[5] = (byte)((value >>> 35) | ~0x7Fu); + buffer[6] = (byte)(value >>> 42); + Advance(7); + break; + } + case < 1ul << 56: + { + var buffer = GetSpan(8); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer[2] = (byte)((value >>> 14) | ~0x7Fu); + buffer[3] = (byte)((value >>> 21) | ~0x7Fu); + buffer[4] = (byte)((value >>> 28) | ~0x7Fu); + buffer[5] = (byte)((value >>> 35) | ~0x7Fu); + buffer[6] = (byte)((value >>> 42) | ~0x7Fu); + buffer[7] = (byte)(value >>> 49); + Advance(8); + break; + } + case < 1ul << 63: + { + var buffer = GetSpan(9); + buffer[0] = (byte)(value | ~0x7Fu); + buffer[1] = (byte)((value >>> 7) | ~0x7Fu); + buffer[2] = (byte)((value >>> 14) | ~0x7Fu); + buffer[3] = (byte)((value >>> 21) | ~0x7Fu); + buffer[4] = (byte)((value >>> 28) | ~0x7Fu); + buffer[5] = (byte)((value >>> 35) | ~0x7Fu); + buffer[6] = (byte)((value >>> 42) | ~0x7Fu); + buffer[7] = (byte)((value >>> 49) | ~0x7Fu); + buffer[8] = (byte)(value >>> 56); + Advance(9); + break; + } + } + } + + public void WriteCount(int length) + { + Write7BitEncodedUint((uint)length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void Write(ReferenceFlag flag) + { + Write((sbyte)flag); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void Write(RefId refId) + { + Write7BitEncodedUint((uint)refId.Value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void Write(TypeId typeId) + { + Write7BitEncodedUint((uint)typeId.Value); + } +} diff --git a/csharp/Fury/BitUtility.cs b/csharp/Fury/BitUtility.cs new file mode 100644 index 0000000000..43ef2f6290 --- /dev/null +++ b/csharp/Fury/BitUtility.cs @@ -0,0 +1,43 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Fury; + +internal static class BitUtility +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetBitMask32(int bitsCount) => (1 << bitsCount) - 1; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long GetBitMask64(int bitsCount) => (1L << bitsCount) - 1; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte ClearLowBits(byte value, int lowBitsCount) => (byte)(value & ~GetBitMask32(lowBitsCount)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long ClearLowBits(long value, int lowBitsCount) => value & ~GetBitMask64(lowBitsCount); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte ClearHighBits(byte value, int highBitsCount) => (byte)(value & GetBitMask32(8 - highBitsCount)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte KeepLowBits(byte value, int lowBitsCount) => (byte)(value & GetBitMask32(lowBitsCount)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte KeepHighBits(byte value, int highBitsCount) => (byte)(value & ~GetBitMask32(8 - highBitsCount)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte ReadBits(byte b1, int bitOffset, int bitCount) + { + return (byte)((b1 >>> (8 - bitCount - bitOffset)) & GetBitMask32(bitCount)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte ReadBits(byte b1, byte b2, int bitOffset, int bitCount) + { + var byteFromB1 = b1 << (bitOffset + bitCount - 8); + var byteFromB2 = b2 >>> (8 * 2 - bitCount - bitOffset); + return (byte)((byteFromB1 | byteFromB2) & GetBitMask32(bitCount)); + } +} diff --git a/csharp/Fury/Box.cs b/csharp/Fury/Box.cs new file mode 100644 index 0000000000..fad71f3fe4 --- /dev/null +++ b/csharp/Fury/Box.cs @@ -0,0 +1,49 @@ +using System.Runtime.CompilerServices; + +namespace Fury; + +public readonly struct Box(object value) +{ + public object? Value { get; init; } = value; + + public Box AsTyped() + where T : notnull + { + return new Box { InternalValue = Value }; + } +} + +public struct Box(in T value) + where T : notnull +{ + internal object? InternalValue = value; + + public T? Value + { + get => (T?)InternalValue; + set => InternalValue = value; + } + + public Box AsUntyped() + { + return new Box { Value = InternalValue }; + } + + public static implicit operator Box(in T boxed) + { + return new Box(in boxed); + } +} + +public static class BoxExtensions +{ + // Users may not know Unsafe.Unbox(ref T) or be afraid of "Unsafe" in the name. + + /// + public static ref T Unbox(this Box box) + where T : struct + { + box.InternalValue ??= new T(); + return ref Unsafe.Unbox(box.InternalValue); + } +} diff --git a/csharp/Fury/Buffers/IArrayPoolProvider.cs b/csharp/Fury/Buffers/IArrayPoolProvider.cs new file mode 100644 index 0000000000..070f16a155 --- /dev/null +++ b/csharp/Fury/Buffers/IArrayPoolProvider.cs @@ -0,0 +1,8 @@ +using System.Buffers; + +namespace Fury.Buffers; + +public interface IArrayPoolProvider +{ + ArrayPool GetArrayPool(); +} diff --git a/csharp/Fury/Buffers/ObjectPool.cs b/csharp/Fury/Buffers/ObjectPool.cs new file mode 100644 index 0000000000..94f0127f83 --- /dev/null +++ b/csharp/Fury/Buffers/ObjectPool.cs @@ -0,0 +1,32 @@ +using System; +using Fury.Collections; + +namespace Fury.Buffers; + +/// +/// A simple object pool. +/// +/// +internal readonly struct ObjectPool(IArrayPoolProvider poolProvider, Func factory) + where T : class +{ + private readonly PooledList _objects = new(poolProvider); + + public T Rent() + { + var lastIndex = _objects.Count - 1; + if (lastIndex < 0) + { + return factory(); + } + + var obj = _objects[lastIndex]; + _objects.RemoveAt(lastIndex); + return obj; + } + + public void Return(T obj) + { + _objects.Add(obj); + } +} diff --git a/csharp/Fury/Buffers/ShardArrayPoolProvider.cs b/csharp/Fury/Buffers/ShardArrayPoolProvider.cs new file mode 100644 index 0000000000..a02efd42ad --- /dev/null +++ b/csharp/Fury/Buffers/ShardArrayPoolProvider.cs @@ -0,0 +1,10 @@ +using System.Buffers; + +namespace Fury.Buffers; + +internal sealed class ShardArrayPoolProvider : IArrayPoolProvider +{ + public static readonly ShardArrayPoolProvider Instance = new(); + + public ArrayPool GetArrayPool() => ArrayPool.Shared; +} diff --git a/csharp/Fury/BuiltIns.cs b/csharp/Fury/BuiltIns.cs new file mode 100644 index 0000000000..85c9a5471d --- /dev/null +++ b/csharp/Fury/BuiltIns.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using Fury.Serializer; + +namespace Fury; + +public static class BuiltIns +{ + public static IReadOnlyDictionary BuiltInTypeToSerializers { get; } = + new Dictionary + { + [typeof(bool)] = PrimitiveSerializer.Instance, + [typeof(sbyte)] = PrimitiveSerializer.Instance, + [typeof(byte)] = PrimitiveSerializer.Instance, + [typeof(short)] = PrimitiveSerializer.Instance, + [typeof(ushort)] = PrimitiveSerializer.Instance, + [typeof(int)] = PrimitiveSerializer.Instance, + [typeof(uint)] = PrimitiveSerializer.Instance, + [typeof(long)] = PrimitiveSerializer.Instance, + [typeof(ulong)] = PrimitiveSerializer.Instance, + [typeof(float)] = PrimitiveSerializer.Instance, + [typeof(double)] = PrimitiveSerializer.Instance, + [typeof(string)] = StringSerializer.Instance, + [typeof(bool[])] = PrimitiveArraySerializer.Instance, + [typeof(byte[])] = PrimitiveArraySerializer.Instance, + [typeof(short[])] = PrimitiveArraySerializer.Instance, + [typeof(int[])] = PrimitiveArraySerializer.Instance, + [typeof(long[])] = PrimitiveArraySerializer.Instance, + [typeof(float[])] = PrimitiveArraySerializer.Instance, + [typeof(double[])] = PrimitiveArraySerializer.Instance, + [typeof(string[])] = new ArraySerializer(StringSerializer.Instance) + }; + + public static IReadOnlyDictionary BuiltInTypeToDeserializers { get; } = + new Dictionary + { + [typeof(bool)] = PrimitiveDeserializer.Instance, + [typeof(sbyte)] = PrimitiveDeserializer.Instance, + [typeof(byte)] = PrimitiveDeserializer.Instance, + [typeof(short)] = PrimitiveDeserializer.Instance, + [typeof(ushort)] = PrimitiveDeserializer.Instance, + [typeof(int)] = PrimitiveDeserializer.Instance, + [typeof(uint)] = PrimitiveDeserializer.Instance, + [typeof(long)] = PrimitiveDeserializer.Instance, + [typeof(ulong)] = PrimitiveDeserializer.Instance, + [typeof(float)] = PrimitiveDeserializer.Instance, + [typeof(double)] = PrimitiveDeserializer.Instance, + [typeof(string)] = StringDeserializer.Instance, + [typeof(bool[])] = PrimitiveArrayDeserializer.Instance, + [typeof(byte[])] = PrimitiveArrayDeserializer.Instance, + [typeof(short[])] = PrimitiveArrayDeserializer.Instance, + [typeof(int[])] = PrimitiveArrayDeserializer.Instance, + [typeof(long[])] = PrimitiveArrayDeserializer.Instance, + [typeof(float[])] = PrimitiveArrayDeserializer.Instance, + [typeof(double[])] = PrimitiveArrayDeserializer.Instance, + [typeof(string[])] = new ArrayDeserializer(StringDeserializer.Instance) + }; + + public static IReadOnlyDictionary BuiltInTypeToTypeInfos { get; } = + new Dictionary + { + [typeof(bool)] = new(TypeId.Bool, typeof(bool)), + [typeof(sbyte)] = new(TypeId.Int8, typeof(sbyte)), + [typeof(byte)] = new(TypeId.Int8, typeof(byte)), + [typeof(short)] = new(TypeId.Int16, typeof(short)), + [typeof(ushort)] = new(TypeId.Int16, typeof(ushort)), + [typeof(int)] = new(TypeId.Int32, typeof(int)), + [typeof(uint)] = new(TypeId.Int32, typeof(uint)), + [typeof(long)] = new(TypeId.Int64, typeof(long)), + [typeof(ulong)] = new(TypeId.Int64, typeof(ulong)), + [typeof(float)] = new(TypeId.Float32, typeof(float)), + [typeof(double)] = new(TypeId.Float64, typeof(double)), + [typeof(string)] = new(TypeId.String, typeof(string)), + [typeof(bool[])] = new(TypeId.BoolArray, typeof(bool[])), + [typeof(byte[])] = new(TypeId.Int8Array, typeof(byte[])), + [typeof(short[])] = new(TypeId.Int16Array, typeof(short[])), + [typeof(int[])] = new(TypeId.Int32Array, typeof(int[])), + [typeof(long[])] = new(TypeId.Int64Array, typeof(long[])), + [typeof(float[])] = new(TypeId.Float32Array, typeof(float[])), + [typeof(double[])] = new(TypeId.Float64Array, typeof(double[])) + }; +} diff --git a/csharp/Fury/Collections/PooledList.cs b/csharp/Fury/Collections/PooledList.cs new file mode 100644 index 0000000000..06f7ded36c --- /dev/null +++ b/csharp/Fury/Collections/PooledList.cs @@ -0,0 +1,177 @@ +using System; +using System.Buffers; +using System.Collections; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using Fury.Buffers; + +namespace Fury.Collections; + +/// +/// A list that uses pooled arrays to reduce allocations. +/// +/// +/// The pool provider to use for array pooling. +/// +/// +/// The type of elements in the list. +/// +internal sealed class PooledList(IArrayPoolProvider poolProvider) : IList, IDisposable + where TElement : class +{ + // Use object instead of TElement to improve possibility of reusing pooled objects. + private readonly ArrayPool _pool = poolProvider.GetArrayPool(); + private object?[] _elements = []; + public int Count { get; private set; } + + public Enumerator GetEnumerator() => new(this); + + public void Add(TElement? item) + { + var length = _elements.Length; + if (Count == length) + { + var newLength = Math.Max(length * 2, StaticConfigs.BuiltInListDefaultCapacity); + var newElements = _pool.Rent(newLength); + _elements.CopyTo(newElements, 0); + ClearElements(); + _pool.Return(_elements); + _elements = newElements; + } + _elements[Count++] = item; + } + + public void Clear() + { + ClearElements(); + Count = 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ClearElements() + { + Array.Clear(_elements, 0, _elements.Length); + } + + public bool Contains(TElement? item) => Array.IndexOf(_elements, item) != -1; + + public void CopyTo(TElement?[] array, int arrayIndex) => _elements.CopyTo(array, arrayIndex); + + public bool Remove(TElement? item) + { + var index = Array.IndexOf(_elements, item); + if (index == -1) + { + return false; + } + + RemoveAt(index); + return true; + } + + public bool IsReadOnly => _elements.IsReadOnly; + + public int IndexOf(TElement? item) => Array.IndexOf(_elements, item); + + public void Insert(int index, TElement? item) + { + if (index < 0 || index > Count) + { + ThrowHelper.ThrowArgumentOutOfRangeException(nameof(index), index); + } + + var length = _elements.Length; + if (Count == length) + { + var newLength = Math.Max(length * 2, StaticConfigs.BuiltInListDefaultCapacity); + var newElements = _pool.Rent(newLength); + _elements.CopyTo(newElements, 0); + Array.Copy(_elements, 0, newElements, 0, index); + newElements[index] = item; + Array.Copy(_elements, index, newElements, index + 1, Count - index); + ClearElements(); + _pool.Return(_elements); + _elements = newElements; + } + else + { + Array.Copy(_elements, index, _elements, index + 1, Count - index); + _elements[index] = item; + } + Count++; + } + + public void RemoveAt(int index) + { + ThrowIfOutOfRange(index, nameof(index)); + + if (index < Count - 1) + { + Array.Copy(_elements, index + 1, _elements, index, Count - index - 1); + } + Count--; + _elements[Count] = default!; + } + + public TElement? this[int index] + { + get + { + ThrowIfOutOfRange(index, nameof(index)); + return Unsafe.As(_elements[index]); + } + set + { + ThrowIfOutOfRange(index, nameof(index)); + _elements[index] = value; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ThrowIfOutOfRange(int index, string paramName) + { + if (index < 0 || index >= Count) + { + ThrowHelper.ThrowArgumentOutOfRangeException(paramName, index); + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public struct Enumerator(PooledList list) : IEnumerator + { + private int _count = list.Count; + private int _current = 0; + + public bool MoveNext() + { + return _current++ < _count; + } + + public void Reset() + { + _count = list.Count; + _current = 0; + } + + public TElement? Current => Unsafe.As(list._elements[_current]); + + object? IEnumerator.Current => Current; + + public void Dispose() { } + } + + public void Dispose() + { + if (_elements.Length <= 0) + { + return; + } + + ClearElements(); + _pool.Return(_elements); + _elements = []; + } +} diff --git a/csharp/Fury/CompatibleMode.cs b/csharp/Fury/CompatibleMode.cs new file mode 100644 index 0000000000..85a8130e83 --- /dev/null +++ b/csharp/Fury/CompatibleMode.cs @@ -0,0 +1,18 @@ +namespace Fury; + +/// +/// Type forward/backward compatibility config. +/// +public enum CompatibleMode +{ + /// + /// Class schema must be consistent between serialization peer and deserialization peer. + /// + SchemaConsistent, + + /// + /// Class schema can be different between serialization peer and deserialization peer. They can + /// add/delete fields independently. + /// + Compatible +} diff --git a/csharp/Fury/Config.cs b/csharp/Fury/Config.cs new file mode 100644 index 0000000000..42d99494b7 --- /dev/null +++ b/csharp/Fury/Config.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; +using Fury.Buffers; +using Fury.Serializer.Provider; + +namespace Fury; + +public sealed record Config( + ReferenceTrackingPolicy ReferenceTracking, + IEnumerable SerializerProviders, + IEnumerable DeserializerProviders, + IArrayPoolProvider ArrayPoolProvider +); + +/// +/// Specifies how reference information will be written when serializing referenceable objects. +/// +public enum ReferenceTrackingPolicy +{ + /// + /// All referenceable objects will be written as referenceable serialization data. + /// + Enabled, + + /// + /// All referenceable objects will be written as unreferenceable serialization data. + /// Referenceable objects may be written multiple times. + /// Throws if a circular dependency is detected. + /// + Disabled, + + /// + /// Similar to but all referenceable objects will be written as referenceable serialization data. + /// When a circular dependency is detected, only the reference information will be written. + /// This policy may be slower than when deserializing because reference tracking is still needed. + /// + OnlyCircularDependency +} diff --git a/csharp/Fury/DeserializationContext.cs b/csharp/Fury/DeserializationContext.cs new file mode 100644 index 0000000000..bdfd57ce10 --- /dev/null +++ b/csharp/Fury/DeserializationContext.cs @@ -0,0 +1,168 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Fury.Meta; +using Fury.Serializer; + +namespace Fury; + +// Async methods do not work with ref, so DeserializationContext is a class + +public sealed class DeserializationContext +{ + public Fury Fury { get; } + public BatchReader Reader { get; } + private readonly RefContext _refContext; + private readonly MetaStringResolver _metaStringResolver; + private readonly TypeResolver _typeResolver; + + internal DeserializationContext( + Fury fury, + BatchReader reader, + RefContext refContext, + MetaStringResolver metaStringResolver + ) + { + Fury = fury; + Reader = reader; + _refContext = refContext; + _metaStringResolver = metaStringResolver; + } + + public bool TryGetDeserializer([NotNullWhen(true)] out IDeserializer? deserializer) + { + return Fury.TypeResolver.TryGetOrCreateDeserializer(typeof(TValue), out deserializer); + } + + public IDeserializer GetDeserializer() + { + if (!TryGetDeserializer(out var deserializer)) + { + ThrowHelper.ThrowDeserializerNotFoundException_DeserializerNotFound(typeof(TValue)); + } + return deserializer; + } + + public async ValueTask ReadAsync( + IDeserializer? deserializer = null, + CancellationToken cancellationToken = default + ) + where TValue : notnull + { + var refFlag = await Reader.ReadReferenceFlagAsync(cancellationToken); + if (refFlag == ReferenceFlag.Null) + { + return default; + } + if (refFlag == ReferenceFlag.Ref) + { + var refId = await Reader.ReadRefIdAsync(cancellationToken); + if (!_refContext.TryGetReadValue(refId, out var readObject)) + { + ThrowHelper.ThrowBadDeserializationInputException_ReferencedObjectNotFound(refId); + } + + return (TValue)readObject; + } + + if (refFlag == ReferenceFlag.RefValue) + { + return (TValue)await DoReadReferenceableAsync(deserializer, cancellationToken); + } + + return await DoReadUnreferenceableAsync(deserializer, cancellationToken); + } + + public async ValueTask ReadNullableAsync( + IDeserializer? deserializer = null, + CancellationToken cancellationToken = default + ) + where TValue : struct + { + var refFlag = await Reader.ReadReferenceFlagAsync(cancellationToken); + if (refFlag == ReferenceFlag.Null) + { + return null; + } + if (refFlag == ReferenceFlag.Ref) + { + var refId = await Reader.ReadRefIdAsync(cancellationToken); + if (!_refContext.TryGetReadValue(refId, out var readObject)) + { + ThrowHelper.ThrowBadDeserializationInputException_ReferencedObjectNotFound(refId); + } + + return (TValue?)readObject; + } + + if (refFlag == ReferenceFlag.RefValue) + { + return (TValue?)await DoReadReferenceableAsync(deserializer, cancellationToken); + } + + return await DoReadUnreferenceableAsync(deserializer, cancellationToken); + } + + private async ValueTask DoReadUnreferenceableAsync( + IDeserializer? deserializer, + CancellationToken cancellationToken = default + ) + where TValue : notnull + { + var declaredType = typeof(TValue); + var typeInfo = await ReadTypeMetaAsync(cancellationToken); + deserializer ??= GetPreferredDeserializer(typeInfo.Type); + if (typeInfo.Type == declaredType && deserializer is IDeserializer typedDeserializer) + { + return await typedDeserializer.ReadAndCreateAsync(this, cancellationToken); + } + var newObj = await deserializer.CreateInstanceAsync(this, cancellationToken); + await deserializer.ReadAndFillAsync(this, newObj, cancellationToken); + return (TValue)newObj.Value!; + } + + private async ValueTask DoReadReferenceableAsync( + IDeserializer? deserializer, + CancellationToken cancellationToken = default + ) + { + var typeInfo = await ReadTypeMetaAsync(cancellationToken); + deserializer ??= GetPreferredDeserializer(typeInfo.Type); + var newObj = await deserializer.CreateInstanceAsync(this, cancellationToken); + _refContext.PushReferenceableObject(newObj); + await deserializer.ReadAndFillAsync(this, newObj, cancellationToken); + return newObj; + } + + private async ValueTask ReadTypeMetaAsync(CancellationToken cancellationToken = default) + { + var typeId = await Reader.ReadTypeIdAsync(cancellationToken); + TypeInfo typeInfo; + if (typeId.IsNamed()) + { + var namespaceBytes = await _metaStringResolver.ReadMetaStringBytesAsync(Reader, cancellationToken); + var typeNameBytes = await _metaStringResolver.ReadMetaStringBytesAsync(Reader, cancellationToken); + } + switch (typeId) + { + // TODO: Read namespace + default: + if (!Fury.TypeResolver.TryGetTypeInfo(typeId, out typeInfo)) + { + ThrowHelper.ThrowBadDeserializationInputException_TypeInfoNotFound(typeId); + } + break; + } + return typeInfo; + } + + private IDeserializer GetPreferredDeserializer(Type typeOfDeserializedObject) + { + if (!Fury.TypeResolver.TryGetOrCreateDeserializer(typeOfDeserializedObject, out var deserializer)) + { + ThrowHelper.ThrowDeserializerNotFoundException_DeserializerNotFound(typeOfDeserializedObject); + } + return deserializer; + } +} diff --git a/csharp/Fury/Exceptions/ArgumentException.cs b/csharp/Fury/Exceptions/ArgumentException.cs new file mode 100644 index 0000000000..2d69ef3675 --- /dev/null +++ b/csharp/Fury/Exceptions/ArgumentException.cs @@ -0,0 +1,19 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury; + +internal static partial class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowArgumentException(string? message = null, string? paramName = null) + { + throw new ArgumentException(message, paramName); + } + + [DoesNotReturn] + public static void ThrowArgumentException_InsufficientSpaceInTheOutputBuffer(string? paramName = null) + { + throw new ArgumentException("Insufficient space in the output buffer.", paramName); + } +} diff --git a/csharp/Fury/Exceptions/ArgumentOutOfRangeException.cs b/csharp/Fury/Exceptions/ArgumentOutOfRangeException.cs new file mode 100644 index 0000000000..d84c757213 --- /dev/null +++ b/csharp/Fury/Exceptions/ArgumentOutOfRangeException.cs @@ -0,0 +1,17 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury; + +internal static partial class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowArgumentOutOfRangeException( + string paramName, + object? actualValue = null, + string? message = null + ) + { + throw new ArgumentOutOfRangeException(paramName, actualValue, message); + } +} diff --git a/csharp/Fury/Exceptions/Backports/UnreachableException.cs b/csharp/Fury/Exceptions/Backports/UnreachableException.cs new file mode 100644 index 0000000000..9b5a17783e --- /dev/null +++ b/csharp/Fury/Exceptions/Backports/UnreachableException.cs @@ -0,0 +1,28 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +#if !NET8_0_OR_GREATER +// ReSharper disable once CheckNamespace +namespace System.Diagnostics +{ + internal sealed class UnreachableException(string? message = null) : Exception(message); +} +#endif + +namespace Fury +{ + internal static partial class ThrowHelper + { + [DoesNotReturn] + public static void ThrowUnreachableException(string? message = null) + { + throw new UnreachableException(message); + } + + [DoesNotReturn] + [Conditional("DEBUG")] + public static void ThrowUnreachableExceptionDebugOnly(string? message = null) + { + throw new UnreachableException(message); + } + } +} diff --git a/csharp/Fury/Exceptions/BadDeserializationInputException.cs b/csharp/Fury/Exceptions/BadDeserializationInputException.cs new file mode 100644 index 0000000000..5d177af8b1 --- /dev/null +++ b/csharp/Fury/Exceptions/BadDeserializationInputException.cs @@ -0,0 +1,84 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using Fury.Meta; + +namespace Fury; + +public class BadDeserializationInputException(string? message = null) : Exception(message); + +internal static partial class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowBadDeserializationInputException(string? message = null) + { + throw new BadDeserializationInputException(message); + } + + [DoesNotReturn] + public static TReturn ThrowBadDeserializationInputException(string? message = null) + { + throw new BadDeserializationInputException(message); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_UnrecognizedMetaStringCodePoint(byte codePoint) + { + throw new BadDeserializationInputException($"Unrecognized MetaString code point: {codePoint}"); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_UpperCaseFlagCannotAppearConsecutively() + { + throw new BadDeserializationInputException( + $"The '{AllToLowerSpecialEncoding.UpperCaseFlag}' cannot appear consecutively" + ); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_UnknownMetaStringId(int id) + { + throw new BadDeserializationInputException($"Unknown MetaString ID: {id}"); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_TypeInfoNotFound(TypeId id) + { + throw new BadDeserializationInputException($"No type info found for type id '{id}'."); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_ReferencedObjectNotFound(RefId refId) + { + throw new BadDeserializationInputException($"Referenced object not found for ref id '{refId}'."); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_InsufficientData() + { + throw new BadDeserializationInputException("Insufficient data."); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_VarInt32Overflow() + { + throw new BadDeserializationInputException("VarInt32 overflow."); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_InvalidMagicNumber() + { + throw new BadDeserializationInputException("Invalid magic number."); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_NotCrossLanguage() + { + throw new BadDeserializationInputException("Not cross language."); + } + + [DoesNotReturn] + public static void ThrowBadDeserializationInputException_NotLittleEndian() + { + throw new BadDeserializationInputException("Not little endian."); + } +} diff --git a/csharp/Fury/Exceptions/BadSerializationInputException.cs b/csharp/Fury/Exceptions/BadSerializationInputException.cs new file mode 100644 index 0000000000..98f3424629 --- /dev/null +++ b/csharp/Fury/Exceptions/BadSerializationInputException.cs @@ -0,0 +1,39 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury; + +public sealed class BadSerializationInputException(string? message = null) : Exception(message); + +internal static partial class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowBadSerializationInputException(string? message = null) + { + throw new BadSerializationInputException(message); + } + + [DoesNotReturn] + public static TReturn ThrowBadSerializationInputException(string? message = null) + { + throw new BadSerializationInputException(message); + } + + [DoesNotReturn] + public static void ThrowBadSerializationInputException_UnsupportedMetaStringChar(char c) + { + throw new BadSerializationInputException($"Unsupported MetaString character: '{c}'"); + } + + [DoesNotReturn] + public static void ThrowBadSerializationInputException_UnregisteredType(Type type) + { + throw new BadSerializationInputException($"Type '{type.FullName}' is not registered."); + } + + [DoesNotReturn] + public static void ThrowBadSerializationInputException_CircularDependencyDetected() + { + throw new BadSerializationInputException("Circular dependency detected."); + } +} diff --git a/csharp/Fury/Exceptions/CircularDependencyException.cs b/csharp/Fury/Exceptions/CircularDependencyException.cs new file mode 100644 index 0000000000..9e95f8fbc9 --- /dev/null +++ b/csharp/Fury/Exceptions/CircularDependencyException.cs @@ -0,0 +1,13 @@ +using System; + +namespace Fury; + +public class CircularDependencyException(string? message = null) : Exception(message); + +internal static partial class ThrowHelper +{ + public static void ThrowCircularDependencyException(string? message = null) + { + throw new CircularDependencyException(message); + } +} diff --git a/csharp/Fury/Exceptions/DeserializerNotFoundException.cs b/csharp/Fury/Exceptions/DeserializerNotFoundException.cs new file mode 100644 index 0000000000..4715d73dfe --- /dev/null +++ b/csharp/Fury/Exceptions/DeserializerNotFoundException.cs @@ -0,0 +1,36 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury; + +public class DeserializerNotFoundException(Type? objectType, TypeId? typeId, string? message = null) + : Exception(message) +{ + public Type? TypeOfDeserializedObject { get; } = objectType; + public TypeId? TypeIdOfDeserializedObject { get; } = typeId; + + public DeserializerNotFoundException(Type objectType, string? message = null) + : this(objectType, null, message) { } + + public DeserializerNotFoundException(TypeId typeId, string? message = null) + : this(null, typeId, message) { } +} + +internal static partial class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowDeserializerNotFoundException( + Type? type = null, + TypeId? typeId = null, + string? message = null + ) + { + throw new DeserializerNotFoundException(type, typeId, message); + } + + [DoesNotReturn] + public static void ThrowDeserializerNotFoundException_DeserializerNotFound(Type type) + { + throw new DeserializerNotFoundException(type, $"No deserializer found for type '{type.FullName}'."); + } +} diff --git a/csharp/Fury/Exceptions/InvalidOperationException.cs b/csharp/Fury/Exceptions/InvalidOperationException.cs new file mode 100644 index 0000000000..62fae470b3 --- /dev/null +++ b/csharp/Fury/Exceptions/InvalidOperationException.cs @@ -0,0 +1,19 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury; + +internal static partial class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowInvalidOperationException(string message) + { + throw new InvalidOperationException(message); + } + + [DoesNotReturn] + public static void ThrowInvalidOperationException_AttemptedToWriteToReadOnlyCollection() + { + throw new InvalidOperationException("Attempted to write to a read-only collection."); + } +} diff --git a/csharp/Fury/Exceptions/NotSupportedException.cs b/csharp/Fury/Exceptions/NotSupportedException.cs new file mode 100644 index 0000000000..4e80ef721f --- /dev/null +++ b/csharp/Fury/Exceptions/NotSupportedException.cs @@ -0,0 +1,25 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury; + +internal static partial class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowNotSupportedException(string? message = null) + { + throw new NotSupportedException(message); + } + + [DoesNotReturn] + public static TReturn ThrowNotSupportedException(string? message = null) + { + throw new NotSupportedException(message); + } + + [DoesNotReturn] + public static TReturn ThrowNotSupportedException_EncoderNotSupportedForThisEncoding(string? encodingName) + { + throw new NotSupportedException($"The encoder is not supported for the encoding '{encodingName}'."); + } +} diff --git a/csharp/Fury/Exceptions/ReferencedObjectNotFoundException.cs b/csharp/Fury/Exceptions/ReferencedObjectNotFoundException.cs new file mode 100644 index 0000000000..f06f179f63 --- /dev/null +++ b/csharp/Fury/Exceptions/ReferencedObjectNotFoundException.cs @@ -0,0 +1,15 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury; + +public class ReferencedObjectNotFoundException(string? message = null) : Exception(message); + +internal static partial class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowReferencedObjectNotFoundException(string? message = null) + { + throw new ReferencedObjectNotFoundException(message); + } +} diff --git a/csharp/Fury/Exceptions/SerializerNotFoundException.cs b/csharp/Fury/Exceptions/SerializerNotFoundException.cs new file mode 100644 index 0000000000..4fead8374d --- /dev/null +++ b/csharp/Fury/Exceptions/SerializerNotFoundException.cs @@ -0,0 +1,35 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury; + +public class SerializerNotFoundException(Type? objectType, TypeId? typeId, string? message = null) : Exception(message) +{ + public Type? TypeOfSerializedObject { get; } = objectType; + public TypeId? TypeIdOfSerializedObject { get; } = typeId; + + public SerializerNotFoundException(Type objectType, string? message = null) + : this(objectType, null, message) { } + + public SerializerNotFoundException(TypeId typeId, string? message = null) + : this(null, typeId, message) { } +} + +internal static partial class ThrowHelper +{ + [DoesNotReturn] + public static void ThrowSerializerNotFoundException( + Type? type = null, + TypeId? typeId = null, + string? message = null + ) + { + throw new SerializerNotFoundException(type, typeId, message); + } + + [DoesNotReturn] + public static void ThrowSerializerNotFoundException_SerializerNotFound(Type type) + { + throw new SerializerNotFoundException(type, $"No serializer found for type '{type.FullName}'."); + } +} diff --git a/csharp/Fury/Fury.cs b/csharp/Fury/Fury.cs new file mode 100644 index 0000000000..6acedf1def --- /dev/null +++ b/csharp/Fury/Fury.cs @@ -0,0 +1,150 @@ +using System.IO.Pipelines; +using System.Threading; +using System.Threading.Tasks; +using Fury.Buffers; +using Fury.Meta; + +namespace Fury; + +public sealed class Fury(Config config) +{ + public Config Config { get; } = config; + + private const short MagicNumber = 0x62D4; + + public TypeResolver TypeResolver { get; } = + new(config.SerializerProviders, config.DeserializerProviders, config.ArrayPoolProvider); + + private readonly ObjectPool _refResolverPool = + new(config.ArrayPoolProvider, () => new RefContext(config.ArrayPoolProvider)); + + public void Serialize(PipeWriter writer, in T? value) + where T : notnull + { + var refResolver = _refResolverPool.Rent(); + try + { + if (SerializeCommon(new BatchWriter(writer), in value, refResolver, out var context)) + { + context.Write(in value); + } + } + finally + { + _refResolverPool.Return(refResolver); + } + } + + public void Serialize(PipeWriter writer, in T? value) + where T : struct + { + var refResolver = _refResolverPool.Rent(); + try + { + if (SerializeCommon(new BatchWriter(writer), in value, refResolver, out var context)) + { + context.Write(in value); + } + } + finally + { + _refResolverPool.Return(refResolver); + } + } + + private bool SerializeCommon( + BatchWriter writer, + in T? value, + RefContext refContext, + out SerializationContext context + ) + { + writer.Write(MagicNumber); + var headerFlag = HeaderFlag.LittleEndian | HeaderFlag.CrossLanguage; + if (value is null) + { + headerFlag |= HeaderFlag.NullRootObject; + writer.Write((byte)headerFlag); + context = default; + return false; + } + writer.Write((byte)headerFlag); + writer.Write((byte)Language.Csharp); + context = new SerializationContext(this, writer, refContext); + return true; + } + + public async ValueTask DeserializeAsync(PipeReader reader, CancellationToken cancellationToken = default) + where T : notnull + { + var refResolver = _refResolverPool.Rent(); + T? result = default; + try + { + var context = await DeserializeCommonAsync(new BatchReader(reader), refResolver); + if (context is not null) + { + result = await context.ReadAsync(cancellationToken: cancellationToken); + } + } + finally + { + _refResolverPool.Return(refResolver); + } + + return result; + } + + public async ValueTask DeserializeNullableAsync( + PipeReader reader, + CancellationToken cancellationToken = default + ) + where T : struct + { + var refResolver = _refResolverPool.Rent(); + T? result = default; + try + { + var context = await DeserializeCommonAsync(new BatchReader(reader), refResolver); + if (context is not null) + { + result = await context.ReadNullableAsync(cancellationToken: cancellationToken); + } + } + finally + { + _refResolverPool.Return(refResolver); + } + + return result; + } + + private async ValueTask DeserializeCommonAsync(BatchReader reader, RefContext refContext) + { + var magicNumber = await reader.ReadAsync(); + if (magicNumber != MagicNumber) + { + ThrowHelper.ThrowBadDeserializationInputException_InvalidMagicNumber(); + return default; + } + var headerFlag = (HeaderFlag)await reader.ReadAsync(); + if (headerFlag.HasFlag(HeaderFlag.NullRootObject)) + { + return null; + } + if (!headerFlag.HasFlag(HeaderFlag.CrossLanguage)) + { + ThrowHelper.ThrowBadDeserializationInputException_NotCrossLanguage(); + return default; + } + if (!headerFlag.HasFlag(HeaderFlag.LittleEndian)) + { + ThrowHelper.ThrowBadDeserializationInputException_NotLittleEndian(); + return default; + } + await reader.ReadAsync(); + var metaStringResolver = new MetaStringResolver(Config.ArrayPoolProvider); + var context = new DeserializationContext(this, reader, refContext, metaStringResolver); + return context; + } +} diff --git a/csharp/Fury/Fury.csproj b/csharp/Fury/Fury.csproj new file mode 100644 index 0000000000..1e0e044ed5 --- /dev/null +++ b/csharp/Fury/Fury.csproj @@ -0,0 +1,19 @@ + + + + netstandard2.0;net8.0 + 12 + enable + true + Debug;Release;ReleaseAot + AnyCPU + + + + + + + + + + diff --git a/csharp/Fury/Global.cs b/csharp/Fury/Global.cs new file mode 100644 index 0000000000..80752dd862 --- /dev/null +++ b/csharp/Fury/Global.cs @@ -0,0 +1,16 @@ +using System.Runtime.CompilerServices; + +// ReSharper disable once CheckNamespace +// ReSharper disable UnusedType.Global +// ReSharper disable UnusedAutoPropertyAccessor.Global + +[assembly: InternalsVisibleTo("Fury.Testing")] + + +#if !NET8_0_OR_GREATER +// ReSharper disable once CheckNamespace +namespace System.Runtime.CompilerServices +{ + internal class IsExternalInit; +} +#endif diff --git a/csharp/Fury/HashHelper.cs b/csharp/Fury/HashHelper.cs new file mode 100644 index 0000000000..626de4e80d --- /dev/null +++ b/csharp/Fury/HashHelper.cs @@ -0,0 +1,139 @@ +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Fury; + +internal static class HashHelper +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong FinalizationMix(ulong k) + { + k ^= k >> 33; + k *= 0xff51afd7ed558ccd; + k ^= k >> 33; + k *= 0xc4ceb9fe1a85ec53; + k ^= k >> 33; + return k; + } + + public static void MurmurHash3_x64_128(ReadOnlySpan key, uint seed, out ulong out1, out ulong out2) + { + const int blockSize = sizeof(ulong) * 2; + + const ulong c1 = 0x87c37b91114253d5; + const ulong c2 = 0x4cf5ad432745937f; + + var length = key.Length; + + ulong h1 = seed; + ulong h2 = seed; + + ulong k1; + ulong k2; + + var blocks = MemoryMarshal.Cast(key); + var nBlocks = length / blockSize; + for (var i = 0; i < nBlocks; i++) + { + k1 = blocks[i * 2]; + k2 = blocks[i * 2 + 1]; + + k1 *= c1; + k1 = BitOperations.RotateLeft(k1, 31); + k1 *= c2; + h1 ^= k1; + + h1 = BitOperations.RotateLeft(h1, 27); + h1 += h2; + h1 = h1 * 5 + 0x52dce729; + + k2 *= c2; + k2 = BitOperations.RotateLeft(k2, 33); + k2 *= c1; + h2 ^= k2; + + h2 = BitOperations.RotateLeft(h2, 31); + h2 += h1; + h2 = h2 * 5 + 0x38495ab5; + } + + var tail = key.Slice(nBlocks * blockSize); + + k1 = 0; + k2 = 0; + + switch (length & 15) + { + case 15: + k2 ^= (ulong)tail[14] << 48; + goto case 14; + case 14: + k2 ^= (ulong)tail[13] << 40; + goto case 13; + case 13: + k2 ^= (ulong)tail[12] << 32; + goto case 12; + case 12: + k2 ^= (ulong)tail[11] << 24; + goto case 11; + case 11: + k2 ^= (ulong)tail[10] << 16; + goto case 10; + case 10: + k2 ^= (ulong)tail[9] << 8; + goto case 9; + case 9: + k2 ^= tail[8]; + k2 *= c2; + k2 = BitOperations.RotateLeft(k2, 33); + k2 *= c1; + h2 ^= k2; + goto case 8; + case 8: + k1 ^= (ulong)tail[7] << 56; + goto case 7; + case 7: + k1 ^= (ulong)tail[6] << 48; + goto case 6; + case 6: + k1 ^= (ulong)tail[5] << 40; + goto case 5; + case 5: + k1 ^= (ulong)tail[4] << 32; + goto case 4; + case 4: + k1 ^= (ulong)tail[3] << 24; + goto case 3; + case 3: + k1 ^= (ulong)tail[2] << 16; + goto case 2; + case 2: + k1 ^= (ulong)tail[1] << 8; + goto case 1; + case 1: + k1 ^= tail[0]; + k1 *= c1; + k1 = BitOperations.RotateLeft(k1, 31); + k1 *= c2; + h1 ^= k1; + break; + } + + h1 ^= (ulong)length; + h2 ^= (ulong)length; + + h1 += h2; + h2 += h1; + + h1 = FinalizationMix(h1); + h2 = FinalizationMix(h2); + + h1 += h2; + h2 += h1; + + out1 = h1; + out2 = h2; + } +} diff --git a/csharp/Fury/HeaderFlag.cs b/csharp/Fury/HeaderFlag.cs new file mode 100644 index 0000000000..12e9055fc3 --- /dev/null +++ b/csharp/Fury/HeaderFlag.cs @@ -0,0 +1,12 @@ +using System; + +namespace Fury; + +[Flags] +public enum HeaderFlag : byte +{ + NullRootObject = 1, + LittleEndian = 1 << 1, + CrossLanguage = 1 << 2, + OutOfBand = 1 << 3, +} diff --git a/csharp/Fury/Language.cs b/csharp/Fury/Language.cs new file mode 100644 index 0000000000..d1cf868b4b --- /dev/null +++ b/csharp/Fury/Language.cs @@ -0,0 +1,13 @@ +namespace Fury; + +public enum Language : byte +{ + Xlang, + Java, + Python, + Cpp, + Go, + Javascript, + Rust, + Csharp, +} diff --git a/csharp/Fury/Meta/AbstractLowerSpecialEncoding.cs b/csharp/Fury/Meta/AbstractLowerSpecialEncoding.cs new file mode 100644 index 0000000000..1011ed5de5 --- /dev/null +++ b/csharp/Fury/Meta/AbstractLowerSpecialEncoding.cs @@ -0,0 +1,60 @@ +using System.Text; + +namespace Fury.Meta; + +internal abstract class AbstractLowerSpecialEncoding(MetaString.Encoding encoding) : MetaStringEncoding(encoding) +{ + internal const int BitsPerChar = 5; + + public sealed override Encoder GetEncoder() => + ThrowHelper.ThrowNotSupportedException_EncoderNotSupportedForThisEncoding(GetType().Name); + + internal static bool TryEncodeChar(char c, out byte b) + { + var (success, encoded) = c switch + { + >= 'a' and <= 'z' => (true, (byte)(c - 'a')), + '.' => (true, NumberOfEnglishLetters), + '_' => (true, NumberOfEnglishLetters + 1), + '$' => (true, NumberOfEnglishLetters + 2), + '|' => (true, NumberOfEnglishLetters + 3), + _ => (false, default) + }; + b = (byte)encoded; + return success; + } + + internal static byte EncodeChar(char c) + { + if (!TryEncodeChar(c, out var b)) + { + ThrowHelper.ThrowBadSerializationInputException_UnsupportedMetaStringChar(c); + } + + return b; + } + + internal static bool TryDecodeByte(byte b, out char c) + { + (var success, c) = b switch + { + < NumberOfEnglishLetters => (true, (char)(b + 'a')), + NumberOfEnglishLetters => (true, '.'), + NumberOfEnglishLetters + 1 => (true, '_'), + NumberOfEnglishLetters + 2 => (true, '$'), + NumberOfEnglishLetters + 3 => (true, '|'), + _ => (false, default) + }; + return success; + } + + internal static char DecodeByte(byte b) + { + if (!TryDecodeByte(b, out var c)) + { + ThrowHelper.ThrowBadDeserializationInputException_UnrecognizedMetaStringCodePoint(b); + } + + return c; + } +} diff --git a/csharp/Fury/Meta/AllToLowerSpecialDecoder.cs b/csharp/Fury/Meta/AllToLowerSpecialDecoder.cs new file mode 100644 index 0000000000..45375426b5 --- /dev/null +++ b/csharp/Fury/Meta/AllToLowerSpecialDecoder.cs @@ -0,0 +1,51 @@ +using System; + +namespace Fury.Meta; + +internal sealed class AllToLowerSpecialDecoder : MetaStringDecoder +{ + internal bool WasLastCharUpperCaseFlag; + + public override void Convert( + ReadOnlySpan bytes, + Span chars, + bool flush, + out int bytesUsed, + out int charsUsed, + out bool completed + ) + { + MustFlush = flush; + AllToLowerSpecialEncoding.GetChars(bytes, chars, this, out bytesUsed, out charsUsed); + completed = bytesUsed == bytes.Length && !HasLeftoverData; + if (flush) + { + Reset(); + } + } + + public override int GetCharCount(ReadOnlySpan bytes, bool flush) + { + MustFlush = flush; + var charCount = AllToLowerSpecialEncoding.GetCharCount(bytes, this); + if (flush) + { + Reset(); + } + return charCount; + } + + public override int GetChars(ReadOnlySpan bytes, Span chars, bool flush) + { + Convert(bytes, chars, flush, out _, out var charsUsed, out _); + return charsUsed; + } + + public override void Reset() + { + MustFlush = false; + LeftoverBits = 0; + LeftoverBitCount = 0; + HasState = false; + } +} diff --git a/csharp/Fury/Meta/AllToLowerSpecialEncoding.cs b/csharp/Fury/Meta/AllToLowerSpecialEncoding.cs new file mode 100644 index 0000000000..238ed27bfe --- /dev/null +++ b/csharp/Fury/Meta/AllToLowerSpecialEncoding.cs @@ -0,0 +1,264 @@ +using System; +using System.Text; + +namespace Fury.Meta; + +internal sealed class AllToLowerSpecialEncoding() : AbstractLowerSpecialEncoding(MetaString.Encoding.AllToLowerSpecial) +{ + public static readonly AllToLowerSpecialEncoding Instance = new(); + + internal const char UpperCaseFlag = '|'; + private static readonly byte EncodedUpperCaseFlag = EncodeChar(UpperCaseFlag); + + private static readonly AllToLowerSpecialDecoder SharedDecoder = new(); + + public override bool CanEncode(ReadOnlySpan chars) + { + foreach (var t in chars) + { + var c = char.ToLowerInvariant(t); + if (!TryEncodeChar(c, out _)) + { + return false; + } + } + + return true; + } + + public override int GetByteCount(ReadOnlySpan chars) + { + var bitCount = 0; + foreach (var c in chars) + { + bitCount += char.IsUpper(c) ? BitsPerChar * 2 : BitsPerChar; + } + return bitCount / BitsOfByte + 1; + } + + public override int GetCharCount(ReadOnlySpan bytes) + { + var charCount = SharedDecoder.GetCharCount(bytes, true); + SharedDecoder.Reset(); + return charCount; + } + + public override int GetMaxByteCount(int charCount) + { + return charCount * BitsPerChar * 2 / BitsOfByte + 1; + } + + public override int GetMaxCharCount(int byteCount) + { + return LowerSpecialEncoding.Instance.GetMaxCharCount(byteCount); + } + + public override int GetBytes(ReadOnlySpan chars, Span bytes) + { + var bitsWriter = new BitsWriter(bytes); + var charsReader = new CharsReader(chars); + + bitsWriter.Advance(1); + while (charsReader.TryReadChar(out var c)) + { + if (char.IsUpper(c)) + { + if (bitsWriter.TryWriteBits(BitsPerChar, EncodedUpperCaseFlag)) + { + bitsWriter.Advance(BitsPerChar); + } + else + { + break; + } + c = char.ToLowerInvariant(c); + } + + var charByte = EncodeChar(c); + + if (bitsWriter.TryWriteBits(BitsPerChar, charByte)) + { + charsReader.Advance(); + bitsWriter.Advance(BitsPerChar); + } + else + { + break; + } + } + + if (charsReader.CharsUsed < chars.Length) + { + ThrowHelper.ThrowArgumentException_InsufficientSpaceInTheOutputBuffer(nameof(bytes)); + } + + if (bitsWriter.UnusedBitCountInLastUsedByte >= BitsPerChar) + { + bitsWriter[0] = true; + } + + return bitsWriter.BytesUsed; + } + + public override int GetChars(ReadOnlySpan bytes, Span chars) + { + GetChars(bytes, chars, SharedDecoder, out _, out var charsUsed); + SharedDecoder.Reset(); + return charsUsed; + } + + private static bool TryWriteChar( + ref CharsWriter writer, + byte charByte, + AllToLowerSpecialDecoder decoder, + out bool writtenChar + ) + { + if (charByte == EncodedUpperCaseFlag) + { + if (decoder.WasLastCharUpperCaseFlag) + { + ThrowHelper.ThrowBadDeserializationInputException_UpperCaseFlagCannotAppearConsecutively(); + } + writtenChar = false; + return true; + } + + var decodedChar = DecodeByte(charByte); + if (decoder.WasLastCharUpperCaseFlag) + { + decodedChar = char.ToUpperInvariant(decodedChar); + } + + writtenChar = writer.TryWriteChar(decodedChar); + return writtenChar; + } + + internal static void GetChars( + ReadOnlySpan bytes, + Span chars, + AllToLowerSpecialDecoder decoder, + out int bytesUsed, + out int charsUsed + ) + { + var bitsReader = new BitsReader(bytes); + var charsWriter = new CharsWriter(chars); + if (!decoder.HasState) + { + decoder.HasState = true; + if (bitsReader.TryReadBits(1, out var stripLastCharFlag)) + { + bitsReader.Advance(1); + decoder.StripLastChar = stripLastCharFlag != 0; + } + } + else + { + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out var charByte, out var bitsUsedFromBitsReader)) + { + if (TryWriteChar(ref charsWriter, charByte, decoder, out var writtenChar)) + { + decoder.WasLastCharUpperCaseFlag = charByte == EncodedUpperCaseFlag; + bitsReader.Advance(bitsUsedFromBitsReader); + if (writtenChar) + { + charsWriter.Advance(); + } + + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out charByte, out bitsUsedFromBitsReader)) + { + if (TryWriteChar(ref charsWriter, charByte, decoder, out writtenChar)) + { + decoder.WasLastCharUpperCaseFlag = charByte == EncodedUpperCaseFlag; + bitsReader.Advance(bitsUsedFromBitsReader); + if (writtenChar) + { + charsWriter.Advance(); + } + } + } + } + } + } + + while (bitsReader.TryReadBits(BitsPerChar, out var charByte)) + { + if (bitsReader.GetRemainingCount(BitsPerChar) == 1 && decoder is { MustFlush: true, StripLastChar: true }) + { + break; + } + if (TryWriteChar(ref charsWriter, charByte, decoder, out var writtenChar)) + { + decoder.WasLastCharUpperCaseFlag = charByte == EncodedUpperCaseFlag; + bitsReader.Advance(BitsPerChar); + if (writtenChar) + { + charsWriter.Advance(); + } + } + else + { + break; + } + } + + decoder.SetLeftoverData(bitsReader.UnusedBitsInLastUsedByte, bitsReader.UnusedBitCountInLastUsedByte); + + bytesUsed = bitsReader.BytesUsed; + charsUsed = charsWriter.CharsUsed; + } + + internal static int GetCharCount(ReadOnlySpan bytes, AllToLowerSpecialDecoder decoder) + { + var bitsReader = new BitsReader(bytes); + var charCount = 0; + if (!decoder.HasState) + { + decoder.HasState = true; + if (bitsReader.TryReadBits(1, out var stripLastCharFlag)) + { + bitsReader.Advance(1); + decoder.StripLastChar = stripLastCharFlag != 0; + } + } + else + { + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out var charByte, out var bitsUsedFromBitsReader)) + { + bitsReader.Advance(bitsUsedFromBitsReader); + if (charByte != EncodedUpperCaseFlag) + { + charCount++; + } + + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out charByte, out bitsUsedFromBitsReader)) + { + bitsReader.Advance(bitsUsedFromBitsReader); + if (charByte != EncodedUpperCaseFlag) + { + charCount++; + } + } + } + } + + while (bitsReader.TryReadBits(BitsPerChar, out var charByte)) + { + if (bitsReader.GetRemainingCount(BitsPerChar) == 1 && decoder is { MustFlush: true, StripLastChar: true }) + { + break; + } + bitsReader.Advance(BitsPerChar); + if (charByte != EncodedUpperCaseFlag) + { + charCount++; + } + } + + decoder.SetLeftoverData(bitsReader.UnusedBitsInLastUsedByte, bitsReader.UnusedBitCountInLastUsedByte); + return charCount; + } + + public override Decoder GetDecoder() => new AllToLowerSpecialDecoder(); +} diff --git a/csharp/Fury/Meta/BitsReader.cs b/csharp/Fury/Meta/BitsReader.cs new file mode 100644 index 0000000000..bdadaf2e23 --- /dev/null +++ b/csharp/Fury/Meta/BitsReader.cs @@ -0,0 +1,87 @@ +using System; + +namespace Fury.Meta; + +internal ref struct BitsReader(ReadOnlySpan bytes) +{ + private const int BitsOfByte = sizeof(byte) * 8; + + private readonly ReadOnlySpan _bytes = bytes; + + private int _currentBitIndex; + private int CurrentByteIndex => _currentBitIndex / BitsOfByte; + + internal int BytesUsed => (_currentBitIndex + BitsOfByte - 1) / BitsOfByte; + internal int UnusedBitCountInLastUsedByte => (BitsOfByte - _currentBitIndex % BitsOfByte) % BitsOfByte; + + internal byte UnusedBitsInLastUsedByte + { + get + { + var unusedBitCountInLastUsedByte = UnusedBitCountInLastUsedByte; + if (unusedBitCountInLastUsedByte == 0) + { + return 0; + } + + var currentByte = _bytes[CurrentByteIndex]; + return BitUtility.KeepLowBits(currentByte, unusedBitCountInLastUsedByte); + } + } + + internal bool HasNext(int bitCount) => _currentBitIndex + bitCount <= _bytes.Length * BitsOfByte; + + internal int GetRemainingCount(int bitCount) => (_bytes.Length * BitsOfByte - _currentBitIndex) / bitCount; + + internal bool TryReadBits(int bitCount, out byte bits) + { + if (!HasNext(bitCount)) + { + bits = default; + return false; + } + var currentByteIndex = CurrentByteIndex; + if (currentByteIndex >= _bytes.Length) + { + bits = default; + return false; + } + + var bitOffsetInCurrentByte = _currentBitIndex % BitsOfByte; + var bitsLeftInCurrentByte = BitsOfByte - bitOffsetInCurrentByte; + if (bitsLeftInCurrentByte >= bitCount) + { + bits = BitUtility.ReadBits(_bytes[currentByteIndex], bitOffsetInCurrentByte, bitCount); + return true; + } + + if (currentByteIndex + 1 >= _bytes.Length) + { + bits = default; + return false; + } + + bits = BitUtility.ReadBits( + _bytes[currentByteIndex], + _bytes[currentByteIndex + 1], + bitOffsetInCurrentByte, + bitCount + ); + return true; + } + + internal void Advance(int bitCount) + { + _currentBitIndex += bitCount; + } + + internal bool this[int bitIndex] + { + get + { + var byteIndex = bitIndex / BitsOfByte; + var bitOffset = bitIndex % BitsOfByte; + return (_bytes[byteIndex] & (1 << (BitsOfByte - bitOffset - 1))) != 0; + } + } +} diff --git a/csharp/Fury/Meta/BitsWriter.cs b/csharp/Fury/Meta/BitsWriter.cs new file mode 100644 index 0000000000..bd7fc3432d --- /dev/null +++ b/csharp/Fury/Meta/BitsWriter.cs @@ -0,0 +1,135 @@ +using System; + +namespace Fury.Meta; + +public ref struct BitsWriter(Span bytes) +{ + private const int BitsOfByte = sizeof(byte) * 8; + + private readonly Span _bytes = bytes; + + private int _currentBitIndex; + private int CurrentByteIndex => _currentBitIndex / BitsOfByte; + + internal int BytesUsed => (_currentBitIndex + BitsOfByte - 1) / BitsOfByte; + internal int UnusedBitCountInLastUsedByte => (BitsOfByte - _currentBitIndex % BitsOfByte) % BitsOfByte; + + internal byte UnusedBitInLastUsedByte + { + get + { + var unusedBitCountInLastUsedByte = UnusedBitCountInLastUsedByte; + if (unusedBitCountInLastUsedByte == 0) + { + return 0; + } + + var currentByte = _bytes[CurrentByteIndex]; + return BitUtility.KeepLowBits(currentByte, unusedBitCountInLastUsedByte); + } + } + + internal bool HasNext(int bitCount) => _currentBitIndex + bitCount <= _bytes.Length * BitsOfByte; + + internal bool TryReadBits(int bitCount, out byte bits) + { + if (!HasNext(bitCount)) + { + bits = default; + return false; + } + var currentByteIndex = CurrentByteIndex; + if (currentByteIndex >= _bytes.Length) + { + bits = default; + return false; + } + + var bitOffsetInCurrentByte = _currentBitIndex % BitsOfByte; + var bitsLeftInCurrentByte = BitsOfByte - bitOffsetInCurrentByte; + if (bitsLeftInCurrentByte >= bitCount) + { + bits = BitUtility.ReadBits(_bytes[currentByteIndex], bitOffsetInCurrentByte, bitCount); + return true; + } + + if (currentByteIndex + 1 >= _bytes.Length) + { + bits = default; + return false; + } + + bits = BitUtility.ReadBits( + _bytes[currentByteIndex], + _bytes[currentByteIndex + 1], + bitOffsetInCurrentByte, + bitCount + ); + return true; + } + + internal bool TryWriteBits(int bitCount, byte bits) + { + if (!HasNext(bitCount)) + { + return false; + } + bits = (byte)(bits & BitUtility.GetBitMask32(bitCount)); + var currentByteIndex = CurrentByteIndex; + if (currentByteIndex >= _bytes.Length) + { + return false; + } + + var bitOffsetInCurrentByte = _currentBitIndex % BitsOfByte; + var bitsLeftInCurrentByte = BitsOfByte - bitOffsetInCurrentByte; + byte currentByte; + if (bitsLeftInCurrentByte >= bitCount) + { + currentByte = BitUtility.ClearLowBits(_bytes[currentByteIndex], bitsLeftInCurrentByte); + _bytes[currentByteIndex] = (byte)(currentByte | (bits << (bitsLeftInCurrentByte - bitCount))); + return true; + } + + if (currentByteIndex + 1 >= _bytes.Length) + { + return false; + } + + var bitsToWriteInCurrentByte = bits >>> (bitCount - bitsLeftInCurrentByte); + var bitsToWriteInNextByte = bits & BitUtility.GetBitMask32(bitCount - bitsLeftInCurrentByte); + currentByte = BitUtility.ClearLowBits(_bytes[currentByteIndex], bitsLeftInCurrentByte); + _bytes[currentByteIndex] = (byte)(currentByte | bitsToWriteInCurrentByte); + _bytes[currentByteIndex + 1] = (byte)(bitsToWriteInNextByte << (BitsOfByte - bitCount + bitsLeftInCurrentByte)); + + return true; + } + + internal void Advance(int bitCount) + { + _currentBitIndex += bitCount; + } + + internal bool this[int bitIndex] + { + get + { + var byteIndex = bitIndex / BitsOfByte; + var bitOffset = bitIndex % BitsOfByte; + return (_bytes[byteIndex] & (1 << (BitsOfByte - bitOffset - 1))) != 0; + } + set + { + var byteIndex = bitIndex / BitsOfByte; + var bitOffset = bitIndex % BitsOfByte; + if (value) + { + _bytes[byteIndex] |= (byte)(1 << (BitsOfByte - bitOffset - 1)); + } + else + { + _bytes[byteIndex] &= (byte)~(1 << (BitsOfByte - bitOffset - 1)); + } + } + } +} diff --git a/csharp/Fury/Meta/CharsReader.cs b/csharp/Fury/Meta/CharsReader.cs new file mode 100644 index 0000000000..a709e5dd25 --- /dev/null +++ b/csharp/Fury/Meta/CharsReader.cs @@ -0,0 +1,29 @@ +using System; + +namespace Fury.Meta; + +public ref struct CharsReader(ReadOnlySpan chars) +{ + private readonly ReadOnlySpan _chars = chars; + + private int _currentIndex; + + internal int CharsUsed => _currentIndex; + + internal bool TryReadChar(out char c) + { + if (_currentIndex >= _chars.Length) + { + c = default; + return false; + } + + c = _chars[_currentIndex]; + return true; + } + + internal void Advance() + { + _currentIndex++; + } +} diff --git a/csharp/Fury/Meta/CharsWriter.cs b/csharp/Fury/Meta/CharsWriter.cs new file mode 100644 index 0000000000..a36cf5847e --- /dev/null +++ b/csharp/Fury/Meta/CharsWriter.cs @@ -0,0 +1,40 @@ +using System; + +namespace Fury.Meta; + +internal ref struct CharsWriter(Span chars) +{ + private readonly Span _chars = chars; + + private int _currentIndex; + + internal int CharsUsed => _currentIndex; + + internal bool TryReadChar(out char c) + { + if (_currentIndex >= _chars.Length) + { + c = default; + return false; + } + + c = _chars[_currentIndex]; + return true; + } + + internal bool TryWriteChar(char c) + { + if (_currentIndex >= _chars.Length) + { + return false; + } + + _chars[_currentIndex] = c; + return true; + } + + internal void Advance() + { + _currentIndex++; + } +} diff --git a/csharp/Fury/Meta/Encodings.cs b/csharp/Fury/Meta/Encodings.cs new file mode 100644 index 0000000000..bf79b646c4 --- /dev/null +++ b/csharp/Fury/Meta/Encodings.cs @@ -0,0 +1,22 @@ +namespace Fury.Meta; + +internal sealed class Encodings +{ + public static readonly HybridMetaStringEncoding CommonEncoding = new('.', '_'); + public static readonly HybridMetaStringEncoding NamespaceEncoding = CommonEncoding; + public static readonly HybridMetaStringEncoding TypeNameEncoding = new('$', '_'); + + private static readonly MetaString.Encoding[] NamespaceEncodings = + [ + MetaString.Encoding.Utf8, + MetaString.Encoding.AllToLowerSpecial, + MetaString.Encoding.LowerUpperDigitSpecial + ]; + private static readonly MetaString.Encoding[] TypeNameEncodings = + [ + MetaString.Encoding.Utf8, + MetaString.Encoding.LowerUpperDigitSpecial, + MetaString.Encoding.FirstToLowerSpecial, + MetaString.Encoding.AllToLowerSpecial + ]; +} diff --git a/csharp/Fury/Meta/FirstToLowerSpecialDecoder.cs b/csharp/Fury/Meta/FirstToLowerSpecialDecoder.cs new file mode 100644 index 0000000000..eea031c919 --- /dev/null +++ b/csharp/Fury/Meta/FirstToLowerSpecialDecoder.cs @@ -0,0 +1,49 @@ +using System; + +namespace Fury.Meta; + +internal sealed class FirstToLowerSpecialDecoder : MetaStringDecoder +{ + internal bool WrittenFirstChar { get; set; } + + public override void Convert( + ReadOnlySpan bytes, + Span chars, + bool flush, + out int bytesUsed, + out int charsUsed, + out bool completed + ) + { + MustFlush = flush; + FirstToLowerSpecialEncoding.GetChars(bytes, chars, this, out bytesUsed, out charsUsed); + completed = bytesUsed == bytes.Length && !HasLeftoverData; + if (flush) + { + Reset(); + } + } + + public override int GetCharCount(ReadOnlySpan bytes, bool flush) + { + MustFlush = flush; + var charCount = MetaStringEncoding.GetCharCount(bytes, AbstractLowerSpecialEncoding.BitsPerChar, this); + if (flush) + { + Reset(); + } + return charCount; + } + + public override int GetChars(ReadOnlySpan bytes, Span chars, bool flush) + { + Convert(bytes, chars, flush, out _, out var charsUsed, out _); + return charsUsed; + } + + public override void Reset() + { + WrittenFirstChar = false; + base.Reset(); + } +} diff --git a/csharp/Fury/Meta/FirstToLowerSpecialEncoding.cs b/csharp/Fury/Meta/FirstToLowerSpecialEncoding.cs new file mode 100644 index 0000000000..83bb84c9b6 --- /dev/null +++ b/csharp/Fury/Meta/FirstToLowerSpecialEncoding.cs @@ -0,0 +1,183 @@ +using System; +using System.Text; + +namespace Fury.Meta; + +internal sealed class FirstToLowerSpecialEncoding() + : AbstractLowerSpecialEncoding(MetaString.Encoding.FirstToLowerSpecial) +{ + public static readonly FirstToLowerSpecialEncoding Instance = new(); + + private static readonly FirstToLowerSpecialDecoder SharedDecoder = new(); + + public override bool CanEncode(ReadOnlySpan chars) + { + if (chars.Length == 0) + { + return true; + } + + if (!TryEncodeChar(char.ToLowerInvariant(chars[0]), out _)) + { + return false; + } + + foreach (var c in chars) + { + if (!TryEncodeChar(c, out _)) + { + return false; + } + } + + return true; + } + + public override int GetByteCount(ReadOnlySpan chars) + { + return LowerSpecialEncoding.Instance.GetByteCount(chars); + } + + public override int GetCharCount(ReadOnlySpan bytes) + { + return LowerSpecialEncoding.Instance.GetCharCount(bytes); + } + + public override int GetMaxByteCount(int charCount) + { + return LowerSpecialEncoding.Instance.GetMaxByteCount(charCount); + } + + public override int GetMaxCharCount(int byteCount) + { + return LowerSpecialEncoding.Instance.GetMaxCharCount(byteCount); + } + + public override int GetBytes(ReadOnlySpan chars, Span bytes) + { + var bitsWriter = new BitsWriter(bytes); + var charsReader = new CharsReader(chars); + + bitsWriter.Advance(1); + var writtenFirstCharBits = false; + while (charsReader.TryReadChar(out var c)) + { + if (!writtenFirstCharBits) + { + c = char.ToLowerInvariant(c); + writtenFirstCharBits = true; + } + + var charByte = EncodeChar(c); + + if (bitsWriter.TryWriteBits(BitsPerChar, charByte)) + { + charsReader.Advance(); + bitsWriter.Advance(BitsPerChar); + } + else + { + break; + } + } + + if (charsReader.CharsUsed < chars.Length) + { + ThrowHelper.ThrowArgumentException_InsufficientSpaceInTheOutputBuffer(nameof(bytes)); + } + + if (bitsWriter.UnusedBitCountInLastUsedByte >= BitsPerChar) + { + bitsWriter[0] = true; + } + + return bitsWriter.BytesUsed; + } + + public override int GetChars(ReadOnlySpan bytes, Span chars) + { + SharedDecoder.Convert(bytes, chars, true, out _, out var charsUsed, out _); + SharedDecoder.Reset(); + return charsUsed; + } + + private static bool TryWriteChar(ref CharsWriter writer, byte charByte, FirstToLowerSpecialDecoder decoder) + { + var decodedChar = DecodeByte(charByte); + if (!decoder.WrittenFirstChar) + { + decodedChar = char.ToUpperInvariant(decodedChar); + } + + return writer.TryWriteChar(decodedChar); + } + + internal static void GetChars( + ReadOnlySpan bytes, + Span chars, + FirstToLowerSpecialDecoder decoder, + out int bytesUsed, + out int charsUsed + ) + { + var bitsReader = new BitsReader(bytes); + var charsWriter = new CharsWriter(chars); + if (!decoder.HasState) + { + decoder.HasState = true; + if (bitsReader.TryReadBits(1, out var stripLastCharFlag)) + { + bitsReader.Advance(1); + decoder.StripLastChar = stripLastCharFlag != 0; + } + } + else + { + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out var charByte, out var bitsUsedFromBitsReader)) + { + if (TryWriteChar(ref charsWriter, charByte, decoder)) + { + decoder.WrittenFirstChar = true; + bitsReader.Advance(bitsUsedFromBitsReader); + charsWriter.Advance(); + + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out charByte, out bitsUsedFromBitsReader)) + { + if (TryWriteChar(ref charsWriter, charByte, decoder)) + { + decoder.WrittenFirstChar = true; + bitsReader.Advance(bitsUsedFromBitsReader); + charsWriter.Advance(); + } + } + } + } + } + + while (bitsReader.TryReadBits(BitsPerChar, out var charByte)) + { + if (bitsReader.GetRemainingCount(BitsPerChar) == 1 && decoder is { MustFlush: true, StripLastChar: true }) + { + break; + } + + if (TryWriteChar(ref charsWriter, charByte, decoder)) + { + decoder.WrittenFirstChar = true; + bitsReader.Advance(BitsPerChar); + charsWriter.Advance(); + } + else + { + break; + } + } + + decoder.SetLeftoverData(bitsReader.UnusedBitsInLastUsedByte, bitsReader.UnusedBitCountInLastUsedByte); + + bytesUsed = bitsReader.BytesUsed; + charsUsed = charsWriter.CharsUsed; + } + + public override Decoder GetDecoder() => new FirstToLowerSpecialDecoder(); +} diff --git a/csharp/Fury/Meta/HybridMetaStringEncoding.cs b/csharp/Fury/Meta/HybridMetaStringEncoding.cs new file mode 100644 index 0000000000..638b076be6 --- /dev/null +++ b/csharp/Fury/Meta/HybridMetaStringEncoding.cs @@ -0,0 +1,55 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury.Meta; + +internal sealed class HybridMetaStringEncoding(char specialChar1, char specialChar2) +{ + public LowerUpperDigitSpecialEncoding LowerUpperDigit { get; } = new(specialChar1, specialChar2); + + public bool TryGetEncoding(MetaString.Encoding encoding, [NotNullWhen(true)] out MetaStringEncoding? result) + { + result = encoding switch + { + MetaString.Encoding.LowerSpecial => LowerSpecialEncoding.Instance, + MetaString.Encoding.FirstToLowerSpecial => FirstToLowerSpecialEncoding.Instance, + MetaString.Encoding.AllToLowerSpecial => AllToLowerSpecialEncoding.Instance, + MetaString.Encoding.LowerUpperDigitSpecial => LowerUpperDigit, + MetaString.Encoding.Utf8 => Utf8Encoding.Instance, + _ => null + }; + + return result is not null; + } + + public bool TryGetMetaString(string chars, MetaString.Encoding encoding, out MetaString output) + { + if (!TryGetEncoding(encoding, out var e)) + { + output = default; + return false; + } + + var byteCount = e.GetByteCount(chars); + var bytes = new byte[byteCount]; + e.GetBytes(chars.AsSpan(), bytes); + output = new MetaString(chars, encoding, specialChar1, specialChar2, bytes); + return true; + } + + public bool TryGetString( + ReadOnlySpan bytes, + MetaString.Encoding encoding, + [NotNullWhen(true)] out string? output + ) + { + if (!TryGetEncoding(encoding, out var e)) + { + output = default; + return false; + } + + output = e.GetString(bytes); + return true; + } +} diff --git a/csharp/Fury/Meta/LowerSpecialDecoder.cs b/csharp/Fury/Meta/LowerSpecialDecoder.cs new file mode 100644 index 0000000000..49067d184a --- /dev/null +++ b/csharp/Fury/Meta/LowerSpecialDecoder.cs @@ -0,0 +1,41 @@ +using System; + +namespace Fury.Meta; + +internal sealed class LowerSpecialDecoder : MetaStringDecoder +{ + public override void Convert( + ReadOnlySpan bytes, + Span chars, + bool flush, + out int bytesUsed, + out int charsUsed, + out bool completed + ) + { + MustFlush = flush; + LowerSpecialEncoding.GetChars(bytes, chars, this, out bytesUsed, out charsUsed); + completed = bytesUsed == bytes.Length && !HasLeftoverData; + if (flush) + { + Reset(); + } + } + + public override int GetCharCount(ReadOnlySpan bytes, bool flush) + { + MustFlush = flush; + var charCount = MetaStringEncoding.GetCharCount(bytes, AbstractLowerSpecialEncoding.BitsPerChar, this); + if (flush) + { + Reset(); + } + return charCount; + } + + public override int GetChars(ReadOnlySpan bytes, Span chars, bool flush) + { + Convert(bytes, chars, flush, out _, out var charsUsed, out _); + return charsUsed; + } +} diff --git a/csharp/Fury/Meta/LowerSpecialEncoding.cs b/csharp/Fury/Meta/LowerSpecialEncoding.cs new file mode 100644 index 0000000000..15e2a25372 --- /dev/null +++ b/csharp/Fury/Meta/LowerSpecialEncoding.cs @@ -0,0 +1,158 @@ +using System; +using System.Text; + +namespace Fury.Meta; + +internal sealed class LowerSpecialEncoding() : AbstractLowerSpecialEncoding(MetaString.Encoding.LowerSpecial) +{ + public static readonly LowerSpecialEncoding Instance = new(); + + private static readonly LowerSpecialDecoder SharedDecoder = new(); + + public override bool CanEncode(ReadOnlySpan chars) + { + foreach (var c in chars) + { + if (!TryEncodeChar(c, out _)) + { + return false; + } + } + return true; + } + + public override int GetByteCount(ReadOnlySpan chars) + { + return GetMaxByteCount(chars.Length); + } + + public override int GetCharCount(ReadOnlySpan bytes) + { + if (bytes.Length == 0) + { + return 0; + } + + var firstByte = bytes[0]; + var stripLastChar = (firstByte & StripLastCharFlagMask) != 0; + return GetMaxCharCount(bytes.Length) - (stripLastChar ? 1 : 0); + } + + public override int GetMaxByteCount(int charCount) + { + return charCount * BitsPerChar / BitsOfByte + 1; + } + + public override int GetMaxCharCount(int byteCount) + { + return (byteCount * BitsOfByte - 1) / BitsPerChar; + } + + public override int GetBytes(ReadOnlySpan chars, Span bytes) + { + var bitsWriter = new BitsWriter(bytes); + var charsReader = new CharsReader(chars); + + bitsWriter.Advance(1); + while (charsReader.TryReadChar(out var c)) + { + var charByte = EncodeChar(c); + + if (bitsWriter.TryWriteBits(BitsPerChar, charByte)) + { + charsReader.Advance(); + bitsWriter.Advance(BitsPerChar); + } + else + { + break; + } + } + + if (charsReader.CharsUsed < chars.Length) + { + ThrowHelper.ThrowArgumentException_InsufficientSpaceInTheOutputBuffer(nameof(bytes)); + } + + if (bitsWriter.UnusedBitCountInLastUsedByte >= BitsPerChar) + { + bitsWriter[0] = true; + } + + return bitsWriter.BytesUsed; + } + + public override int GetChars(ReadOnlySpan bytes, Span chars) + { + SharedDecoder.Convert(bytes, chars, true, out _, out var charsUsed, out _); + SharedDecoder.Reset(); + return charsUsed; + } + + internal static void GetChars( + ReadOnlySpan bytes, + Span chars, + LowerSpecialDecoder decoder, + out int bytesUsed, + out int charsUsed + ) + { + var bitsReader = new BitsReader(bytes); + var charsWriter = new CharsWriter(chars); + if (!decoder.HasState) + { + decoder.HasState = true; + if (bitsReader.TryReadBits(1, out var stripLastCharFlag)) + { + bitsReader.Advance(1); + decoder.StripLastChar = stripLastCharFlag != 0; + } + } + else + { + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out var charByte, out var bitsUsedFromBitsReader)) + { + var decodedChar = DecodeByte(charByte); + if (charsWriter.TryWriteChar(decodedChar)) + { + bitsReader.Advance(bitsUsedFromBitsReader); + charsWriter.Advance(); + + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out charByte, out bitsUsedFromBitsReader)) + { + if (charsWriter.TryWriteChar(decodedChar)) + { + bitsReader.Advance(bitsUsedFromBitsReader); + charsWriter.Advance(); + } + } + } + } + } + + while (bitsReader.TryReadBits(BitsPerChar, out var charByte)) + { + if (bitsReader.GetRemainingCount(BitsPerChar) == 1 && decoder is { MustFlush: true, StripLastChar: true }) + { + break; + } + var decodedChar = DecodeByte(charByte); + if (charsWriter.TryWriteChar(decodedChar)) + { + bitsReader.Advance(BitsPerChar); + charsWriter.Advance(); + } + else + { + break; + } + } + + decoder.SetLeftoverData(bitsReader.UnusedBitsInLastUsedByte, bitsReader.UnusedBitCountInLastUsedByte); + + bytesUsed = bitsReader.BytesUsed; + charsUsed = charsWriter.CharsUsed; + } + + public override Decoder GetDecoder() => new LowerSpecialDecoder(); +} diff --git a/csharp/Fury/Meta/LowerUpperDigitSpecialDecoder.cs b/csharp/Fury/Meta/LowerUpperDigitSpecialDecoder.cs new file mode 100644 index 0000000000..ee95c7c1a7 --- /dev/null +++ b/csharp/Fury/Meta/LowerUpperDigitSpecialDecoder.cs @@ -0,0 +1,41 @@ +using System; + +namespace Fury.Meta; + +internal sealed class LowerUpperDigitSpecialDecoder(LowerUpperDigitSpecialEncoding encoding) : MetaStringDecoder +{ + public override void Convert( + ReadOnlySpan bytes, + Span chars, + bool flush, + out int bytesUsed, + out int charsUsed, + out bool completed + ) + { + MustFlush = flush; + encoding.GetChars(bytes, chars, this, out bytesUsed, out charsUsed); + completed = bytesUsed == bytes.Length && !HasLeftoverData; + if (flush) + { + Reset(); + } + } + + public override int GetCharCount(ReadOnlySpan bytes, bool flush) + { + MustFlush = flush; + var charCount = encoding.GetCharCount(bytes); + if (flush) + { + Reset(); + } + return charCount; + } + + public override int GetChars(ReadOnlySpan bytes, Span chars, bool flush) + { + Convert(bytes, chars, flush, out _, out var charsUsed, out _); + return charsUsed; + } +} diff --git a/csharp/Fury/Meta/LowerUpperDigitSpecialEncoding.cs b/csharp/Fury/Meta/LowerUpperDigitSpecialEncoding.cs new file mode 100644 index 0000000000..499ae20d68 --- /dev/null +++ b/csharp/Fury/Meta/LowerUpperDigitSpecialEncoding.cs @@ -0,0 +1,264 @@ +using System; +using System.Text; + +namespace Fury.Meta; + +internal sealed class LowerUpperDigitSpecialEncoding(char specialChar1, char specialChar2) + : MetaStringEncoding(MetaString.Encoding.LowerUpperDigitSpecial) +{ + internal const int BitsPerChar = 6; + private const int UnusedBitsPerChar = BitsOfByte - BitsPerChar; + private const int MaxRepresentableChar = (1 << BitsPerChar) - 1; + + public override bool CanEncode(ReadOnlySpan chars) + { + foreach (var c in chars) + { + if (!TryEncodeChar(c, out _)) + { + return false; + } + } + return true; + } + + public override int GetByteCount(ReadOnlySpan chars) + { + return GetMaxByteCount(chars.Length); + } + + public override int GetCharCount(ReadOnlySpan bytes) + { + if (bytes.Length == 0) + { + return 0; + } + + var firstByte = bytes[0]; + var stripLastChar = (firstByte & StripLastCharFlagMask) != 0; + return GetMaxCharCount(bytes.Length) - (stripLastChar ? 1 : 0); + } + + public override int GetMaxByteCount(int charCount) + { + return charCount * BitsPerChar / BitsOfByte + 1; + } + + public override int GetMaxCharCount(int byteCount) + { + return (byteCount * BitsOfByte - 1) / BitsPerChar; + } + + public override int GetBytes(ReadOnlySpan chars, Span bytes) + { + var bitsWriter = new BitsWriter(bytes); + var charsReader = new CharsReader(chars); + + bitsWriter.Advance(1); + while (charsReader.TryReadChar(out var c)) + { + var charByte = EncodeChar(c); + + if (bitsWriter.TryWriteBits(BitsPerChar, charByte)) + { + charsReader.Advance(); + bitsWriter.Advance(BitsPerChar); + } + else + { + break; + } + } + + if (charsReader.CharsUsed < chars.Length) + { + ThrowHelper.ThrowArgumentException_InsufficientSpaceInTheOutputBuffer(nameof(bytes)); + } + + if (bitsWriter.UnusedBitCountInLastUsedByte >= BitsPerChar) + { + bitsWriter[0] = true; + } + + return bitsWriter.BytesUsed; + } + + public override int GetChars(ReadOnlySpan bytes, Span chars) + { + const byte bitMask = MaxRepresentableChar; + + var charCount = GetCharCount(bytes); + if (chars.Length < charCount) + { + ThrowHelper.ThrowArgumentException(paramName: nameof(chars)); + } + for (var i = 0; i < charCount; i++) + { + var currentBit = i * BitsPerChar + 1; + var byteIndex = currentBit / BitsOfByte; + var bitOffset = currentBit % BitsOfByte; + + byte charByte; + if (bitOffset <= UnusedBitsPerChar) + { + // bitOffset locations read locations + // x _ _ _ _ _ _ _ x x x x x x _ _ + // _ x _ _ _ _ _ _ _ x x x x x x _ + // _ _ x _ _ _ _ _ _ _ x x x x x x + + charByte = (byte)((bytes[byteIndex] >>> (UnusedBitsPerChar - bitOffset)) & bitMask); + } + else + { + // bitOffset locations read locations + // _ _ _ x _ _ _ _ _ _ _ x x x x x | x _ _ _ _ _ _ _ + // _ _ _ _ x _ _ _ _ _ _ _ x x x x | x x _ _ _ _ _ _ + // _ _ _ _ _ x _ _ _ _ _ _ _ x x x | x x x _ _ _ _ _ + // _ _ _ _ _ _ x _ _ _ _ _ _ _ x x | x x x x _ _ _ _ + // _ _ _ _ _ _ _ x _ _ _ _ _ _ _ x | x x x x x _ _ _ + + charByte = (byte)( + ( + bytes[byteIndex] << (bitOffset - UnusedBitsPerChar) + | bytes[byteIndex + 1] >>> (BitsOfByte + UnusedBitsPerChar - bitOffset) + ) & bitMask + ); + } + + if (!TryDecodeByte(charByte, out var c)) + { + ThrowHelper.ThrowArgumentOutOfRangeException(nameof(bytes)); + } + chars[i] = c; + } + + return charCount; + } + + internal void GetChars( + ReadOnlySpan bytes, + Span chars, + LowerUpperDigitSpecialDecoder decoder, + out int bytesUsed, + out int charsUsed + ) + { + var bitsReader = new BitsReader(bytes); + var charsWriter = new CharsWriter(chars); + if (!decoder.HasState) + { + decoder.HasState = true; + if (bitsReader.TryReadBits(1, out var stripLastCharFlag)) + { + bitsReader.Advance(1); + decoder.StripLastChar = stripLastCharFlag != 0; + } + } + else + { + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out var charByte, out var bitsUsedFromBitsReader)) + { + var decodedChar = DecodeByte(charByte); + if (charsWriter.TryWriteChar(decodedChar)) + { + bitsReader.Advance(bitsUsedFromBitsReader); + charsWriter.Advance(); + + if (TryReadLeftOver(decoder, ref bitsReader, BitsPerChar, out charByte, out bitsUsedFromBitsReader)) + { + if (charsWriter.TryWriteChar(decodedChar)) + { + bitsReader.Advance(bitsUsedFromBitsReader); + charsWriter.Advance(); + } + } + } + } + } + + while (bitsReader.TryReadBits(BitsPerChar, out var charByte)) + { + if (bitsReader.GetRemainingCount(BitsPerChar) == 1 && decoder is { MustFlush: true, StripLastChar: true }) + { + break; + } + var decodedChar = DecodeByte(charByte); + if (charsWriter.TryWriteChar(decodedChar)) + { + bitsReader.Advance(BitsPerChar); + charsWriter.Advance(); + } + else + { + break; + } + } + + decoder.SetLeftoverData(bitsReader.UnusedBitsInLastUsedByte, bitsReader.UnusedBitCountInLastUsedByte); + + bytesUsed = bitsReader.BytesUsed; + charsUsed = charsWriter.CharsUsed; + } + + private bool TryEncodeChar(char c, out byte b) + { + var success = true; + if (c == specialChar1) + { + b = MaxRepresentableChar - 1; + } + else if (c == specialChar2) + { + b = MaxRepresentableChar; + } + else + { + (success, b) = c switch + { + >= 'a' and <= 'z' => (true, (byte)(c - 'a')), + >= 'A' and <= 'Z' => (true, (byte)(c - 'A' + NumberOfEnglishLetters)), + >= '0' and <= '9' => (true, (byte)(c - '0' + NumberOfEnglishLetters * 2)), + _ => (false, default), + }; + } + + return success; + } + + private byte EncodeChar(char c) + { + if (!TryEncodeChar(c, out var b)) + { + ThrowHelper.ThrowBadSerializationInputException_UnsupportedMetaStringChar(c); + } + + return b; + } + + internal bool TryDecodeByte(byte b, out char c) + { + (var success, c) = b switch + { + < NumberOfEnglishLetters => (true, (char)(b + 'a')), + < NumberOfEnglishLetters * 2 => (true, (char)(b - NumberOfEnglishLetters + 'A')), + < NumberOfEnglishLetters * 2 + 10 => (true, (char)(b - NumberOfEnglishLetters * 2 + '0')), + MaxRepresentableChar - 1 => (true, specialChar1), + MaxRepresentableChar => (true, specialChar2), + _ => (false, default), + }; + + return success; + } + + private char DecodeByte(byte b) + { + if (!TryDecodeByte(b, out var c)) + { + ThrowHelper.ThrowBadDeserializationInputException_UnrecognizedMetaStringCodePoint(b); + } + + return c; + } + + public override Decoder GetDecoder() => new LowerUpperDigitSpecialDecoder(this); +} diff --git a/csharp/Fury/Meta/MetaString.cs b/csharp/Fury/Meta/MetaString.cs new file mode 100644 index 0000000000..4a91bd4494 --- /dev/null +++ b/csharp/Fury/Meta/MetaString.cs @@ -0,0 +1,44 @@ +namespace Fury.Meta; + +internal struct MetaString +{ + public enum Encoding : byte + { + Utf8 = 0, + LowerSpecial = 1, + LowerUpperDigitSpecial = 2, + FirstToLowerSpecial = 3, + AllToLowerSpecial = 4, + } + + private readonly string _value; + private readonly Encoding _encoding; + private readonly char _specialChar1; + private readonly char _specialChar2; + private readonly byte[] _bytes; + private readonly bool _stripLastChar; + + public MetaString(string value, Encoding encoding, char specialChar1, char specialChar2, byte[] bytes) + { + _value = value; + _encoding = encoding; + _specialChar1 = specialChar1; + _specialChar2 = specialChar2; + _bytes = bytes; + if (encoding != Encoding.Utf8) + { + if (bytes.Length <= 0) + { + ThrowHelper.ThrowArgumentException( + message: "At least one byte must be provided.", + paramName: nameof(bytes) + ); + } + _stripLastChar = (bytes[0] & 0x80) != 0; + } + else + { + _stripLastChar = false; + } + } +} diff --git a/csharp/Fury/Meta/MetaStringBytes.cs b/csharp/Fury/Meta/MetaStringBytes.cs new file mode 100644 index 0000000000..8f5b2e76a2 --- /dev/null +++ b/csharp/Fury/Meta/MetaStringBytes.cs @@ -0,0 +1,22 @@ +using System; + +namespace Fury.Meta; + +internal sealed class MetaStringBytes +{ + private const byte EncodingMask = 0xff; + + private readonly byte[] _bytes; + private readonly long _hashCode; + private MetaString.Encoding _encoding; + + public long HashCode => _hashCode; + public ReadOnlySpan Bytes => _bytes; + + public MetaStringBytes(byte[] bytes, long hashCode) + { + _bytes = bytes; + _hashCode = hashCode; + _encoding = (MetaString.Encoding)(hashCode & EncodingMask); + } +} diff --git a/csharp/Fury/Meta/MetaStringDecoder.cs b/csharp/Fury/Meta/MetaStringDecoder.cs new file mode 100644 index 0000000000..f6ac3c53c4 --- /dev/null +++ b/csharp/Fury/Meta/MetaStringDecoder.cs @@ -0,0 +1,127 @@ +using System; +using System.Text; + +namespace Fury.Meta; + +// The StripLastChar flag need to be set in the first byte of the encoded data, +// so that wo can not implement a dotnet-style encoder. +// However, implementing a decoder is possible. + +internal abstract class MetaStringDecoder : Decoder +{ + internal bool StripLastChar { get; set; } + internal bool HasState { get; set; } + protected byte LeftoverBits { get; set; } + protected int LeftoverBitCount { get; set; } + internal bool MustFlush { get; private protected set; } + + internal bool HasLeftoverData => LeftoverBitCount > 0; + + internal void SetLeftoverData(byte bits, int bitCount) + { + LeftoverBits = (byte)(bits & ((1 << bitCount) - 1)); + LeftoverBitCount = bitCount; + } + + internal (byte bits, int bitCount) GetLeftoverData() + { + return (LeftoverBits, LeftoverBitCount); + } + + public override void Reset() + { + LeftoverBits = 0; + LeftoverBitCount = 0; + HasState = false; + StripLastChar = false; + MustFlush = false; + } + +#if !NET8_0_OR_GREATER + public abstract void Convert( + ReadOnlySpan bytes, + Span chars, + bool flush, + out int bytesUsed, + out int charsUsed, + out bool completed + ); + + public abstract int GetCharCount(ReadOnlySpan bytes, bool flush); + + public abstract int GetChars(ReadOnlySpan bytes, Span chars, bool flush); +#endif + + public sealed override unsafe void Convert( + byte* bytes, + int byteCount, + char* chars, + int charCount, + bool flush, + out int bytesUsed, + out int charsUsed, + out bool completed + ) + { + var byteSpan = new ReadOnlySpan(bytes, byteCount); + var charSpan = new Span(chars, charCount); + Convert(byteSpan, charSpan, flush, out bytesUsed, out charsUsed, out completed); + } + + public sealed override void Convert( + byte[] bytes, + int byteIndex, + int byteCount, + char[] chars, + int charIndex, + int charCount, + bool flush, + out int bytesUsed, + out int charsUsed, + out bool completed + ) + { + var byteSpan = new ReadOnlySpan(bytes, byteIndex, byteCount); + var charSpan = new Span(chars, charIndex, charCount); + Convert(byteSpan, charSpan, flush, out bytesUsed, out charsUsed, out completed); + } + + public sealed override unsafe int GetCharCount(byte* bytes, int count, bool flush) + { + return GetCharCount(new ReadOnlySpan(bytes, count), flush); + } + + public sealed override int GetCharCount(byte[] bytes, int index, int count) + { + return GetCharCount(bytes, index, count, true); + } + + public sealed override int GetCharCount(byte[] bytes, int index, int count, bool flush) + { + return GetCharCount(bytes.AsSpan(index, count), flush); + } + + public sealed override unsafe int GetChars(byte* bytes, int byteCount, char* chars, int charCount, bool flush) + { + var byteSpan = new ReadOnlySpan(bytes, byteCount); + var charSpan = new Span(chars, charCount); + return GetChars(byteSpan, charSpan, flush); + } + + public sealed override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) + { + return GetChars(bytes, byteIndex, byteCount, chars, charIndex, true); + } + + public sealed override int GetChars( + byte[] bytes, + int byteIndex, + int byteCount, + char[] chars, + int charIndex, + bool flush + ) + { + return GetChars(bytes.AsSpan(byteIndex, byteCount), chars.AsSpan(charIndex), flush); + } +} diff --git a/csharp/Fury/Meta/MetaStringEncoding.cs b/csharp/Fury/Meta/MetaStringEncoding.cs new file mode 100644 index 0000000000..1e4cf405b3 --- /dev/null +++ b/csharp/Fury/Meta/MetaStringEncoding.cs @@ -0,0 +1,260 @@ +using System; +using System.Text; + +namespace Fury.Meta; + +internal abstract class MetaStringEncoding(MetaString.Encoding encoding) : Encoding +{ + protected const int BitsOfByte = sizeof(byte) * 8; + protected const int NumberOfEnglishLetters = 26; + protected const int StripLastCharFlagMask = 1 << (BitsOfByte - 1); + + public MetaString.Encoding Encoding { get; } = encoding; + + public abstract bool CanEncode(ReadOnlySpan chars); + + protected static void WriteByte(byte input, ref byte b1, int bitOffset, int bitsPerChar) + { + var unusedBitsPerChar = BitsOfByte - bitsPerChar; + b1 |= (byte)(input << (unusedBitsPerChar - bitOffset)); + } + + protected static void WriteByte(byte input, ref byte b1, ref byte b2, int bitOffset, int bitsPerChar) + { + var unusedBitsPerChar = BitsOfByte - bitsPerChar; + b1 |= (byte)(input >>> (bitOffset - unusedBitsPerChar)); + b2 |= (byte)(input << (BitsOfByte + unusedBitsPerChar - bitOffset)); + } + + protected static byte ReadByte(byte b1, int bitOffset, int bitsPerChar) + { + var bitMask = (1 << bitsPerChar) - 1; + var unusedBitsPerChar = BitsOfByte - bitsPerChar; + return (byte)((b1 >>> (unusedBitsPerChar - bitOffset)) & bitMask); + } + + protected static byte ReadByte(byte b1, byte b2, int bitOffset, int bitsPerChar) + { + var bitMask = (1 << bitsPerChar) - 1; + var unusedBitsPerChar = BitsOfByte - bitsPerChar; + return (byte)( + (b1 << (bitOffset - unusedBitsPerChar) | b2 >>> (BitsOfByte + unusedBitsPerChar - bitOffset)) & bitMask + ); + } + + protected static bool TryReadLeftOver( + MetaStringDecoder decoder, + ref BitsReader bitsReader, + int bitsPerChar, + out byte bits, + out int bitsUsedFromBitsReader + ) + { + if (!decoder.HasLeftoverData) + { + bits = default; + bitsUsedFromBitsReader = default; + return false; + } + var (leftOverBits, leftOverBitsCount) = decoder.GetLeftoverData(); + if (leftOverBitsCount >= bitsPerChar) + { + leftOverBitsCount -= bitsPerChar; + bits = BitUtility.KeepLowBits((byte)(leftOverBits >>> leftOverBitsCount), bitsPerChar); + leftOverBits = BitUtility.KeepLowBits(leftOverBits, leftOverBitsCount); + decoder.SetLeftoverData(leftOverBits, leftOverBitsCount); + bitsUsedFromBitsReader = 0; + return true; + } + + bitsUsedFromBitsReader = bitsPerChar - leftOverBitsCount; + if (!bitsReader.TryReadBits(bitsUsedFromBitsReader, out var bitsFromNextByte)) + { + bits = default; + bitsUsedFromBitsReader = 0; + return false; + } + + var bitsFromLeftOver = leftOverBits << bitsUsedFromBitsReader; + bits = BitUtility.KeepLowBits((byte)(bitsFromLeftOver | bitsFromNextByte), bitsPerChar); + decoder.SetLeftoverData(0, 0); + return true; + } + + internal static int GetCharCount(ReadOnlySpan bytes, int bitsPerChar, MetaStringDecoder decoder) + { + if (bytes.Length == 0) + { + return 0; + } + + var charCount = 0; + var currentBit = 0; + if (!decoder.HasState) + { + decoder.StripLastChar = (bytes[0] & StripLastCharFlagMask) != 0; + currentBit = 1; + } + + if (decoder.HasLeftoverData) + { + var (_, bitCount) = decoder.GetLeftoverData(); + currentBit += bitsPerChar - bitCount; + charCount++; + } + + var bitsAvailable = bytes.Length * BitsOfByte - currentBit; + var charsAvailable = bitsAvailable / bitsPerChar; + charCount += charsAvailable; + var leftOverBitCount = bitsAvailable % bitsPerChar; + decoder.SetLeftoverData(default, leftOverBitCount); + if (decoder is { MustFlush: true, StripLastChar: true }) + { + charCount--; + } + + return charCount; + } + +#if !NET8_0_OR_GREATER + public abstract int GetByteCount(ReadOnlySpan chars); + public abstract int GetCharCount(ReadOnlySpan bytes); + public abstract int GetBytes(ReadOnlySpan chars, Span bytes); + public abstract int GetChars(ReadOnlySpan bytes, Span chars); +#endif + + public +#if NET8_0_OR_GREATER + sealed override +#endif + bool TryGetBytes(ReadOnlySpan chars, Span bytes, out int bytesWritten) + { + var byteCount = GetByteCount(chars); + if (bytes.Length < byteCount) + { + bytesWritten = 0; + return false; + } + + bytesWritten = GetBytes(chars, bytes); + return true; + } + + public +#if NET8_0_OR_GREATER + sealed override +#endif + bool TryGetChars(ReadOnlySpan bytes, Span chars, out int charsWritten) + { + var charCount = GetCharCount(bytes); + if (chars.Length < charCount) + { + charsWritten = 0; + return false; + } + + charsWritten = GetChars(bytes, chars); + return true; + } + + public sealed override unsafe int GetByteCount(char* chars, int count) => + GetByteCount(new ReadOnlySpan(chars, count)); + + public sealed override int GetByteCount(char[] chars) => GetByteCount(chars.AsSpan()); + + public sealed override int GetByteCount(char[] chars, int index, int count) => + GetByteCount(chars.AsSpan(index, count)); + + public sealed override int GetByteCount(string s) => GetByteCount(s.AsSpan()); + + public sealed override unsafe int GetBytes(char* chars, int charCount, byte* bytes, int byteCount) => + GetBytes(new ReadOnlySpan(chars, charCount), new Span(bytes, byteCount)); + + public sealed override byte[] GetBytes(char[] chars) => GetBytes(chars, 0, chars.Length); + + public sealed override byte[] GetBytes(char[] chars, int index, int count) + { + var span = chars.AsSpan().Slice(index, count); + var byteCount = GetByteCount(chars); + var bytes = new byte[byteCount]; + GetBytes(span, bytes); + return bytes; + } + + public sealed override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) => + GetBytes(chars.AsSpan(charIndex, charCount), bytes.AsSpan(byteIndex)); + + public sealed override byte[] GetBytes(string s) + { + var span = s.AsSpan(); + var byteCount = GetByteCount(span); + var bytes = new byte[byteCount]; + GetBytes(span, bytes); + return bytes; + } + + public sealed override int GetBytes(string s, int charIndex, int charCount, byte[] bytes, int byteIndex) + { + var span = s.AsSpan().Slice(charIndex, charCount); + var byteCount = GetByteCount(span); + if (bytes.Length - byteIndex < byteCount) + { + ThrowHelper.ThrowArgumentException(paramName: nameof(bytes)); + } + return GetBytes(span, bytes.AsSpan(byteIndex)); + } + + public sealed override unsafe int GetCharCount(byte* bytes, int count) => + GetCharCount(new ReadOnlySpan(bytes, count)); + + public sealed override int GetCharCount(byte[] bytes) => GetCharCount(bytes.AsSpan()); + + public sealed override int GetCharCount(byte[] bytes, int index, int count) => + GetCharCount(bytes.AsSpan(index, count)); + + public sealed override unsafe int GetChars(byte* bytes, int byteCount, char* chars, int charCount) + { + var byteSpan = new ReadOnlySpan(bytes, byteCount); + var charSpan = new Span(chars, charCount); + return GetChars(byteSpan, charSpan); + } + + public sealed override char[] GetChars(byte[] bytes) + { + var charCount = GetCharCount(bytes); + var chars = new char[charCount]; + GetChars(bytes, chars); + return chars; + } + + public sealed override char[] GetChars(byte[] bytes, int index, int count) + { + var span = bytes.AsSpan(index, count); + var charCount = GetCharCount(span); + var chars = new char[charCount]; + GetChars(span, chars); + return chars; + } + + public sealed override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) + { + var byteSpan = bytes.AsSpan(byteIndex, byteCount); + var charSpan = chars.AsSpan(charIndex); + return GetChars(byteSpan, charSpan); + } + +#if !NET8_0_OR_GREATER + public string GetString(ReadOnlySpan bytes) + { + var charCount = GetCharCount(bytes); + Span chars = stackalloc char[charCount]; + GetChars(bytes, chars); + return chars.ToString(); + } +#endif + + public sealed override string GetString(byte[] bytes) => GetString(bytes.AsSpan()); + + public sealed override string GetString(byte[] bytes, int index, int count) => + GetString(bytes.AsSpan().Slice(index, count)); +} diff --git a/csharp/Fury/Meta/MetaStringResolver.cs b/csharp/Fury/Meta/MetaStringResolver.cs new file mode 100644 index 0000000000..339dd46700 --- /dev/null +++ b/csharp/Fury/Meta/MetaStringResolver.cs @@ -0,0 +1,130 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Fury.Buffers; +using Fury.Collections; + +namespace Fury.Meta; + +internal sealed class MetaStringResolver(IArrayPoolProvider poolProvider) +{ + public const int SmallStringThreshold = sizeof(long) * 2; + + private readonly Dictionary _smallStrings = new(); + private readonly Dictionary _bigStrings = new(); + + private readonly PooledList _readMetaStrings = new(poolProvider); + + public async ValueTask ReadMetaStringBytesAsync( + BatchReader reader, + CancellationToken cancellationToken = default + ) + { + var header = (int)await reader.Read7BitEncodedUintAsync(cancellationToken); + var isMetaStringId = (header & 0b1) != 0; + if (isMetaStringId) + { + var id = header >>> 1; + if (id > _readMetaStrings.Count || id <= 0) + { + ThrowHelper.ThrowBadDeserializationInputException_UnknownMetaStringId(id); + } + return _readMetaStrings[id - 1]!; + } + + var length = header >>> 1; + MetaStringBytes byteString; + if (length <= SmallStringThreshold) + { + byteString = await ReadSmallMetaStringBytesAsync(reader, length, cancellationToken); + } + else + { + byteString = await ReadBigMetaStringBytesAsync(reader, length, cancellationToken); + } + _readMetaStrings.Add(byteString); + return byteString; + } + + private async ValueTask ReadSmallMetaStringBytesAsync( + BatchReader reader, + int length, + CancellationToken cancellationToken = default + ) + { + var encoding = await reader.ReadAsync(cancellationToken); + ulong v1; + ulong v2 = 0; + if (length <= sizeof(long)) + { + v1 = await reader.ReadAsAsync(length, cancellationToken); + } + else + { + v1 = await reader.ReadAsync(cancellationToken); + v2 = await reader.ReadAsAsync(length - sizeof(long), cancellationToken); + } + return GetOrCreateSmallMetaStringBytes(v1, v2, length, encoding); + } + + private MetaStringBytes GetOrCreateSmallMetaStringBytes(ulong v1, ulong v2, int length, byte encoding) + { + var key = new UInt128(v1, v2); +#if NET8_0_OR_GREATER + ref var byteString = ref CollectionsMarshal.GetValueRefOrAddDefault(_smallStrings, key, out var exists); +#else + var exists = _smallStrings.TryGetValue(key, out var byteString); +#endif + if (!exists || byteString is null) + { + Span data = stackalloc ulong[2]; + data[0] = v1; + data[1] = v2; + var bytes = MemoryMarshal.Cast(data); + HashHelper.MurmurHash3_x64_128(bytes, 47, out var out1, out _); + var hashCode = Math.Abs((long)out1); + hashCode = (hashCode & unchecked((long)0xffff_ffff_ffff_ff00L)) | encoding; + byteString = new MetaStringBytes(bytes.Slice(0, length).ToArray(), hashCode); +#if !NET8_0_OR_GREATER + _smallStrings.Add(key, byteString); +#endif + } + + return byteString; + } + + private async ValueTask ReadBigMetaStringBytesAsync( + BatchReader reader, + int length, + CancellationToken cancellationToken = default + ) + { + var hashCode = await reader.ReadAsync(cancellationToken); + var readResult = await reader.ReadAtLeastAsync(length, cancellationToken); + var byteString = GetOrCreateBigMetaStringBytes(readResult.Buffer, length, hashCode); + reader.AdvanceTo(length); + return byteString; + } + + private MetaStringBytes GetOrCreateBigMetaStringBytes(ReadOnlySequence buffer, int length, long hashCode) + { +#if NET8_0_OR_GREATER + ref var byteString = ref CollectionsMarshal.GetValueRefOrAddDefault(_bigStrings, hashCode, out var exists); +#else + var exists = _bigStrings.TryGetValue(hashCode, out var byteString); +#endif + if (!exists || byteString is null) + { + var bytes = buffer.Slice(0, length).ToArray(); + byteString = new MetaStringBytes(bytes, hashCode); +#if !NET8_0_OR_GREATER + _bigStrings.Add(hashCode, byteString); +#endif + } + + return byteString; + } +} diff --git a/csharp/Fury/Meta/Utf8Encoding.cs b/csharp/Fury/Meta/Utf8Encoding.cs new file mode 100644 index 0000000000..4caf71aa46 --- /dev/null +++ b/csharp/Fury/Meta/Utf8Encoding.cs @@ -0,0 +1,54 @@ +using System; +using System.Text; + +namespace Fury.Meta; + +internal sealed class Utf8Encoding() : MetaStringEncoding(MetaString.Encoding.Utf8) +{ + public static readonly Utf8Encoding Instance = new(); + + public override bool CanEncode(ReadOnlySpan chars) => true; + + public override int GetMaxByteCount(int charCount) => UTF8.GetMaxByteCount(charCount); + + public override int GetMaxCharCount(int byteCount) => UTF8.GetMaxCharCount(byteCount); + + public override unsafe int GetByteCount(ReadOnlySpan chars) + { + fixed (char* p = chars) + { + return UTF8.GetByteCount(p, chars.Length); + } + } + + public override unsafe int GetCharCount(ReadOnlySpan bytes) + { + fixed (byte* p = bytes) + { + return UTF8.GetCharCount(p, bytes.Length); + } + } + + public override unsafe int GetBytes(ReadOnlySpan chars, Span bytes) + { + fixed (char* pChars = chars) + fixed (byte* pBytes = bytes) + { + return UTF8.GetBytes(pChars, chars.Length, pBytes, bytes.Length); + } + } + + public override unsafe int GetChars(ReadOnlySpan bytes, Span chars) + { + fixed (byte* pBytes = bytes) + fixed (char* pChars = chars) + { + return UTF8.GetChars(pBytes, bytes.Length, pChars, chars.Length); + } + } + + public override Encoder GetEncoder() + { + return UTF8.GetEncoder(); + } +} diff --git a/csharp/Fury/RefContext.cs b/csharp/Fury/RefContext.cs new file mode 100644 index 0000000000..6d88e4822f --- /dev/null +++ b/csharp/Fury/RefContext.cs @@ -0,0 +1,123 @@ +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; +using Fury.Buffers; +using Fury.Collections; + +namespace Fury; + +internal sealed class RefContext(IArrayPoolProvider poolProvider) +{ + private readonly Dictionary _objectsToRefId = new(); + private readonly PooledList _readObjects = new(poolProvider); + private readonly HashSet _partiallyProcessedRefIds = []; + + public bool Contains(RefId refId) => refId.IsValid && refId.Value < _readObjects.Count; + + public enum ObjectProcessingState + { + FullyProcessed, + PartiallyProcessed, + Unprocessed, + } + + public bool TryGetReadValue(RefId refId, [NotNullWhen(true)] out TValue value) + { + if (!Contains(refId)) + { + value = default!; + return false; + } + + var writtenValue = _readObjects[refId.Value]; + if (writtenValue is TValue typedValue) + { + value = typedValue; + return true; + } + + value = default!; + return false; + } + + public bool TryGetReadValue(RefId refId, [NotNullWhen(true)] out object? value) + { + if (!Contains(refId)) + { + value = null; + return false; + } + + value = _readObjects[refId.Value]; + return value is not null; + } + + public void PushReferenceableObject(object value) + { + var refId = _readObjects.Count; + _readObjects.Add(value); + _objectsToRefId[value] = refId; + } + + /// + /// This method pops the last pushed referenceable object. + /// It is used to support . + /// + public void PopReferenceableObject() + { + var refId = _readObjects.Count - 1; + var value = _readObjects[refId]; + _readObjects.RemoveAt(refId); + _partiallyProcessedRefIds.Remove(refId); + if (value is not null) + { + _objectsToRefId.Remove(value); + } + } + + /// + /// Gets the existing refId of the specified object or pushes the object and returns a new refId. + /// + /// + /// The object to get or push. + /// + /// + /// The processing state of the object. + /// + /// , if the object is newly pushed. + /// , if the object is pushed and being processed. + /// , if the object is processed completely. + /// + /// + /// + public RefId GetOrPushRefId(object value, out ObjectProcessingState processingState) + { +#if NET8_0_OR_GREATER + ref var refId = ref CollectionsMarshal.GetValueRefOrAddDefault(_objectsToRefId, value, out var exists); +#else + var exists = _objectsToRefId.TryGetValue(value, out var refId); +#endif + if (exists) + { + processingState = _partiallyProcessedRefIds.Contains(refId) + ? ObjectProcessingState.PartiallyProcessed + : ObjectProcessingState.FullyProcessed; + return new RefId(refId); + } + + processingState = ObjectProcessingState.Unprocessed; + + refId = _readObjects.Count; + _readObjects.Add(value); +#if !NET8_0_OR_GREATER + _objectsToRefId.Add(value, refId); +#endif + _partiallyProcessedRefIds.Add(refId); + return new RefId(refId); + } + + public void MarkFullyProcessed(RefId refId) + { + _partiallyProcessedRefIds.Remove(refId.Value); + } +} diff --git a/csharp/Fury/RefId.cs b/csharp/Fury/RefId.cs new file mode 100644 index 0000000000..69e9d2b4ab --- /dev/null +++ b/csharp/Fury/RefId.cs @@ -0,0 +1,12 @@ +namespace Fury; + +public readonly struct RefId(int value) +{ + public static readonly RefId Invalid = new(-1); + + internal int Value { get; } = value; + + public bool IsValid => Value >= 0; + + public override string ToString() => Value.ToString(); +} diff --git a/csharp/Fury/ReferenceFlag.cs b/csharp/Fury/ReferenceFlag.cs new file mode 100644 index 0000000000..30116887e0 --- /dev/null +++ b/csharp/Fury/ReferenceFlag.cs @@ -0,0 +1,22 @@ +namespace Fury; + +internal enum ReferenceFlag : sbyte +{ + Null = -3, + + /// + /// This flag indicates that object is a not-null value. + /// We don't use another byte to indicate REF, so that we can save one byte. + /// + Ref = -2, + + /// + /// this flag indicates that the object is a non-null value. + /// + NotNullValue = -1, + + /// + /// this flag indicates that the object is a referencable and first write. + /// + RefValue = 0, +} diff --git a/csharp/Fury/SerializationContext.cs b/csharp/Fury/SerializationContext.cs new file mode 100644 index 0000000000..9fe9c90c48 --- /dev/null +++ b/csharp/Fury/SerializationContext.cs @@ -0,0 +1,197 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Fury.Serializer; + +namespace Fury; + +// BatchWriter is ref struct, so SerializationContext must be ref struct too + +public ref struct SerializationContext +{ + public Fury Fury { get; } + public BatchWriter Writer; + private RefContext RefContext { get; } + + internal SerializationContext(Fury fury, BatchWriter writer, RefContext refContext) + { + Fury = fury; + Writer = writer; + RefContext = refContext; + } + + public bool TryGetSerializer([NotNullWhen(true)] out ISerializer? serializer) + { + return Fury.TypeResolver.TryGetOrCreateSerializer(typeof(TValue), out serializer); + } + + public ISerializer GetSerializer() + { + if (!TryGetSerializer(out var serializer)) + { + ThrowHelper.ThrowSerializerNotFoundException_SerializerNotFound(typeof(TValue)); + } + return serializer; + } + + public void Write(in TValue? value, ReferenceTrackingPolicy referenceable, ISerializer? serializer = null) + where TValue : notnull + { + if (value is null) + { + Writer.Write(ReferenceFlag.Null); + return; + } + + if (TypeHelper.IsValueType) + { + // Objects declared as ValueType are not possible to be referenced + + Writer.Write(ReferenceFlag.NotNullValue); + DoWriteValueType(in value, serializer); + return; + } + + if (referenceable == ReferenceTrackingPolicy.Enabled) + { + var refId = RefContext.GetOrPushRefId(value, out var processingState); + if (processingState == RefContext.ObjectProcessingState.Unprocessed) + { + // A new referenceable object + + Writer.Write(ReferenceFlag.RefValue); + DoWriteReferenceType(value, serializer); + RefContext.MarkFullyProcessed(refId); + } + else + { + // A referenceable object that has been recorded + + Writer.Write(ReferenceFlag.Ref); + Writer.Write(refId); + } + } + else + { + var refId = RefContext.GetOrPushRefId(value, out var processingState); + if (processingState == RefContext.ObjectProcessingState.PartiallyProcessed) + { + // A referenceable object that has been recorded but not fully processed, + // which means it is the ancestor of the current object. + + // Circular dependency detected + if (referenceable == ReferenceTrackingPolicy.OnlyCircularDependency) + { + Writer.Write(ReferenceFlag.Ref); + Writer.Write(refId); + } + else + { + ThrowHelper.ThrowBadSerializationInputException_CircularDependencyDetected(); + } + return; + } + + // ProcessingState should not be FullyProcessed, + // because we pop the referenceable object after writing it + + // For the possible circular dependency in the future, + // we need to write RefValue instead of NotNullValue + + var flag = + referenceable == ReferenceTrackingPolicy.OnlyCircularDependency + ? ReferenceFlag.RefValue + : ReferenceFlag.NotNullValue; + Writer.Write(flag); + DoWriteReferenceType(value, serializer); + RefContext.PopReferenceableObject(); + } + } + + public void Write(in TValue? value, ReferenceTrackingPolicy referenceable, ISerializer? serializer = null) + where TValue : struct + { + if (value is null) + { + Writer.Write(ReferenceFlag.Null); + return; + } + + // Objects declared as ValueType are not possible to be referenced + Writer.Write(ReferenceFlag.NotNullValue); +#if NET8_0_OR_GREATER + DoWriteValueType(in Nullable.GetValueRefOrDefaultRef(in value), serializer); +#else + DoWriteValueType(value.Value, serializer); +#endif + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Write(in TValue? value, ISerializer? serializer = null) + where TValue : notnull + { + Write(value, Fury.Config.ReferenceTracking, serializer); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Write(in TValue? value, ISerializer? serializer = null) + where TValue : struct + { + Write(value, Fury.Config.ReferenceTracking, serializer); + } + + private void DoWriteValueType(in TValue value, ISerializer? serializer) + where TValue : notnull + { + var type = typeof(TValue); + var typeInfo = GetOrRegisterTypeInfo(type); + WriteTypeMeta(typeInfo); + + switch (typeInfo.TypeId) { + // TODO: Fast path for primitive types + } + + var typedSerializer = (ISerializer)(serializer ?? GetPreferredSerializer(type)); + typedSerializer.Write(this, in value); + } + + private void DoWriteReferenceType(object value, ISerializer? serializer) + { + var type = value.GetType(); + var typeInfo = GetOrRegisterTypeInfo(type); + WriteTypeMeta(typeInfo); + + switch (typeInfo.TypeId) { + // TODO: Fast path for string, string array and primitive arrays + } + + serializer ??= GetPreferredSerializer(type); + serializer.Write(this, value); + } + + private TypeInfo GetOrRegisterTypeInfo(Type typeOfSerializedObject) + { + if (!Fury.TypeResolver.TryGetTypeInfo(typeOfSerializedObject, out var typeInfo)) + { + ThrowHelper.ThrowBadSerializationInputException_UnregisteredType(typeOfSerializedObject); + } + + return typeInfo; + } + + private void WriteTypeMeta(TypeInfo typeInfo) + { + var typeId = typeInfo.TypeId; + Writer.Write(typeId); + if (typeId.IsNamed()) { } + } + + private ISerializer GetPreferredSerializer(Type typeOfSerializedObject) + { + if (!Fury.TypeResolver.TryGetOrCreateSerializer(typeOfSerializedObject, out var serializer)) + { + ThrowHelper.ThrowSerializerNotFoundException_SerializerNotFound(typeOfSerializedObject); + } + return serializer; + } +} diff --git a/csharp/Fury/Serializer/AbstractSerializer.cs b/csharp/Fury/Serializer/AbstractSerializer.cs new file mode 100644 index 0000000000..baeec25047 --- /dev/null +++ b/csharp/Fury/Serializer/AbstractSerializer.cs @@ -0,0 +1,60 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Fury.Serializer; + +public abstract class AbstractSerializer : ISerializer + where TValue : notnull +{ + public abstract void Write(SerializationContext context, in TValue value); + + public virtual void Write(SerializationContext context, object value) + { + var typedValue = (TValue)value; + Write(context, in typedValue); + } +} + +public abstract class AbstractDeserializer : IDeserializer + where TValue : notnull +{ + public abstract ValueTask> CreateInstanceAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ); + + public abstract ValueTask ReadAndFillAsync( + DeserializationContext context, + Box instance, + CancellationToken cancellationToken = default + ); + + public virtual async ValueTask ReadAndCreateAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + var instance = await CreateInstanceAsync(context, cancellationToken); + await ReadAndFillAsync(context, instance, cancellationToken); + return instance.Value!; + } + + async ValueTask IDeserializer.CreateInstanceAsync( + DeserializationContext context, + CancellationToken cancellationToken + ) + { + var instance = await CreateInstanceAsync(context, cancellationToken); + return instance.AsUntyped(); + } + + async ValueTask IDeserializer.ReadAndFillAsync( + DeserializationContext context, + Box instance, + CancellationToken cancellationToken + ) + { + var typedInstance = instance.AsTyped(); + await ReadAndFillAsync(context, typedInstance, cancellationToken); + } +} diff --git a/csharp/Fury/Serializer/ArraySerializers.cs b/csharp/Fury/Serializer/ArraySerializers.cs new file mode 100644 index 0000000000..617c39d0ed --- /dev/null +++ b/csharp/Fury/Serializer/ArraySerializers.cs @@ -0,0 +1,141 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Fury.Serializer; + +internal class ArraySerializer(ISerializer? elementSerializer) : AbstractSerializer + where TElement : notnull +{ + // ReSharper disable once UnusedMember.Global + public ArraySerializer() + : this(null) { } + + public override void Write(SerializationContext context, in TElement?[] value) + { + context.Writer.WriteCount(value.Length); + foreach (var element in value) + { + context.Write(element, elementSerializer); + } + } +} + +internal class NullableArraySerializer(ISerializer? elementSerializer) + : AbstractSerializer + where TElement : struct +{ + // ReSharper disable once UnusedMember.Global + public NullableArraySerializer() + : this(null) { } + + public override void Write(SerializationContext context, in TElement?[] value) + { + context.Writer.WriteCount(value.Length); + foreach (var element in value) + { + context.Write(element, elementSerializer); + } + } +} + +internal class ArrayDeserializer(IDeserializer? elementDeserializer) + : AbstractDeserializer + where TElement : notnull +{ + public ArrayDeserializer() + : this(null) { } + + public override async ValueTask> CreateInstanceAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + var length = await context.Reader.ReadCountAsync(cancellationToken); + return new TElement?[length]; + } + + public override async ValueTask ReadAndFillAsync( + DeserializationContext context, + Box box, + CancellationToken cancellationToken = default + ) + { + var instance = box.Value!; + for (var i = 0; i < instance.Length; i++) + { + instance[i] = await context.ReadAsync(elementDeserializer, cancellationToken); + } + } + + public override async ValueTask ReadAndCreateAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + var length = await context.Reader.ReadCountAsync(cancellationToken); + var result = new TElement?[length]; + for (var i = 0; i < result.Length; i++) + { + result[i] = await context.ReadAsync(elementDeserializer, cancellationToken); + } + return result; + } +} + +internal class NullableArrayDeserializer(IDeserializer? elementDeserializer) + : AbstractDeserializer + where TElement : struct +{ + public NullableArrayDeserializer() + : this(null) { } + + public override async ValueTask> CreateInstanceAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + var length = await context.Reader.ReadCountAsync(cancellationToken); + return new TElement?[length]; + } + + public override async ValueTask ReadAndFillAsync( + DeserializationContext context, + Box box, + CancellationToken cancellationToken = default + ) + { + var instance = box.Value!; + for (var i = 0; i < instance.Length; i++) + { + instance[i] = await context.ReadNullableAsync(elementDeserializer, cancellationToken); + } + } +} + +internal sealed class PrimitiveArraySerializer : AbstractSerializer + where TElement : unmanaged +{ + public static PrimitiveArraySerializer Instance { get; } = new(); + + public override void Write(SerializationContext context, in TElement[] value) + { + context.Writer.WriteCount(value.Length); + context.Writer.Write(value); + } +} + +internal sealed class PrimitiveArrayDeserializer : ArrayDeserializer + where TElement : unmanaged +{ + public static PrimitiveArrayDeserializer Instance { get; } = new(); + + public override async ValueTask ReadAndFillAsync( + DeserializationContext context, + Box box, + CancellationToken cancellationToken = default + ) + { + var instance = box.Value!; + await context.Reader.ReadMemoryAsync(instance, cancellationToken); + } +} diff --git a/csharp/Fury/Serializer/CollectionDeserializer.cs b/csharp/Fury/Serializer/CollectionDeserializer.cs new file mode 100644 index 0000000000..14023a2dc2 --- /dev/null +++ b/csharp/Fury/Serializer/CollectionDeserializer.cs @@ -0,0 +1,79 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Fury.Serializer; + +public abstract class CollectionDeserializer(IDeserializer? elementSerializer) + : AbstractDeserializer + where TElement : notnull + where TEnumerable : class, ICollection +{ + private IDeserializer? _elementDeserializer = elementSerializer; + + public override async ValueTask ReadAndFillAsync( + DeserializationContext context, + Box box, + CancellationToken cancellationToken = default + ) + { + var instance = box.Value!; + var count = instance.Count; + if (count <= 0) + { + return; + } + var typedElementSerializer = _elementDeserializer; + if (typedElementSerializer is null) + { + if (TypeHelper.IsSealed) + { + typedElementSerializer = (IDeserializer)context.GetDeserializer(); + _elementDeserializer = typedElementSerializer; + } + } + + for (var i = 0; i < instance.Count; i++) + { + var item = await context.ReadAsync(typedElementSerializer, cancellationToken); + instance.Add(item); + } + } +} + +public abstract class NullableCollectionDeserializer(IDeserializer? elementSerializer) + : AbstractDeserializer + where TElement : struct + where TEnumerable : class, ICollection +{ + private IDeserializer? _elementDeserializer = elementSerializer; + + public override async ValueTask ReadAndFillAsync( + DeserializationContext context, + Box box, + CancellationToken cancellationToken = default + ) + { + var instance = box.Value!; + var count = instance.Count; + if (count <= 0) + { + return; + } + var typedElementSerializer = _elementDeserializer; + if (typedElementSerializer is null) + { + if (TypeHelper.IsSealed) + { + typedElementSerializer = (IDeserializer)context.GetDeserializer(); + _elementDeserializer = typedElementSerializer; + } + } + + for (var i = 0; i < instance.Count; i++) + { + var item = await context.ReadNullableAsync(typedElementSerializer, cancellationToken); + instance.Add(item); + } + } +} diff --git a/csharp/Fury/Serializer/EnumSerializer.cs b/csharp/Fury/Serializer/EnumSerializer.cs new file mode 100644 index 0000000000..8c34bd8b34 --- /dev/null +++ b/csharp/Fury/Serializer/EnumSerializer.cs @@ -0,0 +1,39 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Fury.Serializer; + +internal sealed class EnumSerializer : AbstractSerializer + where TEnum : Enum +{ + public override void Write(SerializationContext context, in TEnum value) + { + // TODO: Serialize by name + + var v = Convert.ToUInt32(value); + context.Writer.Write7BitEncodedUint(v); + } +} + +internal sealed class EnumDeserializer : AbstractDeserializer + where TEnum : Enum +{ + public override ValueTask> CreateInstanceAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + return new ValueTask>(new Box(default!)); + } + + public override async ValueTask ReadAndFillAsync( + DeserializationContext context, + Box instance, + CancellationToken cancellationToken = default + ) + { + var v = await context.Reader.Read7BitEncodedUintAsync(cancellationToken); + instance.Value = (TEnum)Enum.ToObject(typeof(TEnum), v); + } +} diff --git a/csharp/Fury/Serializer/EnumerableSerializer.cs b/csharp/Fury/Serializer/EnumerableSerializer.cs new file mode 100644 index 0000000000..c4f5768cff --- /dev/null +++ b/csharp/Fury/Serializer/EnumerableSerializer.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Fury.Serializer.Provider; +#if NET8_0_OR_GREATER +using System.Collections.Immutable; +using System.Runtime.InteropServices; +#endif + +namespace Fury.Serializer; + +// TEnumerable is required, because we need to assign the serializer to ISerializer. + +public class EnumerableSerializer(ISerializer? elementSerializer) + : AbstractSerializer + where TElement : notnull + where TEnumerable : IEnumerable +{ + private ISerializer? _elementSerializer = elementSerializer; + + // ReSharper disable once UnusedMember.Global + /// + /// This constructor is required for Activator.CreateInstance. See . + /// + public EnumerableSerializer() + : this(null) { } + + public override void Write(SerializationContext context, in TEnumerable value) + { + var count = value.Count(); + context.Writer.WriteCount(count); + if (count <= 0) + { + return; + } + var typedElementSerializer = _elementSerializer; + if (typedElementSerializer is null) + { + if (TypeHelper.IsSealed) + { + typedElementSerializer = (ISerializer)context.GetSerializer(); + _elementSerializer = typedElementSerializer; + } + } + if (TryGetSpan(value, out var elements)) + { + foreach (ref readonly var element in elements) + { + context.Write(in element, typedElementSerializer); + } + return; + } + + foreach (var element in value) + { + context.Write(in element, typedElementSerializer); + } + } + + protected virtual bool TryGetSpan(TEnumerable value, out ReadOnlySpan span) + { + switch (value) + { + case TElement[] elements: + span = elements; + return true; +#if NET8_0_OR_GREATER + case List elements: + span = CollectionsMarshal.AsSpan(elements); + return true; + case ImmutableArray elements: + span = ImmutableCollectionsMarshal.AsArray(elements); + return true; +#endif + default: + span = ReadOnlySpan.Empty; + return false; + } + } +} + +public class NullableEnumerableSerializer(ISerializer? elementSerializer) + : AbstractSerializer + where TElement : struct + where TEnumerable : IEnumerable +{ + private ISerializer? _elementSerializer = elementSerializer; + + // ReSharper disable once UnusedMember.Global + /// + /// This constructor is required for Activator.CreateInstance. See . + /// + public NullableEnumerableSerializer() + : this(null) { } + + public override void Write(SerializationContext context, in TEnumerable value) + { + var count = value.Count(); + context.Writer.WriteCount(count); + if (count <= 0) + { + return; + } + var typedElementSerializer = _elementSerializer; + if (typedElementSerializer is null) + { + if (TypeHelper.IsSealed) + { + typedElementSerializer = (ISerializer)context.GetSerializer(); + _elementSerializer = typedElementSerializer; + } + } + if (TryGetSpan(value, out var elements)) + { + foreach (ref readonly var element in elements) + { + context.Write(in element, typedElementSerializer); + } + return; + } + + foreach (var element in value) + { + context.Write(in element, typedElementSerializer); + } + } + + protected virtual bool TryGetSpan(TEnumerable value, out ReadOnlySpan span) + { + switch (value) + { + case TElement?[] elements: + span = elements; + return true; +#if NET8_0_OR_GREATER + case List elements: + span = CollectionsMarshal.AsSpan(elements); + return true; + case ImmutableArray elements: + span = ImmutableCollectionsMarshal.AsArray(elements); + return true; +#endif + default: + span = ReadOnlySpan.Empty; + return false; + } + } +} diff --git a/csharp/Fury/Serializer/ISerializer.cs b/csharp/Fury/Serializer/ISerializer.cs new file mode 100644 index 0000000000..f3e6384b77 --- /dev/null +++ b/csharp/Fury/Serializer/ISerializer.cs @@ -0,0 +1,73 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Fury.Serializer; + +// This interface is used to support polymorphism. +public interface ISerializer +{ + void Write(SerializationContext context, object value); +} + +public interface IDeserializer +{ + // It is very common that the data is not all available at once, so we need to read it asynchronously. + + /// + /// Create an instance of the object which will be deserialized. + /// + /// + /// An instance of the object which is not deserialized yet. + /// + /// + /// + /// This method is used to solve the circular reference problem. + /// When deserializing an object which may be referenced by itself or its child objects, + /// we need to create an instance before reading its fields. + /// So that we can reference it before it is fully deserialized. + /// + /// + /// You can read some necessary data from the context to create the instance, e.g. the length of an array. + /// + /// + /// If the object certainly does not have circular references, you can return a fully deserialized object + /// and keep the method empty.
+ /// Be careful that the default implementation of + /// in use this method to create an instance.
+ /// If you want to do all the deserialization here, it is recommended to override + /// and call it in this method. + /// + /// + ValueTask CreateInstanceAsync(DeserializationContext context, CancellationToken cancellationToken = default); + + /// + /// Read the serialized data and populate the given object. + /// + /// + /// The context which contains the state of the deserialization process. + /// + /// + /// The object which is not deserialized yet. It is created by . + /// + /// + /// + /// The object which is deserialized from the serialized data. This should be the inputted instance. + /// + ValueTask ReadAndFillAsync( + DeserializationContext context, + Box instance, + CancellationToken cancellationToken = default + ); +} + +public interface ISerializer : ISerializer + where TValue : notnull +{ + void Write(SerializationContext context, in TValue value); +} + +public interface IDeserializer : IDeserializer + where TValue : notnull +{ + ValueTask ReadAndCreateAsync(DeserializationContext context, CancellationToken cancellationToken = default); +} diff --git a/csharp/Fury/Serializer/NotSupportedSerializer.cs b/csharp/Fury/Serializer/NotSupportedSerializer.cs new file mode 100644 index 0000000000..c381521556 --- /dev/null +++ b/csharp/Fury/Serializer/NotSupportedSerializer.cs @@ -0,0 +1,48 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Fury.Serializer; + +public sealed class NotSupportedSerializer : ISerializer + where TValue : notnull +{ + public void Write(SerializationContext context, in TValue value) + { + throw new NotSupportedException(); + } + + public void Write(SerializationContext context, object value) + { + throw new NotSupportedException(); + } +} + +public sealed class NotSupportedDeserializer : IDeserializer + where TValue : notnull +{ + public ValueTask ReadAndCreateAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public ValueTask CreateInstanceAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public ValueTask ReadAndFillAsync( + DeserializationContext context, + Box instance, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } +} diff --git a/csharp/Fury/Serializer/PrimitiveSerializers.cs b/csharp/Fury/Serializer/PrimitiveSerializers.cs new file mode 100644 index 0000000000..fe046ed818 --- /dev/null +++ b/csharp/Fury/Serializer/PrimitiveSerializers.cs @@ -0,0 +1,48 @@ +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Fury.Serializer; + +internal sealed class PrimitiveSerializer : AbstractSerializer + where T : unmanaged +{ + public static PrimitiveSerializer Instance { get; } = new(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Write(SerializationContext context, in T value) + { + context.Writer.Write(value); + } +} + +internal sealed class PrimitiveDeserializer : AbstractDeserializer + where T : unmanaged +{ + public static PrimitiveDeserializer Instance { get; } = new(); + + public override async ValueTask> CreateInstanceAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + return await ReadAndCreateAsync(context, cancellationToken); + } + + public override ValueTask ReadAndFillAsync( + DeserializationContext context, + Box instance, + CancellationToken cancellationToken = default + ) + { + return TaskHelper.CompletedValueTask; + } + + public override async ValueTask ReadAndCreateAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + return await context.Reader.ReadAsync(cancellationToken: cancellationToken); + } +} diff --git a/csharp/Fury/Serializer/Provider/ArraySerializerProvider.cs b/csharp/Fury/Serializer/Provider/ArraySerializerProvider.cs new file mode 100644 index 0000000000..bb3efe16c4 --- /dev/null +++ b/csharp/Fury/Serializer/Provider/ArraySerializerProvider.cs @@ -0,0 +1,94 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury.Serializer.Provider; + +internal sealed class ArraySerializerProvider : ISerializerProvider +{ + public bool TryCreateSerializer(TypeResolver resolver, Type type, [NotNullWhen(true)] out ISerializer? serializer) + { + serializer = null; + if (!type.IsArray) + { + return false; + } + + var elementType = type.GetElementType(); + if (elementType is null) + { + return false; + } + + var underlyingType = Nullable.GetUnderlyingType(elementType); + Type serializerType; + if (underlyingType is null) + { + serializerType = typeof(ArraySerializer<>).MakeGenericType(elementType); + } + else + { + serializerType = typeof(NullableArraySerializer<>).MakeGenericType(underlyingType); + elementType = underlyingType; + } + + ISerializer? elementSerializer = null; + if (elementType.IsSealed) + { + if (!resolver.TryGetOrCreateSerializer(elementType, out elementSerializer)) + { + return false; + } + } + + serializer = (ISerializer?)Activator.CreateInstance(serializerType, elementSerializer); + + return serializer is not null; + } +} + +internal class ArrayDeserializerProvider : IDeserializerProvider +{ + public bool TryCreateDeserializer( + TypeResolver resolver, + Type type, + [NotNullWhen(true)] out IDeserializer? deserializer + ) + { + deserializer = null; + if (!type.IsArray) + { + return false; + } + + var elementType = type.GetElementType(); + if (elementType is null) + { + return false; + } + + var underlyingType = Nullable.GetUnderlyingType(elementType); + Type deserializerType; + if (underlyingType is null) + { + deserializerType = typeof(ArrayDeserializer<>).MakeGenericType(elementType); + } + else + { + deserializerType = typeof(NullableArrayDeserializer<>).MakeGenericType(underlyingType); + elementType = underlyingType; + } + + IDeserializer? elementDeserializer = null; + if (elementType.IsSealed) + { + if (!resolver.TryGetOrCreateDeserializer(elementType, out elementDeserializer)) + { + return false; + } + } + + deserializer = (IDeserializer?)Activator.CreateInstance(deserializerType, elementDeserializer); + + return deserializer is not null; + } +} diff --git a/csharp/Fury/Serializer/Provider/CollectionDeserializerProvider.cs b/csharp/Fury/Serializer/Provider/CollectionDeserializerProvider.cs new file mode 100644 index 0000000000..2a6a87826d --- /dev/null +++ b/csharp/Fury/Serializer/Provider/CollectionDeserializerProvider.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Fury.Serializer.Provider; + +internal sealed class CollectionDeserializerProvider : IDeserializerProvider +{ + private static readonly string CollectionInterfaceName = typeof(ICollection<>).Name; + + public bool TryCreateDeserializer( + TypeResolver resolver, + Type type, + [NotNullWhen(true)] out IDeserializer? deserializer + ) + { + deserializer = null; + if (type.IsAbstract) + { + return false; + } + if (type.GetInterface(CollectionInterfaceName) is not { } collectionInterface) + { + return false; + } + + var elementType = collectionInterface.GetGenericArguments()[0]; + if (elementType.IsGenericParameter) + { + return false; + } + + var underlyingType = Nullable.GetUnderlyingType(elementType); + Type deserializerType; + if (underlyingType is null) + { + deserializerType = typeof(CollectionDeserializer<,>).MakeGenericType(elementType, type); + } + else + { + deserializerType = typeof(NullableCollectionDeserializer<,>).MakeGenericType(underlyingType, type); + elementType = underlyingType; + } + + IDeserializer? elementDeserializer = null; + if (elementType.IsSealed) + { + if (!resolver.TryGetOrCreateDeserializer(elementType, out elementDeserializer)) + { + return false; + } + } + + deserializer = (IDeserializer?)Activator.CreateInstance(deserializerType, elementDeserializer); + + return deserializer is not null; + } +} diff --git a/csharp/Fury/Serializer/Provider/EnumSerializerProvider.cs b/csharp/Fury/Serializer/Provider/EnumSerializerProvider.cs new file mode 100644 index 0000000000..08483d63a8 --- /dev/null +++ b/csharp/Fury/Serializer/Provider/EnumSerializerProvider.cs @@ -0,0 +1,42 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury.Serializer.Provider; + +public sealed class EnumSerializerProvider : ISerializerProvider +{ + public bool TryCreateSerializer(TypeResolver resolver, Type type, [NotNullWhen(true)] out ISerializer? serializer) + { + if (!type.IsEnum) + { + serializer = null; + return false; + } + + var serializerType = typeof(EnumSerializer<>).MakeGenericType(type); + serializer = (ISerializer?)Activator.CreateInstance(serializerType); + + return serializer is not null; + } +} + +public sealed class EnumDeserializerProvider : IDeserializerProvider +{ + public bool TryCreateDeserializer( + TypeResolver resolver, + Type type, + [NotNullWhen(true)] out IDeserializer? deserializer + ) + { + if (!type.IsEnum) + { + deserializer = default; + return false; + } + + var deserializerType = typeof(EnumDeserializer<>).MakeGenericType(type); + deserializer = (IDeserializer?)Activator.CreateInstance(deserializerType); + + return deserializer is not null; + } +} diff --git a/csharp/Fury/Serializer/Provider/EnumerableSerializerProvider.cs b/csharp/Fury/Serializer/Provider/EnumerableSerializerProvider.cs new file mode 100644 index 0000000000..9a3077cd71 --- /dev/null +++ b/csharp/Fury/Serializer/Provider/EnumerableSerializerProvider.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Fury.Serializer.Provider; + +internal sealed class EnumerableSerializerProvider : ISerializerProvider +{ + private static readonly string EnumerableInterfaceName = typeof(IEnumerable<>).Name; + + public bool TryCreateSerializer(TypeResolver resolver, Type type, [NotNullWhen(true)] out ISerializer? serializer) + { + serializer = null; + if (type.IsAbstract) + { + return false; + } + if (type.GetInterface(EnumerableInterfaceName) is not { } enumerableInterface) + { + return false; + } + + var elementType = enumerableInterface.GetGenericArguments()[0]; + if (elementType.IsGenericParameter) + { + return false; + } + + var underlyingType = Nullable.GetUnderlyingType(elementType); + Type serializerType; + if (underlyingType is null) + { + serializerType = typeof(EnumerableSerializer<,>).MakeGenericType(elementType, type); + } + else + { + serializerType = typeof(NullableEnumerableSerializer<,>).MakeGenericType(underlyingType, type); + elementType = underlyingType; + } + + ISerializer? elementSerializer = null; + if (elementType.IsSealed) + { + if (!resolver.TryGetOrCreateSerializer(elementType, out elementSerializer)) + { + return false; + } + } + + serializer = (ISerializer?)Activator.CreateInstance(serializerType, elementSerializer); + + return serializer is not null; + } +} diff --git a/csharp/Fury/Serializer/Provider/ISerializerProvider.cs b/csharp/Fury/Serializer/Provider/ISerializerProvider.cs new file mode 100644 index 0000000000..0e988c6ec8 --- /dev/null +++ b/csharp/Fury/Serializer/Provider/ISerializerProvider.cs @@ -0,0 +1,14 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Fury.Serializer.Provider; + +public interface ISerializerProvider +{ + bool TryCreateSerializer(TypeResolver resolver, Type type, [NotNullWhen(true)] out ISerializer? serializer); +} + +public interface IDeserializerProvider +{ + bool TryCreateDeserializer(TypeResolver resolver, Type type, [NotNullWhen(true)] out IDeserializer? deserializer); +} diff --git a/csharp/Fury/Serializer/StringSerializer.cs b/csharp/Fury/Serializer/StringSerializer.cs new file mode 100644 index 0000000000..e0e6eb059d --- /dev/null +++ b/csharp/Fury/Serializer/StringSerializer.cs @@ -0,0 +1,51 @@ +using System; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Fury.Serializer; + +internal sealed class StringSerializer : AbstractSerializer +{ + public static StringSerializer Instance { get; } = new(); + + public override void Write(SerializationContext context, in string value) + { + // TODO: write encoding flags + var byteCount = Encoding.UTF8.GetByteCount(value); + context.Writer.WriteCount(byteCount); + context.Writer.Write(value.AsSpan(), Encoding.UTF8, byteCount); + } +} + +internal sealed class StringDeserializer : AbstractDeserializer +{ + public static StringDeserializer Instance { get; } = new(); + + public override async ValueTask> CreateInstanceAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + return await ReadAndCreateAsync(context, cancellationToken); + } + + public override ValueTask ReadAndFillAsync( + DeserializationContext context, + Box instance, + CancellationToken cancellationToken = default + ) + { + return TaskHelper.CompletedValueTask; + } + + public override async ValueTask ReadAndCreateAsync( + DeserializationContext context, + CancellationToken cancellationToken = default + ) + { + // TODO: read encoding flags + var byteCount = await context.Reader.ReadCountAsync(cancellationToken); + return await context.Reader.ReadStringAsync(byteCount, Encoding.UTF8, cancellationToken); + } +} diff --git a/csharp/Fury/StaticConfigs.cs b/csharp/Fury/StaticConfigs.cs new file mode 100644 index 0000000000..c6cd43a1e8 --- /dev/null +++ b/csharp/Fury/StaticConfigs.cs @@ -0,0 +1,8 @@ +namespace Fury; + +internal static class StaticConfigs +{ + public const int StackAllocLimit = 1024; + + public const int BuiltInListDefaultCapacity = 16; +} diff --git a/csharp/Fury/TaskHelper.cs b/csharp/Fury/TaskHelper.cs new file mode 100644 index 0000000000..4296cf7825 --- /dev/null +++ b/csharp/Fury/TaskHelper.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; + +namespace Fury; + +internal class TaskHelper +{ + // ValueTask.CompletedTask is not available in .NET Standard 2.0 + + public static readonly ValueTask CompletedValueTask = default; +} diff --git a/csharp/Fury/TypeHelper.cs b/csharp/Fury/TypeHelper.cs new file mode 100644 index 0000000000..fff7d0153e --- /dev/null +++ b/csharp/Fury/TypeHelper.cs @@ -0,0 +1,72 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using System.Runtime.CompilerServices; + +namespace Fury; + +internal static class TypeHelper +{ + public static readonly bool IsSealed = typeof(T).IsSealed; + public static readonly bool IsValueType = typeof(T).IsValueType; + public static readonly int Size = Unsafe.SizeOf(); + public static readonly bool IsReferenceOrContainsReferences = TypeHelper.CheckIsReferenceOrContainsReferences(); +} + +internal static class TypeHelper +{ + public static bool TryGetUnderlyingElementType( + Type arrayType, + [NotNullWhen(true)] out Type? elementType, + out int rank + ) + { + // TODO: Multi-dimensional arrays are not supported yet. + rank = 0; + var currentType = arrayType; + while (currentType.IsArray) + { + elementType = currentType.GetElementType(); + if (elementType is null) + { + return false; + } + currentType = elementType; + rank++; + } + elementType = currentType; + return true; + } + + public static bool CheckIsReferenceOrContainsReferences(Type type) + { + if (!type.IsValueType) + { + return true; + } + + if (type.IsPrimitive || type.IsEnum) + { + return false; + } + + foreach (var field in type.GetFields(BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public)) + { + if (CheckIsReferenceOrContainsReferences(field.FieldType)) + { + return true; + } + } + + return false; + } + + public static bool CheckIsReferenceOrContainsReferences() + { +#if NET8_0_OR_GREATER + return RuntimeHelpers.IsReferenceOrContainsReferences(); +#else + return CheckIsReferenceOrContainsReferences(typeof(T)); +#endif + } +} diff --git a/csharp/Fury/TypeId.cs b/csharp/Fury/TypeId.cs new file mode 100644 index 0000000000..fbcd899a83 --- /dev/null +++ b/csharp/Fury/TypeId.cs @@ -0,0 +1,297 @@ +using System; +using System.Collections.Generic; + +namespace Fury; + +public readonly struct TypeId : IEquatable +{ + internal int Value { get; } + + internal TypeId(int value) + { + Value = value; + } + + public bool Equals(TypeId other) + { + return Value == other.Value; + } + + public override bool Equals(object? obj) + { + return obj is TypeId other && Equals(other); + } + + public override int GetHashCode() + { + return Value; + } + + public static bool operator ==(TypeId left, TypeId right) + { + return left.Equals(right); + } + + public static bool operator !=(TypeId left, TypeId right) + { + return !left.Equals(right); + } + + /// + /// bool: a boolean value (true or false). + /// + public static readonly TypeId Bool = new(1); + + /// + /// int8: an 8-bit signed integer. + /// + public static readonly TypeId Int8 = new(2); + + /// + /// int16: a 16-bit signed integer. + /// + public static readonly TypeId Int16 = new(3); + + /// + /// int32: a 32-bit signed integer. + /// + public static readonly TypeId Int32 = new(4); + + /// + /// var\_int32: a 32-bit signed integer which uses Fury var\_int32 encoding. + /// + public static readonly TypeId VarInt32 = new(5); + + /// + /// int64: a 64-bit signed integer. + /// + public static readonly TypeId Int64 = new(6); + + /// + /// var\_int64: a 64-bit signed integer which uses Fury PVL encoding. + /// + public static readonly TypeId VarInt64 = new(7); + + /// + /// sli\_int64: a 64-bit signed integer which uses Fury SLI encoding. + /// + public static readonly TypeId SliInt64 = new(8); + + /// + /// float16: a 16-bit floating point number. + /// + public static readonly TypeId Float16 = new(9); + + /// + /// float32: a 32-bit floating point number. + /// + public static readonly TypeId Float32 = new(10); + + /// + /// float64: a 64-bit floating point number including NaN and Infinity. + /// + public static readonly TypeId Float64 = new(11); + + /// + /// string: a text string encoded using Latin1/UTF16/UTF-8 encoding. + /// + public static readonly TypeId String = new(12); + + /// + /// enum: a data type consisting of a set of named values. + /// Rust enum with non-predefined field values are not supported as an enum. + /// + public static readonly TypeId Enum = new(13); + + /// + /// named_enum: an enum whose value will be serialized as the registered name. + /// + public static readonly TypeId NamedEnum = new(14); + + /// + /// a morphic (sealed) type serialized by Fury Struct serializer. i.e. it doesn't have subclasses. + /// Suppose we're deserializing , we can save dynamic serializer dispatch since T is morphic (sealed). + /// + public static readonly TypeId Struct = new(15); + + /// + /// a type which is polymorphic (not sealed). i.e. it has subclasses. + /// Suppose we're deserializing , we must dispatch serializer dynamically since T is polymorphic (non-sealed). + /// + public static readonly TypeId PolymorphicStruct = new(16); + + /// + /// a morphic (sealed) type serialized by Fury compatible Struct serializer. + /// + public static readonly TypeId CompatibleStruct = new(17); + + /// + /// a non-morphic (non-sealed) type serialized by Fury compatible Struct serializer. + /// + public static readonly TypeId PolymorphicCompatibleStruct = new(18); + + /// + /// a whose type mapping will be encoded as a name. + /// + public static readonly TypeId NamedStruct = new(19); + + /// + /// a whose type mapping will be encoded as a name. + /// + public static readonly TypeId NamedPolymorphicStruct = new(20); + + /// + /// a whose type mapping will be encoded as a name. + /// + public static readonly TypeId NamedCompatibleStruct = new(21); + + /// + /// a whose type mapping will be encoded as a name. + /// + public static readonly TypeId NamedPolymorphicCompatibleStruct = new(22); + + /// + /// a type which will be serialized by a customized serializer. + /// + public static readonly TypeId Ext = new(23); + + /// + /// an type which is not morphic (not sealed). + /// + public static readonly TypeId PolymorphicExt = new(24); + + /// + /// an type whose type mapping will be encoded as a name. + /// + public static readonly TypeId NamedExt = new(25); + + /// + /// an type whose type mapping will be encoded as a name. + /// + public static readonly TypeId NamedPolymorphicExt = new(26); + + /// + /// a sequence of objects. + /// + public static readonly TypeId List = new(27); + + /// + /// an unordered set of unique elements. + /// + public static readonly TypeId Set = new(28); + + /// + /// a map of key-value pairs. Mutable types such as list/map/set/array/tensor/arrow are not allowed as key of map. + /// + public static readonly TypeId Map = new(29); + + /// + /// an absolute length of time, independent of any calendar/timezone, as a count of nanoseconds. + /// + public static readonly TypeId Duration = new(30); + + /// + /// timestamp: a point in time, independent of any calendar/timezone, as a count of nanoseconds. The count is relative to an epoch at UTC midnight on January 1, 1970. + /// + public static readonly TypeId Timestamp = new(31); + + /// + /// a naive date without timezone. The count is days relative to an epoch at UTC midnight on Jan 1, 1970. + /// + public static readonly TypeId LocalDate = new(32); + + /// + /// exact decimal value represented as an integer value in two's complement. + /// + public static readonly TypeId Decimal = new(33); + + /// + /// a variable-length array of bytes. + /// + public static readonly TypeId Binary = new(34); + + /// + /// a multidimensional array which every sub-array can have different sizes but all have same type. only allow numeric components. + /// Other arrays will be taken as List. The implementation should support the interoperability between array and list. + /// + public static readonly TypeId Array = new(35); + + /// + /// one dimensional int16 array. + /// + public static readonly TypeId BoolArray = new(36); + + /// + /// one dimensional int8 array. + /// + public static readonly TypeId Int8Array = new(37); + + /// + /// one dimensional int16 array. + /// + public static readonly TypeId Int16Array = new(38); + + /// + /// one dimensional int32 array. + /// + public static readonly TypeId Int32Array = new(39); + + /// + /// one dimensional int64 array. + /// + public static readonly TypeId Int64Array = new(40); + + /// + /// one dimensional half\_float\_16 array. + /// + public static readonly TypeId Float16Array = new(41); + + /// + /// one dimensional float32 array. + /// + public static readonly TypeId Float32Array = new(42); + + /// + /// one dimensional float64 array. + /// + public static readonly TypeId Float64Array = new(43); + + /// + /// an arrow record batch object. + /// + public static readonly TypeId ArrowRecordBatch = new(44); + + /// + /// an arrow table object. + /// + public static readonly TypeId ArrowTable = new(45); + + /// + /// Checks if this type is a struct type. + /// + /// + /// True if this type is a struct type; otherwise, false. + /// + public bool IsStructType() + { + return this == Struct + || this == PolymorphicStruct + || this == CompatibleStruct + || this == PolymorphicCompatibleStruct + || this == NamedStruct + || this == NamedPolymorphicStruct + || this == NamedCompatibleStruct + || this == NamedPolymorphicCompatibleStruct; + } + + internal bool IsNamed() + { + return this == NamedEnum + || this == NamedStruct + || this == NamedPolymorphicStruct + || this == NamedCompatibleStruct + || this == NamedPolymorphicCompatibleStruct + || this == NamedExt + || this == NamedPolymorphicExt; + } +} diff --git a/csharp/Fury/TypeInfo.cs b/csharp/Fury/TypeInfo.cs new file mode 100644 index 0000000000..fb4b34dd5c --- /dev/null +++ b/csharp/Fury/TypeInfo.cs @@ -0,0 +1,5 @@ +using System; + +namespace Fury; + +public record struct TypeInfo(TypeId TypeId, Type Type); diff --git a/csharp/Fury/TypeResolver.cs b/csharp/Fury/TypeResolver.cs new file mode 100644 index 0000000000..b63535d1ff --- /dev/null +++ b/csharp/Fury/TypeResolver.cs @@ -0,0 +1,175 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Fury.Buffers; +using Fury.Collections; +using Fury.Meta; +using Fury.Serializer; +using Fury.Serializer.Provider; + +namespace Fury; + +public sealed class TypeResolver +{ + private readonly Dictionary _typeToSerializers = new(); + private readonly Dictionary _typeToDeserializers = new(); + private readonly Dictionary _typeToTypeInfos = new(); + private readonly Dictionary _fullNameHashToTypeInfos = new(); + private readonly PooledList _types; + + private readonly ISerializerProvider[] _serializerProviders; + private readonly IDeserializerProvider[] _deserializerProviders; + + internal TypeResolver( + IEnumerable serializerProviders, + IEnumerable deserializerProviders, + IArrayPoolProvider poolProvider + ) + { + _serializerProviders = serializerProviders.ToArray(); + _deserializerProviders = deserializerProviders.ToArray(); + _types = new PooledList(poolProvider); + } + + public bool TryGetOrCreateSerializer(Type type, [NotNullWhen(true)] out ISerializer? serializer) + { +#if NET8_0_OR_GREATER + ref var registeredSerializer = ref CollectionsMarshal.GetValueRefOrAddDefault( + _typeToSerializers, + type, + out var exists + ); +#else + var exists = _typeToSerializers.TryGetValue(type, out var registeredSerializer); +#endif + + if (!exists || registeredSerializer == null) + { + TryCreateSerializer(type, out registeredSerializer); +#if !NET8_0_OR_GREATER + if (registeredSerializer is not null) + { + _typeToSerializers[type] = registeredSerializer; + } +#endif + } + + serializer = registeredSerializer; + return serializer is not null; + } + + public bool TryGetOrCreateDeserializer(Type type, [NotNullWhen(true)] out IDeserializer? deserializer) + { +#if NET8_0_OR_GREATER + ref var registeredDeserializer = ref CollectionsMarshal.GetValueRefOrAddDefault( + _typeToDeserializers, + type, + out var exists + ); +#else + var exists = _typeToDeserializers.TryGetValue(type, out var registeredDeserializer); +#endif + + if (!exists || registeredDeserializer == null) + { + TryCreateDeserializer(type, out registeredDeserializer); +#if !NET8_0_OR_GREATER + if (registeredDeserializer is not null) + { + _typeToDeserializers[type] = registeredDeserializer; + } +#endif + } + + deserializer = registeredDeserializer; + return deserializer is not null; + } + + public bool TryGetTypeInfo(Type type, out TypeInfo typeInfo) + { +#if NET8_0_OR_GREATER + ref var registeredTypeInfo = ref CollectionsMarshal.GetValueRefOrAddDefault( + _typeToTypeInfos, + type, + out var exists + ); +#else + var exists = _typeToTypeInfos.TryGetValue(type, out var registeredTypeInfo); +#endif + + if (!exists) + { + var newId = _types.Count; + _types.Add(type); + registeredTypeInfo = new TypeInfo(new TypeId(newId), type); +#if !NET8_0_OR_GREATER + _typeToTypeInfos[type] = registeredTypeInfo; +#endif + } + + typeInfo = registeredTypeInfo; + return true; + } + + public bool TryGetTypeInfo(TypeId typeId, out TypeInfo typeInfo) + { + var id = typeId.Value; + if (id < 0 || id >= _types.Count) + { + typeInfo = default; + return false; + } + + typeInfo = new TypeInfo(typeId, _types[id]); + return true; + } + + private bool TryCreateSerializer(Type type, [NotNullWhen(true)] out ISerializer? serializer) + { + for (var i = _serializerProviders.Length - 1; i >= 0; i--) + { + var provider = _serializerProviders[i]; + if (provider.TryCreateSerializer(this, type, out serializer)) + { + return true; + } + } + + serializer = null; + return false; + } + + private bool TryCreateDeserializer(Type type, [NotNullWhen(true)] out IDeserializer? deserializer) + { + for (var i = _deserializerProviders.Length - 1; i >= 0; i--) + { + var provider = _deserializerProviders[i]; + if (provider.TryCreateDeserializer(this, type, out deserializer)) + { + return true; + } + } + + deserializer = null; + return false; + } + + internal void GetOrRegisterTypeInfo(TypeId typeId, MetaStringBytes namespaceBytes, MetaStringBytes typeNameBytes) + { + var hashCode = new UInt128((ulong)namespaceBytes.HashCode, (ulong)typeNameBytes.HashCode); +#if NET8_0_OR_GREATER + ref var typeInfo = ref CollectionsMarshal.GetValueRefOrAddDefault( + _fullNameHashToTypeInfos, + hashCode, + out var exists + ); +#else + var exists = _fullNameHashToTypeInfos.TryGetValue(hashCode, out var typeInfo); +#endif + if (!exists) { } + } +} diff --git a/csharp/global.json b/csharp/global.json new file mode 100644 index 0000000000..dad2db5efd --- /dev/null +++ b/csharp/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "8.0.0", + "rollForward": "latestMajor", + "allowPrerelease": true + } +} \ No newline at end of file