diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index d9ddf7c639..76fd5f5990 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -26,8 +26,8 @@ import { CodegenRegistry } from "./router"; import { BaseSerializerGenerator, RefState } from "./serializer"; import SerializerResolver from "../classResolver"; import { MetaString } from "../meta/MetaString"; +import { FieldInfo, TypeMeta } from '../meta/TypeMeta'; -// Ensure MetaString methods are correctly implemented const computeMetaInformation = (description: any) => { const metaInfo = JSON.stringify(description); return MetaString.encode(metaInfo); @@ -81,10 +81,16 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const options = this.description.options; const expectHash = computeStructHash(this.description); const metaInformation = Buffer.from(computeMetaInformation(this.description)); + const decodedInformation = decodeMetaInformation(metaInformation); + const fields = Object.entries(options.props).map(([key, value]) => { + return new FieldInfo(key, value.type); + }); + const binary = TypeMeta.from_fields(256, fields).to_bytes(); return ` ${this.builder.writer.int32(expectHash)}; ${this.builder.writer.buffer(`Buffer.from("${metaInformation.toString("base64")}", "base64")`)}; + ${this.builder.writer.buffer(`Buffer.from("${binary}","base64")`)}; ${Object.entries(options.props).sort().map(([key, inner]) => { const InnerGeneratorClass = CodegenRegistry.get(inner.type); if (!InnerGeneratorClass) { @@ -100,8 +106,12 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const options = this.description.options; const expectHash = computeStructHash(this.description); const encodedMetaInformation = computeMetaInformation(this.description); + const encodedPropsInformation = computeMetaInformation(options.props); + const metaInformation = decodeMetaInformation(encodedMetaInformation); const result = this.scope.uniqueName("result"); const pass = this.builder.reader.int32(); + // const handler = this.scope.declare("handler",""); + return ` if (${this.builder.reader.int32()} !== ${expectHash}) { throw new Error("got ${this.builder.reader.int32()} validate hash failed: ${this.safeTag()}. expect ${expectHash}"); @@ -125,8 +135,6 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { `; } - // /8 /7 /20 % 2 - // is there a ratio from length / deserializer private safeTag() { return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag); } diff --git a/javascript/packages/fury/lib/meta/TypeMeta.ts b/javascript/packages/fury/lib/meta/TypeMeta.ts new file mode 100644 index 0000000000..1096ba5065 --- /dev/null +++ b/javascript/packages/fury/lib/meta/TypeMeta.ts @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { BinaryWriter } from "../writer"; +import { BinaryReader } from "../reader"; +import { ObjectTypeDescription, Type } from "../description"; +import { MetaString } from "./MetaString"; + +enum Encoding { + Utf8, + AllToLowerSpecial, + LowerUpperDigitSpecial, +} + +class FieldInfo { + constructor(private field_name: string, private field_id: number) { + } + + static u8_to_encoding(value: number) { + switch (value) { + case 0x00: + return Encoding.Utf8; + case 0x01: + return Encoding.AllToLowerSpecial; + case 0x02: + return Encoding.LowerUpperDigitSpecial; + } + } + + static uint8ArrayToBinary(array: Uint8Array): string { + return Array.from(array) + .map(byte => byte.toString(2).padStart(8, "0")) + .join(""); + } + + static from_bytes(reader: BinaryReader) { + const header = reader.uint8(); + const encoding = this.u8_to_encoding((header & 0b11000) >> 3); + let size = (header & 0b11100000) >> 5; + size = (size === 0b111) ? reader.varInt32() + 7 : size; + const type_id = reader.int16(); + const field_name = MetaString.decode(reader.buffer(size)); + return new FieldInfo(field_name, type_id); + } + + to_bytes() { + const writer = new BinaryWriter({}); + const meta_string = MetaString.encode(this.field_name); + let header = 1 << 2; + const encoded = FieldInfo.uint8ArrayToBinary(meta_string); + const size = encoded.length; + header |= MetaString.inferBitsPerChar(this.field_name) << 3; + const big_size = size >= 7; + if (big_size) { + header |= 0b11100000; + writer.uint8(header); + writer.varInt32(size - 7); + } else { + header |= size << 5; + writer.uint8(header); + } + writer.int16(this.field_id); + writer.buffer(meta_string); + return writer.dump(); + } +} + +// Using classes to emulate struct methods in Rust +class TypeMetaLayer { + constructor(private type_id: number, private field_info: FieldInfo[]) { + } + + get_type_id() { + return this.type_id; + } + + get_field_info() { + return this.field_info; + } + + to_bytes() { + const writer = new BinaryWriter({}); + writer.varInt32(this.field_info.length); + writer.varInt32(this.type_id); + for (const field of this.field_info) { + writer.buffer(field.to_bytes()); + } + return writer.dump(); + } + + static from_bytes(reader: BinaryReader) { + const field_num = reader.varInt32(); + const type_id = reader.varInt32(); + const field_info = []; + for (let i = 0; i < field_num; i++) { + field_info.push(FieldInfo.from_bytes(reader)); + } + return new TypeMetaLayer(type_id, field_info); + } +} + +class TypeMeta { + constructor(private hash: bigint, private layers: TypeMetaLayer[]) { + } + + get_field_info() { + return this.layers[0].get_field_info(); + } + + get_type_id() { + return this.layers[0].get_type_id(); + } + + static from_fields(type_id: number, field_info: FieldInfo[]) { + return new TypeMeta(BigInt(0), [new TypeMetaLayer(type_id, field_info)]); + } + + static from_bytes(reader: BinaryReader) { + const header = reader.uint64(); + const hash = header >> BigInt(8); + const layer_count = header & BigInt(0b1111); + const layers = []; + for (let i = 0; i < layer_count; i++) { + layers.push(TypeMetaLayer.from_bytes(reader)); + } + return new TypeMeta(hash, layers); + } + + to_bytes() { + const writer = new BinaryWriter({}); + writer.uint64(BigInt((this.hash << BigInt(8)) | BigInt((this.layers.length & 0b1111)))); + for (const layer of this.layers) { + writer.buffer(layer.to_bytes()); + } + return writer.dump(); + } +} diff --git a/javascript/test/typeMeta.test.ts b/javascript/test/typeMeta.test.ts new file mode 100644 index 0000000000..09f7cd6b6c --- /dev/null +++ b/javascript/test/typeMeta.test.ts @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fury, { TypeDescription, InternalSerializerType, ObjectTypeDescription, Type } from '../packages/fury/index'; +import { describe, expect, test } from '@jest/globals'; + +describe('TypeMeta', () => { + test('should TypeMeta work', () => { + const c: ObjectTypeDescription = Type.object("hello", { + a: Type.string(), + b: Type.int32(), + }); + const fields = Object.entries(c.options.props).map(([key, value]) => { + return new FieldInfo(key, value.type); + }); + const binary = TypeMeta.from_fields(256, fields).to_bytes(); + }); +}); + +