From 31ab26441163024c9982fda45323b16d4b822fd2 Mon Sep 17 00:00:00 2001 From: FORCHA PEARL Date: Thu, 1 Aug 2024 14:21:58 +0100 Subject: [PATCH 1/9] meta type decoding --- javascript/packages/fury/lib/gen/object.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index d9ddf7c639..da128897ca 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -102,10 +102,14 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const encodedMetaInformation = computeMetaInformation(this.description); const result = this.scope.uniqueName("result"); const pass = this.builder.reader.int32(); + const handler = this.scope.declare("handler",""); return ` if (${this.builder.reader.int32()} !== ${expectHash}) { throw new Error("got ${this.builder.reader.int32()} validate hash failed: ${this.safeTag()}. expect ${expectHash}"); } + if(${handler} == ""){ + ${handler} = ${this.builder.classResolver.getSerializerByTag(decodeMetaInformation(encodedMetaInformation))}; + } const ${result} = { ${Object.entries(options.props).sort().map(([key]) => { return `${CodecBuilder.safePropName(key)}: null`; @@ -125,8 +129,6 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { `; } - // /8 /7 /20 % 2 - // is there a ratio from length / deserializer private safeTag() { return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag); } From ce41f3e90fddffc919ba515fa09d171e1c7b023f Mon Sep 17 00:00:00 2001 From: FORCHA PEARL Date: Wed, 7 Aug 2024 22:57:13 +0100 Subject: [PATCH 2/9] Encoded props parameter --- javascript/packages/fury/lib/gen/object.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index da128897ca..61daeffed7 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -57,6 +57,7 @@ const computeStringHash = (str: string) => { return hash; }; + const computeStructHash = (description: TypeDescription) => { let hash = 17; for (const [, value] of Object.entries((description).options.props).sort()) { @@ -81,10 +82,13 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const options = this.description.options; const expectHash = computeStructHash(this.description); const metaInformation = Buffer.from(computeMetaInformation(this.description)); + const decodedInformation = decodeMetaInformation(metaInformation); + const propsInformation = Buffer.from(computeMetaInformation(options.props)); return ` ${this.builder.writer.int32(expectHash)}; ${this.builder.writer.buffer(`Buffer.from("${metaInformation.toString("base64")}", "base64")`)}; + ${this.builder.writer.buffer(`Buffer.from("${propsInformation.toString("base64")}", "base64")`)} ${Object.entries(options.props).sort().map(([key, inner]) => { const InnerGeneratorClass = CodegenRegistry.get(inner.type); if (!InnerGeneratorClass) { @@ -100,16 +104,16 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const options = this.description.options; const expectHash = computeStructHash(this.description); const encodedMetaInformation = computeMetaInformation(this.description); + const encodedPropsInformation = computeMetaInformation(options.props); + const metaInformation = decodeMetaInformation(encodedMetaInformation); const result = this.scope.uniqueName("result"); const pass = this.builder.reader.int32(); - const handler = this.scope.declare("handler",""); + //const handler = this.scope.declare("handler",""); + return ` if (${this.builder.reader.int32()} !== ${expectHash}) { throw new Error("got ${this.builder.reader.int32()} validate hash failed: ${this.safeTag()}. expect ${expectHash}"); } - if(${handler} == ""){ - ${handler} = ${this.builder.classResolver.getSerializerByTag(decodeMetaInformation(encodedMetaInformation))}; - } const ${result} = { ${Object.entries(options.props).sort().map(([key]) => { return `${CodecBuilder.safePropName(key)}: null`; @@ -117,6 +121,7 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { }; ${this.maybeReference(result, refState)} ${this.builder.reader.buffer(encodedMetaInformation.byteLength)} + ${this.builder.reader.buffer(encodedPropsInformation.byteLength)} ${Object.entries(options.props).sort().map(([key, inner]) => { const InnerGeneratorClass = CodegenRegistry.get(inner.type); if (!InnerGeneratorClass) { @@ -129,6 +134,7 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { `; } + private safeTag() { return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag); } From a49ed3ff76572c54cfa58e249406c71fee86e00f Mon Sep 17 00:00:00 2001 From: FORCHA PEARL Date: Wed, 14 Aug 2024 08:20:53 +0100 Subject: [PATCH 3/9] reverted Changes --- javascript/packages/fury/lib/gen/object.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index 61daeffed7..a99b8fea73 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -82,13 +82,10 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const options = this.description.options; const expectHash = computeStructHash(this.description); const metaInformation = Buffer.from(computeMetaInformation(this.description)); - const decodedInformation = decodeMetaInformation(metaInformation); - const propsInformation = Buffer.from(computeMetaInformation(options.props)); return ` ${this.builder.writer.int32(expectHash)}; ${this.builder.writer.buffer(`Buffer.from("${metaInformation.toString("base64")}", "base64")`)}; - ${this.builder.writer.buffer(`Buffer.from("${propsInformation.toString("base64")}", "base64")`)} ${Object.entries(options.props).sort().map(([key, inner]) => { const InnerGeneratorClass = CodegenRegistry.get(inner.type); if (!InnerGeneratorClass) { @@ -104,8 +101,6 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const options = this.description.options; const expectHash = computeStructHash(this.description); const encodedMetaInformation = computeMetaInformation(this.description); - const encodedPropsInformation = computeMetaInformation(options.props); - const metaInformation = decodeMetaInformation(encodedMetaInformation); const result = this.scope.uniqueName("result"); const pass = this.builder.reader.int32(); //const handler = this.scope.declare("handler",""); @@ -121,7 +116,6 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { }; ${this.maybeReference(result, refState)} ${this.builder.reader.buffer(encodedMetaInformation.byteLength)} - ${this.builder.reader.buffer(encodedPropsInformation.byteLength)} ${Object.entries(options.props).sort().map(([key, inner]) => { const InnerGeneratorClass = CodegenRegistry.get(inner.type); if (!InnerGeneratorClass) { From 22b21f224ee374fb0b83afd4960e34b0f42a7b30 Mon Sep 17 00:00:00 2001 From: FORCHA PEARL Date: Wed, 14 Aug 2024 08:22:30 +0100 Subject: [PATCH 4/9] removed unused constant --- javascript/packages/fury/lib/gen/object.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index a99b8fea73..39501ba0ec 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -103,7 +103,6 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const encodedMetaInformation = computeMetaInformation(this.description); const result = this.scope.uniqueName("result"); const pass = this.builder.reader.int32(); - //const handler = this.scope.declare("handler",""); return ` if (${this.builder.reader.int32()} !== ${expectHash}) { From 9215d65c7c4337102cc7eb1b0dd4db6679e5a2c5 Mon Sep 17 00:00:00 2001 From: FORCHA PEARL Date: Wed, 14 Aug 2024 08:27:07 +0100 Subject: [PATCH 5/9] lint fixing --- javascript/packages/fury/lib/gen/object.ts | 251 +++++++++++---------- 1 file changed, 132 insertions(+), 119 deletions(-) diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index 39501ba0ec..b4afe153be 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -17,142 +17,155 @@ * under the License. */ -import { InternalSerializerType, MaxInt32 } from "../type"; -import { Scope } from "./scope"; -import { CodecBuilder } from "./builder"; -import { ObjectTypeDescription, TypeDescription } from "../description"; -import { fromString } from "../platformBuffer"; -import { CodegenRegistry } from "./router"; -import { BaseSerializerGenerator, RefState } from "./serializer"; -import SerializerResolver from "../classResolver"; -import { MetaString } from "../meta/MetaString"; - -// Ensure MetaString methods are correctly implemented -const computeMetaInformation = (description: any) => { - const metaInfo = JSON.stringify(description); - return MetaString.encode(metaInfo); -}; - -const decodeMetaInformation = (encodedMetaInfo: Uint8Array) => { - return MetaString.decode(encodedMetaInfo); -}; - -function computeFieldHash(hash: number, id: number): number { - let newHash = (hash) * 31 + (id); - while (newHash >= MaxInt32) { - newHash = Math.floor(newHash / 7); - } - return newHash; -} - -const computeStringHash = (str: string) => { - const bytes = fromString(str); - let hash = 17; - bytes.forEach((b) => { - hash = hash * 31 + b; - while (hash >= MaxInt32) { - hash = Math.floor(hash / 7); +export class MetaString { + static LOWER_SPECIAL = 5; + static LOWER_UPPER_DIGIT_SPECIAL = 6; + static UTF_8 = 8; + + // Encode function that infers the bits per character + static encode(str: string): Uint8Array { + const bitsPerChar = MetaString.inferBitsPerChar(str); + const totalBits = str.length * bitsPerChar + 8; // Adjusted for metadata bits + const byteLength = Math.ceil(totalBits / 8); + const bytes = new Uint8Array(byteLength); + let currentBit = 8; // Start after the first 8 metadata bits + + for (const char of str) { + const value = bitsPerChar === MetaString.LOWER_SPECIAL + ? MetaString.charToValueLowerSpecial(char) + : bitsPerChar === MetaString.LOWER_UPPER_DIGIT_SPECIAL + ? MetaString.charToValueLowerUpperDigitSpecial(char) + : MetaString.charToValueUTF8(char); + + for (let i = bitsPerChar - 1; i >= 0; i--) { + if ((value & (1 << i)) !== 0) { + const bytePos = Math.floor(currentBit / 8); + const bitPos = currentBit % 8; + + if (bytePos >= byteLength) { + throw new RangeError("Offset is outside the bounds of the DataView"); + } + bytes[bytePos] |= (1 << (7 - bitPos)); + } + currentBit++; + } } - }); - return hash; -}; + // Store bitsPerChar in the first byte + bytes[0] = bitsPerChar; -const computeStructHash = (description: TypeDescription) => { - let hash = 17; - for (const [, value] of Object.entries((description).options.props).sort()) { - let id = SerializerResolver.getTypeIdByInternalSerializerType(value.type); - if (value.type === InternalSerializerType.OBJECT) { - id = computeStringHash((value).options.tag); - } - hash = computeFieldHash(hash, id); + return bytes; } - return hash; -}; - -class ObjectSerializerGenerator extends BaseSerializerGenerator { - description: ObjectTypeDescription; - constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { - super(description, builder, scope); - this.description = description; - } + // Decoding function that extracts bits per character from the first byte + static decode(bytes: Uint8Array): string { + const bitsPerChar = bytes[0] & 0x0F; + const totalBits = (bytes.length * 8); // Adjusted for metadata bits + const chars: string[] = []; + let currentBit = 8; // Start after the first 8 metadata bits + + while (currentBit < totalBits) { + let value = 0; + for (let i = 0; i < bitsPerChar; i++) { + const bytePos = Math.floor(currentBit / 8); + const bitPos = currentBit % 8; + + if (bytePos >= bytes.length) { + throw new RangeError("Offset is outside the bounds of the DataView"); + } - writeStmt(accessor: string): string { - const options = this.description.options; - const expectHash = computeStructHash(this.description); - const metaInformation = Buffer.from(computeMetaInformation(this.description)); - - return ` - ${this.builder.writer.int32(expectHash)}; - ${this.builder.writer.buffer(`Buffer.from("${metaInformation.toString("base64")}", "base64")`)}; - ${Object.entries(options.props).sort().map(([key, inner]) => { - const InnerGeneratorClass = CodegenRegistry.get(inner.type); - if (!InnerGeneratorClass) { - throw new Error(`${inner.type} generator not exists`); + if (bytes[bytePos] & (1 << (7 - bitPos))) { + value |= (1 << (bitsPerChar - i - 1)); } - const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope); - return innerGenerator.toWriteEmbed(`${accessor}${CodecBuilder.safePropAccessor(key)}`); - }).join(";\n")} - `; - } + currentBit++; + } - readStmt(accessor: (expr: string) => string, refState: RefState): string { - const options = this.description.options; - const expectHash = computeStructHash(this.description); - const encodedMetaInformation = computeMetaInformation(this.description); - const result = this.scope.uniqueName("result"); - const pass = this.builder.reader.int32(); + chars.push(bitsPerChar === MetaString.LOWER_SPECIAL + ? MetaString.valueToCharLowerSpecial(value) + : bitsPerChar === MetaString.LOWER_UPPER_DIGIT_SPECIAL + ? MetaString.valueToCharLowerUpperDigitSpecial(value) + : MetaString.valueToCharUTF8(value)); + } - return ` - if (${this.builder.reader.int32()} !== ${expectHash}) { - throw new Error("got ${this.builder.reader.int32()} validate hash failed: ${this.safeTag()}. expect ${expectHash}"); - } - const ${result} = { - ${Object.entries(options.props).sort().map(([key]) => { - return `${CodecBuilder.safePropName(key)}: null`; - }).join(",\n")} - }; - ${this.maybeReference(result, refState)} - ${this.builder.reader.buffer(encodedMetaInformation.byteLength)} - ${Object.entries(options.props).sort().map(([key, inner]) => { - const InnerGeneratorClass = CodegenRegistry.get(inner.type); - if (!InnerGeneratorClass) { - throw new Error(`${inner.type} generator not exists`); - } - const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope); - return innerGenerator.toReadEmbed(expr => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`); - }).join(";\n")} - ${accessor(result)} - `; + return chars.join(""); } + // Infer bits per character based on the content of the string + static inferBitsPerChar(str: string): number { + if (/^[a-z._$|]+$/.test(str)) { + return MetaString.LOWER_SPECIAL; + } else if (/^[a-zA-Z0-9._]+$/.test(str)) { + return MetaString.LOWER_UPPER_DIGIT_SPECIAL; + } + return MetaString.UTF_8; // Default to UTF-8 + } - private safeTag() { - return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag); + // Convert a character to its value for LOWER_SPECIAL encoding + static charToValueLowerSpecial(char: string): number { + if (char >= "a" && char <= "z") { + return char.charCodeAt(0) - "a".charCodeAt(0); + } else if (char === ".") { + return 26; + } else if (char === "_") { + return 27; + } else if (char === "$") { + return 28; + } else if (char === "|") { + return 29; + } + throw new Error(`Invalid character for LOWER_SPECIAL: ${char}`); } - toReadEmbed(accessor: (expr: string) => string, excludeHead?: boolean, refState?: RefState): string { - const name = this.scope.declare( - "tag_ser", - `fury.classResolver.getSerializerByTag("${this.safeTag()}")` - ); - if (!excludeHead) { - return accessor(`${name}.read()`); + static valueToCharLowerSpecial(value: number): string { + if (value >= 0 && value <= 25) { + return String.fromCharCode("a".charCodeAt(0) + value); + } else if (value === 26) { + return "."; + } else if (value === 27) { + return "_"; + } else if (value === 28) { + return "$"; + } else if (value === 29) { + return "|"; } - return accessor(`${name}.readInner(${refState!.toConditionExpr()})`); + throw new Error(`Invalid value for LOWER_SPECIAL: ${value}`); } - toWriteEmbed(accessor: string, excludeHead?: boolean): string { - const name = this.scope.declare( - "tag_ser", - `fury.classResolver.getSerializerByTag("${this.safeTag()}")` - ); - if (!excludeHead) { - return `${name}.write(${accessor})`; + static charToValueLowerUpperDigitSpecial(char: string): number { + if (char >= "a" && char <= "z") { + return char.charCodeAt(0) - "a".charCodeAt(0); + } else if (char >= "A" && char <= "Z") { + return char.charCodeAt(0) - "A".charCodeAt(0) + 26; + } else if (char >= "0" && char <= "9") { + return char.charCodeAt(0) - "0".charCodeAt(0) + 52; + } else if (char === ".") { + return 62; + } else if (char === "_") { + return 63; } - return `${name}.writeInner(${accessor})`; + throw new Error(`Invalid character for LOWER_UPPER_DIGIT_SPECIAL: ${char}`); + } + + static valueToCharLowerUpperDigitSpecial(value: number): string { + if (value >= 0 && value <= 25) { + return String.fromCharCode("a".charCodeAt(0) + value); + } else if (value >= 26 && value <= 51) { + return String.fromCharCode("A".charCodeAt(0) + value - 26); + } else if (value >= 52 && value <= 61) { + return String.fromCharCode("0".charCodeAt(0) + value - 52); + } else if (value === 62) { + return "."; + } else if (value === 63) { + return "_"; + } + throw new Error(`Invalid value for LOWER_UPPER_DIGIT_SPECIAL: ${value}`); + } + + static charToValueUTF8(char: string): number { + return char.charCodeAt(0); } -} -CodegenRegistry.register(InternalSerializerType.OBJECT, ObjectSerializerGenerator); + static valueToCharUTF8(value: number): string { + return String.fromCharCode(value); + } +} From 8b3d79f83f351ecc0839ebd12e5b40189771d7e0 Mon Sep 17 00:00:00 2001 From: FORCHA PEARL Date: Wed, 14 Aug 2024 08:29:34 +0100 Subject: [PATCH 6/9] reverted Changes --- javascript/packages/fury/lib/gen/object.ts | 250 ++++++++++----------- 1 file changed, 117 insertions(+), 133 deletions(-) diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index b4afe153be..7196625fbf 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -17,155 +17,139 @@ * under the License. */ -export class MetaString { - static LOWER_SPECIAL = 5; - static LOWER_UPPER_DIGIT_SPECIAL = 6; - static UTF_8 = 8; - - // Encode function that infers the bits per character - static encode(str: string): Uint8Array { - const bitsPerChar = MetaString.inferBitsPerChar(str); - const totalBits = str.length * bitsPerChar + 8; // Adjusted for metadata bits - const byteLength = Math.ceil(totalBits / 8); - const bytes = new Uint8Array(byteLength); - let currentBit = 8; // Start after the first 8 metadata bits - - for (const char of str) { - const value = bitsPerChar === MetaString.LOWER_SPECIAL - ? MetaString.charToValueLowerSpecial(char) - : bitsPerChar === MetaString.LOWER_UPPER_DIGIT_SPECIAL - ? MetaString.charToValueLowerUpperDigitSpecial(char) - : MetaString.charToValueUTF8(char); - - for (let i = bitsPerChar - 1; i >= 0; i--) { - if ((value & (1 << i)) !== 0) { - const bytePos = Math.floor(currentBit / 8); - const bitPos = currentBit % 8; +import { InternalSerializerType, MaxInt32 } from "../type"; +import { Scope } from "./scope"; +import { CodecBuilder } from "./builder"; +import { ObjectTypeDescription, TypeDescription } from "../description"; +import { fromString } from "../platformBuffer"; +import { CodegenRegistry } from "./router"; +import { BaseSerializerGenerator, RefState } from "./serializer"; +import SerializerResolver from "../classResolver"; +import { MetaString } from "../meta/MetaString"; + +// Ensure MetaString methods are correctly implemented +const computeMetaInformation = (description: any) => { + const metaInfo = JSON.stringify(description); + return MetaString.encode(metaInfo); +}; + +const decodeMetaInformation = (encodedMetaInfo: Uint8Array) => { + return MetaString.decode(encodedMetaInfo); +}; + +function computeFieldHash(hash: number, id: number): number { + let newHash = (hash) * 31 + (id); + while (newHash >= MaxInt32) { + newHash = Math.floor(newHash / 7); + } + return newHash; +} - if (bytePos >= byteLength) { - throw new RangeError("Offset is outside the bounds of the DataView"); - } - bytes[bytePos] |= (1 << (7 - bitPos)); - } - currentBit++; - } +const computeStringHash = (str: string) => { + const bytes = fromString(str); + let hash = 17; + bytes.forEach((b) => { + hash = hash * 31 + b; + while (hash >= MaxInt32) { + hash = Math.floor(hash / 7); } - - // Store bitsPerChar in the first byte - bytes[0] = bitsPerChar; - - return bytes; + }); + return hash; +}; + +const computeStructHash = (description: TypeDescription) => { + let hash = 17; + for (const [, value] of Object.entries((description).options.props).sort()) { + let id = SerializerResolver.getTypeIdByInternalSerializerType(value.type); + if (value.type === InternalSerializerType.OBJECT) { + id = computeStringHash((value).options.tag); + } + hash = computeFieldHash(hash, id); } + return hash; +}; - // Decoding function that extracts bits per character from the first byte - static decode(bytes: Uint8Array): string { - const bitsPerChar = bytes[0] & 0x0F; - const totalBits = (bytes.length * 8); // Adjusted for metadata bits - const chars: string[] = []; - let currentBit = 8; // Start after the first 8 metadata bits +class ObjectSerializerGenerator extends BaseSerializerGenerator { + description: ObjectTypeDescription; - while (currentBit < totalBits) { - let value = 0; - for (let i = 0; i < bitsPerChar; i++) { - const bytePos = Math.floor(currentBit / 8); - const bitPos = currentBit % 8; - - if (bytePos >= bytes.length) { - throw new RangeError("Offset is outside the bounds of the DataView"); - } - - if (bytes[bytePos] & (1 << (7 - bitPos))) { - value |= (1 << (bitsPerChar - i - 1)); - } - currentBit++; - } - - chars.push(bitsPerChar === MetaString.LOWER_SPECIAL - ? MetaString.valueToCharLowerSpecial(value) - : bitsPerChar === MetaString.LOWER_UPPER_DIGIT_SPECIAL - ? MetaString.valueToCharLowerUpperDigitSpecial(value) - : MetaString.valueToCharUTF8(value)); - } - - return chars.join(""); + constructor(description: TypeDescription, builder: CodecBuilder, scope: Scope) { + super(description, builder, scope); + this.description = description; } - // Infer bits per character based on the content of the string - static inferBitsPerChar(str: string): number { - if (/^[a-z._$|]+$/.test(str)) { - return MetaString.LOWER_SPECIAL; - } else if (/^[a-zA-Z0-9._]+$/.test(str)) { - return MetaString.LOWER_UPPER_DIGIT_SPECIAL; - } - return MetaString.UTF_8; // Default to UTF-8 + writeStmt(accessor: string): string { + const options = this.description.options; + const expectHash = computeStructHash(this.description); + const metaInformation = Buffer.from(computeMetaInformation(this.description)); + + return ` + ${this.builder.writer.int32(expectHash)}; + ${this.builder.writer.buffer(`Buffer.from("${metaInformation.toString("base64")}", "base64")`)}; + ${Object.entries(options.props).sort().map(([key, inner]) => { + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope); + return innerGenerator.toWriteEmbed(`${accessor}${CodecBuilder.safePropAccessor(key)}`); + }).join(";\n")} + `; } - // Convert a character to its value for LOWER_SPECIAL encoding - static charToValueLowerSpecial(char: string): number { - if (char >= "a" && char <= "z") { - return char.charCodeAt(0) - "a".charCodeAt(0); - } else if (char === ".") { - return 26; - } else if (char === "_") { - return 27; - } else if (char === "$") { - return 28; - } else if (char === "|") { - return 29; - } - throw new Error(`Invalid character for LOWER_SPECIAL: ${char}`); + readStmt(accessor: (expr: string) => string, refState: RefState): string { + const options = this.description.options; + const expectHash = computeStructHash(this.description); + const encodedMetaInformation = computeMetaInformation(this.description); + const result = this.scope.uniqueName("result"); + const pass = this.builder.reader.int32(); + return ` + if (${this.builder.reader.int32()} !== ${expectHash}) { + throw new Error("got ${this.builder.reader.int32()} validate hash failed: ${this.safeTag()}. expect ${expectHash}"); + } + const ${result} = { + ${Object.entries(options.props).sort().map(([key]) => { + return `${CodecBuilder.safePropName(key)}: null`; + }).join(",\n")} + }; + ${this.maybeReference(result, refState)} + ${this.builder.reader.buffer(encodedMetaInformation.byteLength)} + ${Object.entries(options.props).sort().map(([key, inner]) => { + const InnerGeneratorClass = CodegenRegistry.get(inner.type); + if (!InnerGeneratorClass) { + throw new Error(`${inner.type} generator not exists`); + } + const innerGenerator = new InnerGeneratorClass(inner, this.builder, this.scope); + return innerGenerator.toReadEmbed(expr => `${result}${CodecBuilder.safePropAccessor(key)} = ${expr}`); + }).join(";\n")} + ${accessor(result)} + `; } - static valueToCharLowerSpecial(value: number): string { - if (value >= 0 && value <= 25) { - return String.fromCharCode("a".charCodeAt(0) + value); - } else if (value === 26) { - return "."; - } else if (value === 27) { - return "_"; - } else if (value === 28) { - return "$"; - } else if (value === 29) { - return "|"; - } - throw new Error(`Invalid value for LOWER_SPECIAL: ${value}`); + private safeTag() { + return CodecBuilder.replaceBackslashAndQuote(this.description.options.tag); } - static charToValueLowerUpperDigitSpecial(char: string): number { - if (char >= "a" && char <= "z") { - return char.charCodeAt(0) - "a".charCodeAt(0); - } else if (char >= "A" && char <= "Z") { - return char.charCodeAt(0) - "A".charCodeAt(0) + 26; - } else if (char >= "0" && char <= "9") { - return char.charCodeAt(0) - "0".charCodeAt(0) + 52; - } else if (char === ".") { - return 62; - } else if (char === "_") { - return 63; + toReadEmbed(accessor: (expr: string) => string, excludeHead?: boolean, refState?: RefState): string { + const name = this.scope.declare( + "tag_ser", + `fury.classResolver.getSerializerByTag("${this.safeTag()}")` + ); + if (!excludeHead) { + return accessor(`${name}.read()`); } - throw new Error(`Invalid character for LOWER_UPPER_DIGIT_SPECIAL: ${char}`); + return accessor(`${name}.readInner(${refState!.toConditionExpr()})`); } - static valueToCharLowerUpperDigitSpecial(value: number): string { - if (value >= 0 && value <= 25) { - return String.fromCharCode("a".charCodeAt(0) + value); - } else if (value >= 26 && value <= 51) { - return String.fromCharCode("A".charCodeAt(0) + value - 26); - } else if (value >= 52 && value <= 61) { - return String.fromCharCode("0".charCodeAt(0) + value - 52); - } else if (value === 62) { - return "."; - } else if (value === 63) { - return "_"; + toWriteEmbed(accessor: string, excludeHead?: boolean): string { + const name = this.scope.declare( + "tag_ser", + `fury.classResolver.getSerializerByTag("${this.safeTag()}")` + ); + if (!excludeHead) { + return `${name}.write(${accessor})`; } - throw new Error(`Invalid value for LOWER_UPPER_DIGIT_SPECIAL: ${value}`); - } - - static charToValueUTF8(char: string): number { - return char.charCodeAt(0); - } - - static valueToCharUTF8(value: number): string { - return String.fromCharCode(value); + return `${name}.writeInner(${accessor})`; } } + +CodegenRegistry.register(InternalSerializerType.OBJECT, ObjectSerializerGenerator); From 14e96dfa911e335f87f4c0ef596e379517ed4271 Mon Sep 17 00:00:00 2001 From: FORCHA PEARL Date: Wed, 14 Aug 2024 08:32:06 +0100 Subject: [PATCH 7/9] Type Meta Encodng --- javascript/packages/fury/lib/meta/TypeMeta.ts | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 javascript/packages/fury/lib/meta/TypeMeta.ts diff --git a/javascript/packages/fury/lib/meta/TypeMeta.ts b/javascript/packages/fury/lib/meta/TypeMeta.ts new file mode 100644 index 0000000000..1096ba5065 --- /dev/null +++ b/javascript/packages/fury/lib/meta/TypeMeta.ts @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { BinaryWriter } from "../writer"; +import { BinaryReader } from "../reader"; +import { ObjectTypeDescription, Type } from "../description"; +import { MetaString } from "./MetaString"; + +enum Encoding { + Utf8, + AllToLowerSpecial, + LowerUpperDigitSpecial, +} + +class FieldInfo { + constructor(private field_name: string, private field_id: number) { + } + + static u8_to_encoding(value: number) { + switch (value) { + case 0x00: + return Encoding.Utf8; + case 0x01: + return Encoding.AllToLowerSpecial; + case 0x02: + return Encoding.LowerUpperDigitSpecial; + } + } + + static uint8ArrayToBinary(array: Uint8Array): string { + return Array.from(array) + .map(byte => byte.toString(2).padStart(8, "0")) + .join(""); + } + + static from_bytes(reader: BinaryReader) { + const header = reader.uint8(); + const encoding = this.u8_to_encoding((header & 0b11000) >> 3); + let size = (header & 0b11100000) >> 5; + size = (size === 0b111) ? reader.varInt32() + 7 : size; + const type_id = reader.int16(); + const field_name = MetaString.decode(reader.buffer(size)); + return new FieldInfo(field_name, type_id); + } + + to_bytes() { + const writer = new BinaryWriter({}); + const meta_string = MetaString.encode(this.field_name); + let header = 1 << 2; + const encoded = FieldInfo.uint8ArrayToBinary(meta_string); + const size = encoded.length; + header |= MetaString.inferBitsPerChar(this.field_name) << 3; + const big_size = size >= 7; + if (big_size) { + header |= 0b11100000; + writer.uint8(header); + writer.varInt32(size - 7); + } else { + header |= size << 5; + writer.uint8(header); + } + writer.int16(this.field_id); + writer.buffer(meta_string); + return writer.dump(); + } +} + +// Using classes to emulate struct methods in Rust +class TypeMetaLayer { + constructor(private type_id: number, private field_info: FieldInfo[]) { + } + + get_type_id() { + return this.type_id; + } + + get_field_info() { + return this.field_info; + } + + to_bytes() { + const writer = new BinaryWriter({}); + writer.varInt32(this.field_info.length); + writer.varInt32(this.type_id); + for (const field of this.field_info) { + writer.buffer(field.to_bytes()); + } + return writer.dump(); + } + + static from_bytes(reader: BinaryReader) { + const field_num = reader.varInt32(); + const type_id = reader.varInt32(); + const field_info = []; + for (let i = 0; i < field_num; i++) { + field_info.push(FieldInfo.from_bytes(reader)); + } + return new TypeMetaLayer(type_id, field_info); + } +} + +class TypeMeta { + constructor(private hash: bigint, private layers: TypeMetaLayer[]) { + } + + get_field_info() { + return this.layers[0].get_field_info(); + } + + get_type_id() { + return this.layers[0].get_type_id(); + } + + static from_fields(type_id: number, field_info: FieldInfo[]) { + return new TypeMeta(BigInt(0), [new TypeMetaLayer(type_id, field_info)]); + } + + static from_bytes(reader: BinaryReader) { + const header = reader.uint64(); + const hash = header >> BigInt(8); + const layer_count = header & BigInt(0b1111); + const layers = []; + for (let i = 0; i < layer_count; i++) { + layers.push(TypeMetaLayer.from_bytes(reader)); + } + return new TypeMeta(hash, layers); + } + + to_bytes() { + const writer = new BinaryWriter({}); + writer.uint64(BigInt((this.hash << BigInt(8)) | BigInt((this.layers.length & 0b1111)))); + for (const layer of this.layers) { + writer.buffer(layer.to_bytes()); + } + return writer.dump(); + } +} From e8cf33f18d4f12d94b3271eb6603b2076abc865f Mon Sep 17 00:00:00 2001 From: FORCHA PEARL Date: Wed, 14 Aug 2024 16:25:50 +0100 Subject: [PATCH 8/9] Add files via upload --- javascript/test/typeMeta.test.ts | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 javascript/test/typeMeta.test.ts diff --git a/javascript/test/typeMeta.test.ts b/javascript/test/typeMeta.test.ts new file mode 100644 index 0000000000..09f7cd6b6c --- /dev/null +++ b/javascript/test/typeMeta.test.ts @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fury, { TypeDescription, InternalSerializerType, ObjectTypeDescription, Type } from '../packages/fury/index'; +import { describe, expect, test } from '@jest/globals'; + +describe('TypeMeta', () => { + test('should TypeMeta work', () => { + const c: ObjectTypeDescription = Type.object("hello", { + a: Type.string(), + b: Type.int32(), + }); + const fields = Object.entries(c.options.props).map(([key, value]) => { + return new FieldInfo(key, value.type); + }); + const binary = TypeMeta.from_fields(256, fields).to_bytes(); + }); +}); + + From 947689aff130176c5b6d564f0c85cac5a8c68a91 Mon Sep 17 00:00:00 2001 From: FORCHA PEARL Date: Fri, 16 Aug 2024 06:27:33 +0100 Subject: [PATCH 9/9] called Type Meta on options props --- javascript/packages/fury/lib/gen/object.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/javascript/packages/fury/lib/gen/object.ts b/javascript/packages/fury/lib/gen/object.ts index 7196625fbf..76fd5f5990 100644 --- a/javascript/packages/fury/lib/gen/object.ts +++ b/javascript/packages/fury/lib/gen/object.ts @@ -26,8 +26,8 @@ import { CodegenRegistry } from "./router"; import { BaseSerializerGenerator, RefState } from "./serializer"; import SerializerResolver from "../classResolver"; import { MetaString } from "../meta/MetaString"; +import { FieldInfo, TypeMeta } from '../meta/TypeMeta'; -// Ensure MetaString methods are correctly implemented const computeMetaInformation = (description: any) => { const metaInfo = JSON.stringify(description); return MetaString.encode(metaInfo); @@ -81,10 +81,16 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const options = this.description.options; const expectHash = computeStructHash(this.description); const metaInformation = Buffer.from(computeMetaInformation(this.description)); + const decodedInformation = decodeMetaInformation(metaInformation); + const fields = Object.entries(options.props).map(([key, value]) => { + return new FieldInfo(key, value.type); + }); + const binary = TypeMeta.from_fields(256, fields).to_bytes(); return ` ${this.builder.writer.int32(expectHash)}; ${this.builder.writer.buffer(`Buffer.from("${metaInformation.toString("base64")}", "base64")`)}; + ${this.builder.writer.buffer(`Buffer.from("${binary}","base64")`)}; ${Object.entries(options.props).sort().map(([key, inner]) => { const InnerGeneratorClass = CodegenRegistry.get(inner.type); if (!InnerGeneratorClass) { @@ -100,8 +106,12 @@ class ObjectSerializerGenerator extends BaseSerializerGenerator { const options = this.description.options; const expectHash = computeStructHash(this.description); const encodedMetaInformation = computeMetaInformation(this.description); + const encodedPropsInformation = computeMetaInformation(options.props); + const metaInformation = decodeMetaInformation(encodedMetaInformation); const result = this.scope.uniqueName("result"); const pass = this.builder.reader.int32(); + // const handler = this.scope.declare("handler",""); + return ` if (${this.builder.reader.int32()} !== ${expectHash}) { throw new Error("got ${this.builder.reader.int32()} validate hash failed: ${this.safeTag()}. expect ${expectHash}");