Skip to content

Commit b07d0f6

Browse files
mcdurdinmcdurdin
authored
Merge pull request #9691 from keymanapp/feat/common/9640-precompile-schemas
feat(common): precompile ajv schemas 🎺
2 parents a9ea0fb + b8b0e56 commit b07d0f6

25 files changed

+298
-156
lines changed

‎common/web/types/.eslintrc.cjs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ module.exports = {
88
"coverage/*",
99
"node_modules/*",
1010
"test/fixtures/*",
11+
"tools/*",
12+
"src/schemas/*"
1113
],
1214
overrides: [
1315
{

‎common/web/types/.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
src/schemas/
1+
src/schemas/
2+
obj/

‎common/web/types/build.sh

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,35 @@ function compile_schemas() {
4040
"$KEYMAN_ROOT/common/schemas/keyboard_info/keyboard_info.schema.json"
4141
)
4242

43+
rm -rf "$THIS_SCRIPT_PATH/obj/schemas"
44+
mkdir -p "$THIS_SCRIPT_PATH/obj/schemas"
4345
rm -rf "$THIS_SCRIPT_PATH/src/schemas"
4446
mkdir -p "$THIS_SCRIPT_PATH/src/schemas"
4547
cp "${schemas[@]}" "$THIS_SCRIPT_PATH/src/schemas/"
4648

4749
# TODO: use https://github.com/tc39/proposal-json-modules instead of this once it stablises
4850
for schema in "${schemas[@]}"; do
49-
local fn="$THIS_SCRIPT_PATH/src/schemas/$(basename "$schema" .json)"
51+
local schema_base="$(basename "$schema" .json)"
52+
local fn="$THIS_SCRIPT_PATH/src/schemas/$schema_base"
53+
local out="$THIS_SCRIPT_PATH/obj/schemas/$schema_base.validator.cjs"
54+
55+
# emit a .ts wrapper for the schema file
56+
57+
builder_echo "Compiling schema $schema_base.json"
5058
echo 'export default ' > "$fn.ts"
5159
cat "$fn.json" >> "$fn.ts"
60+
61+
# emit a compiled validator for the schema file
62+
63+
# While would seem obvious to just run 'ajv' directly here, somewhere node
64+
# is picking up the wrong path for the build and breaking the formats
65+
# imports. So it is essential to use `npm run` at this point, even though it
66+
# is painfully slower, at least until we figure out the path discrepancy.
67+
npm run build:schema -- -c ./tools/formats.cjs -s "$fn.json" --strict-types false -o "$out"
5268
done
69+
70+
# the validators now need to be compiled to esm
71+
node tools/schema-bundler.js
5372
}
5473

5574
function copy_cldr_imports() {

‎common/web/types/package.json

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
],
1717
"scripts": {
1818
"build": "tsc -b",
19+
"build:schema": "ajv compile",
1920
"lint": "eslint .",
2021
"test": "npm run lint && cd test && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha",
2122
"prepublishOnly": "npm run build"
@@ -27,7 +28,6 @@
2728
},
2829
"dependencies": {
2930
"@keymanapp/keyman-version": "*",
30-
"ajv": "^8.11.0",
3131
"restructure": "git+https://github.com/keymanapp/dependency-restructure.git#7a188a1e26f8f36a175d95b67ffece8702363dfc",
3232
"semver": "^7.5.2",
3333
"xml2js": "git+https://github.com/keymanapp/dependency-node-xml2js#535fe732dc408d697e0f847c944cc45f0baf0829"
@@ -39,6 +39,9 @@
3939
"@types/node": "^20.4.1",
4040
"@types/semver": "^7.3.12",
4141
"@types/xml2js": "^0.4.5",
42+
"ajv": "^8.12.0",
43+
"ajv-cli": "^5.0.0",
44+
"ajv-formats": "^2.1.1",
4245
"c8": "^7.12.0",
4346
"chai": "^4.3.4",
4447
"chalk": "^2.4.2",
@@ -74,6 +77,7 @@
7477
"src/ldml-keyboard/unicodeset-parser-api.ts",
7578
"src/keyman-touch-layout/keyman-touch-layout-file-writer.ts",
7679
"src/osk/osk.ts",
80+
"src/schemas/*",
7781
"test/"
7882
]
7983
}

‎common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-reader.ts

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
1-
import { default as AjvModule } from 'ajv';
2-
const Ajv = AjvModule.default; // The actual expected Ajv type.
31
import { TouchLayoutFile } from "./keyman-touch-layout-file.js";
4-
import Schemas from '../../src/schemas.js';
2+
import SchemaValidators from '../schema-validators.js';
53

64
export class TouchLayoutFileReader {
75
public read(source: Uint8Array): TouchLayoutFile {
@@ -69,11 +67,10 @@ export class TouchLayoutFileReader {
6967
}
7068

7169
public validate(source: TouchLayoutFile): void {
72-
const ajv = new Ajv();
73-
if(!ajv.validate(Schemas.touchLayoutClean, source))
70+
if(!SchemaValidators.touchLayoutClean(source))
7471
/* c8 ignore next 3 */
7572
{
76-
throw new Error(ajv.errorsText());
73+
throw new Error((<any>SchemaValidators.touchLayoutClean).errors);
7774
}
7875
}
7976

‎common/web/types/src/kpj/kpj-file-reader.ts

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
import * as xml2js from 'xml2js';
22
import { KPJFile, KPJFileProject } from './kpj-file.js';
3-
import { default as AjvModule } from 'ajv';
4-
const Ajv = AjvModule.default; // The actual expected Ajv type.
53
import { boxXmlArray } from '../util/util.js';
64
import { KeymanDeveloperProject, KeymanDeveloperProjectFile10, KeymanDeveloperProjectType } from './keyman-developer-project.js';
75
import { CompilerCallbacks } from '../util/compiler-interfaces.js';
8-
import Schemas from '../schemas.js';
6+
import SchemaValidators from '../schema-validators.js';
97

108
export class KPJFileReader {
119
constructor(private callbacks: CompilerCallbacks) {
@@ -35,13 +33,11 @@ export class KPJFileReader {
3533
}
3634

3735
public validate(source: KPJFile): void {
38-
const ajv = new Ajv();
39-
if(!ajv.validate(Schemas.kpj, source)) {
40-
const ajvLegacy = new Ajv();
41-
if(!ajvLegacy.validate(Schemas.kpj90, source)) {
36+
if(!SchemaValidators.kpj(source)) {
37+
if(!SchemaValidators.kpj90(source)) {
4238
// If the legacy schema also does not validate, then we will only report
4339
// the errors against the modern schema
44-
throw new Error(ajv.errorsText());
40+
throw new Error((<any>SchemaValidators.kpj).errors);
4541
}
4642
}
4743
}

‎common/web/types/src/kvk/kvks-file-reader.ts

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
import * as xml2js from 'xml2js';
22
import KVKSourceFile from './kvks-file.js';
3-
import { default as AjvModule } from 'ajv';
4-
const Ajv = AjvModule.default; // The actual expected Ajv type.
53
import { boxXmlArray } from '../util/util.js';
64
import { DEFAULT_KVK_FONT, VisualKeyboard, VisualKeyboardHeaderFlags, VisualKeyboardKey, VisualKeyboardKeyFlags, VisualKeyboardLegalShiftStates, VisualKeyboardShiftState } from './visual-keyboard.js';
75
import { USVirtualKeyCodes } from '../consts/virtual-key-constants.js';
86
import { BUILDER_KVK_HEADER_VERSION, KVK_HEADER_IDENTIFIER_BYTES } from './kvk-file.js';
9-
import Schemas from '../schemas.js';
7+
import SchemaValidators from '../schema-validators.js';
108

119

1210
export default class KVKSFileReader {
@@ -85,9 +83,8 @@ export default class KVKSFileReader {
8583
}
8684

8785
public validate(source: KVKSourceFile): void {
88-
const ajv = new Ajv();
89-
if(!ajv.validate(Schemas.kvks, source)) {
90-
throw new Error(ajv.errorsText());
86+
if(!SchemaValidators.kvks(source)) {
87+
throw new Error((<any>SchemaValidators.kvks).errorsText());
9188
}
9289
}
9390

‎common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
import * as xml2js from 'xml2js';
22
import { LDMLKeyboardXMLSourceFile, LKImport, ImportStatus } from './ldml-keyboard-xml.js';
3-
import { default as AjvModule } from 'ajv';
4-
const Ajv = AjvModule.default; // The actual expected Ajv type.
53
import { boxXmlArray } from '../util/util.js';
64
import { CompilerCallbacks } from '../util/compiler-interfaces.js';
75
import { constants } from '@keymanapp/ldml-keyboard-constants';
86
import { CommonTypesMessages } from '../util/common-events.js';
97
import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js';
10-
import Schemas from '../schemas.js';
8+
import SchemaValidators from '../schema-validators.js';
119

1210
interface NameAndProps {
1311
'$'?: any; // content
@@ -237,9 +235,8 @@ export class LDMLKeyboardXMLSourceFileReader {
237235
* @returns true if valid, false if invalid
238236
*/
239237
public validate(source: LDMLKeyboardXMLSourceFile | LDMLKeyboardTestDataXMLSourceFile): boolean {
240-
const ajv = new Ajv();
241-
if(!ajv.validate(Schemas.ldmlKeyboard3, source)) {
242-
for (let err of ajv.errors) {
238+
if(!SchemaValidators.ldmlKeyboard3(source)) {
239+
for (let err of (<any>SchemaValidators.ldmlKeyboard3).errors) {
243240
this.callbacks.reportMessage(CommonTypesMessages.Error_SchemaValidationError({
244241
instancePath: err.instancePath,
245242
keyword: err.keyword,

‎common/web/types/src/main.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,5 @@ export * as KeymanFileTypes from './util/file-types.js';
4747

4848
export * as Osk from './osk/osk.js';
4949

50-
export * as Schemas from './schemas.js';
50+
export * as Schemas from './schemas.js';
51+
export * as SchemaValidators from './schema-validators.js';

‎common/web/types/src/osk/osk.ts

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
import { TouchLayoutFile, TouchLayoutFlick, TouchLayoutKey, TouchLayoutPlatform, TouchLayoutSubKey } from "src/keyman-touch-layout/keyman-touch-layout-file.js";
22
import { VisualKeyboard } from "../kvk/visual-keyboard.js";
3-
import { default as AjvModule } from 'ajv';
4-
import Schemas from "../schemas.js";
5-
const Ajv = AjvModule.default; // The actual expected Ajv type.
3+
import SchemaValidators from "../schema-validators.js";
64

75
export interface StringRefUsage {
86
filename: string;
@@ -24,11 +22,10 @@ export interface StringResult {
2422
export type PuaMap = {[index:string]: string};
2523

2624
export function parseMapping(mapping: any) {
27-
const ajv = new Ajv();
28-
if(!ajv.validate(Schemas.displayMap, <any>mapping))
25+
if(!SchemaValidators.displayMap(<any>mapping))
2926
/* c8 ignore next 3 */
3027
{
31-
throw new Error(ajv.errorsText());
28+
throw new Error((<any>SchemaValidators.displayMap).errorsText());
3229
}
3330

3431
let map: PuaMap = {};

‎common/web/types/src/schema-validators.ts

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import kpj from './schemas/kpj.schema.validator.mjs';
2+
import kpj90 from './schemas/kpj-9.0.schema.validator.mjs';
3+
import kvks from './schemas/kvks.schema.validator.mjs';
4+
import ldmlKeyboard3 from './schemas/ldml-keyboard3.schema.validator.mjs';
5+
import ldmlKeyboardTest3 from './schemas/ldml-keyboardtest3.schema.validator.mjs';
6+
import displayMap from './schemas/displaymap.schema.validator.mjs';
7+
import touchLayoutClean from './schemas/keyman-touch-layout.clean.spec.validator.mjs';
8+
import touchLayout from './schemas/keyman-touch-layout.spec.validator.mjs';
9+
import keyboard_info from './schemas/keyboard_info.schema.validator.mjs';
10+
11+
// How to use: https://ajv.js.org/standalone.html#using-the-validation-function-s
12+
// See also existing uses (search for `SchemaValidators`) for examples.
13+
const SchemaValidators = {
14+
kpj,
15+
kpj90,
16+
kvks,
17+
ldmlKeyboard3,
18+
ldmlKeyboardTest3,
19+
displayMap,
20+
touchLayoutClean,
21+
touchLayout,
22+
keyboard_info,
23+
};
24+
25+
export default SchemaValidators;

‎common/web/types/test/helpers/index.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import path from "path";
2-
import fs from "fs";
1+
import * as path from "path";
2+
import * as fs from "fs";
33
import { fileURLToPath } from "url";
44

55
/**

‎common/web/types/test/tsconfig.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"outDir": "../build/test",
88
"baseUrl": ".",
99
"strictNullChecks": false, // TODO: get rid of this as some point
10-
"allowSyntheticDefaultImports": true // for ajv
10+
"allowSyntheticDefaultImports": true
1111
},
1212
"include": [
1313
"**/test-*.ts",

‎common/web/types/tools/formats.cjs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/*
2+
* This somewhat peculiar function is used in `build.sh configure` when
3+
* precompiling the validators and makes it possible to use the extended formats
4+
* in ajv-formats.
5+
*/
6+
function formats(ajv) {
7+
require("ajv-formats")(ajv);
8+
}
9+
10+
module.exports = formats;

‎common/web/types/tools/schema-bundler.js

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* Bundle schema validation files (from .cjs) and make them available as ES modules
3+
*/
4+
5+
import esbuild from 'esbuild';
6+
7+
await esbuild.build({
8+
entryPoints: [
9+
'obj/schemas/kpj.schema.validator.cjs',
10+
'obj/schemas/kpj-9.0.schema.validator.cjs',
11+
'obj/schemas/kvks.schema.validator.cjs',
12+
'obj/schemas/ldml-keyboard3.schema.validator.cjs',
13+
'obj/schemas/ldml-keyboardtest3.schema.validator.cjs',
14+
'obj/schemas/displaymap.schema.validator.cjs',
15+
'obj/schemas/keyman-touch-layout.clean.spec.validator.cjs',
16+
'obj/schemas/keyman-touch-layout.spec.validator.cjs',
17+
'obj/schemas/keyboard_info.schema.validator.cjs',
18+
],
19+
bundle: true,
20+
format: 'esm',
21+
target: 'es2022',
22+
outdir: 'src/schemas/',
23+
sourcemap: false,
24+
25+
// We want a .mjs extension to force node into ESM module mode
26+
outExtension: { '.js': '.mjs' },
27+
});

‎common/web/types/tsconfig.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@
55
"outDir": "build/src/",
66
"rootDir": "src/",
77
"baseUrl": ".",
8-
"allowSyntheticDefaultImports": true, // for ajv
9-
"resolveJsonModule": true,
8+
"allowSyntheticDefaultImports": true,
109
},
1110
"include": [
12-
"src/**/*.ts"
11+
"src/**/*.ts",
12+
"src/schemas/*.mjs", // Import the validators
1313
],
1414
"references": [
1515
{ "path": "../keyman-version" },

‎common/web/utils/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,13 @@
2323
},
2424
"homepage": "https://github.com/keymanapp/keyman#readme",
2525
"devDependencies": {
26-
"@keymanapp/resources-gosh": "*",
2726
"@keymanapp/keyman-version": "*",
27+
"@keymanapp/resources-gosh": "*",
28+
"@types/node": "^14.0.5",
2829
"c8": "^7.12.0",
2930
"chai": "^4.3.4",
3031
"mocha": "^10.0.0",
3132
"mocha-teamcity-reporter": "^4.0.0",
32-
"@types/node": "^14.0.5",
3333
"typescript": "^4.9.5"
3434
},
3535
"type": "module",

‎developer/src/kmc-keyboard-info/build.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ cd "$THIS_SCRIPT_PATH"
1111

1212
builder_describe "Build Keyman kmc keyboard-info Compiler module" \
1313
"@/common/web/types" \
14+
"@/developer/src/common/web/utils" \
1415
"clean" \
1516
"configure" \
1617
"build" \

‎developer/src/kmc-keyboard-info/package.json

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,8 @@
2525
},
2626
"dependencies": {
2727
"@keymanapp/common-types": "*",
28-
"@keymanapp/kmc-package": "*",
2928
"@keymanapp/developer-utils": "*",
30-
"ajv": "^8.11.0",
31-
"ajv-formats": "^2.1.1"
29+
"@keymanapp/kmc-package": "*"
3230
},
3331
"bundleDependencies": [
3432
"@keymanapp/developer-utils"

‎developer/src/kmc-keyboard-info/src/index.ts

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,7 @@ import langtags from "./imports/langtags.js";
1111
import { validateMITLicense } from "@keymanapp/developer-utils";
1212
import { KmpCompiler } from "@keymanapp/kmc-package";
1313

14-
import AjvModule from 'ajv';
15-
import AjvFormatsModule from 'ajv-formats';
16-
const Ajv = AjvModule.default; // The actual expected Ajv type.
17-
const ajvFormats = AjvFormatsModule.default;
18-
19-
import { Schemas } from "@keymanapp/common-types";
14+
import { SchemaValidators } from "@keymanapp/common-types";
2015
import { packageKeysExamplesToKeyboardInfo } from "./example-keys.js";
2116

2217
const regionNames = new Intl.DisplayNames(['en'], { type: "region" });
@@ -290,17 +285,10 @@ export class KeyboardInfoCompiler {
290285

291286
const jsonOutput = JSON.stringify(keyboard_info, null, 2);
292287

293-
// TODO: look at performance improvements by precompiling Ajv schemas on first use
294-
const ajv = new Ajv({ logger: {
295-
log: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_OutputValidation({message})),
296-
warn: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Warn_OutputValidation({message})),
297-
error: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Error_OutputValidation({message})),
298-
}});
299-
ajvFormats.default(ajv);
300-
if(!ajv.validate(Schemas.default.keyboard_info, keyboard_info)) {
288+
if(!SchemaValidators.default.keyboard_info(keyboard_info)) {
301289
// This is an internal fatal error; we should not be capable of producing
302290
// invalid output, so it is best to throw and die
303-
throw new Error(ajv.errorsText());
291+
throw new Error((<any>SchemaValidators.default.keyboard_info).errorsText());
304292
}
305293

306294
return new TextEncoder().encode(jsonOutput);

‎developer/src/kmc-ldml/package.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
"@keymanapp/keyman-version": "*",
3030
"@keymanapp/kmc-kmn": "*",
3131
"@keymanapp/ldml-keyboard-constants": "*",
32-
"ajv": "^8.11.0",
3332
"restructure": "git+https://github.com/keymanapp/dependency-restructure.git#7a188a1e26f8f36a175d95b67ffece8702363dfc",
3433
"semver": "^7.5.2",
3534
"xml2js": "git+https://github.com/keymanapp/dependency-node-xml2js#535fe732dc408d697e0f847c944cc45f0baf0829"

0 commit comments

Comments
 (0)