Skip to content

Commit 1d52bbd

Browse files
jdaltonjdalton
committed
Initial commit
- Fixed WASM sync wrapper generation for minified production code - Three-pass AST transformation to avoid MagicString split point errors - Properly handle WebAssembly.instantiate gutting for Emscripten modules - Reorganized build-infra structure - Moved wasm-sync-wrapper.mjs from lib/ to wasm-synced/ following checkpoint naming convention - Updated all imports and package.json exports - Pinned Emscripten version to 4.0.20 for reproducible builds - Added to build-infra/package.json externalTools - Updated CI workflows to use pinned version - ONNX Runtime and Yoga Layout builders read version from package.json - Ensures identical builds across local and CI environments - Version 4.0+ required (3.1.x produces incompatible WASM modules) - Fixed cache invalidation - Added wasm-sync-wrapper.mjs to cache hash calculation - Added CACHE_VERSION for force-invalidating stale caches - Cumulative hash chains ensure all dependencies are tracked
1 parent a50ef63 commit 1d52bbd

File tree

10 files changed

+64
-29
lines changed

10 files changed

+64
-29
lines changed

‎.github/workflows/onnxruntime.yml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,13 +90,16 @@ jobs:
9090
fi
9191
}
9292
93+
# Cache version - bump to force-invalidate all caches
94+
CACHE_VERSION="v1"
95+
9396
COMMON=$(hash_dir packages/onnxruntime-builder/scripts/common)
9497
PACKAGE_JSON=$(shasum -a 256 packages/onnxruntime-builder/package.json | cut -d' ' -f1)
9598
BUILD_MJS=$(shasum -a 256 packages/onnxruntime-builder/scripts/build.mjs | cut -d' ' -f1)
9699
97-
# source-cloned
100+
# source-cloned: cache-version + common + source-cloned + build.mjs + package.json
98101
SOURCE_CLONED_DIR=$(hash_dir packages/onnxruntime-builder/scripts/source-cloned)
99-
SOURCE_CLONED_HASH=$(echo "${COMMON}${SOURCE_CLONED_DIR}${BUILD_MJS}${PACKAGE_JSON}" | shasum -a 256 | cut -d' ' -f1)
102+
SOURCE_CLONED_HASH=$(echo "${CACHE_VERSION}${COMMON}${SOURCE_CLONED_DIR}${BUILD_MJS}${PACKAGE_JSON}" | shasum -a 256 | cut -d' ' -f1)
100103
101104
# wasm-compiled
102105
WASM_COMPILED_DIR=$(hash_dir packages/onnxruntime-builder/scripts/wasm-compiled)
@@ -110,9 +113,10 @@ jobs:
110113
WASM_OPTIMIZED_DIR=$(hash_dir packages/onnxruntime-builder/scripts/wasm-optimized)
111114
WASM_OPTIMIZED_HASH=$(echo "${WASM_RELEASE_HASH}${WASM_OPTIMIZED_DIR}" | shasum -a 256 | cut -d' ' -f1)
112115
113-
# wasm-synced
116+
# wasm-synced: wasm-optimized + wasm-synced + wasm-sync-wrapper.mjs (shared infra)
114117
WASM_SYNC_DIR=$(hash_dir packages/onnxruntime-builder/scripts/wasm-synced)
115-
WASM_SYNC_HASH=$(echo "${WASM_OPTIMIZED_HASH}${WASM_SYNC_DIR}" | shasum -a 256 | cut -d' ' -f1)
118+
WASM_SYNC_WRAPPER=$(shasum -a 256 packages/build-infra/wasm-synced/wasm-sync-wrapper.mjs | cut -d' ' -f1)
119+
WASM_SYNC_HASH=$(echo "${WASM_OPTIMIZED_HASH}${WASM_SYNC_DIR}${WASM_SYNC_WRAPPER}" | shasum -a 256 | cut -d' ' -f1)
116120
117121
# wasm-finalized
118122
WASM_FINAL_DIR=$(hash_dir packages/onnxruntime-builder/scripts/wasm-finalized)

‎.github/workflows/yoga-layout.yml‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,19 +94,25 @@ jobs:
9494
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
9595
with:
9696
path: ~/emsdk
97-
key: emsdk-${{ runner.os }}-3.1.68
97+
key: emsdk-${{ runner.os }}-4.0.20
9898

9999
- name: Setup Emscripten
100100
run: |
101+
# Pinned version from build-infra/package.json externalTools.emscripten.versions.emsdk
102+
EMSCRIPTEN_VERSION="4.0.20"
103+
101104
if [ ! -d ~/emsdk ]; then
102105
git clone https://github.com/emscripten-core/emsdk.git ~/emsdk
103106
cd ~/emsdk
104-
./emsdk install 3.1.68
107+
./emsdk install ${EMSCRIPTEN_VERSION}
105108
fi
106109
cd ~/emsdk
107-
./emsdk activate 3.1.68
110+
./emsdk activate ${EMSCRIPTEN_VERSION}
108111
source ./emsdk_env.sh
109112
113+
echo "Emscripten version: ${EMSCRIPTEN_VERSION}"
114+
emcc --version
115+
110116
- name: Generate Yoga cache key
111117
id: cache-key
112118
run: |
@@ -123,14 +129,17 @@ jobs:
123129
fi
124130
}
125131
132+
# Cache version - bump to force-invalidate all caches
133+
CACHE_VERSION="v1"
134+
126135
# Common scripts (used by all checkpoints)
127136
COMMON=$(hash_dir packages/yoga-layout-builder/scripts/common)
128137
PACKAGE_JSON=$(shasum -a 256 packages/yoga-layout-builder/package.json | cut -d' ' -f1)
129138
BUILD_MJS=$(shasum -a 256 packages/yoga-layout-builder/scripts/build.mjs | cut -d' ' -f1)
130139
131-
# source-cloned: common + source-cloned + build.mjs + package.json
140+
# source-cloned: cache-version + common + source-cloned + build.mjs + package.json
132141
SOURCE_CLONED_DIR=$(hash_dir packages/yoga-layout-builder/scripts/source-cloned)
133-
SOURCE_CLONED_HASH=$(echo "${COMMON}${SOURCE_CLONED_DIR}${BUILD_MJS}${PACKAGE_JSON}" | shasum -a 256 | cut -d' ' -f1)
142+
SOURCE_CLONED_HASH=$(echo "${CACHE_VERSION}${COMMON}${SOURCE_CLONED_DIR}${BUILD_MJS}${PACKAGE_JSON}" | shasum -a 256 | cut -d' ' -f1)
134143
135144
# source-configured: source-cloned + source-configured
136145
SOURCE_CONFIGURED_DIR=$(hash_dir packages/yoga-layout-builder/scripts/source-configured)
@@ -148,9 +157,10 @@ jobs:
148157
WASM_OPTIMIZED_DIR=$(hash_dir packages/yoga-layout-builder/scripts/wasm-optimized)
149158
WASM_OPTIMIZED_HASH=$(echo "${WASM_RELEASE_HASH}${WASM_OPTIMIZED_DIR}" | shasum -a 256 | cut -d' ' -f1)
150159
151-
# wasm-synced: wasm-optimized + wasm-synced
160+
# wasm-synced: wasm-optimized + wasm-synced + wasm-sync-wrapper.mjs (shared infra)
152161
WASM_SYNC_DIR=$(hash_dir packages/yoga-layout-builder/scripts/wasm-synced)
153-
WASM_SYNC_HASH=$(echo "${WASM_OPTIMIZED_HASH}${WASM_SYNC_DIR}" | shasum -a 256 | cut -d' ' -f1)
162+
WASM_SYNC_WRAPPER=$(shasum -a 256 packages/build-infra/wasm-synced/wasm-sync-wrapper.mjs | cut -d' ' -f1)
163+
WASM_SYNC_HASH=$(echo "${WASM_OPTIMIZED_HASH}${WASM_SYNC_DIR}${WASM_SYNC_WRAPPER}" | shasum -a 256 | cut -d' ' -f1)
154164
155165
# wasm-finalized: wasm-synced + wasm-finalized (most complete hash)
156166
WASM_FINAL_DIR=$(hash_dir packages/yoga-layout-builder/scripts/wasm-finalized)

‎packages/build-infra/package.json‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
"./lib/test-helpers": "./lib/test-helpers.mjs",
2929
"./lib/tool-installer": "./lib/tool-installer.mjs",
3030
"./lib/unicode-property-escape-transform": "./lib/unicode-property-escape-transform.mjs",
31-
"./lib/wasm-sync-wrapper": "./lib/wasm-sync-wrapper.mjs"
31+
"./wasm-synced/wasm-sync-wrapper": "./wasm-synced/wasm-sync-wrapper.mjs"
3232
},
3333
"scripts": {
3434
"test": "vitest run",
@@ -236,6 +236,14 @@
236236
"github-runner": "14.2.0"
237237
},
238238
"notes": "GitHub Actions windows-2022 runner includes g++ 14.2.0 by default from MinGW-w64"
239+
},
240+
"emscripten": {
241+
"description": "Emscripten WASM compiler toolchain (for yoga-layout and onnxruntime WASM builds)",
242+
"packages": {},
243+
"versions": {
244+
"emsdk": "4.0.20"
245+
},
246+
"notes": "Installed via emsdk. Version 4.0+ required for correct WASM module generation with wasm-sync-wrapper transformation. CI and local must use same version for reproducible builds."
239247
}
240248
},
241249
"dependencies": {

‎packages/build-infra/test-transform.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { fileURLToPath } from 'node:url'
99

1010
import { getDefaultLogger } from '@socketsecurity/lib/logger'
1111

12-
import { generateWasmSyncWrapper } from './lib/wasm-sync-wrapper.mjs'
12+
import { generateWasmSyncWrapper } from './wasm-synced/wasm-sync-wrapper.mjs'
1313

1414
const require = createRequire(import.meta.url)
1515
const logger = getDefaultLogger()

‎packages/build-infra/test-wasm-sync-generation.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { fileURLToPath } from 'node:url'
99

1010
import { getDefaultLogger } from '@socketsecurity/lib/logger'
1111

12-
import { generateWasmSyncWrapper } from './lib/wasm-sync-wrapper.mjs'
12+
import { generateWasmSyncWrapper } from './wasm-synced/wasm-sync-wrapper.mjs'
1313

1414
const __filename = fileURLToPath(import.meta.url)
1515
const __dirname = path.dirname(__filename)

‎packages/build-infra/lib/wasm-sync-wrapper.mjs‎ renamed to ‎packages/build-infra/wasm-synced/wasm-sync-wrapper.mjs‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ export async function generateWasmSyncWrapper(options) {
5454
outputSyncJs,
5555
packageName,
5656
wasmFile,
57-
} = options
57+
} = { __proto__: null, ...options }
5858

5959
if (!existsSync(wasmFile)) {
6060
throw new Error(`WASM file not found: ${wasmFile}`)
@@ -116,19 +116,19 @@ export async function generateWasmSyncWrapper(options) {
116116
s = new MagicString(mjsContent)
117117

118118
// Helpers: Safe transformations for minified code (may have overlapping mods)
119-
const safeRemove = (start, end) => {
119+
const safeOverwrite = (start, end, content) => {
120120
try {
121-
s.remove(start, end)
121+
s.overwrite(start, end, content)
122122
} catch {
123-
// Minified code - skip conflicting removes, leave as dead code
123+
// Minified code - skip conflicting overwrites
124124
}
125125
}
126126

127-
const safeOverwrite = (start, end, content) => {
127+
const safeRemove = (start, end) => {
128128
try {
129-
s.overwrite(start, end, content)
129+
s.remove(start, end)
130130
} catch {
131-
// Minified code - skip conflicting overwrites
131+
// Minified code - skip conflicting removes, leave as dead code
132132
}
133133
}
134134

@@ -141,19 +141,16 @@ export async function generateWasmSyncWrapper(options) {
141141
// 5. Remove top-level statements that fetch/load WASM asynchronously
142142

143143
const topLevelStatementsToRemove = []
144-
const _variableDeclaratorsToRemove = []
145144
const requireDeclaratorsToRemove = []
146145
const returnModuleToFix = []
147146
// Track functions that need WebAssembly.instantiate replacement
148147
const functionsToGut = []
149-
let _exportDefaultFound = false
150148
let exportDefaultIndex = -1
151149

152150
traverse(ast, {
153151
// Remove ALL export statements (convert ESM to CommonJS)
154152
ExportDefaultDeclaration(path) {
155153
if (path.parent.type === 'Program') {
156-
_exportDefaultFound = true
157154
// Find the index of this export in the program body
158155
const programBody = path.parent.body
159156
exportDefaultIndex = programBody.indexOf(path.node)

‎packages/onnxruntime-builder/scripts/build.mjs‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,16 @@ if (!eigenSource) {
8888
const EIGEN_COMMIT = eigenSource.ref
8989
const EIGEN_SHA1 = eigenSource.sha1
9090

91+
// Read Emscripten version from build-infra package.json
92+
const buildInfraPackageJson = JSON.parse(
93+
await fs.readFile(
94+
path.join(PACKAGE_ROOT, '../build-infra/package.json'),
95+
'utf-8',
96+
),
97+
)
98+
const emscriptenVersion =
99+
buildInfraPackageJson.externalTools?.emscripten?.versions?.emsdk || 'latest'
100+
91101
// Get paths from source of truth
92102
const {
93103
buildDir: SHARED_BUILD_DIR,
@@ -157,6 +167,7 @@ async function buildWasm() {
157167
isCI: IS_CI,
158168
buildOutputPaths: getBuildOutputPaths(MODE_SOURCE_DIR),
159169
forceRebuild: FORCE_BUILD,
170+
emscriptenVersion,
160171
})
161172
}
162173

@@ -274,9 +285,11 @@ async function main() {
274285
}
275286

276287
// Ensure Emscripten SDK is available.
277-
logger.substep('Checking for Emscripten SDK...')
288+
logger.substep(
289+
`Checking for Emscripten SDK (version ${emscriptenVersion})...`,
290+
)
278291
const emscriptenResult = await ensureEmscripten({
279-
version: 'latest',
292+
version: emscriptenVersion,
280293
autoInstall: true,
281294
quiet: false,
282295
})

‎packages/onnxruntime-builder/scripts/wasm-compiled/shared/compile-wasm.mjs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,15 @@ const logger = getDefaultLogger()
3232
* @param {boolean} options.isCI - Is running in CI environment
3333
* @param {object} options.buildOutputPaths - Build output paths (buildWasmFile, buildCmakeCacheFile, buildPostBuildScriptFile)
3434
* @param {boolean} options.forceRebuild - Force rebuild (ignore checkpoints)
35+
* @param {string} options.emscriptenVersion - Emscripten version to use (from build-infra package.json)
3536
*/
3637
export async function compileWasm(options) {
3738
const {
3839
buildDir,
3940
buildMode,
4041
buildOutputPaths,
4142
buildScriptFile,
43+
emscriptenVersion = 'latest',
4244
forceRebuild,
4345
isCI,
4446
modeSourceDir,
@@ -51,8 +53,9 @@ export async function compileWasm(options) {
5153
logger.step('Building ONNX Runtime with Emscripten')
5254

5355
// Auto-detect and activate Emscripten SDK.
56+
logger.substep(`Using Emscripten version ${emscriptenVersion}`)
5457
const emscriptenResult = await ensureEmscripten({
55-
version: 'latest',
58+
version: emscriptenVersion,
5659
autoInstall: false,
5760
quiet: true,
5861
})

‎packages/onnxruntime-builder/scripts/wasm-synced/shared/generate-sync.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import path from 'node:path'
1010

1111
import { getFileSize } from 'build-infra/lib/build-helpers'
1212
import { createCheckpoint, shouldRun } from 'build-infra/lib/checkpoint-manager'
13-
import { generateWasmSyncWrapper } from 'build-infra/lib/wasm-sync-wrapper'
13+
import { generateWasmSyncWrapper } from 'build-infra/wasm-synced/wasm-sync-wrapper'
1414

1515
import { getDefaultLogger } from '@socketsecurity/lib/logger'
1616
import { hasKeys, isObjectObject } from '@socketsecurity/lib/objects'

‎packages/yoga-layout-builder/scripts/wasm-synced/shared/generate-sync.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import path from 'node:path'
1010

1111
import { getFileSize } from 'build-infra/lib/build-helpers'
1212
import { createCheckpoint, shouldRun } from 'build-infra/lib/checkpoint-manager'
13-
import { generateWasmSyncWrapper } from 'build-infra/lib/wasm-sync-wrapper'
13+
import { generateWasmSyncWrapper } from 'build-infra/wasm-synced/wasm-sync-wrapper'
1414

1515
import { getDefaultLogger } from '@socketsecurity/lib/logger'
1616
import { hasKeys, isObjectObject } from '@socketsecurity/lib/objects'

0 commit comments

Comments
 (0)