Skip to content

Commit 3cee5f3

Browse files
jdaltonjdalton
committed
Initial commit
1 parent 6608f30 commit 3cee5f3

File tree

37 files changed

+398
-335
lines changed

37 files changed

+398
-335
lines changed

‎.github/workflows/node-smol.yml‎

Lines changed: 29 additions & 138 deletions
Original file line numberDiff line numberDiff line change
@@ -238,13 +238,9 @@ jobs:
238238
fi
239239
240240
# Helper function to hash files, returns empty string if no files found
241+
# Callers must pass -type f and optionally -name "*.mjs" to filter appropriately
241242
hash_files() {
242-
local files=$(find "$@" 2>/dev/null | sort)
243-
if [ -z "$files" ]; then
244-
echo ""
245-
else
246-
echo "$files" | xargs $hash_cmd 2>/dev/null | $hash_cmd | cut -d' ' -f1
247-
fi
243+
find "$@" 2>/dev/null | sort | xargs $hash_cmd 2>/dev/null | $hash_cmd | cut -d' ' -f1 || echo ""
248244
}
249245
250246
# Helper function to get hierarchical paths for a category/phase/platform/arch
@@ -269,6 +265,8 @@ jobs:
269265
# Now with hierarchical paths: shared/ → platform/shared/ → platform/arch/
270266
271267
# Common scripts (used by all phases) - hierarchical
268+
# Each checkpoint now has its own paths.mjs that re-exports from scripts/paths.mjs
269+
# This makes dependencies explicit and ensures changes to paths.mjs are tracked per-checkpoint
272270
COMMON_PATHS=$(get_hierarchical_paths scripts common "$PLATFORM" "$ARCH")
273271
COMMON_SCRIPTS=$(hash_files $COMMON_PATHS -type f -name "*.mjs")
274272
@@ -334,61 +332,6 @@ jobs:
334332
echo "final_hash=${FINAL_KEY}" >> $GITHUB_OUTPUT
335333
echo "build_mode=${BUILD_MODE}" >> $GITHUB_OUTPUT
336334
337-
- name: Restore node-source cache
338-
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
339-
id: node-source-cache
340-
if: ${{ !inputs.force }}
341-
with:
342-
path: packages/node-smol-builder/build/${{ steps.smol-cache-key.outputs.build_mode }}/source
343-
key: node-source-${{ env.CACHE_VERSION }}-${{ steps.tool-versions.outputs.node-build-version }}-${{ steps.smol-cache-key.outputs.source_cloned_hash }}
344-
345-
- name: Clean stale ninja files from cache
346-
shell: bash
347-
run: |
348-
BUILD_MODE="${STEPS_SMOL_CACHE_KEY_OUTPUTS_BUILD_MODE}"
349-
OUT_DIR="packages/node-smol-builder/build/${BUILD_MODE}/source/out"
350-
if [ -d "$OUT_DIR" ]; then
351-
echo "Cleaning stale ninja files from cached source to free disk space..."
352-
rm -rf "$OUT_DIR"
353-
echo "✅ Cleaned $OUT_DIR"
354-
else
355-
echo "No out/ directory to clean"
356-
fi
357-
env:
358-
STEPS_SMOL_CACHE_KEY_OUTPUTS_BUILD_MODE: ${{ steps.smol-cache-key.outputs.build_mode }}
359-
360-
- name: Restore node Release cache
361-
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
362-
id: node-release-cache
363-
if: ${{ !inputs.force }}
364-
with:
365-
path: packages/node-smol-builder/build/${{ steps.smol-cache-key.outputs.build_mode }}/out/Release
366-
key: node-release-${{ steps.tool-versions.outputs.node-build-version }}-${{ matrix.platform }}-${{ matrix.arch }}-${{ steps.smol-cache-key.outputs.build_mode }}-${{ steps.smol-cache-key.outputs.binary_released_hash }}
367-
368-
- name: Restore node Stripped cache
369-
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
370-
id: node-stripped-cache
371-
if: ${{ !inputs.force }}
372-
with:
373-
path: packages/node-smol-builder/build/${{ steps.smol-cache-key.outputs.build_mode }}/out/Stripped
374-
key: node-stripped-${{ steps.tool-versions.outputs.node-build-version }}-${{ matrix.platform }}-${{ matrix.arch }}-${{ steps.smol-cache-key.outputs.build_mode }}-${{ steps.smol-cache-key.outputs.binary_stripped_hash }}
375-
376-
- name: Restore node Compressed cache
377-
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
378-
id: node-compressed-cache
379-
if: ${{ !inputs.force }}
380-
with:
381-
path: packages/node-smol-builder/build/${{ steps.smol-cache-key.outputs.build_mode }}/out/Compressed
382-
key: node-compressed-${{ steps.tool-versions.outputs.node-build-version }}-${{ matrix.platform }}-${{ matrix.arch }}-${{ steps.smol-cache-key.outputs.build_mode }}-${{ steps.smol-cache-key.outputs.binary_compressed_hash }}
383-
384-
- name: Restore node Final cache
385-
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
386-
id: node-final-cache
387-
if: ${{ !inputs.force }}
388-
with:
389-
path: packages/node-smol-builder/build/${{ steps.smol-cache-key.outputs.build_mode }}/out/Final
390-
key: node-final-${{ steps.tool-versions.outputs.node-build-version }}-${{ matrix.platform }}-${{ matrix.arch }}-${{ steps.smol-cache-key.outputs.build_mode }}-${{ steps.smol-cache-key.outputs.final_hash }}
391-
392335
- name: Restore checkpoint cache
393336
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
394337
id: checkpoint-cache
@@ -397,102 +340,50 @@ jobs:
397340
path: packages/node-smol-builder/build/${{ steps.smol-cache-key.outputs.build_mode }}/checkpoints
398341
key: node-checkpoints-${{ steps.tool-versions.outputs.node-build-version }}-${{ matrix.platform }}-${{ matrix.arch }}-${{ steps.smol-cache-key.outputs.build_mode }}-${{ steps.smol-cache-key.outputs.final_hash }}
399342

400-
- name: Validate build cache integrity
343+
- name: Validate checkpoint cache integrity
401344
id: validate-cache
402-
if: steps.node-final-cache.outputs.cache-hit == 'true'
345+
if: steps.checkpoint-cache.outputs.cache-hit == 'true'
403346
shell: bash
404347
run: |
405348
BUILD_MODE="${STEPS_SMOL_CACHE_KEY_OUTPUTS_BUILD_MODE}"
406-
FINAL_DIR="packages/node-smol-builder/build/${BUILD_MODE}/out/Final"
407349
CHECKPOINT_DIR="packages/node-smol-builder/build/${BUILD_MODE}/checkpoints"
408350
409-
echo "Validating cached build for ${{ matrix.platform }}-${{ matrix.arch }}..."
410-
411-
# Determine expected binary name
412-
if [ "${{ matrix.os }}" = "windows" ]; then
413-
BINARY_NAME="node.exe"
414-
else
415-
BINARY_NAME="node"
416-
fi
417-
418-
# Check if binary exists
419-
if [ ! -f "${FINAL_DIR}/${BINARY_NAME}" ]; then
420-
echo "❌ Binary missing: ${FINAL_DIR}/${BINARY_NAME}"
421-
rm -rf "$FINAL_DIR" "$CHECKPOINT_DIR"
422-
echo "cache_valid=false" >> $GITHUB_OUTPUT
423-
exit 0
424-
fi
425-
426-
# Check binary size (minimum 50MB for node binary)
427-
if [ "${{ matrix.os }}" = "windows" ]; then
428-
BINARY_SIZE=$(stat -c%s "${FINAL_DIR}/${BINARY_NAME}" 2>/dev/null || powershell -Command "(Get-Item '${FINAL_DIR}/${BINARY_NAME}').Length")
429-
else
430-
BINARY_SIZE=$(stat -f%z "${FINAL_DIR}/${BINARY_NAME}" 2>/dev/null || stat -c%s "${FINAL_DIR}/${BINARY_NAME}")
431-
fi
351+
echo "Validating cached checkpoints for ${{ matrix.platform }}-${{ matrix.arch }}..."
432352
433-
MIN_SIZE=52428800 # 50MB in bytes
434-
if [ "$BINARY_SIZE" -lt "$MIN_SIZE" ]; then
435-
echo "❌ Binary too small: $BINARY_SIZE bytes (minimum $MIN_SIZE)"
436-
rm -rf "$FINAL_DIR" "$CHECKPOINT_DIR"
353+
# Check if required checkpoint files exist (phase-based structure, flat directory)
354+
if [ ! -f "${CHECKPOINT_DIR}/source-cloned.json" ] || \
355+
[ ! -f "${CHECKPOINT_DIR}/binary-released.json" ] || \
356+
[ ! -f "${CHECKPOINT_DIR}/binary-stripped.json" ] || \
357+
[ ! -f "${CHECKPOINT_DIR}/binary-compressed.json" ] || \
358+
[ ! -f "${CHECKPOINT_DIR}/finalized.json" ]; then
359+
echo "❌ Checkpoint files incomplete"
360+
rm -rf "$CHECKPOINT_DIR"
437361
echo "cache_valid=false" >> $GITHUB_OUTPUT
438362
exit 0
439363
fi
440-
echo "✓ Binary size OK: $BINARY_SIZE bytes"
441364
442-
# Check if checkpoint files exist (phase-based structure, flat directory)
443-
if [ ! -f "${CHECKPOINT_DIR}/binary-released.json" ] || \
444-
[ ! -f "${CHECKPOINT_DIR}/binary-stripped.json" ] || \
445-
[ ! -f "${CHECKPOINT_DIR}/binary-compressed.json" ]; then
446-
echo "❌ Checkpoint files incomplete"
447-
rm -rf "$FINAL_DIR" "$CHECKPOINT_DIR"
365+
# Check if checkpoint tarballs exist
366+
if [ ! -f "${CHECKPOINT_DIR}/source-cloned.tar.gz" ] || \
367+
[ ! -f "${CHECKPOINT_DIR}/finalized.tar.gz" ]; then
368+
echo "❌ Checkpoint tarballs incomplete"
369+
rm -rf "$CHECKPOINT_DIR"
448370
echo "cache_valid=false" >> $GITHUB_OUTPUT
449371
exit 0
450372
fi
451373
452-
# Validate checksum if available in checkpoint
453-
if command -v jq &> /dev/null; then
454-
EXPECTED_CHECKSUM=$(jq -r '.checksum // empty' "${CHECKPOINT_DIR}/binary-compressed.json")
455-
if [ -n "$EXPECTED_CHECKSUM" ]; then
456-
if command -v shasum &> /dev/null; then
457-
ACTUAL_CHECKSUM=$(shasum -a 256 "${FINAL_DIR}/${BINARY_NAME}" | cut -d' ' -f1)
458-
else
459-
ACTUAL_CHECKSUM=$(sha256sum "${FINAL_DIR}/${BINARY_NAME}" | cut -d' ' -f1)
460-
fi
461-
462-
if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]; then
463-
echo "❌ Checksum mismatch (cache corruption detected)"
464-
echo " Expected: $EXPECTED_CHECKSUM"
465-
echo " Actual: $ACTUAL_CHECKSUM"
466-
rm -rf "$FINAL_DIR" "$CHECKPOINT_DIR"
374+
# Check tarball integrity
375+
for tarball in "${CHECKPOINT_DIR}"/*.tar.gz; do
376+
if [ -f "$tarball" ]; then
377+
if ! gzip -t "$tarball" 2>/dev/null; then
378+
echo "❌ Corrupted tarball: $(basename "$tarball")"
379+
rm -rf "$CHECKPOINT_DIR"
467380
echo "cache_valid=false" >> $GITHUB_OUTPUT
468381
exit 0
469382
fi
470-
echo "✓ Checksum validation passed"
471383
fi
472-
fi
473-
474-
# Smoke test: check version (only for native builds, skip cross-compiled)
475-
# We build natively on each architecture except Windows ARM64 (cross-compiled on x64)
476-
if [ "${{ matrix.cross_compile }}" != "true" ]; then
477-
"${FINAL_DIR}/${BINARY_NAME}" --version > /dev/null 2>&1
478-
if [ $? -ne 0 ]; then
479-
echo "❌ Binary smoke test failed"
480-
rm -rf "$FINAL_DIR" "$CHECKPOINT_DIR"
481-
echo "cache_valid=false" >> $GITHUB_OUTPUT
482-
exit 0
483-
fi
484-
485-
# Verify Node.js version matches
486-
NODE_VER=$("${FINAL_DIR}/${BINARY_NAME}" --version | sed 's/v//')
487-
if [ "$NODE_VER" != "${{ steps.tool-versions.outputs.node-build-version }}" ]; then
488-
echo "❌ Version mismatch: expected ${{ steps.tool-versions.outputs.node-build-version }}, got $NODE_VER"
489-
rm -rf "$FINAL_DIR" "$CHECKPOINT_DIR"
490-
echo "cache_valid=false" >> $GITHUB_OUTPUT
491-
exit 0
492-
fi
493-
fi
384+
done
494385
495-
echo "✅ Cache validation passed"
386+
echo "✅ Checkpoint cache validation passed"
496387
echo "cache_valid=true" >> $GITHUB_OUTPUT
497388
env:
498389
STEPS_SMOL_CACHE_KEY_OUTPUTS_BUILD_MODE: ${{ steps.smol-cache-key.outputs.build_mode }}
@@ -526,7 +417,7 @@ jobs:
526417
fi
527418
528419
- name: Build Node.js smol
529-
if: steps.node-final-cache.outputs.cache-hit != 'true' || steps.validate-cache.outputs.cache_valid == 'false'
420+
if: steps.checkpoint-cache.outputs.cache-hit != 'true' || steps.validate-cache.outputs.cache_valid == 'false'
530421
shell: bash
531422
env:
532423
BUILD_MODE: ${{ inputs.build_mode || 'prod' }}

‎.github/workflows/onnxruntime.yml‎

Lines changed: 33 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ jobs:
8484
}
8585
8686
# Cache version - bump to force-invalidate all caches
87-
CACHE_VERSION="v2"
87+
CACHE_VERSION="v3"
8888
8989
COMMON=$(hash_dir packages/onnxruntime-builder/scripts/common)
9090
PACKAGE_JSON=$(shasum -a 256 packages/onnxruntime-builder/package.json | cut -d' ' -f1)
@@ -123,26 +123,8 @@ jobs:
123123
sudo apt-get update
124124
# Load versions from external-tools.json (single source of truth)
125125
NINJA_VERSION=$(node packages/build-infra/scripts/get-tool-version.mjs ninja apt)
126-
CCACHE_VERSION=$(node packages/build-infra/scripts/get-tool-version.mjs ccache apt)
127-
sudo apt-get install -y ninja-build=${NINJA_VERSION}-* ccache=${CCACHE_VERSION}-*
128-
echo "Installed ninja-build ${NINJA_VERSION} and ccache ${CCACHE_VERSION} for faster builds"
129-
130-
- name: Setup ccache
131-
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
132-
with:
133-
path: ~/.ccache
134-
key: ccache-${{ runner.os }}-onnx-v${{ steps.tool-versions.outputs.onnx-version }}-${{ steps.cache-key.outputs.source_cloned_hash }}
135-
restore-keys: |
136-
ccache-${{ runner.os }}-onnx-v${{ steps.tool-versions.outputs.onnx-version }}-
137-
ccache-${{ runner.os }}-onnx-
138-
139-
- name: Configure ccache
140-
run: |
141-
ccache --set-config=max_size=10G
142-
ccache --set-config=compression=true
143-
ccache --set-config=compression_level=6
144-
ccache --zero-stats
145-
ccache --show-stats
126+
sudo apt-get install -y ninja-build=${NINJA_VERSION}-*
127+
echo "Installed ninja-build ${NINJA_VERSION} for faster builds"
146128
147129
- name: Set build mode
148130
id: build-mode
@@ -158,24 +140,6 @@ jobs:
158140
echo "mode=$BUILD_MODE" >> $GITHUB_OUTPUT
159141
echo "Build mode: $BUILD_MODE"
160142
161-
- name: Restore ONNX Runtime source cache
162-
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
163-
id: onnx-source-cache
164-
if: ${{ !inputs.force }}
165-
with:
166-
path: packages/onnxruntime-builder/build/shared/source
167-
key: onnx-source-v${{ steps.tool-versions.outputs.onnx-version }}-${{ steps.cache-key.outputs.source_cloned_hash }}
168-
restore-keys: |
169-
onnx-source-v${{ steps.tool-versions.outputs.onnx-version }}-
170-
171-
- name: Restore ONNX Runtime Final cache
172-
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
173-
id: onnx-final-cache
174-
if: ${{ !inputs.force }}
175-
with:
176-
path: packages/onnxruntime-builder/build/${{ steps.build-mode.outputs.mode }}/out/Final
177-
key: onnx-final-v${{ steps.tool-versions.outputs.onnx-version }}-${{ runner.os }}-${{ steps.build-mode.outputs.mode }}-${{ steps.cache-key.outputs.wasm_final_hash }}
178-
179143
- name: Restore ONNX Runtime checkpoint cache
180144
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
181145
id: onnx-checkpoint-cache
@@ -184,58 +148,59 @@ jobs:
184148
path: packages/onnxruntime-builder/build/${{ steps.build-mode.outputs.mode }}/checkpoints
185149
key: onnx-checkpoints-v${{ steps.tool-versions.outputs.onnx-version }}-${{ runner.os }}-${{ steps.build-mode.outputs.mode }}-${{ steps.cache-key.outputs.wasm_final_hash }}
186150

187-
- name: Validate build cache integrity
151+
- name: Validate checkpoint cache integrity
188152
id: validate-cache
189-
if: steps.onnx-final-cache.outputs.cache-hit == 'true'
153+
if: steps.onnx-checkpoint-cache.outputs.cache-hit == 'true'
190154
env:
191155
BUILD_MODE: ${{ steps.build-mode.outputs.mode }}
192156
run: |
193-
echo "Validating cached ONNX Runtime build..."
157+
echo "Validating cached ONNX Runtime checkpoints..."
194158
195-
FINAL_DIR="packages/onnxruntime-builder/build/${BUILD_MODE}/out/Final"
196159
CHECKPOINT_DIR="packages/onnxruntime-builder/build/${BUILD_MODE}/checkpoints"
197160
198-
# Check if WASM file exists
199-
if [ ! -f "${FINAL_DIR}/ort.wasm" ]; then
200-
echo "❌ WASM file missing: ${FINAL_DIR}/ort.wasm"
201-
rm -rf "$FINAL_DIR" "$CHECKPOINT_DIR"
161+
# Check if required checkpoint files exist
162+
if [ ! -f "${CHECKPOINT_DIR}/source-cloned.json" ] || \
163+
[ ! -f "${CHECKPOINT_DIR}/wasm-compiled.json" ] || \
164+
[ ! -f "${CHECKPOINT_DIR}/wasm-released.json" ] || \
165+
[ ! -f "${CHECKPOINT_DIR}/wasm-optimized.json" ] || \
166+
[ ! -f "${CHECKPOINT_DIR}/wasm-synced.json" ] || \
167+
[ ! -f "${CHECKPOINT_DIR}/wasm-finalized.json" ]; then
168+
echo "❌ Checkpoint files incomplete"
169+
rm -rf "$CHECKPOINT_DIR"
202170
echo "cache_valid=false" >> $GITHUB_OUTPUT
203171
exit 0
204172
fi
205173
206-
# Check if MJS file exists
207-
if [ ! -f "${FINAL_DIR}/ort.mjs" ]; then
208-
echo "❌ MJS file missing: ${FINAL_DIR}/ort.mjs"
209-
rm -rf "$FINAL_DIR" "$CHECKPOINT_DIR"
174+
# Check if checkpoint tarballs exist
175+
if [ ! -f "${CHECKPOINT_DIR}/source-cloned.tar.gz" ] || \
176+
[ ! -f "${CHECKPOINT_DIR}/wasm-finalized.tar.gz" ]; then
177+
echo "❌ Checkpoint tarballs incomplete"
178+
rm -rf "$CHECKPOINT_DIR"
210179
echo "cache_valid=false" >> $GITHUB_OUTPUT
211180
exit 0
212181
fi
213182
214-
# Check WASM file size (minimum 1MB)
215-
WASM_SIZE=$(stat -c%s "${FINAL_DIR}/ort.wasm" 2>/dev/null || stat -f%z "${FINAL_DIR}/ort.wasm")
216-
MIN_SIZE=1000000 # 1MB
217-
if [ "$WASM_SIZE" -lt "$MIN_SIZE" ]; then
218-
echo "❌ WASM file too small: $WASM_SIZE bytes (minimum $MIN_SIZE)"
219-
rm -rf "$FINAL_DIR" "$CHECKPOINT_DIR"
220-
echo "cache_valid=false" >> $GITHUB_OUTPUT
221-
exit 0
222-
fi
183+
# Check tarball integrity
184+
for tarball in "${CHECKPOINT_DIR}"/*.tar.gz; do
185+
if [ -f "$tarball" ]; then
186+
if ! gzip -t "$tarball" 2>/dev/null; then
187+
echo "❌ Corrupted tarball: $(basename "$tarball")"
188+
rm -rf "$CHECKPOINT_DIR"
189+
echo "cache_valid=false" >> $GITHUB_OUTPUT
190+
exit 0
191+
fi
192+
fi
193+
done
223194
224-
echo "✅ Cache validation passed"
195+
echo "✅ Checkpoint cache validation passed"
225196
echo "cache_valid=true" >> $GITHUB_OUTPUT
226197
227198
- name: Build ONNX Runtime WASM
228-
if: steps.onnx-final-cache.outputs.cache-hit != 'true' || steps.validate-cache.outputs.cache_valid == 'false'
199+
if: steps.onnx-checkpoint-cache.outputs.cache-hit != 'true' || steps.validate-cache.outputs.cache_valid == 'false'
229200
env:
230201
BUILD_MODE: ${{ steps.build-mode.outputs.mode }}
231202
run: pnpm --filter onnxruntime-builder build --$BUILD_MODE
232203

233-
- name: Show ccache statistics
234-
if: steps.onnx-checkpoint-cache.outputs.cache-hit != 'true' || steps.validate-cache.outputs.cache_valid == 'false'
235-
run: |
236-
echo "ccache statistics after build:"
237-
ccache --show-stats
238-
239204
- name: Validate build output
240205
env:
241206
BUILD_MODE: ${{ steps.build-mode.outputs.mode }}

0 commit comments

Comments
 (0)