Skip to content

Commit b9953dd

Browse files
aduh95aduh95
committed
tools: cache V8 on test-shared workflow
1 parent 7215ea4 commit b9953dd

File tree

5 files changed

+188
-37
lines changed

5 files changed

+188
-37
lines changed

‎.github/workflows/test-shared.yml‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -182,25 +182,20 @@ jobs:
182182
core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
183183
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
184184
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
185-
186-
- name: Load shell.nix
187-
if: github.event_name != 'workflow_dispatch'
188-
run: |
189-
mv "$TAR_DIR"/*.nix .
190-
mkdir tools
191-
mv "$TAR_DIR"/tools/nix tools/.
185+
core.exportVariable('NIX_SCCACHE', '(import <nixpkgs> {}).sccache');
192186
193187
- name: Build Node.js and run tests
194188
run: |
195189
nix-shell \
196-
-I nixpkgs=./tools/nix/pkgs.nix \
190+
-I "nixpkgs=$TAR_DIR/tools/nix/pkgs.nix" \
197191
--pure --keep TAR_DIR --keep FLAKY_TESTS \
198192
--keep SCCACHE_GHA_ENABLED --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
199193
--arg loadJSBuiltinsDynamically false \
200-
--arg ccache '(import <nixpkgs> {}).sccache' \
194+
--arg cachedV8 'import ./tools/nix/v8.nix {}' \
195+
--arg ccache "${NIX_SCCACHE:-null}" \
201196
--arg devTools '[]' \
202197
--arg benchmarkTools '[]' \
203198
${{ endsWith(matrix.system, '-darwin') && '--arg withAmaro false --arg withLief false --arg withSQLite false --arg extraConfigFlags ''["--without-inspector" "--without-node-options"]'' \' || '\' }}
204199
--run '
205200
make -C "$TAR_DIR" run-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS"
206-
'
201+
' "$TAR_DIR/shell.nix"

‎configure.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1090,6 +1090,13 @@
10901090
help='do not use V8 includes from the bundled deps folder. ' +
10911091
'(This mode is not officially supported for regular applications)')
10921092

1093+
parser.add_argument('--external-v8',
1094+
action='store',
1095+
dest='external_v8_path',
1096+
default=False,
1097+
help='Path to shared V8. ' +
1098+
'(This mode is not officially supported for regular applications)')
1099+
10931100
parser.add_argument('--verbose',
10941101
action='store_true',
10951102
dest='verbose',
@@ -2068,6 +2075,11 @@ def configure_v8(o, configs):
20682075
raise Exception('--enable-d8 is incompatible with --without-bundled-v8.')
20692076
if options.without_bundled_v8 and options.enable_v8windbg:
20702077
raise Exception('--enable-v8windbg is incompatible with --without-bundled-v8.')
2078+
if options.external_v8_path:
2079+
if not options.without_bundled_v8:
2080+
raise Exception('--external-v8 requires --without-bundled-v8.')
2081+
output['libraries'] += [f"-L{options.external_v8_path}/lib", '-lv8']
2082+
output['include_dirs'] += [f"-L{options.external_v8_path}/include"]
20712083
if options.static_zoslib_gyp:
20722084
o['variables']['static_zoslib_gyp'] = options.static_zoslib_gyp
20732085
if flavor != 'linux' and options.v8_enable_hugepage:

‎shell.nix‎

Lines changed: 17 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,6 @@
55
ccache ? pkgs.ccache,
66
loadJSBuiltinsDynamically ? true, # Load `lib/**.js` from disk instead of embedding
77
ninja ? pkgs.ninja,
8-
extraConfigFlags ? [
9-
"--debug-node"
10-
],
118

129
# Build options
1310
icu ? pkgs.icu,
@@ -27,6 +24,22 @@
2724
withTemporal
2825
;
2926
},
27+
cachedV8 ? null,
28+
29+
configureFlags ? import ./tools/nix/configureFlags.nix {
30+
inherit
31+
cachedV8
32+
icu
33+
loadJSBuiltinsDynamically
34+
ninja
35+
pkgs
36+
withLief
37+
withQuic
38+
withSQLite
39+
withSSL
40+
withTemporal
41+
;
42+
},
3043

3144
# dev tools (not needed to build Node.js, useful to maintain it)
3245
ncu-path ? null, # Provide this if you want to use a local version of NCU
@@ -73,29 +86,6 @@ pkgs.mkShell {
7386
"test-strace-openat-openssl"
7487
]
7588
);
76-
CONFIG_FLAGS = builtins.toString (
77-
[
78-
(
79-
if icu == null then
80-
"--without-intl"
81-
else
82-
"--with-intl=${if useSharedICU then "system" else icu}-icu"
83-
)
84-
]
85-
++ extraConfigFlags
86-
++ pkgs.lib.optional (!withAmaro) "--without-amaro"
87-
++ pkgs.lib.optional (!withLief) "--without-lief"
88-
++ pkgs.lib.optional withQuic "--experimental-quic"
89-
++ pkgs.lib.optional (!withSQLite) "--without-sqlite"
90-
++ pkgs.lib.optional (!withSSL) "--without-ssl"
91-
++ pkgs.lib.optional withTemporal "--v8-enable-temporal-support"
92-
++ pkgs.lib.optional (ninja != null) "--ninja"
93-
++ pkgs.lib.optional loadJSBuiltinsDynamically "--node-builtin-modules-path=${builtins.toString ./.}"
94-
++ pkgs.lib.concatMap (name: [
95-
"--shared-${name}"
96-
"--shared-${name}-libpath=${pkgs.lib.getLib sharedLibDeps.${name}}/lib"
97-
"--shared-${name}-include=${pkgs.lib.getInclude sharedLibDeps.${name}}/include"
98-
]) (builtins.attrNames sharedLibDeps)
99-
);
89+
CONFIG_FLAGS = builtins.toString (configureFlags);
10090
NOSQLITE = pkgs.lib.optionalString (!withSQLite) "1";
10191
}

‎tools/nix/configureFlags.nix‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{
2+
pkgs ? import ./pkgs.nix { },
3+
4+
# Optional build tools / config
5+
cachedV8 ? null,
6+
loadJSBuiltinsDynamically ? true, # Load `lib/**.js` from disk instead of embedding
7+
ninja ? pkgs.ninja,
8+
9+
# Build options
10+
icu ? pkgs.icu,
11+
withAmaro ? true,
12+
withLief ? true,
13+
withQuic ? false,
14+
withSQLite ? true,
15+
withSSL ? true,
16+
withTemporal ? false,
17+
sharedLibDeps ? import ./sharedLibDeps.nix {
18+
inherit
19+
pkgs
20+
withLief
21+
withQuic
22+
withSQLite
23+
withSSL
24+
withTemporal
25+
;
26+
},
27+
}:
28+
let
29+
useSharedICU = if builtins.isString icu then icu == "system" else icu != null;
30+
in
31+
[
32+
(
33+
if icu == null then
34+
"--without-intl"
35+
else
36+
"--with-intl=${if useSharedICU then "system" else icu}-icu"
37+
)
38+
]
39+
++ pkgs.lib.optionals (cachedV8 != null) [
40+
"--without-bundled-v8"
41+
"--external-v8=${cachedV8}"
42+
]
43+
++ pkgs.lib.optional (!withAmaro) "--without-amaro"
44+
++ pkgs.lib.optional (!withLief) "--without-lief"
45+
++ pkgs.lib.optional withQuic "--experimental-quic"
46+
++ pkgs.lib.optional (!withSQLite) "--without-sqlite"
47+
++ pkgs.lib.optional (!withSSL) "--without-ssl"
48+
++ pkgs.lib.optional withTemporal "--v8-enable-temporal-support"
49+
++ pkgs.lib.optional (ninja != null) "--ninja"
50+
++ pkgs.lib.optional loadJSBuiltinsDynamically "--node-builtin-modules-path=${builtins.toString ./.}"
51+
++ pkgs.lib.concatMap (name: [
52+
"--shared-${name}"
53+
"--shared-${name}-libpath=${pkgs.lib.getLib sharedLibDeps.${name}}/lib"
54+
"--shared-${name}-include=${pkgs.lib.getInclude sharedLibDeps.${name}}/include"
55+
]) (builtins.attrNames sharedLibDeps)

‎tools/nix/v8.nix‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
{
2+
pkgs ? import ./pkgs.nix { },
3+
}:
4+
5+
let
6+
inherit (pkgs) writeScript icu;
7+
8+
nodejs = pkgs.nodejs-slim_latest;
9+
sharedLibDeps = import ./sharedLibDeps.nix { inherit pkgs; };
10+
configureFlags = import ./configureFlags.nix {
11+
inherit pkgs icu sharedLibDeps;
12+
cachedV8 = null;
13+
loadJSBuiltinsDynamically = false;
14+
};
15+
in
16+
pkgs.stdenv.mkDerivation (finalAttrs: {
17+
pname = "libv8";
18+
version = "0.0.0-unstable";
19+
src =
20+
let
21+
inherit (pkgs.lib) fileset;
22+
in
23+
fileset.toSource {
24+
root = ../../.;
25+
fileset = fileset.unions [
26+
../../common.gypi
27+
../../configure
28+
../../configure.py
29+
../../deps/inspector_protocol/inspector_protocol.gyp
30+
../../deps/ncrypto/ncrypto.gyp
31+
../../deps/v8
32+
../../node.gyp
33+
../../node.gypi
34+
../../src/inspector/node_inspector.gypi
35+
../../src/node_version.h
36+
../../tools/configure.d
37+
../../tools/getmoduleversion.py
38+
../../tools/getnapibuildversion.py
39+
../../tools/gyp
40+
../../tools/gyp_node.py
41+
../../tools/icu/icu_versions.json
42+
../../tools/icu/icu-system.gyp
43+
../../tools/utils.py
44+
../../tools/v8_gypfiles
45+
];
46+
};
47+
48+
prePatch = ''
49+
# Filter patches to only include changes under deps/npm/ and strip that prefix
50+
patches=()
51+
for patch in ${pkgs.lib.concatStringsSep " " nodejs.patches}; do
52+
filtered=$(mktemp)
53+
filterdiff -p1 -i 'tools/gyp/*' "$patch" > "$filtered"
54+
if [ -s "$filtered" ]; then
55+
patches+=("$filtered")
56+
fi
57+
done
58+
'';
59+
60+
inherit (nodejs) configureScript;
61+
inherit configureFlags;
62+
63+
nativeBuildInputs = nodejs.nativeBuildInputs ++ [ pkgs.patchutils ];
64+
buildInputs = [ icu ] ++ builtins.attrValues sharedLibDeps;
65+
66+
buildPhase = ''
67+
ninja -C out/Release v8_snapshot v8_libplatform
68+
'';
69+
installPhase = ''
70+
# assemble a static v8 library and put it in the 'out' output
71+
mkdir -p $out/lib
72+
pushd out/Release/obj
73+
find . -path "**/torque_*/**/*.o" -or -path "**/v8*/**/*.o" \
74+
-and -not -name "torque.*" \
75+
-and -not -name "mksnapshot.*" \
76+
-and -not -name "gen-regexp-special-case.*" \
77+
-and -not -name "bytecode_builtins_list_generator.*" \
78+
| sort -u >files
79+
test -s files # ensure that the list is not empty
80+
$AR -cqs $out/lib/out.a @files
81+
popd
82+
83+
# copy v8 headers
84+
cp -r deps/v8/include $out/
85+
86+
# create a pkgconfig file for v8
87+
major=$(grep V8_MAJOR_VERSION deps/v8/include/v8-version.h | cut -d ' ' -f 3)
88+
minor=$(grep V8_MINOR_VERSION deps/v8/include/v8-version.h | cut -d ' ' -f 3)
89+
patch=$(grep V8_PATCH_LEVEL deps/v8/include/v8-version.h | cut -d ' ' -f 3)
90+
mkdir -p $out/lib/pkgconfig
91+
cat > $out/lib/pkgconfig/v8.pc << EOF
92+
Name: v8
93+
Description: V8 JavaScript Engine
94+
Version: $major.$minor.$patch
95+
Libs: -L$out/lib -lv8 -pthread -licui18n -licuuc
96+
Cflags: -I$out/include
97+
EOF
98+
'';
99+
})

0 commit comments

Comments
 (0)