Skip to content

Commit f7179e4

Browse files
sbc100sbc100
authored
Simplify MEMFS to always use TypedArray for file data. NFC (#26398)
As far as I can tell we have not actually support anything else for a long time so I think comment that I'm updating here was inaccurate. There are several places where `node.contents.subarray` (only available on TypedArrays) assumed in the code. For example the existing `expandFileStorage` and `resizeFileStorage` both only operator on TypedArrays.
1 parent 0d24370 commit f7179e4

14 files changed

+109
-122
lines changed

‎src/lib/libmemfs.js‎

Lines changed: 36 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -70,11 +70,14 @@ addToLibrary({
7070
} else if (FS.isFile(node.mode)) {
7171
node.node_ops = MEMFS.ops_table.file.node;
7272
node.stream_ops = MEMFS.ops_table.file.stream;
73-
node.usedBytes = 0; // The actual number of bytes used in the typed array, as opposed to contents.length which gives the whole capacity.
74-
// When the byte data of the file is populated, this will point to either a typed array, or a normal JS array. Typed arrays are preferred
75-
// for performance, and used by default. However, typed arrays are not resizable like normal JS arrays are, so there is a small disk size
76-
// penalty involved for appending file writes that continuously grow a file similar to std::vector capacity vs used -scheme.
77-
node.contents = null;
73+
// The actual number of bytes used in the typed array, as opposed to
74+
// contents.length which gives the whole capacity.
75+
node.usedBytes = 0;
76+
// The byte data of the file is stored in a typed array.
77+
// Note: typed arrays are not resizable like normal JS arrays are, so
78+
// there is a small penalty involved for appending file writes that
79+
// continuously grow a file similar to std::vector capacity vs used.
80+
node.contents = MEMFS.emptyFileContents ??= new Uint8Array(0);
7881
} else if (FS.isLink(node.mode)) {
7982
node.node_ops = MEMFS.ops_table.link.node;
8083
node.stream_ops = MEMFS.ops_table.link.stream;
@@ -93,42 +96,41 @@ addToLibrary({
9396

9497
// Given a file node, returns its file data converted to a typed array.
9598
getFileDataAsTypedArray(node) {
96-
if (!node.contents) return new Uint8Array(0);
97-
if (node.contents.subarray) return node.contents.subarray(0, node.usedBytes); // Make sure to not return excess unused bytes.
98-
return new Uint8Array(node.contents);
99+
#if ASSERTIONS
100+
assert(FS.isFile(node.mode), 'getFileDataAsTypedArray called on non-file');
101+
#endif
102+
return node.contents.subarray(0, node.usedBytes); // Make sure to not return excess unused bytes.
99103
},
100104

101-
// Allocates a new backing store for the given node so that it can fit at least newSize amount of bytes.
102-
// May allocate more, to provide automatic geometric increase and amortized linear performance appending writes.
105+
// Allocates a new backing store for the given node so that it can fit at
106+
// least newSize amount of bytes.
107+
// May allocate more, to provide automatic geometric increase and amortized
108+
// linear performance appending writes.
103109
// Never shrinks the storage.
104110
expandFileStorage(node, newCapacity) {
105-
var prevCapacity = node.contents ? node.contents.length : 0;
111+
var prevCapacity = node.contents.length;
106112
if (prevCapacity >= newCapacity) return; // No need to expand, the storage was already large enough.
107-
// Don't expand strictly to the given requested limit if it's only a very small increase, but instead geometrically grow capacity.
108-
// For small filesizes (<1MB), perform size*2 geometric increase, but for large sizes, do a much more conservative size*1.125 increase to
109-
// avoid overshooting the allocation cap by a very large margin.
113+
// Don't expand strictly to the given requested limit if it's only a very
114+
// small increase, but instead geometrically grow capacity.
115+
// For small filesizes (<1MB), perform size*2 geometric increase, but for
116+
// large sizes, do a much more conservative size*1.125 increase to avoid
117+
// overshooting the allocation cap by a very large margin.
110118
var CAPACITY_DOUBLING_MAX = 1024 * 1024;
111119
newCapacity = Math.max(newCapacity, (prevCapacity * (prevCapacity < CAPACITY_DOUBLING_MAX ? 2.0 : 1.125)) >>> 0);
112-
if (prevCapacity != 0) newCapacity = Math.max(newCapacity, 256); // At minimum allocate 256b for each file when expanding.
113-
var oldContents = node.contents;
120+
if (prevCapacity) newCapacity = Math.max(newCapacity, 256); // At minimum allocate 256b for each file when expanding.
121+
var oldContents = MEMFS.getFileDataAsTypedArray(node);
114122
node.contents = new Uint8Array(newCapacity); // Allocate new storage.
115-
if (node.usedBytes > 0) node.contents.set(oldContents.subarray(0, node.usedBytes), 0); // Copy old data over to the new storage.
123+
node.contents.set(oldContents);
116124
},
117125

118-
// Performs an exact resize of the backing file storage to the given size, if the size is not exactly this, the storage is fully reallocated.
126+
// Performs an exact resize of the backing file storage to the given size,
127+
// if the size is not exactly this, the storage is fully reallocated.
119128
resizeFileStorage(node, newSize) {
120129
if (node.usedBytes == newSize) return;
121-
if (newSize == 0) {
122-
node.contents = null; // Fully decommit when requesting a resize to zero.
123-
node.usedBytes = 0;
124-
} else {
125-
var oldContents = node.contents;
126-
node.contents = new Uint8Array(newSize); // Allocate new storage.
127-
if (oldContents) {
128-
node.contents.set(oldContents.subarray(0, Math.min(newSize, node.usedBytes))); // Copy old data over to the new storage.
129-
}
130-
node.usedBytes = newSize;
131-
}
130+
var oldContents = node.contents;
131+
node.contents = new Uint8Array(newSize); // Allocate new storage.
132+
node.contents.set(oldContents.subarray(0, Math.min(newSize, node.usedBytes))); // Copy old data over to the new storage.
133+
node.usedBytes = newSize;
132134
},
133135

134136
node_ops: {
@@ -242,11 +244,7 @@ addToLibrary({
242244
#if ASSERTIONS
243245
assert(size >= 0);
244246
#endif
245-
if (size > 8 && contents.subarray) { // non-trivial, and typed array
246-
buffer.set(contents.subarray(position, position + size), offset);
247-
} else {
248-
for (var i = 0; i < size; i++) buffer[offset + i] = contents[position + i];
249-
}
247+
buffer.set(contents.subarray(position, position + size), offset);
250248
return size;
251249
},
252250

@@ -275,7 +273,7 @@ addToLibrary({
275273
var node = stream.node;
276274
node.mtime = node.ctime = Date.now();
277275

278-
if (buffer.subarray && (!node.contents || node.contents.subarray)) { // This write is from a typed array to a typed array?
276+
if (buffer.subarray) { // This write is from a typed array to a typed array?
279277
if (canOwn) {
280278
#if ASSERTIONS
281279
assert(position === 0, 'canOwn must imply no weird position inside the file');
@@ -295,12 +293,12 @@ addToLibrary({
295293

296294
// Appending to an existing file and we need to reallocate, or source data did not come as a typed array.
297295
MEMFS.expandFileStorage(node, position+length);
298-
if (node.contents.subarray && buffer.subarray) {
296+
if (buffer.subarray) {
299297
// Use typed array write which is available.
300298
node.contents.set(buffer.subarray(offset, offset + length), position);
301299
} else {
302300
for (var i = 0; i < length; i++) {
303-
node.contents[position + i] = buffer[offset + i]; // Or fall back to manual write if not.
301+
node.contents[position + i] = buffer[offset + i]; // Or fall back to manual write if not.
304302
}
305303
}
306304
node.usedBytes = Math.max(node.usedBytes, position + length);
@@ -329,7 +327,7 @@ addToLibrary({
329327
var allocated;
330328
var contents = stream.node.contents;
331329
// Only make a new copy when MAP_PRIVATE is specified.
332-
if (!(flags & {{{ cDefs.MAP_PRIVATE }}}) && contents && contents.buffer === HEAP8.buffer) {
330+
if (!(flags & {{{ cDefs.MAP_PRIVATE }}}) && contents.buffer === HEAP8.buffer) {
333331
// We can't emulate MAP_SHARED when the file is not backed by the
334332
// buffer we're mapping to (e.g. the HEAP buffer).
335333
allocated = false;

‎test/codesize/test_codesize_cxx_ctors1.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 19555,
3-
"a.out.js.gz": 8102,
2+
"a.out.js": 19344,
3+
"a.out.js.gz": 8016,
44
"a.out.nodebug.wasm": 132813,
55
"a.out.nodebug.wasm.gz": 49862,
6-
"total": 152368,
7-
"total_gz": 57964,
6+
"total": 152157,
7+
"total_gz": 57878,
88
"sent": [
99
"__cxa_throw",
1010
"_abort_js",

‎test/codesize/test_codesize_cxx_ctors2.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 19532,
3-
"a.out.js.gz": 8087,
2+
"a.out.js": 19321,
3+
"a.out.js.gz": 8003,
44
"a.out.nodebug.wasm": 132233,
55
"a.out.nodebug.wasm.gz": 49515,
6-
"total": 151765,
7-
"total_gz": 57602,
6+
"total": 151554,
7+
"total_gz": 57518,
88
"sent": [
99
"__cxa_throw",
1010
"_abort_js",

‎test/codesize/test_codesize_cxx_except.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 23216,
3-
"a.out.js.gz": 9081,
2+
"a.out.js": 23005,
3+
"a.out.js.gz": 8994,
44
"a.out.nodebug.wasm": 172743,
55
"a.out.nodebug.wasm.gz": 57374,
6-
"total": 195959,
7-
"total_gz": 66455,
6+
"total": 195748,
7+
"total_gz": 66368,
88
"sent": [
99
"__cxa_begin_catch",
1010
"__cxa_end_catch",

‎test/codesize/test_codesize_cxx_except_wasm.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 19366,
3-
"a.out.js.gz": 8022,
2+
"a.out.js": 19155,
3+
"a.out.js.gz": 7938,
44
"a.out.nodebug.wasm": 148138,
55
"a.out.nodebug.wasm.gz": 55261,
6-
"total": 167504,
7-
"total_gz": 63283,
6+
"total": 167293,
7+
"total_gz": 63199,
88
"sent": [
99
"_abort_js",
1010
"_tzset_js",

‎test/codesize/test_codesize_cxx_except_wasm_legacy.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 19440,
3-
"a.out.js.gz": 8042,
2+
"a.out.js": 19229,
3+
"a.out.js.gz": 7959,
44
"a.out.nodebug.wasm": 145944,
55
"a.out.nodebug.wasm.gz": 54887,
6-
"total": 165384,
7-
"total_gz": 62929,
6+
"total": 165173,
7+
"total_gz": 62846,
88
"sent": [
99
"_abort_js",
1010
"_tzset_js",

‎test/codesize/test_codesize_cxx_lto.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 18902,
3-
"a.out.js.gz": 7782,
2+
"a.out.js": 18692,
3+
"a.out.js.gz": 7702,
44
"a.out.nodebug.wasm": 102066,
55
"a.out.nodebug.wasm.gz": 39421,
6-
"total": 120968,
7-
"total_gz": 47203,
6+
"total": 120758,
7+
"total_gz": 47123,
88
"sent": [
99
"a (emscripten_resize_heap)",
1010
"b (_setitimer_js)",

‎test/codesize/test_codesize_cxx_mangle.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 23266,
3-
"a.out.js.gz": 9101,
2+
"a.out.js": 23055,
3+
"a.out.js.gz": 9014,
44
"a.out.nodebug.wasm": 239177,
55
"a.out.nodebug.wasm.gz": 79774,
6-
"total": 262443,
7-
"total_gz": 88875,
6+
"total": 262232,
7+
"total_gz": 88788,
88
"sent": [
99
"__cxa_begin_catch",
1010
"__cxa_end_catch",

‎test/codesize/test_codesize_cxx_noexcept.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 19555,
3-
"a.out.js.gz": 8102,
2+
"a.out.js": 19344,
3+
"a.out.js.gz": 8016,
44
"a.out.nodebug.wasm": 134820,
55
"a.out.nodebug.wasm.gz": 50699,
6-
"total": 154375,
7-
"total_gz": 58801,
6+
"total": 154164,
7+
"total_gz": 58715,
88
"sent": [
99
"__cxa_throw",
1010
"_abort_js",

‎test/codesize/test_codesize_file_preload.expected.js‎

Lines changed: 27 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1033,12 +1033,14 @@ var MEMFS = {
10331033
} else if (FS.isFile(node.mode)) {
10341034
node.node_ops = MEMFS.ops_table.file.node;
10351035
node.stream_ops = MEMFS.ops_table.file.stream;
1036+
// The actual number of bytes used in the typed array, as opposed to
1037+
// contents.length which gives the whole capacity.
10361038
node.usedBytes = 0;
1037-
// The actual number of bytes used in the typed array, as opposed to contents.length which gives the whole capacity.
1038-
// When the byte data of the file is populated, this will point to either a typed array, or a normal JS array. Typed arrays are preferred
1039-
// for performance, and used by default. However, typed arrays are not resizable like normal JS arrays are, so there is a small disk size
1040-
// penalty involved for appending file writes that continuously grow a file similar to std::vector capacity vs used -scheme.
1041-
node.contents = null;
1039+
// The byte data of the file is stored in a typed array.
1040+
// Note: typed arrays are not resizable like normal JS arrays are, so
1041+
// there is a small penalty involved for appending file writes that
1042+
// continuously grow a file similar to std::vector capacity vs used.
1043+
node.contents = MEMFS.emptyFileContents ??= new Uint8Array(0);
10421044
} else if (FS.isLink(node.mode)) {
10431045
node.node_ops = MEMFS.ops_table.link.node;
10441046
node.stream_ops = MEMFS.ops_table.link.stream;
@@ -1055,42 +1057,34 @@ var MEMFS = {
10551057
return node;
10561058
},
10571059
getFileDataAsTypedArray(node) {
1058-
if (!node.contents) return new Uint8Array(0);
1059-
if (node.contents.subarray) return node.contents.subarray(0, node.usedBytes);
1060-
// Make sure to not return excess unused bytes.
1061-
return new Uint8Array(node.contents);
1060+
return node.contents.subarray(0, node.usedBytes);
10621061
},
10631062
expandFileStorage(node, newCapacity) {
1064-
var prevCapacity = node.contents ? node.contents.length : 0;
1063+
var prevCapacity = node.contents.length;
10651064
if (prevCapacity >= newCapacity) return;
10661065
// No need to expand, the storage was already large enough.
1067-
// Don't expand strictly to the given requested limit if it's only a very small increase, but instead geometrically grow capacity.
1068-
// For small filesizes (<1MB), perform size*2 geometric increase, but for large sizes, do a much more conservative size*1.125 increase to
1069-
// avoid overshooting the allocation cap by a very large margin.
1066+
// Don't expand strictly to the given requested limit if it's only a very
1067+
// small increase, but instead geometrically grow capacity.
1068+
// For small filesizes (<1MB), perform size*2 geometric increase, but for
1069+
// large sizes, do a much more conservative size*1.125 increase to avoid
1070+
// overshooting the allocation cap by a very large margin.
10701071
var CAPACITY_DOUBLING_MAX = 1024 * 1024;
10711072
newCapacity = Math.max(newCapacity, (prevCapacity * (prevCapacity < CAPACITY_DOUBLING_MAX ? 2 : 1.125)) >>> 0);
1072-
if (prevCapacity != 0) newCapacity = Math.max(newCapacity, 256);
1073+
if (prevCapacity) newCapacity = Math.max(newCapacity, 256);
10731074
// At minimum allocate 256b for each file when expanding.
1074-
var oldContents = node.contents;
1075+
var oldContents = MEMFS.getFileDataAsTypedArray(node);
10751076
node.contents = new Uint8Array(newCapacity);
10761077
// Allocate new storage.
1077-
if (node.usedBytes > 0) node.contents.set(oldContents.subarray(0, node.usedBytes), 0);
1078+
node.contents.set(oldContents);
10781079
},
10791080
resizeFileStorage(node, newSize) {
10801081
if (node.usedBytes == newSize) return;
1081-
if (newSize == 0) {
1082-
node.contents = null;
1083-
// Fully decommit when requesting a resize to zero.
1084-
node.usedBytes = 0;
1085-
} else {
1086-
var oldContents = node.contents;
1087-
node.contents = new Uint8Array(newSize);
1088-
// Allocate new storage.
1089-
if (oldContents) {
1090-
node.contents.set(oldContents.subarray(0, Math.min(newSize, node.usedBytes)));
1091-
}
1092-
node.usedBytes = newSize;
1093-
}
1082+
var oldContents = node.contents;
1083+
node.contents = new Uint8Array(newSize);
1084+
// Allocate new storage.
1085+
node.contents.set(oldContents.subarray(0, Math.min(newSize, node.usedBytes)));
1086+
// Copy old data over to the new storage.
1087+
node.usedBytes = newSize;
10941088
},
10951089
node_ops: {
10961090
getattr(node) {
@@ -1195,19 +1189,14 @@ var MEMFS = {
11951189
var contents = stream.node.contents;
11961190
if (position >= stream.node.usedBytes) return 0;
11971191
var size = Math.min(stream.node.usedBytes - position, length);
1198-
if (size > 8 && contents.subarray) {
1199-
// non-trivial, and typed array
1200-
buffer.set(contents.subarray(position, position + size), offset);
1201-
} else {
1202-
for (var i = 0; i < size; i++) buffer[offset + i] = contents[position + i];
1203-
}
1192+
buffer.set(contents.subarray(position, position + size), offset);
12041193
return size;
12051194
},
12061195
write(stream, buffer, offset, length, position, canOwn) {
12071196
if (!length) return 0;
12081197
var node = stream.node;
12091198
node.mtime = node.ctime = Date.now();
1210-
if (buffer.subarray && (!node.contents || node.contents.subarray)) {
1199+
if (buffer.subarray) {
12111200
// This write is from a typed array to a typed array?
12121201
if (canOwn) {
12131202
node.contents = buffer.subarray(offset, offset + length);
@@ -1226,7 +1215,7 @@ var MEMFS = {
12261215
}
12271216
// Appending to an existing file and we need to reallocate, or source data did not come as a typed array.
12281217
MEMFS.expandFileStorage(node, position + length);
1229-
if (node.contents.subarray && buffer.subarray) {
1218+
if (buffer.subarray) {
12301219
// Use typed array write which is available.
12311220
node.contents.set(buffer.subarray(offset, offset + length), position);
12321221
} else {
@@ -1259,7 +1248,7 @@ var MEMFS = {
12591248
var allocated;
12601249
var contents = stream.node.contents;
12611250
// Only make a new copy when MAP_PRIVATE is specified.
1262-
if (!(flags & 2) && contents && contents.buffer === HEAP8.buffer) {
1251+
if (!(flags & 2) && contents.buffer === HEAP8.buffer) {
12631252
// We can't emulate MAP_SHARED when the file is not backed by the
12641253
// buffer we're mapping to (e.g. the HEAP buffer).
12651254
allocated = false;

0 commit comments

Comments
 (0)