Skip to content

Commit 22b499d

Browse files
kripkenkripken
authored
[WasmFS] Add JSImplBackend (emscripten-core#16209)
This refactors the existing JSFileBackend to use a new model for creating JS backends. A JSImplBackend is defined, which adds a layer of indirection in JS, allowing a backend to be implemented almost 100% in JS. That is, JSImplBackend is a new backend that forwards operations to an implemenation in JS. See full docs in js_impl_backend.h. Note that the diffs here may not be as useful as reading the files in their final state. In particular, look at the how the JSFileBackend is now implemented in a minimal way in C++ in js_file_backend.cpp. This is a step towards it being practical to make backends like Node etc. None of the actual JSFileBackend logic is changed, just moved around.
1 parent e60664d commit 22b499d

5 files changed

Lines changed: 213 additions & 112 deletions

File tree

‎src/library_wasmfs.js‎

Lines changed: 42 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
1-
var WasmfsLibrary = {
2-
$wasmFS$JSMemoryFiles : [],
3-
$wasmFS$JSMemoryFreeList: [],
1+
var WasmFSLibrary = {
42
$wasmFS$preloadedFiles: [],
53
$wasmFS$preloadedDirs: [],
64
$FS__deps: [
75
'$wasmFS$preloadedFiles',
86
'$wasmFS$preloadedDirs',
9-
'$wasmFS$JSMemoryFiles',
10-
'$wasmFS$JSMemoryFreeList',
117
'$asyncLoad',
128
#if !MINIMAL_RUNTIME
139
// TODO: when preload-plugins are not used, we do not need this.
@@ -145,64 +141,54 @@ var WasmfsLibrary = {
145141
var len = lengthBytesUTF8(s) + 1;
146142
stringToUTF8(s, fileNameBuffer, len);
147143
},
148-
_wasmfs_write_js_file: function(index, buffer, length, offset) {
149-
try {
150-
if (!wasmFS$JSMemoryFiles[index]) {
151-
// Initialize typed array on first write operation.
152-
wasmFS$JSMemoryFiles[index] = new Uint8Array(offset + length);
153-
}
154-
155-
if (offset + length > wasmFS$JSMemoryFiles[index].length) {
156-
// Resize the typed array if the length of the write buffer exceeds its capacity.
157-
var oldContents = wasmFS$JSMemoryFiles[index];
158-
var newContents = new Uint8Array(offset + length);
159-
newContents.set(oldContents);
160-
wasmFS$JSMemoryFiles[index] = newContents;
161-
}
162-
163-
wasmFS$JSMemoryFiles[index].set(HEAPU8.subarray(buffer, buffer + length), offset);
164-
return 0;
165-
} catch (err) {
166-
return {{{ cDefine('EIO') }}};
167-
}
144+
_wasmfs_get_preloaded_file_size: function(index) {
145+
return wasmFS$preloadedFiles[index].fileData.length;
168146
},
169-
_wasmfs_read_js_file: function(index, buffer, length, offset) {
170-
try {
171-
HEAPU8.set(wasmFS$JSMemoryFiles[index].subarray(offset, offset + length), buffer);
172-
return 0;
173-
} catch (err) {
174-
return {{{ cDefine('EIO') }}};
175-
}
147+
_wasmfs_copy_preloaded_file_data: function(index, buffer) {
148+
HEAPU8.set(wasmFS$preloadedFiles[index].fileData, buffer);
176149
},
177-
_wasmfs_get_js_file_size: function(index) {
178-
return wasmFS$JSMemoryFiles[index] ? wasmFS$JSMemoryFiles[index].length : 0;
150+
151+
// Backend support. wasmFS$backends will contain a mapping of backend IDs to
152+
// the JS code that implements them. This is the JS side of the JSImpl class
153+
// in C++, together with the js_impl calls defined right after it.
154+
$wasmFS$backends: {},
155+
156+
_wasmfs_jsimpl_alloc_file: function(backend, file) {
157+
#if ASSERTIONS
158+
assert(wasmFS$backends[backend]);
159+
#endif
160+
return wasmFS$backends[backend].alloc_file(file);
179161
},
180-
_wasmfs_create_js_file: function() {
181-
// Find a free entry in the $wasmFS$JSMemoryFreeList or append a new entry to
182-
// wasmFS$JSMemoryFiles.
183-
if (wasmFS$JSMemoryFreeList.length) {
184-
// Pop off the top of the free list.
185-
var index = wasmFS$JSMemoryFreeList.pop();
186-
return index;
187-
}
188-
wasmFS$JSMemoryFiles.push(null);
189-
return wasmFS$JSMemoryFiles.length - 1;
162+
163+
_wasmfs_jsimpl_free_file: function(backend, file) {
164+
#if ASSERTIONS
165+
assert(wasmFS$backends[backend]);
166+
#endif
167+
return wasmFS$backends[backend].free_file(file);
190168
},
191-
_wasmfs_remove_js_file: function(index) {
192-
wasmFS$JSMemoryFiles[index] = null;
193-
// Add the index to the free list.
194-
wasmFS$JSMemoryFreeList.push(index);
169+
170+
_wasmfs_jsimpl_write: function(backend, file, buffer, length, offset) {
171+
#if ASSERTIONS
172+
assert(wasmFS$backends[backend]);
173+
#endif
174+
return wasmFS$backends[backend].write(file, buffer, length, offset);
195175
},
196-
_wasmfs_get_preloaded_file_size: function(index) {
197-
return wasmFS$preloadedFiles[index].fileData.length;
176+
177+
_wasmfs_jsimpl_read: function(backend, file, buffer, length, offset) {
178+
#if ASSERTIONS
179+
assert(wasmFS$backends[backend]);
180+
#endif
181+
return wasmFS$backends[backend].read(file, buffer, length, offset);
198182
},
199-
_wasmfs_copy_preloaded_file_data: function(index, buffer) {
200-
HEAPU8.set(wasmFS$preloadedFiles[index].fileData, buffer);
183+
184+
_wasmfs_jsimpl_get_size: function(backend, file) {
185+
#if ASSERTIONS
186+
assert(wasmFS$backends[backend]);
187+
#endif
188+
return wasmFS$backends[backend].getSize(file);
201189
},
202190
}
203191

204-
mergeInto(LibraryManager.library, WasmfsLibrary);
192+
mergeInto(LibraryManager.library, WasmFSLibrary);
205193

206-
if (WASMFS) {
207-
DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$FS');
208-
}
194+
DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$FS');

‎src/library_wasmfs_js_file.js‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
mergeInto(LibraryManager.library, {
2+
// JSFile backend: Store a file's data in JS. We map File objects in C++ to
3+
// entries here that contain typed arrays.
4+
$wasmFS$JSMemoryFiles: {},
5+
6+
_wasmfs_create_js_file_backend_js__deps: [
7+
'$wasmFS$backends',
8+
'$wasmFS$JSMemoryFiles',
9+
],
10+
_wasmfs_create_js_file_backend_js: function(backend) {
11+
wasmFS$backends[backend] = {
12+
alloc_file: function(file) {
13+
// Do nothing: we allocate the typed array lazily, see write()
14+
},
15+
free_file: function(file) {
16+
// Release the memory, as it now has no references to it any more.
17+
wasmFS$JSMemoryFiles[file] = undefined;
18+
},
19+
write: function(file, buffer, length, offset) {
20+
try {
21+
if (!wasmFS$JSMemoryFiles[file]) {
22+
// Initialize typed array on first write operation.
23+
wasmFS$JSMemoryFiles[file] = new Uint8Array(offset + length);
24+
}
25+
26+
if (offset + length > wasmFS$JSMemoryFiles[file].length) {
27+
// Resize the typed array if the length of the write buffer exceeds its capacity.
28+
var oldContents = wasmFS$JSMemoryFiles[file];
29+
var newContents = new Uint8Array(offset + length);
30+
newContents.set(oldContents);
31+
wasmFS$JSMemoryFiles[file] = newContents;
32+
}
33+
34+
wasmFS$JSMemoryFiles[file].set(HEAPU8.subarray(buffer, buffer + length), offset);
35+
return 0;
36+
} catch (err) {
37+
return {{{ cDefine('EIO') }}};
38+
}
39+
},
40+
read: function(file, buffer, length, offset) {
41+
try {
42+
HEAPU8.set(wasmFS$JSMemoryFiles[file].subarray(offset, offset + length), buffer);
43+
return 0;
44+
} catch (err) {
45+
return {{{ cDefine('EIO') }}};
46+
}
47+
},
48+
getSize: function(file) {
49+
return wasmFS$JSMemoryFiles[file] ? wasmFS$JSMemoryFiles[file].length : 0;
50+
},
51+
};
52+
},
53+
});

‎src/modules.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ global.LibraryManager = {
9595
}
9696
} else if (WASMFS) {
9797
libraries.push('library_wasmfs.js');
98+
libraries.push('library_wasmfs_js_file.js');
9899
}
99100

100101
// Additional JS libraries (without AUTO_JS_LIBRARIES, link to these explicitly via -lxxx.js)

‎system/lib/wasmfs/js_file_backend.cpp‎

Lines changed: 6 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -4,74 +4,24 @@
44
// found in the LICENSE file.
55

66
// This file defines the JS file backend and JS file of the new file system.
7-
// Current Status: Work in Progress.
87
// See https://github.com/emscripten-core/emscripten/issues/15041.
98

109
#include "backend.h"
10+
#include "js_impl_backend.h"
1111
#include "wasmfs.h"
1212

13-
using js_index_t = uint32_t;
13+
// See library_wasmfs_js_file.js
1414

1515
extern "C" {
16-
int _wasmfs_write_js_file(js_index_t index,
17-
const uint8_t* buffer,
18-
size_t length,
19-
off_t offset);
20-
int _wasmfs_read_js_file(js_index_t index,
21-
const uint8_t* buffer,
22-
size_t length,
23-
off_t offset);
24-
int _wasmfs_get_js_file_size(js_index_t index);
25-
int _wasmfs_create_js_file();
26-
void _wasmfs_remove_js_file(js_index_t index);
16+
void _wasmfs_create_js_file_backend_js(wasmfs::backend_t);
2717
}
2818

2919
namespace wasmfs {
3020

31-
// This class describes a file that lives in JS Memory
32-
class JSFile : public DataFile {
33-
// This index indicates the location of the JS File in the backing JS array.
34-
js_index_t index;
35-
36-
// JSFiles will write from a Wasm Memory buffer into the backing JS array.
37-
__wasi_errno_t write(const uint8_t* buf, size_t len, off_t offset) override {
38-
return _wasmfs_write_js_file(index, buf, len, offset);
39-
}
40-
41-
// JSFiles will read from the backing JS array into a Wasm Memory buffer.
42-
__wasi_errno_t read(uint8_t* buf, size_t len, off_t offset) override {
43-
// The caller should have already checked that the offset + len does
44-
// not exceed the file's size.
45-
assert(offset + len <= getSize());
46-
return _wasmfs_read_js_file(index, buf, len, offset);
47-
}
48-
49-
void flush() override {}
50-
51-
// The size of the JSFile is defined as the length of the backing JS array.
52-
size_t getSize() override { return _wasmfs_get_js_file_size(index); }
53-
54-
public:
55-
JSFile(mode_t mode, backend_t backend) : DataFile(mode, backend) {
56-
// Create a new file in the backing JS array and store its index.
57-
index = _wasmfs_create_js_file();
58-
}
59-
60-
// Remove the typed array file contents in the backing JS array.
61-
~JSFile() { _wasmfs_remove_js_file(index); }
62-
};
63-
64-
class JSFileBackend : public Backend {
65-
66-
public:
67-
std::shared_ptr<DataFile> createFile(mode_t mode) override {
68-
return std::make_shared<JSFile>(mode, this);
69-
}
70-
};
71-
72-
// This function is exposed to users to instantiate a new JSBackend.
7321
extern "C" backend_t wasmfs_create_js_file_backend() {
74-
return wasmFS.addBackend(std::make_unique<JSFileBackend>());
22+
backend_t backend = wasmFS.addBackend(std::make_unique<JSImplBackend>());
23+
_wasmfs_create_js_file_backend_js(backend);
24+
return backend;
7525
}
7626

7727
} // namespace wasmfs

‎system/lib/wasmfs/js_impl_backend.h‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Copyright 2022 The Emscripten Authors. All rights reserved.
2+
// Emscripten is available under two separate licenses, the MIT license and the
3+
// University of Illinois/NCSA Open Source License. Both these licenses can be
4+
// found in the LICENSE file.
5+
6+
// This file defines the JS file backend and JS file of the new file system.
7+
// Current Status: Work in Progress.
8+
// See https://github.com/emscripten-core/emscripten/issues/15041.
9+
10+
#pragma once
11+
12+
#include "backend.h"
13+
#include "wasmfs.h"
14+
15+
//
16+
// A JS Impl backend has files that are implemented by JS code. Each backend
17+
// object in C++ can call JS to define the proper JS code on that side, and we
18+
// keep a mapping of C++ pointer of backend => JS code in JS. Then, when we do
19+
// something like a read from a JSImplFile we pass it the backend, find the
20+
// proper code to run, and run it. This adds a layer of indirection at the JS
21+
// level that makes it easy to write new backends in 99% JS.
22+
//
23+
// Each file operation in the _wasmfs_jsimpl_* APIs that we call from here take
24+
// the backend and a pointer to this file itself. Those allow the JS to identify
25+
// both the backend and the particular file. TODO: We could use dense indexes
26+
// instead of pointers, and use an array instead of a map.
27+
//
28+
// To write a new backend in JS, you basically do the following:
29+
//
30+
// 1. Add a declaration of the C function to create the backend in the
31+
// "backend creation" section of emscripten/wasmfs.h.
32+
// 2. Add a cpp file for the new backend, and implement the C function from 1,
33+
// which should create it on both the C++ (using JSImplBackend) and JS
34+
// sides. (By convention, the C function should just call into C++ and JS
35+
// which do the interesting work; the C is just a thin wrapper.)
36+
// 3. Write a new JS library, and add the implementation of the JS method just
37+
// mentioned, which should set up the mapping from the C++ backend object's
38+
// address to the JS code containing the hooks to read and write etc.
39+
//
40+
// For a simple example, see js_file_backend.cpp and library_wasmfs_js_file.js
41+
//
42+
43+
using js_index_t = uint32_t;
44+
45+
extern "C" {
46+
// JSImpl API (see below for overview).
47+
void _wasmfs_jsimpl_alloc_file(js_index_t backend, js_index_t index);
48+
void _wasmfs_jsimpl_free_file(js_index_t backend, js_index_t index);
49+
int _wasmfs_jsimpl_write(js_index_t backend,
50+
js_index_t index,
51+
const uint8_t* buffer,
52+
size_t length,
53+
off_t offset);
54+
int _wasmfs_jsimpl_read(js_index_t backend,
55+
js_index_t index,
56+
const uint8_t* buffer,
57+
size_t length,
58+
off_t offset);
59+
int _wasmfs_jsimpl_get_size(js_index_t backend, js_index_t index);
60+
}
61+
62+
namespace wasmfs {
63+
64+
class JSImplFile : public DataFile {
65+
js_index_t getBackendIndex() {
66+
static_assert(sizeof(backend_t) == sizeof(js_index_t), "TODO: wasm64");
67+
return js_index_t(getBackend());
68+
}
69+
70+
js_index_t getFileIndex() {
71+
static_assert(sizeof(this) == sizeof(js_index_t), "TODO: wasm64");
72+
return js_index_t(this);
73+
}
74+
75+
__wasi_errno_t write(const uint8_t* buf, size_t len, off_t offset) override {
76+
return _wasmfs_jsimpl_write(
77+
getBackendIndex(), getFileIndex(), buf, len, offset);
78+
}
79+
80+
__wasi_errno_t read(uint8_t* buf, size_t len, off_t offset) override {
81+
// The caller should have already checked that the offset + len does
82+
// not exceed the file's size.
83+
assert(offset + len <= getSize());
84+
return _wasmfs_jsimpl_read(
85+
getBackendIndex(), getFileIndex(), buf, len, offset);
86+
}
87+
88+
void flush() override {}
89+
90+
size_t getSize() override {
91+
return _wasmfs_jsimpl_get_size(getBackendIndex(), getFileIndex());
92+
}
93+
94+
public:
95+
JSImplFile(mode_t mode, backend_t backend) : DataFile(mode, backend) {
96+
_wasmfs_jsimpl_alloc_file(getBackendIndex(), getFileIndex());
97+
}
98+
99+
~JSImplFile() {
100+
_wasmfs_jsimpl_free_file(getBackendIndex(), getFileIndex());
101+
}
102+
};
103+
104+
class JSImplBackend : public Backend {
105+
public:
106+
std::shared_ptr<DataFile> createFile(mode_t mode) override {
107+
return std::make_shared<JSImplFile>(mode, this);
108+
}
109+
};
110+
111+
} // namespace wasmfs

0 commit comments

Comments
 (0)