Skip to content

Commit 7e6d511

Browse files
ethanaleeethanalee
authored
[WASMFS] JS File Backend Structure (emscripten-core#15562)
Relevant Issue: emscripten-core#15041 Tentative JS File Backend Structure Introduced user-visible header file. Users can create a new JS Backend and create JS Backed Files under both In-Memory and JS Backend directories.
1 parent 63b3f94 commit 7e6d511

14 files changed

Lines changed: 467 additions & 38 deletions

File tree

‎src/library_wasmfs.js‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
var WasmfsLibrary = {
2+
$wasmFS$JSMemoryFiles : [],
3+
$wasmFS$JSMemoryFreeList: [],
24
$wasmFS$preloadedFiles: [],
35
$wasmFS$preloadedDirs: [],
4-
$FS__deps: ['$wasmFS$preloadedFiles', '$wasmFS$preloadedDirs'],
6+
$FS__deps: ['$wasmFS$preloadedFiles', '$wasmFS$preloadedDirs', '$wasmFS$JSMemoryFiles', '$wasmFS$JSMemoryFreeList'],
57
$FS : {
68
// TODO: Clean up the following functions - currently copied from library_fs.js directly.
79
createPreloadedFile: function(parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish) {
@@ -85,6 +87,54 @@ var WasmfsLibrary = {
8587
// For file preloading, cwd should be '/' to begin with.
8688
return '/';
8789
}
90+
},
91+
_emscripten_write_js_file: function(index, buffer, length, offset) {
92+
try {
93+
if (!wasmFS$JSMemoryFiles[index]) {
94+
// Initialize typed array on first write operation.
95+
wasmFS$JSMemoryFiles[index] = new Uint8Array(offset + length);
96+
}
97+
98+
if (offset + length > wasmFS$JSMemoryFiles[index].length) {
99+
// Resize the typed array if the length of the write buffer exceeds its capacity.
100+
var oldContents = wasmFS$JSMemoryFiles[index];
101+
var newContents = new Uint8Array(offset + length);
102+
newContents.set(oldContents);
103+
wasmFS$JSMemoryFiles[index] = newContents;
104+
}
105+
106+
wasmFS$JSMemoryFiles[index].set(HEAPU8.subarray(buffer, buffer + length), offset);
107+
return 0;
108+
} catch (err) {
109+
return {{{ cDefine('EIO') }}};
110+
}
111+
},
112+
_emscripten_read_js_file: function(index, buffer, length, offset) {
113+
try {
114+
HEAPU8.set(wasmFS$JSMemoryFiles[index].subarray(offset, offset + length), buffer);
115+
return 0;
116+
} catch (err) {
117+
return {{{ cDefine('EIO') }}};
118+
}
119+
},
120+
_emscripten_get_js_file_size: function(index) {
121+
return wasmFS$JSMemoryFiles[index] ? wasmFS$JSMemoryFiles[index].length : 0;
122+
},
123+
_emscripten_create_js_file: function() {
124+
// Find a free entry in the $wasmFS$JSMemoryFreeList or append a new entry to
125+
// wasmFS$JSMemoryFiles.
126+
if (wasmFS$JSMemoryFreeList.length) {
127+
// Pop off the top of the free list.
128+
var index = wasmFS$JSMemoryFreeList.pop();
129+
return index;
130+
}
131+
wasmFS$JSMemoryFiles.push(null);
132+
return wasmFS$JSMemoryFiles.length - 1;
133+
},
134+
_emscripten_remove_js_file: function(index) {
135+
wasmFS$JSMemoryFiles[index] = null;
136+
// Add the index to the free list.
137+
wasmFS$JSMemoryFreeList.push(index);
88138
}
89139
}
90140

‎system/include/emscripten/wasmfs.h‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Copyright 2021 The Emscripten Authors. All rights reserved.
3+
* Emscripten is available under two separate licenses, the MIT license and the
4+
* University of Illinois/NCSA Open Source License. Both these licenses can be
5+
* found in the LICENSE file.
6+
*/
7+
8+
#pragma once
9+
10+
#include <stdint.h>
11+
#include <sys/stat.h>
12+
13+
#ifdef __cplusplus
14+
extern "C" {
15+
#endif
16+
17+
typedef struct Backend* backend_t;
18+
19+
// Obtains the backend_t of a specified path.
20+
backend_t wasmfs_get_backend_by_path(char* path);
21+
22+
// Obtains the backend_t of a specified fd.
23+
backend_t wasmfs_get_backend_by_fd(int fd);
24+
25+
// Creates a JSFile Backend in the new file system.
26+
backend_t wasmfs_create_js_file_backend();
27+
28+
// Creates a file in a specific backend and returns an fd to an open file.
29+
uint32_t wasmfs_create_file(char* pathname, mode_t mode, backend_t backend);
30+
31+
// Creates a new directory in the new file system under a specific backend.
32+
long wasmfs_create_directory(char* path, long mode, backend_t backend);
33+
34+
#ifdef __cplusplus
35+
}
36+
#endif

‎system/lib/wasmfs/backend.h‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414

1515
namespace wasmfs {
1616
// A backend (or modular backend) provides a base for the new file system to
17-
// extend its storage capabilities. Files and directories will be represented in
18-
// the file system structure, but their underlying backing could exist in
17+
// extend its storage capabilities. Files and directories will be represented
18+
// in the file system structure, but their underlying backing could exist in
1919
// persistent storage, another thread, etc.
2020
class Backend {
2121

@@ -29,7 +29,5 @@ class Backend {
2929
// Note: Backends will be defined in cpp files, but functions to instantiate
3030
// them will be defined in a header file. This is so that any unused backends
3131
// are not linked in if they are not called.
32-
// TODO: In the next PR, a user-visible header is introduced. Update this
33-
// comment then.
3432
backend_t createMemoryFileBackend();
3533
} // namespace wasmfs

‎system/lib/wasmfs/file.h‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Emscripten is available under two separate licenses, the MIT license and the
33
// University of Illinois/NCSA Open Source License. Both these licenses can be
44
// found in the LICENSE file.
5+
56
// This file defines the file object of the new file system.
67
// Current Status: Work in Progress.
78
// See https://github.com/emscripten-core/emscripten/issues/15041.
@@ -25,6 +26,7 @@ class Backend;
2526
// This represents an opaque pointer to a Backend. A user may use this to
2627
// specify a backend in file operations.
2728
using backend_t = Backend*;
29+
const backend_t NullBackend = nullptr;
2830

2931
class File : public std::enable_shared_from_this<File> {
3032

@@ -62,6 +64,8 @@ class File : public std::enable_shared_from_this<File> {
6264
return (ino_t)this;
6365
}
6466

67+
backend_t getBackend() { return backend; }
68+
6569
class Handle {
6670

6771
protected:
@@ -101,7 +105,8 @@ class File : public std::enable_shared_from_this<File> {
101105
}
102106

103107
protected:
104-
File(FileKind kind, mode_t mode) : kind(kind), mode(mode) {}
108+
File(FileKind kind, mode_t mode, backend_t backend)
109+
: kind(kind), mode(mode), backend(backend) {}
105110
// A mutex is needed for multiple accesses to the same file.
106111
std::recursive_mutex mutex;
107112

@@ -121,6 +126,9 @@ class File : public std::enable_shared_from_this<File> {
121126
// dependencies where the parent and child have shared_ptrs that reference
122127
// each other. This prevents the case in which an uncollectable cycle occurs.
123128
std::weak_ptr<File> parent;
129+
130+
// This specifies which backend a file is associated with.
131+
backend_t backend;
124132
};
125133

126134
class DataFile : public File {
@@ -131,7 +139,8 @@ class DataFile : public File {
131139

132140
public:
133141
static constexpr FileKind expectedKind = File::DataFileKind;
134-
DataFile(mode_t mode) : File(File::DataFileKind, mode) {}
142+
DataFile(mode_t mode, backend_t backend)
143+
: File(File::DataFileKind, mode, backend) {}
135144
virtual ~DataFile() = default;
136145

137146
class Handle : public File::Handle {
@@ -161,15 +170,10 @@ class Directory : public File {
161170
// This value was also copied from the existing file system.
162171
size_t getSize() override { return 4096; }
163172

164-
// This specifies which backend a directory is associated with. By default,
165-
// files and sub-directories added to this directory's entries will be created
166-
// through this same backend unless an alternative is specified.
167-
backend_t backend;
168-
169173
public:
170174
static constexpr FileKind expectedKind = File::DirectoryKind;
171175
Directory(mode_t mode, backend_t backend)
172-
: File(File::DirectoryKind, mode), backend(backend) {}
176+
: File(File::DirectoryKind, mode, backend) {}
173177

174178
struct Entry {
175179
std::string name;
@@ -231,8 +235,6 @@ class Directory : public File {
231235
return entries;
232236
}
233237

234-
backend_t getBackend() { return getDir()->backend; }
235-
236238
#ifdef WASMFS_DEBUG
237239
void printKeys() {
238240
for (auto keyPair : getDir()->entries) {
@@ -253,7 +255,6 @@ class Directory : public File {
253255
}
254256
}
255257
};
256-
257258
// Obtains parent directory of a given pathname.
258259
// Will return a nullptr if the parent is not a directory.
259260
// Will error if the forbiddenAncestor is encountered while processing.
@@ -265,8 +266,8 @@ getDir(std::vector<std::string>::iterator begin,
265266
long& err,
266267
std::shared_ptr<File> forbiddenAncestor = nullptr);
267268

268-
// Return a vector of the '/'-delimited components of a path. The first element
269-
// will be "/" iff the path is an absolute path.
269+
// Return a vector of the '/'-delimited components of a path. The first
270+
// element will be "/" iff the path is an absolute path.
270271
std::vector<std::string> splitPath(char* pathname);
271272

272273
} // namespace wasmfs

‎system/lib/wasmfs/js_file_backend.cpp‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Copyright 2021 The Emscripten Authors. All rights reserved.
2+
// Emscripten is available under two separate licenses, the MIT license and the
3+
// University of Illinois/NCSA Open Source License. Both these licenses can be
4+
// found in the LICENSE file.
5+
6+
// This file defines the JS file backend and JS file of the new file system.
7+
// Current Status: Work in Progress.
8+
// See https://github.com/emscripten-core/emscripten/issues/15041.
9+
10+
#include "backend.h"
11+
#include "wasmfs.h"
12+
13+
using js_index_t = uint32_t;
14+
15+
extern "C" {
16+
int _emscripten_write_js_file(js_index_t index,
17+
const uint8_t* buffer,
18+
size_t length,
19+
off_t offset);
20+
int _emscripten_read_js_file(js_index_t index,
21+
const uint8_t* buffer,
22+
size_t length,
23+
off_t offset);
24+
int _emscripten_get_js_file_size(js_index_t index);
25+
int _emscripten_create_js_file();
26+
void _emscripten_remove_js_file(js_index_t index);
27+
}
28+
29+
namespace wasmfs {
30+
31+
// This class describes a file that lives in JS Memory
32+
class JSFile : public DataFile {
33+
// This index indicates the location of the JS File in the backing JS array.
34+
js_index_t index;
35+
36+
// JSFiles will write from a Wasm Memory buffer into the backing JS array.
37+
__wasi_errno_t write(const uint8_t* buf, size_t len, off_t offset) override {
38+
return _emscripten_write_js_file(index, buf, len, offset);
39+
}
40+
41+
// JSFiles will read from the backing JS array into a Wasm Memory buffer.
42+
__wasi_errno_t read(uint8_t* buf, size_t len, off_t offset) override {
43+
// The caller should have already checked that the offset + len does
44+
// not exceed the file's size.
45+
assert(offset + len <= getSize());
46+
return _emscripten_read_js_file(index, buf, len, offset);
47+
}
48+
49+
// The size of the JSFile is defined as the length of the backing JS array.
50+
size_t getSize() override { return _emscripten_get_js_file_size(index); }
51+
52+
public:
53+
JSFile(mode_t mode, backend_t backend) : DataFile(mode, backend) {
54+
// Create a new file in the backing JS array and store its index.
55+
index = _emscripten_create_js_file();
56+
}
57+
58+
// Remove the typed array file contents in the backing JS array.
59+
~JSFile() { _emscripten_remove_js_file(index); }
60+
};
61+
62+
class JSFileBackend : public Backend {
63+
64+
public:
65+
std::shared_ptr<DataFile> createFile(mode_t mode) override {
66+
return std::make_shared<JSFile>(mode, this);
67+
}
68+
std::shared_ptr<Directory> createDirectory(mode_t mode) override {
69+
return std::make_shared<Directory>(mode, this);
70+
}
71+
};
72+
73+
// This function is exposed to users to instantiate a new JSBackend.
74+
extern "C" backend_t wasmfs_create_js_file_backend() {
75+
return wasmFS.addBackend(std::make_unique<JSFileBackend>());
76+
}
77+
78+
} // namespace wasmfs

‎system/lib/wasmfs/memory_file.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Emscripten is available under two separate licenses, the MIT license and the
33
// University of Illinois/NCSA Open Source License. Both these licenses can be
44
// found in the LICENSE file.
5+
56
// This file defines the memory file class of the new file system.
67
// This should be the only backend file type defined in a header since it is the
78
// default type. Current Status: Work in Progress. See
@@ -23,7 +24,7 @@ class MemoryFile : public DataFile {
2324
size_t getSize() override { return buffer.size(); }
2425

2526
public:
26-
MemoryFile(mode_t mode) : DataFile(mode) {}
27+
MemoryFile(mode_t mode, backend_t backend) : DataFile(mode, backend) {}
2728

2829
class Handle : public DataFile::Handle {
2930

‎system/lib/wasmfs/memory_file_backend.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Emscripten is available under two separate licenses, the MIT license and the
33
// University of Illinois/NCSA Open Source License. Both these licenses can be
44
// found in the LICENSE file.
5+
56
// This file defines the memory file backend of the new file system.
67
// Current Status: Work in Progress.
78
// See https://github.com/emscripten-core/emscripten/issues/15041.
@@ -15,7 +16,7 @@ class MemoryFileBackend : public Backend {
1516

1617
public:
1718
std::shared_ptr<DataFile> createFile(mode_t mode) override {
18-
return std::make_shared<MemoryFile>(mode);
19+
return std::make_shared<MemoryFile>(mode, this);
1920
}
2021
std::shared_ptr<Directory> createDirectory(mode_t mode) override {
2122
return std::make_shared<Directory>(mode, this);

‎system/lib/wasmfs/streams.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ class StdinFile : public DataFile {
2828
size_t getSize() override { return 0; }
2929

3030
public:
31-
StdinFile(mode_t mode) : DataFile(mode) {}
31+
StdinFile(mode_t mode) : DataFile(mode, NullBackend) {}
3232
static std::shared_ptr<StdinFile> getSingleton();
3333
};
3434

@@ -45,7 +45,7 @@ class StdoutFile : public DataFile {
4545
size_t getSize() override { return 0; }
4646

4747
public:
48-
StdoutFile(mode_t mode) : DataFile(mode) {}
48+
StdoutFile(mode_t mode) : DataFile(mode, NullBackend) {}
4949
static std::shared_ptr<StdoutFile> getSingleton();
5050
};
5151

@@ -65,7 +65,7 @@ class StderrFile : public DataFile {
6565
size_t getSize() override { return 0; }
6666

6767
public:
68-
StderrFile(mode_t mode) : DataFile(mode) {}
68+
StderrFile(mode_t mode) : DataFile(mode, NullBackend) {}
6969
static std::shared_ptr<StderrFile> getSingleton();
7070
};
7171
} // namespace wasmfs

0 commit comments

Comments
 (0)