Skip to content

Commit 8e2ae7e

Browse files
jlongsternrakochy
jlongster
authored and
nrakochy
committed
Implement changes required for IndexedDB-backed filesystem
1 parent a161ff4 commit 8e2ae7e

File tree

6 files changed

+163
-12
lines changed

6 files changed

+163
-12
lines changed

‎Makefile‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ EMFLAGS_DEBUG = \
5959
-s ASSERTIONS=1 \
6060
-O1
6161

62-
BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc
62+
BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc out/vfs.bc
6363

6464
OUTPUT_WRAPPER_FILES = src/shell-pre.js src/shell-post.js
6565

@@ -76,13 +76,13 @@ all: optimized debug worker
7676
debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js
7777

7878
dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
79-
$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
79+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
8080
mv $@ out/tmp-raw.js
8181
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
8282
rm out/tmp-raw.js
8383

8484
dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
85-
$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
85+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
8686
mv $@ out/tmp-raw.js
8787
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
8888
rm out/tmp-raw.js
@@ -91,19 +91,19 @@ dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FI
9191
optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js
9292

9393
dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
94-
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
94+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
9595
mv $@ out/tmp-raw.js
9696
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
9797
rm out/tmp-raw.js
9898

9999
dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
100-
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
100+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
101101
mv $@ out/tmp-raw.js
102102
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
103103
rm out/tmp-raw.js
104104

105105
dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
106-
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
106+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
107107
mv $@ out/tmp-raw.js
108108
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
109109
rm out/tmp-raw.js
@@ -153,6 +153,11 @@ out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION)
153153
# Generate llvm bitcode
154154
$(EMCC) $(SQLITE_COMPILATION_FLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@
155155

156+
out/vfs.bc: src/vfs.c sqlite-src/$(SQLITE_AMALGAMATION)
157+
mkdir -p out
158+
# Generate llvm bitcode
159+
$(EMCC) $(CFLAGS) -s LINKABLE=1 -I sqlite-src/$(SQLITE_AMALGAMATION) -c src/vfs.c -o $@
160+
156161
# TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder
157162
# module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js
158163
# tar --create --gzip $^ > $@

‎src/api.js‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -819,13 +819,19 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
819819
* @memberof module:SqlJs
820820
* Open a new database either by creating a new one or opening an existing
821821
* one stored in the byte array passed in first argument
822-
* @param {number[]} data An array of bytes representing
823-
* an SQLite database file
822+
* @param {number[]|string} data An array of bytes representing
823+
* an SQLite database file or a path
824+
* @param {Object} opts Options to specify a filename
824825
*/
825-
function Database(data) {
826-
this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0);
827-
if (data != null) {
826+
function Database(data, { filename = false } = {}) {
827+
if(filename === false) {
828+
this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0);
829+
if (data != null) {
828830
FS.createDataFile("/", this.filename, data, true, true);
831+
}
832+
}
833+
else {
834+
this.filename = data;
829835
}
830836
this.handleError(sqlite3_open(this.filename, apiTemp));
831837
this.db = getValue(apiTemp, "i32");
@@ -1379,4 +1385,44 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
13791385

13801386
// export Database to Module
13811387
Module.Database = Database;
1388+
1389+
// Because emscripten doesn't allow us to handle `ioctl`, we need
1390+
// to manually install lock/unlock methods. Unfortunately we need
1391+
// to keep track of a mapping of `sqlite_file*` pointers to filename
1392+
// so that we can tell our filesystem which files to lock/unlock
1393+
var sqliteFiles = new Map();
1394+
1395+
Module["register_for_idb"] = (customFS) => {
1396+
var SQLITE_BUSY = 5;
1397+
1398+
function open(namePtr, file) {
1399+
var path = UTF8ToString(namePtr);
1400+
sqliteFiles.set(file, path);
1401+
}
1402+
1403+
function lock(file, lockType) {
1404+
var path = sqliteFiles.get(file);
1405+
var success = customFS.lock(path, lockType)
1406+
return success? 0 : SQLITE_BUSY;
1407+
}
1408+
1409+
function unlock(file,lockType) {
1410+
var path = sqliteFiles.get(file);
1411+
customFS.unlock(path, lockType)
1412+
return 0;
1413+
}
1414+
1415+
let lockPtr = addFunction(lock, 'iii');
1416+
let unlockPtr = addFunction(unlock, 'iii');
1417+
let openPtr = addFunction(open, 'vii');
1418+
Module["_register_for_idb"](lockPtr, unlockPtr, openPtr)
1419+
}
1420+
1421+
// TODO: This isn't called from anywhere yet. We need to
1422+
// somehow cleanup closed files from `sqliteFiles`
1423+
Module["cleanup_file"] = (path) => {
1424+
let filesInfo = [...sqliteFiles.entries()]
1425+
let fileInfo = filesInfo.find(f => f[1] === path);
1426+
sqliteFiles.delete(fileInfo[0])
1427+
}
13821428
};

‎src/exported_functions.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,5 +42,7 @@
4242
"_sqlite3_result_int64",
4343
"_sqlite3_result_error",
4444
"_sqlite3_aggregate_context",
45+
"_sqlite3_vfs_find",
46+
"_register_for_idb",
4547
"_RegisterExtensionFunctions"
4648
]

‎src/exported_runtime_methods.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,6 @@
33
"stackAlloc",
44
"stackSave",
55
"stackRestore",
6-
"UTF8ToString"
6+
"UTF8ToString",
7+
"FS"
78
]

‎src/fs-externs.js‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/**
2+
* @externs
3+
*/
4+
5+
Module.FS = class {
6+
constructor() {
7+
this.ErrnoError = class {};
8+
}
9+
mount() {}
10+
isRoot() {}
11+
isFile() {}
12+
isDir() {}
13+
stat() {}
14+
/** @return {FSNode} */
15+
lookupPath() {}
16+
/** @return {FSNode} */
17+
lookupNode() {}
18+
/** @return {FSNode} */
19+
createNode() {}
20+
/** @return {FSNode} */
21+
mknod() {}
22+
};
23+
24+
Module.FS.FSNode = class {
25+
constructor() {
26+
this.node_ops = {
27+
getattr: () => {},
28+
setattr: () => {},
29+
lookup: () => {},
30+
mknod: () => {},
31+
rename: () => {},
32+
unlink: () => {},
33+
rmdir: () => {},
34+
reaaddir: () => {},
35+
symlink: () => {},
36+
readlink: () => {}
37+
};
38+
39+
this.stream_ops = {
40+
llseek: () => {},
41+
read: () => {},
42+
write: () => {},
43+
allocate: () => {},
44+
mmap: () => {},
45+
msync: () => {}
46+
};
47+
}
48+
};

‎src/vfs.c‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#include <sqlite3.h>
2+
#include <stdio.h>
3+
4+
static int (*defaultOpen)(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags);
5+
6+
static void (*fsOpen)(const char *, void*);
7+
static int (*fsLock)(sqlite3_file *file, int);
8+
static int (*fsUnlock)(sqlite3_file *file, int);
9+
10+
static int blockDeviceCharacteristics(sqlite3_file* file) {
11+
return SQLITE_IOCAP_SAFE_APPEND |
12+
SQLITE_IOCAP_SEQUENTIAL |
13+
SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
14+
}
15+
16+
static int block_lock(sqlite3_file *file, int lock) {
17+
return fsLock(file, lock);
18+
}
19+
20+
static int block_unlock(sqlite3_file *file, int lock) {
21+
return fsUnlock(file, lock);
22+
}
23+
24+
static int block_open(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags) {
25+
int res = defaultOpen(vfs, zName, file, flags, pOutFlags);
26+
27+
printf("Opened!\n");
28+
29+
sqlite3_io_methods* methods = (sqlite3_io_methods*)file->pMethods;
30+
methods->xDeviceCharacteristics = blockDeviceCharacteristics;
31+
methods->xLock = block_lock;
32+
methods->xUnlock = block_unlock;
33+
34+
fsOpen(zName, (void*)file);
35+
36+
return res;
37+
}
38+
39+
void register_for_idb(int(*lockFile)(sqlite3_file*,int), int(*unlockFile)(sqlite3_file*,int), void(*openFile)(const char*, void*)) {
40+
sqlite3_vfs *vfs = sqlite3_vfs_find("unix");
41+
defaultOpen = vfs->xOpen;
42+
43+
vfs->xOpen = block_open;
44+
sqlite3_vfs_register(vfs, 1);
45+
46+
fsLock = lockFile;
47+
fsUnlock = unlockFile;
48+
fsOpen = openFile;
49+
}

0 commit comments

Comments
 (0)