Skip to content

Commit 04da985

Browse files
jlongsterjlongster
committed
Implement changes required for IndexedDB-backed filesystem
1 parent 16d9a13 commit 04da985

File tree

6 files changed

+163
-12
lines changed

6 files changed

+163
-12
lines changed

‎Makefile‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ EMFLAGS_DEBUG = \
6262
-s ASSERTIONS=1 \
6363
-O1
6464

65-
BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc
65+
BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc out/vfs.bc
6666

6767
OUTPUT_WRAPPER_FILES = src/shell-pre.js src/shell-post.js
6868

@@ -79,13 +79,13 @@ all: optimized debug worker
7979
debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js
8080

8181
dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
82-
$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
82+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
8383
mv $@ out/tmp-raw.js
8484
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
8585
rm out/tmp-raw.js
8686

8787
dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
88-
$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
88+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
8989
mv $@ out/tmp-raw.js
9090
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
9191
rm out/tmp-raw.js
@@ -94,19 +94,19 @@ dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FI
9494
optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js
9595

9696
dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
97-
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
97+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
9898
mv $@ out/tmp-raw.js
9999
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
100100
rm out/tmp-raw.js
101101

102102
dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
103-
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
103+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
104104
mv $@ out/tmp-raw.js
105105
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
106106
rm out/tmp-raw.js
107107

108108
dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
109-
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
109+
EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
110110
mv $@ out/tmp-raw.js
111111
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
112112
rm out/tmp-raw.js
@@ -156,6 +156,11 @@ out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION)
156156
# Generate llvm bitcode
157157
$(EMCC) $(CFLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@
158158

159+
out/vfs.bc: src/vfs.c sqlite-src/$(SQLITE_AMALGAMATION)
160+
mkdir -p out
161+
# Generate llvm bitcode
162+
$(EMCC) $(CFLAGS) -s LINKABLE=1 -I sqlite-src/$(SQLITE_AMALGAMATION) -c src/vfs.c -o $@
163+
159164
# TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder
160165
# module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js
161166
# tar --create --gzip $^ > $@

‎src/api.js‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -811,13 +811,19 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
811811
* @memberof module:SqlJs
812812
* Open a new database either by creating a new one or opening an existing
813813
* one stored in the byte array passed in first argument
814-
* @param {number[]} data An array of bytes representing
815-
* an SQLite database file
814+
* @param {number[]|string} data An array of bytes representing
815+
* an SQLite database file or a path
816+
* @param {Object} opts Options to specify a filename
816817
*/
817-
function Database(data) {
818-
this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0);
819-
if (data != null) {
818+
function Database(data, { filename = false } = {}) {
819+
if(filename === false) {
820+
this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0);
821+
if (data != null) {
820822
FS.createDataFile("/", this.filename, data, true, true);
823+
}
824+
}
825+
else {
826+
this.filename = data;
821827
}
822828
this.handleError(sqlite3_open(this.filename, apiTemp));
823829
this.db = getValue(apiTemp, "i32");
@@ -1231,4 +1237,44 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
12311237

12321238
// export Database to Module
12331239
Module.Database = Database;
1240+
1241+
// Because emscripten doesn't allow us to handle `ioctl`, we need
1242+
// to manually install lock/unlock methods. Unfortunately we need
1243+
// to keep track of a mapping of `sqlite_file*` pointers to filename
1244+
// so that we can tell our filesystem which files to lock/unlock
1245+
var sqliteFiles = new Map();
1246+
1247+
Module["register_for_idb"] = (customFS) => {
1248+
var SQLITE_BUSY = 5;
1249+
1250+
function open(namePtr, file) {
1251+
var path = UTF8ToString(namePtr);
1252+
sqliteFiles.set(file, path);
1253+
}
1254+
1255+
function lock(file, lockType) {
1256+
var path = sqliteFiles.get(file);
1257+
var success = customFS.lock(path, lockType)
1258+
return success? 0 : SQLITE_BUSY;
1259+
}
1260+
1261+
function unlock(file,lockType) {
1262+
var path = sqliteFiles.get(file);
1263+
customFS.unlock(path, lockType)
1264+
return 0;
1265+
}
1266+
1267+
let lockPtr = addFunction(lock, 'iii');
1268+
let unlockPtr = addFunction(unlock, 'iii');
1269+
let openPtr = addFunction(open, 'vii');
1270+
Module["_register_for_idb"](lockPtr, unlockPtr, openPtr)
1271+
}
1272+
1273+
// TODO: This isn't called from anywhere yet. We need to
1274+
// somehow cleanup closed files from `sqliteFiles`
1275+
Module["cleanup_file"] = (path) => {
1276+
let filesInfo = [...sqliteFiles.entries()]
1277+
let fileInfo = filesInfo.find(f => f[1] === path);
1278+
sqliteFiles.delete(fileInfo[0])
1279+
}
12341280
};

‎src/exported_functions.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,7 @@
4141
"_sqlite3_result_int",
4242
"_sqlite3_result_int64",
4343
"_sqlite3_result_error",
44+
"_sqlite3_vfs_find",
45+
"_register_for_idb",
4446
"_RegisterExtensionFunctions"
4547
]

‎src/exported_runtime_methods.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,6 @@
33
"stackAlloc",
44
"stackSave",
55
"stackRestore",
6-
"UTF8ToString"
6+
"UTF8ToString",
7+
"FS"
78
]

‎src/fs-externs.js‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/**
2+
* @externs
3+
*/
4+
5+
Module.FS = class {
6+
constructor() {
7+
this.ErrnoError = class {};
8+
}
9+
mount() {}
10+
isRoot() {}
11+
isFile() {}
12+
isDir() {}
13+
stat() {}
14+
/** @return {FSNode} */
15+
lookupPath() {}
16+
/** @return {FSNode} */
17+
lookupNode() {}
18+
/** @return {FSNode} */
19+
createNode() {}
20+
/** @return {FSNode} */
21+
mknod() {}
22+
};
23+
24+
Module.FS.FSNode = class {
25+
constructor() {
26+
this.node_ops = {
27+
getattr: () => {},
28+
setattr: () => {},
29+
lookup: () => {},
30+
mknod: () => {},
31+
rename: () => {},
32+
unlink: () => {},
33+
rmdir: () => {},
34+
reaaddir: () => {},
35+
symlink: () => {},
36+
readlink: () => {}
37+
};
38+
39+
this.stream_ops = {
40+
llseek: () => {},
41+
read: () => {},
42+
write: () => {},
43+
allocate: () => {},
44+
mmap: () => {},
45+
msync: () => {}
46+
};
47+
}
48+
};

‎src/vfs.c‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#include <sqlite3.h>
2+
#include <stdio.h>
3+
4+
static int (*defaultOpen)(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags);
5+
6+
static void (*fsOpen)(const char *, void*);
7+
static int (*fsLock)(sqlite3_file *file, int);
8+
static int (*fsUnlock)(sqlite3_file *file, int);
9+
10+
static int blockDeviceCharacteristics(sqlite3_file* file) {
11+
return SQLITE_IOCAP_SAFE_APPEND |
12+
SQLITE_IOCAP_SEQUENTIAL |
13+
SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
14+
}
15+
16+
static int block_lock(sqlite3_file *file, int lock) {
17+
return fsLock(file, lock);
18+
}
19+
20+
static int block_unlock(sqlite3_file *file, int lock) {
21+
return fsUnlock(file, lock);
22+
}
23+
24+
static int block_open(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags) {
25+
int res = defaultOpen(vfs, zName, file, flags, pOutFlags);
26+
27+
printf("Opened!\n");
28+
29+
sqlite3_io_methods* methods = (sqlite3_io_methods*)file->pMethods;
30+
methods->xDeviceCharacteristics = blockDeviceCharacteristics;
31+
methods->xLock = block_lock;
32+
methods->xUnlock = block_unlock;
33+
34+
fsOpen(zName, (void*)file);
35+
36+
return res;
37+
}
38+
39+
void register_for_idb(int(*lockFile)(sqlite3_file*,int), int(*unlockFile)(sqlite3_file*,int), void(*openFile)(const char*, void*)) {
40+
sqlite3_vfs *vfs = sqlite3_vfs_find("unix");
41+
defaultOpen = vfs->xOpen;
42+
43+
vfs->xOpen = block_open;
44+
sqlite3_vfs_register(vfs, 1);
45+
46+
fsLock = lockFile;
47+
fsUnlock = unlockFile;
48+
fsOpen = openFile;
49+
}

0 commit comments

Comments
 (0)