Move some FS methods to FS_foo notation, allowing WasmFS to use them from JS APIs (#20098)

FS.createDataFile, FS.mkdirTree, FS.unlink are all now usable as FS_foo, that is,
without the FS object being included. For the JS FS this is just a slightly odd
refactoring, but for WasmFS it means we can use dependencies on a per-function
basis, both for including them, and for including what they need.

Also in WasmFS do the same for FS.create, FS.mknod, FS.mkdir, FS.writeFile
as those are dependencies of the above.

This may have a slight code size disadvantage in builds that include the FS object
anyhow, but closure should be able to remove that anyhow. For the JS FS, FS is
there all the time so we don't see any benefits really, but for WasmFS this allows us
to use filesystem operations from JS APIs without bringing in all of FS (and without
all the wasm code it calls into). WasmFS builds that do include FS - which is builds
that have FORCE_FILESYSTEM enabled - may increase in size slightly, but those
builds are rarer and large anyhow due to FS.

With this, emscripten_async_wget can be converted to work in WasmFS. To do so
we switch from FS.foo to FS_foo notation and add the proper deps, and then
everything just works.

Implements the ideas discussed in the later part of #20022

Author: kripken

src/library_fs.js

@@ -1869,4 +1869,20 @@ FS.staticInit();` +
     },
 #endif
   },
+
+  $FS_createDataFile__deps: ['$FS'],
+  $FS_createDataFile: (parent, name, fileData, canRead, canWrite, canOwn) => {
+    return FS.createDataFile(parent, name, fileData, canRead, canWrite, canOwn);
+  },
+
+  $FS_unlink__deps: ['$FS'],
+  $FS_unlink: (path) => FS.unlink(path),
+
+  $FS_mkdirTree__docs: `
+  /**
+   * @param {number=} mode Optionally, the mode to create in. Uses mkdir's
+   *                       default if not set.
+   */`,
+  $FS_mkdirTree__deps: ['$FS'],
+  $FS_mkdirTree: (path, mode) => FS.mkdirTree(path, mode),
 });

src/library_fs_shared.js

@@ -45,6 +45,7 @@ addToLibrary({
   $FS_createPreloadedFile__deps: [
     '$asyncLoad',
     '$PATH_FS',
+    '$FS_createDataFile',
 #if !MINIMAL_RUNTIME
     '$FS_handledByPreloadPlugin',
 #endif
@@ -58,7 +59,7 @@ addToLibrary({
       function finish(byteArray) {
         if (preFinish) preFinish();
         if (!dontCreateFile) {
-          FS.createDataFile(parent, name, byteArray, canRead, canWrite, canOwn);
+          FS_createDataFile(parent, name, byteArray, canRead, canWrite, canOwn);
         }
         if (onload) onload();
         removeRunDependency(dep);

src/library_wasmfs.js

@@ -23,7 +23,6 @@ addToLibrary({
 
   $FS__postset: `
 FS.init();
-FS.createPreloadedFile = FS_createPreloadedFile;
 `,
   $FS__deps: [
     '$MEMFS',
@@ -35,6 +34,7 @@ FS.createPreloadedFile = FS_createPreloadedFile;
     '$withStackSave',
     '$readI53FromI64',
     '$readI53FromU64',
+    '$FS_createDataFile',
     '$FS_createPreloadedFile',
     '$FS_getMode',
     // For FS.readFile
@@ -44,6 +44,12 @@ FS.createPreloadedFile = FS_createPreloadedFile;
                                              // up requiring all of our code
                                              // here.
     '$FS_modeStringToFlags',
+    '$FS_create',
+    '$FS_mknod',
+    '$FS_mkdir',
+    '$FS_mkdirTree',
+    '$FS_writeFile',
+    '$FS_unlink',
 #if LibraryManager.has('library_icasefs.js')
     '$ICASEFS',
 #endif
@@ -93,19 +99,7 @@ FS.createPreloadedFile = FS_createPreloadedFile;
       FS.ErrnoError.prototype.constructor = FS.ErrnoError;
     },
     createDataFile(parent, name, fileData, canRead, canWrite, canOwn) {
-      var pathName = name ? parent + '/' + name : parent;
-      var mode = FS_getMode(canRead, canWrite);
-
-      if (!wasmFSPreloadingFlushed) {
-        // WasmFS code in the wasm is not ready to be called yet. Cache the
-        // files we want to create here in JS, and WasmFS will read them
-        // later.
-        wasmFSPreloadedFiles.push({pathName, fileData, mode});
-      } else {
-        // WasmFS is already running, so create the file normally.
-        FS.create(pathName, mode);
-        FS.writeFile(pathName, fileData);
-      }
+      return FS_createDataFile(parent, name, fileData, canRead, canWrite, canOwn);
     },
     createPath(parent, path, canRead, canWrite) {
       // Cache file path directory names.
@@ -124,6 +118,10 @@ FS.createPreloadedFile = FS_createPreloadedFile;
       return current;
     },
 
+    createPreloadedFile(parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish) {
+      return FS_createPreloadedFile(parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish);
+    },
+
 #if hasExportedSymbol('_wasmfs_read_file') // Support the JS function exactly
                                            // when the __wasmfs_* function is
                                            // present to be called (otherwise,
@@ -173,24 +171,8 @@ FS.createPreloadedFile = FS_createPreloadedFile;
 
     // libc methods
 
-    mkdir: (path, mode) => FS.handleError(withStackSave(() => {
-      mode = mode !== undefined ? mode : 511 /* 0777 */;
-      var buffer = stringToUTF8OnStack(path);
-      return __wasmfs_mkdir(buffer, mode);
-    })),
-    mkdirTree(path, mode) {
-      var dirs = path.split('/');
-      var d = '';
-      for (var i = 0; i < dirs.length; ++i) {
-        if (!dirs[i]) continue;
-        d += '/' + dirs[i];
-        try {
-          FS.mkdir(d, mode);
-        } catch(e) {
-          if (e.errno != {{{ cDefs.EEXIST }}}) throw e;
-        }
-      }
-    },
+    mkdir: (path, mode) => FS_mkdir(path, mode),
+    mkdirTree: (path, mode) => FS_mkdirTree(path, mode),
     rmdir: (path) => FS.handleError(
       withStackSave(() => __wasmfs_rmdir(stringToUTF8OnStack(path)))
     ),
@@ -201,18 +183,9 @@ FS.createPreloadedFile = FS_createPreloadedFile;
       var fd = FS.handleError(__wasmfs_open(buffer, flags, mode));
       return { fd : fd };
     }),
-    create(path, mode) {
-      // Default settings copied from the legacy JS FS API.
-      mode = mode !== undefined ? mode : 438 /* 0666 */;
-      mode &= {{{ cDefs.S_IALLUGO }}};
-      mode |= {{{ cDefs.S_IFREG }}};
-      return FS.mknod(path, mode, 0);
-    },
+    create: (path, mode) => FS_create(path, mode),
     close: (stream) => FS.handleError(-__wasmfs_close(stream.fd)),
-    unlink: (path) => withStackSave(() => {
-      var buffer = stringToUTF8OnStack(path);
-      return __wasmfs_unlink(buffer);
-    }),
+    unlink: (path) => FS_unlink(path),
     chdir: (path) => withStackSave(() => {
       var buffer = stringToUTF8OnStack(path);
       return __wasmfs_chdir(buffer);
@@ -260,6 +233,7 @@ FS.createPreloadedFile = FS_createPreloadedFile;
     allocate(stream, offset, length) {
       return FS.handleError(__wasmfs_allocate(stream.fd, {{{ splitI64('offset') }}}, {{{ splitI64('length') }}}));
     },
+    writeFile: (path, data) => FS_writeFile(path, data),
     mmap: (stream, length, offset, prot, flags) => {
       var buf = FS.handleError(__wasmfs_mmap(length, prot, flags, stream.fd, {{{ splitI64('offset') }}}));
       return { ptr: buf, allocated: true };
@@ -273,24 +247,6 @@ FS.createPreloadedFile = FS_createPreloadedFile;
     munmap: (addr, length) => (
       FS.handleError(__wasmfs_munmap(addr, length))
     ),
-    writeFile: (path, data) => withStackSave(() => {
-      var pathBuffer = stringToUTF8OnStack(path);
-      if (typeof data == 'string') {
-        var buf = new Uint8Array(lengthBytesUTF8(data) + 1);
-        var actualNumBytes = stringToUTF8Array(data, buf, 0, buf.length);
-        data = buf.slice(0, actualNumBytes);
-      }
-      var dataBuffer = _malloc(data.length);
-#if ASSERTIONS
-      assert(dataBuffer);
-#endif
-      for (var i = 0; i < data.length; i++) {
-        {{{ makeSetValue('dataBuffer', 'i', 'data[i]', 'i8') }}};
-      }
-      var ret = __wasmfs_write_file(pathBuffer, dataBuffer, data.length);
-      _free(dataBuffer);
-      return ret;
-    }),
     symlink: (target, linkpath) => withStackSave(() => (
       __wasmfs_symlink(stringToUTF8OnStack(target), stringToUTF8OnStack(linkpath))
     )),
@@ -402,12 +358,7 @@ FS.createPreloadedFile = FS_createPreloadedFile;
       FS.handleError(withStackSave(() => __wasmfs_unmount(stringToUTF8OnStack(mountpoint))))
     ),
     // TODO: lookup
-    mknod(path, mode, dev) {
-      return FS.handleError(withStackSave(() => {
-        var pathBuffer = stringToUTF8OnStack(path);
-        return __wasmfs_mknod(pathBuffer, mode, dev);
-      }));
-    },
+    mknod: (path, mode, dev) => FS_mknod(path, mode, dev),
     makedev: (ma, mi) => ((ma) << 8 | (mi)),
     registerDevice(dev, ops) {
       var backendPointer = _wasmfs_create_jsimpl_backend();
@@ -522,6 +473,110 @@ FS.createPreloadedFile = FS_createPreloadedFile;
 #endif
   },
 
+  // Split-out FS.* methods. These are split out for code size reasons, so that
+  // we can include the ones we need on demand, rather than put them all on the
+  // main FS object. As a result the entire FS object is not needed if you just
+  // need some specific FS_* operations. When the FS object is present, it calls
+  // into those FS_* methods as needed.
+  //
+  // In contrast, the old JS FS (library_fs.js) does the opposite: it puts all
+  // things on the FS object, and copies them to FS_* methods for use from JS
+  // library code. Given that the JS FS is implemented entirely in JS, that
+  // makes sense there (as almost all that FS object ends up needed anyhow all
+  // the time).
+
+  $FS_createDataFile__deps: [
+    '$wasmFSPreloadingFlushed', '$wasmFSPreloadedFiles',
+    '$FS_create', '$FS_writeFile',
+  ],
+  $FS_createDataFile: (parent, name, fileData, canRead, canWrite, canOwn) => {
+    var pathName = name ? parent + '/' + name : parent;
+    var mode = FS_getMode(canRead, canWrite);
+
+    if (!wasmFSPreloadingFlushed) {
+      // WasmFS code in the wasm is not ready to be called yet. Cache the
+      // files we want to create here in JS, and WasmFS will read them
+      // later.
+      wasmFSPreloadedFiles.push({pathName, fileData, mode});
+    } else {
+      // WasmFS is already running, so create the file normally.
+      FS_create(pathName, mode);
+      FS_writeFile(pathName, fileData);
+    }
+  },
+
+  $FS_mknod__deps: ['_wasmfs_mknod'],
+  $FS_mknod: (path, mode, dev) => {
+    return FS.handleError(withStackSave(() => {
+      var pathBuffer = stringToUTF8OnStack(path);
+      return __wasmfs_mknod(pathBuffer, mode, dev);
+    }));
+  },
+
+  $FS_create__deps: ['$FS_mknod'],
+  $FS_create: (path, mode) => {
+    // Default settings copied from the legacy JS FS API.
+    mode = mode !== undefined ? mode : 438 /* 0666 */;
+    mode &= {{{ cDefs.S_IALLUGO }}};
+    mode |= {{{ cDefs.S_IFREG }}};
+    return FS_mknod(path, mode, 0);
+  },
+
+  $FS_writeFile__deps: ['_wasmfs_write_file'],
+  $FS_writeFile: (path, data) => withStackSave(() => {
+    var pathBuffer = stringToUTF8OnStack(path);
+    if (typeof data == 'string') {
+      var buf = new Uint8Array(lengthBytesUTF8(data) + 1);
+      var actualNumBytes = stringToUTF8Array(data, buf, 0, buf.length);
+      data = buf.slice(0, actualNumBytes);
+    }
+    var dataBuffer = _malloc(data.length);
+#if ASSERTIONS
+    assert(dataBuffer);
+#endif
+    for (var i = 0; i < data.length; i++) {
+      {{{ makeSetValue('dataBuffer', 'i', 'data[i]', 'i8') }}};
+    }
+    var ret = __wasmfs_write_file(pathBuffer, dataBuffer, data.length);
+    _free(dataBuffer);
+    return ret;
+  }),
+
+  $FS_mkdir__deps: ['_wasmfs_mkdir'],
+  $FS_mkdir: (path, mode) => FS.handleError(withStackSave(() => {
+    mode = mode !== undefined ? mode : 511 /* 0777 */;
+    var buffer = stringToUTF8OnStack(path);
+    return __wasmfs_mkdir(buffer, mode);
+  })),
+
+  $FS_mkdirTree__docs: `
+  /**
+   * @param {number=} mode Optionally, the mode to create in. Uses mkdir's
+   *                       default if not set.
+   */`,
+  $FS_mkdirTree__deps: ['$FS_mkdir'],
+  $FS_mkdirTree: (path, mode) => {
+    var dirs = path.split('/');
+    var d = '';
+    for (var i = 0; i < dirs.length; ++i) {
+      if (!dirs[i]) continue;
+      d += '/' + dirs[i];
+      try {
+        FS_mkdir(d, mode);
+      } catch(e) {
+        if (e.errno != {{{ cDefs.EEXIST }}}) throw e;
+      }
+    }
+  },
+
+  $FS_unlink__deps: ['_wasmfs_unlink'],
+  $FS_unlink: (path) => withStackSave(() => {
+    var buffer = stringToUTF8OnStack(path);
+    return __wasmfs_unlink(buffer);
+  }),
+
+  // Wasm access calls.
+
   _wasmfs_get_num_preloaded_files__deps: [
     '$wasmFSPreloadedFiles',
     '$wasmFSPreloadingFlushed'],

src/library_wget.js

@@ -16,7 +16,13 @@ var LibraryWget = {
     },
   },
 
-  emscripten_async_wget__deps: ['$PATH_FS', '$wget', '$callUserCallback', '$Browser', '$withStackSave', '$stringToUTF8OnStack'],
+  emscripten_async_wget__deps: [
+    '$PATH_FS', '$wget', '$callUserCallback', '$Browser',
+    '$withStackSave', '$stringToUTF8OnStack',
+    '$FS_mkdirTree',
+    '$FS_createPreloadedFile',
+    '$FS_unlink',
+  ],
   emscripten_async_wget__proxy: 'sync',
   emscripten_async_wget: (url, file, onload, onerror) => {
     {{{ runtimeKeepalivePush() }}}
@@ -35,7 +41,7 @@ var LibraryWget = {
       }
     }
     var destinationDirectory = PATH.dirname(_file);
-    FS.createPreloadedFile(
+    FS_createPreloadedFile(
       destinationDirectory,
       PATH.basename(_file),
       _url, true, true,
@@ -50,10 +56,10 @@ var LibraryWget = {
       function() { // preFinish
         // if a file exists there, we overwrite it
         try {
-          FS.unlink(_file);
+          FS_unlink(_file);
         } catch (e) {}
         // if the destination directory does not yet exist, create it
-        FS.mkdirTree(destinationDirectory);
+        FS_mkdirTree(destinationDirectory);
       }
     );
   },

src/modules.js

@@ -371,12 +371,10 @@ function exportRuntime() {
     'removeRunDependency',
     'FS_createFolder',
     'FS_createPath',
-    'FS_createDataFile',
     'FS_createLazyFile',
     'FS_createLink',
     'FS_createDevice',
     'FS_readFile',
-    'FS_unlink',
     'out',
     'err',
     'callMain',
@@ -455,6 +453,9 @@ function exportRuntime() {
   for (const ident of Object.keys(LibraryManager.library)) {
     if (isJsOnlySymbol(ident) && !isDecorator(ident) && !isInternalSymbol(ident)) {
       const jsname = ident.substr(1);
+      // Note that this assertion may be hit when a function is moved into the
+      // JS library. In that case the function should be removed from the list
+      // of runtime elements above.
       assert(!runtimeElementsSet.has(jsname), 'runtimeElements contains library symbol: ' + ident);
       runtimeElements.push(jsname);
     }

test/other/metadce/test_metadce_hello_O0.jssize

@@ -1 +1 @@
-23669
+23682

test/other/metadce/test_metadce_minimal_O0.jssize

@@ -1 +1 @@
-20140
+20153

test/other/test_unoptimized_code_size.js.size

@@ -1 +1 @@
-58694
+58712

test/other/test_unoptimized_code_size_strict.js.size

@@ -1 +1 @@
-57633
+57651

test/test_browser.py

@@ -1915,6 +1915,7 @@ def setup():
   def test_emscripten_api_infloop(self):
     self.btest_exit('emscripten_api_browser_infloop.cpp', assert_returncode=7)
 
+  @also_with_wasmfs
   def test_emscripten_fs_api(self):
     shutil.copyfile(test_file('screenshot.png'), 'screenshot.png') # preloaded *after* run
     self.btest_exit('emscripten_fs_api_browser.c', assert_returncode=1, args=['-lSDL'])