Merge pull request #447 from ruby/katei/dynamic-component

Dynamic Component

Author: kateinoigakukun

lib/ruby_wasm/build/product/crossruby.rb

@@ -42,7 +42,7 @@ def make_args(crossruby)
       make_args
     end
 
-    def build(executor, crossruby)
+    def build(executor, crossruby, extra_mkargs = [])
       objdir = product_build_dir crossruby
       executor.mkdir_p objdir
       do_extconf executor, crossruby
@@ -54,7 +54,8 @@ def build(executor, crossruby)
                       "-C",
                       "#{objdir}",
                       *make_args(crossruby),
-                      build_target
+                      build_target,
+                      *extra_mkargs
       # A ext can provide link args by link.filelist. It contains only built archive file by default.
       unless File.exist?(linklist(crossruby))
         executor.write(
@@ -187,7 +188,7 @@ def need_extinit_obj?
     def build_exts(executor)
       @user_exts.each do |prod|
         executor.begin_section prod.class, prod.name, "Building"
-        prod.build(executor, self)
+        prod.build(executor, self, [])
         executor.end_section prod.class, prod.name
       end
     end

lib/ruby_wasm/build/toolchain/wit_bindgen.rb

@@ -4,7 +4,7 @@ class WitBindgen
 
     def initialize(
       build_dir:,
-      revision: "v0.24.0"
+      revision: "2e8fb8ede8242288d4cc682cd9dff3057ef09a57"
     )
       @build_dir = build_dir
       @tool_dir = File.join(@build_dir, "toolchain", "wit-bindgen-#{revision}")

lib/ruby_wasm/cli.rb

@@ -53,6 +53,8 @@ def build(args)
         target_triplet: "wasm32-unknown-wasip1",
         profile: "full",
         stdlib: true,
+        without_stdlib_components: [],
+        dest_dir: nil,
         disable_gems: false,
         gemfile: nil,
         patches: [],
@@ -104,10 +106,18 @@ def build(args)
             options[:stdlib] = stdlib
           end
 
+          opts.on("--without-stdlib COMPONENT", "Exclude stdlib component") do |component|
+            options[:without_stdlib_components] << component
+          end
+
           opts.on("--disable-gems", "Disable gems") do
             options[:disable_gems] = true
           end
 
+          opts.on("--dest-dir PATH", "(Experimental) Destination directory") do |path|
+            options[:dest_dir] = path
+          end
+
           opts.on("-p", "--patch PATCH", "Apply a patch") do |patch|
             options[:patches] << patch
           end
@@ -149,7 +159,9 @@ def do_build_with_force_ruby_platform(options)
 
       require "tmpdir"
 
-      if options[:save_temps]
+      if dest_dir = options[:dest_dir]
+        self.do_build(executor, dest_dir, packager, options)
+      elsif options[:save_temps]
         tmpdir = Dir.mktmpdir
         self.do_build(executor, tmpdir, packager, options)
         @stderr.puts "Temporary files are saved to #{tmpdir}"

lib/ruby_wasm/packager.rb

@@ -32,19 +32,27 @@ def package(executor, dest_dir, options)
 
     wasm_bytes = File.binread(File.join(fs.ruby_root, "bin", "ruby"))
 
+    ruby_core.build_gem_exts(executor, fs.bundle_dir)
+
     fs.package_gems
     fs.remove_non_runtime_files(executor)
-    fs.remove_stdlib(executor) unless options[:stdlib]
+    if options[:stdlib]
+      options[:without_stdlib_components].each do |component|
+        fs.remove_stdlib_component(executor, component)
+      end
+    else
+      fs.remove_stdlib(executor)
+    end
 
-    if full_build_options[:target] == "wasm32-unknown-wasip1"
+    if full_build_options[:target] == "wasm32-unknown-wasip1" && !features.support_dynamic_linking?
       # wasi-vfs supports only WASI target
       wasi_vfs = RubyWasmExt::WasiVfs.new
       wasi_vfs.map_dir("/bundle", fs.bundle_dir)
       wasi_vfs.map_dir("/usr", File.dirname(fs.ruby_root))
 
       wasm_bytes = wasi_vfs.pack(wasm_bytes)
     end
-    wasm_bytes = ruby_core.build_and_link_exts(executor, wasm_bytes)
+    wasm_bytes = ruby_core.link_gem_exts(executor, fs.ruby_root, fs.bundle_dir, wasm_bytes)
 
     wasm_bytes = RubyWasmExt.preinitialize(wasm_bytes) if options[:optimize]
     wasm_bytes

lib/ruby_wasm/packager/core.rb

@@ -12,7 +12,7 @@ def build(executor, options)
 
   extend Forwardable
 
-  def_delegators :build_strategy, :cache_key, :artifact, :build_and_link_exts
+  def_delegators :build_strategy, :cache_key, :artifact, :build_gem_exts, :link_gem_exts
 
   private
 
@@ -37,7 +37,11 @@ def build(executor, options)
       raise NotImplementedError
     end
 
-    def build_and_link_exts(executor, module_bytes)
+    def build_gem_exts(executor, gem_home)
+      raise NotImplementedError
+    end
+
+    def link_gem_exts(executor, ruby_root, gem_home, module_bytes)
       raise NotImplementedError
     end
 
@@ -55,6 +59,14 @@ def specs_with_extensions
       end
     end
 
+    def wasi_exec_model
+      # TODO: Detect WASI exec-model from binary exports (_start or _initialize)
+      use_js_gem = @packager.specs.any? do |spec|
+        spec.name == "js"
+      end
+      use_js_gem ? "reactor" : "command"
+    end
+
     def cache_key(digest)
       raise NotImplementedError
     end
@@ -93,20 +105,24 @@ def build(executor, options)
       build.crossruby.artifact
     end
 
-    def build_and_link_exts(executor, module_bytes)
+    def build_gem_exts(executor, gem_home)
       build = derive_build
-      self.build_exts(executor, build)
-      self.link_exts(executor, build)
+      self._build_gem_exts(executor, build, gem_home)
     end
 
-    def link_exts(executor, build)
-      ruby_root = build.crossruby.dest_dir
+    def link_gem_exts(executor, ruby_root, gem_home, module_bytes)
+      build = derive_build
+      self._link_gem_exts(executor, build, ruby_root, gem_home, module_bytes)
+    end
 
-      libraries = [File.join(ruby_root, "usr", "local", "bin", "ruby")]
+    def _link_gem_exts(executor, build, ruby_root, gem_home, module_bytes)
+      libraries = []
 
       # TODO: Should be computed from dyinfo of ruby binary
       wasi_libc_shared_libs = [
         "libc.so",
+        "libc++.so",
+        "libc++abi.so",
         "libwasi-emulated-getpid.so",
         "libwasi-emulated-mman.so",
         "libwasi-emulated-process-clocks.so",
@@ -119,13 +135,18 @@ def link_exts(executor, build)
         wasi_sdk_path = toolchain.wasi_sdk_path
         libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
       end
-      wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("command")
+      wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1(wasi_exec_model)
       adapters = [wasi_adapter]
-      dl_openable_libs = Dir.glob(File.join(ruby_root, "usr", "local", "lib", "ruby", "**", "*.so"))
+      dl_openable_libs = []
+      dl_openable_libs << [File.dirname(ruby_root), Dir.glob(File.join(ruby_root, "lib", "ruby", "**", "*.so"))]
+      dl_openable_libs << [gem_home, Dir.glob(File.join(gem_home, "**", "*.so"))]
+
       linker = RubyWasmExt::ComponentLink.new
       linker.use_built_in_libdl(true)
       linker.stub_missing_functions(false)
-      linker.validate(true)
+      linker.validate(ENV["RUBYWASM_SKIP_LINKER_VALIDATION"] != "1")
+
+      linker.library("ruby", module_bytes, false)
 
       libraries.each do |lib|
         # Non-DL openable libraries should be referenced as base name
@@ -135,42 +156,51 @@ def link_exts(executor, build)
         linker.library(lib_name, module_bytes, false)
       end
 
-      dl_openable_libs.each do |lib|
-        # DL openable lib_name should be a relative path from ruby_root
-        lib_name = "/" + Pathname.new(lib).relative_path_from(Pathname.new(ruby_root)).to_s
-        module_bytes = File.binread(lib)
-        RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
-        linker.library(lib_name, module_bytes, true)
+      dl_openable_libs.each do |root, libs|
+        libs.each do |lib|
+          # DL openable lib_name should be a relative path from ruby_root
+          lib_name = "/" + Pathname.new(lib).relative_path_from(Pathname.new(File.dirname(root))).to_s
+          module_bytes = File.binread(lib)
+          RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
+          linker.library(lib_name, module_bytes, true)
+        end
       end
 
       adapters.each do |adapter|
         adapter_name = File.basename(adapter)
         # e.g. wasi_snapshot_preview1.command.wasm -> wasi_snapshot_preview1
         adapter_name = adapter_name.split(".")[0]
         module_bytes = File.binread(adapter)
+        RubyWasm.logger.info "Linking adapter #{adapter_name}=#{adapter} (#{module_bytes.size} bytes)"
         linker.adapter(adapter_name, module_bytes)
       end
       return linker.encode()
     end
 
-    def build_exts(executor, build)
+    def _build_gem_exts(executor, build, gem_home)
       exts = specs_with_extensions.flat_map do |spec, exts|
         exts.map do |ext|
           ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
           ext_srcdir = File.join(spec.full_gem_path, ext_feature)
           ext_relative_path = File.join(spec.full_name, ext_feature)
-          RubyWasm::CrossRubyExtProduct.new(
+          prod = RubyWasm::CrossRubyExtProduct.new(
             ext_srcdir,
             build.toolchain,
             features: @packager.features,
             ext_relative_path: ext_relative_path
           )
+          [prod, spec]
         end
       end
 
-      exts.each do |prod|
+      exts.each do |prod, spec|
+        libdir = File.join(gem_home, "gems", spec.full_name, spec.raw_require_paths.first)
+        extra_mkargs = [
+          "sitearchdir=#{libdir}",
+          "sitelibdir=#{libdir}",
+        ]
         executor.begin_section prod.class, prod.name, "Building"
-        prod.build(executor, build.crossruby)
+        prod.build(executor, build.crossruby, extra_mkargs)
         executor.end_section prod.class, prod.name
       end
     end
@@ -301,15 +331,19 @@ def derive_build
       build
     end
 
-    def build_and_link_exts(executor, module_bytes)
+    def build_gem_exts(executor, gem_home)
+      # No-op because we already built extensions as part of the Ruby build
+    end
+
+    def link_gem_exts(executor, ruby_root, gem_home, module_bytes)
       return module_bytes unless @packager.features.support_component_model?
 
       linker = RubyWasmExt::ComponentEncode.new
-      linker.validate(true)
+      linker.validate(ENV["RUBYWASM_SKIP_LINKER_VALIDATION"] != "1")
       linker.module(module_bytes)
       linker.adapter(
         "wasi_snapshot_preview1",
-        File.binread(RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("reactor"))
+        File.binread(RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1(wasi_exec_model))
       )
 
       linker.encode()

lib/ruby_wasm/packager/file_system.rb

@@ -38,6 +38,23 @@ def remove_stdlib(executor)
     File.write(rbconfig, rbconfig_contents)
   end
 
+  def remove_stdlib_component(executor, component)
+    RubyWasm.logger.info "Removing stdlib component: #{component}"
+    case component
+    when "enc"
+      # Remove all encodings except for encdb.so and transdb.so
+      enc_dir = File.join(@ruby_root, "lib", "ruby", ruby_version, "wasm32-wasi", "enc")
+      puts File.join(enc_dir, "**/*.so")
+      Dir.glob(File.join(enc_dir, "**/*.so")).each do |entry|
+        next if entry.end_with?("encdb.so", "transdb.so")
+        RubyWasm.logger.debug "Removing stdlib encoding: #{entry}"
+        executor.rm_rf entry
+      end
+    else
+      raise "Unknown stdlib component: #{component}"
+    end
+  end
+
   def package_gems
     @packager.specs.each do |spec|
       RubyWasm.logger.info "Packaging gem: #{spec.full_name}"

packages/gems/js/ext/js/bindgen/ext.c

@@ -150,7 +150,7 @@ void ruby_js_js_runtime_list_borrow_js_abi_value_free(ruby_js_js_runtime_list_bo
   }
 }
 
-__attribute__((__import_module__("ruby:js/ruby-runtime"), __import_name__("[resource-drop]rb-iseq")))
+__attribute__((__import_module__("[export]ruby:js/ruby-runtime"), __import_name__("[resource-drop]rb-iseq")))
 extern void __wasm_import_exports_ruby_js_ruby_runtime_rb_iseq_drop(int32_t handle);
 
 void exports_ruby_js_ruby_runtime_rb_iseq_drop_own(exports_ruby_js_ruby_runtime_own_rb_iseq_t handle) {
@@ -176,7 +176,7 @@ void __wasm_export_exports_ruby_js_ruby_runtime_rb_iseq_dtor(exports_ruby_js_rub
   exports_ruby_js_ruby_runtime_rb_iseq_destructor(arg);
 }
 
-__attribute__((__import_module__("ruby:js/ruby-runtime"), __import_name__("[resource-drop]rb-abi-value")))
+__attribute__((__import_module__("[export]ruby:js/ruby-runtime"), __import_name__("[resource-drop]rb-abi-value")))
 extern void __wasm_import_exports_ruby_js_ruby_runtime_rb_abi_value_drop(int32_t handle);
 
 void exports_ruby_js_ruby_runtime_rb_abi_value_drop_own(exports_ruby_js_ruby_runtime_own_rb_abi_value_t handle) {

packages/gems/js/ext/js/bindgen/ext_component_type.o

@@ -168,7 +168,7 @@ module RubyWasm
     def feature_name: (CrossRubyProduct crossruby) -> String
 
     def make_args: (CrossRubyProduct crossruby) -> Array[String]
-    def build: (BuildExecutor executor, CrossRubyProduct crossruby) -> void
+    def build: (BuildExecutor executor, CrossRubyProduct crossruby, Array[String] extra_mkargs) -> void
     def do_extconf: (BuildExecutor executor, CrossRubyProduct crossruby) -> void
     def do_install_rb: (BuildExecutor executor, CrossRubyProduct crossruby) -> void
 

sig/ruby_wasm/build.rbs

@@ -14,10 +14,12 @@ module RubyWasm
       target_triplet: String,
       profile: String,
       stdlib: bool,
+      without_stdlib_components: Array[String],
       disable_gems: bool,
       gemfile: String?,
       patches: Array[String],
       format: String,
+      dest_dir: String,
     }
 
     DEFAULT_RUBIES_DIR: string