From 56e84433b1e93aac25096f4eccc53095c20e0e92 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Sun, 16 Nov 2025 09:16:56 -0800 Subject: [PATCH 1/2] WIP: prototype of wasm host as run_binary wrapper --- MODULE.bazel | 1 + go.mod | 1 + go.sum | 10 ++++++ lib/private/run_binary.bzl | 8 ++++- path/to/package/BUILD | 33 +++++++++++++++++ path/to/package/tool.go | 23 ++++++++++++ path/to/package/tool.sh | 5 +++ tools/run_binary/BUILD.bazel | 16 +++++++++ tools/run_binary/main.go | 69 ++++++++++++++++++++++++++++++++++++ 9 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 path/to/package/BUILD create mode 100644 path/to/package/tool.go create mode 100755 path/to/package/tool.sh create mode 100644 tools/run_binary/BUILD.bazel create mode 100644 tools/run_binary/main.go diff --git a/MODULE.bazel b/MODULE.bazel index 92a801577..937ea2bcc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -62,6 +62,7 @@ go_deps.from_file(go_mod = "//:go.mod") use_repo( go_deps, "com_github_bmatcuk_doublestar_v4", + "com_github_bytecodealliance_wasmtime_go", "org_golang_x_exp", "org_golang_x_sys", ) diff --git a/go.mod b/go.mod index 777cccf44..5a09afbbe 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24.6 require ( github.com/bazelbuild/rules_go v0.55.0 github.com/bmatcuk/doublestar/v4 v4.7.1 + github.com/bytecodealliance/wasmtime-go v1.0.0 golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 golang.org/x/sys v0.30.0 ) diff --git a/go.sum b/go.sum index c454ceb75..b1100cc93 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,17 @@ github.com/bazelbuild/rules_go v0.55.0 h1:S8X/b/Oygw/Dtv7NuyW7ht0QwdynMEdXQqYigX github.com/bazelbuild/rules_go v0.55.0/go.mod h1:T90Gpyq4HDFlsrvtQa2CBdHNJ2P4rAu/uUTmQbanzf0= github.com/bmatcuk/doublestar/v4 v4.7.1 h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0CXv75Q= github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bytecodealliance/wasmtime-go v1.0.0 h1:9u9gqaUiaJeN5IoD1L7egD8atOnTGyJcNp8BhkL9cUU= +github.com/bytecodealliance/wasmtime-go v1.0.0/go.mod h1:jjlqQbWUfVSbehpErw3UoWFndBXRRMvfikYH6KsCwOg= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lib/private/run_binary.bzl b/lib/private/run_binary.bzl index b480a2743..356fee63b 100644 --- a/lib/private/run_binary.bzl +++ b/lib/private/run_binary.bzl @@ -65,7 +65,7 @@ Possible fixes: ctx.actions.run( outputs = outputs, inputs = inputs, - executable = ctx.executable.tool, + executable = ctx.executable.tool_launcher, arguments = [args], resource_set = resource_set(ctx.attr), mnemonic = ctx.attr.mnemonic if ctx.attr.mnemonic else None, @@ -88,6 +88,12 @@ _run_binary = rule( mandatory = True, cfg = "exec", ), + "tool_launcher": attr.label( + default = Label("//tools/run_binary"), + executable = True, + allow_files = True, + cfg = "exec", + ), "env": attr.string_dict(), "srcs": attr.label_list( allow_files = True, diff --git a/path/to/package/BUILD b/path/to/package/BUILD new file mode 100644 index 000000000..16a8358f1 --- /dev/null +++ b/path/to/package/BUILD @@ -0,0 +1,33 @@ +load("@bazel_lib//lib:run_binary.bzl", "run_binary") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") + +sh_binary( + name = "tool", + srcs = ["tool.sh"], + visibility = ["//visibility:public"], +) + +go_binary( + name = "wasm_tool", + srcs = ["tool.go"], + goarch = "wasm", + goos = "wasip1", + # build as a reactor + linkmode = "c-shared", + visibility = ["//visibility:public"], +) + +run_binary( + name = "package", + outs = ["output.txt"], + args = ["$(location output.txt)"], + tool = ":fake_tool", +) + +go_library( + name = "package_lib", + srcs = ["tool.go"], + importpath = "github.com/bazel-contrib/bazel-lib/path/to/package", + visibility = ["//visibility:private"], +) diff --git a/path/to/package/tool.go b/path/to/package/tool.go new file mode 100644 index 000000000..e9951b363 --- /dev/null +++ b/path/to/package/tool.go @@ -0,0 +1,23 @@ +// This is a WASM tool that prints "Hello from a WASM tool!" to a file path passed in as an argument. +// It is compiled to WASM using Bazel. +package main + +import ( + "fmt" + "os" +) + +//go:wasmexport spawn +func spawn(path string) { + f, err := os.Create(path) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create file: %v\n", err) + os.Exit(1) + } + defer f.Close() + _, err = f.Write([]byte("Hello from a WASM tool!\n")) +} + +func main() { + panic("Should be called by WASM runtime") +} diff --git a/path/to/package/tool.sh b/path/to/package/tool.sh new file mode 100755 index 000000000..f164a66d1 --- /dev/null +++ b/path/to/package/tool.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail + +echo "Hello, World!" > "$1" diff --git a/tools/run_binary/BUILD.bazel b/tools/run_binary/BUILD.bazel new file mode 100644 index 000000000..c12d7e7ba --- /dev/null +++ b/tools/run_binary/BUILD.bazel @@ -0,0 +1,16 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "run_binary_lib", + srcs = ["main.go"], + importpath = "github.com/bazel-contrib/bazel-lib/tools/run_binary", + visibility = ["//visibility:private"], + deps = ["@com_github_bytecodealliance_wasmtime_go//:wasmtime"], +) + +go_binary( + name = "run_binary", + embed = [":run_binary_lib"], + pure = "on", + visibility = ["//visibility:public"], +) diff --git a/tools/run_binary/main.go b/tools/run_binary/main.go new file mode 100644 index 000000000..e6ad78ff0 --- /dev/null +++ b/tools/run_binary/main.go @@ -0,0 +1,69 @@ +// This is a wrapper/host process for running tools under a Bazel action. +// The environment it runs in is specified by Bazel: +// - the working directory is the execution root +// - args/env are dropped since the tool is not run directly by the user under Bazel run +// +// It also supports WASM tools. It hosts a WASM runtime and loads the WASM module into it. +// This makes it easier to ship tools from Bazel rules without having to build lots of different target-triple pre-compiled versions. +// +// In the future, this tool can also strace the tool we run to collect telemetry data. +// That would be great for: +// - The tool wrote to a different output path than expected, where did it go? +// - What's making the tool slow? Maybe we can get eBPF timing data under Linux at least - and stitch the profiling into the Bazel profile. +// +// The tool we run can do things like: +// - write output files and output directories under bazel-bin/path/to/package +package main + +import ( + "fmt" + "os" + + // add a high-performance WASM runtime + "github.com/bytecodealliance/wasmtime-go" +) + +func main() { + + engine := wasmtime.NewEngine() + store := wasmtime.NewStore(engine) + // Load wasm code by reading /Users/alexeagle/Projects/bazel-lib/bazel-bin/path/to/package/tool_/tool.wasm + wasmCode, err := os.ReadFile("/Users/alexeagle/Projects/bazel-lib/bazel-bin/path/to/package/tool_/tool.wasm") + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to read wasm code: %v\n", err) + os.Exit(1) + } + module, err := wasmtime.NewModule(engine, []byte(wasmCode)) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create module: %v\n", err) + os.Exit(1) + } + instance, err := wasmtime.NewInstance(store, module, []wasmtime.AsExtern{}) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create instance: %v\n", err) + os.Exit(1) + } + spawnFunc := instance.GetFunc(store, "spawn") + if spawnFunc == nil { + fmt.Fprintf(os.Stderr, "Failed to get spawn function\n") + os.Exit(1) + } + _, err = spawnFunc.Call(store, "bazel-out/darwin_arm64-fastbuild/bin/path/to/package/output.txt") + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to call spawn function: %v\n", err) + os.Exit(1) + } + + // // writes to bazel-bin/path/to/package/output.txt + // f, err := os.Create("bazel-out/darwin_arm64-fastbuild/bin/path/to/package/output.txt") + // if err != nil { + // fmt.Fprintf(os.Stderr, "Failed to create output.txt: %v\n", err) + // os.Exit(1) + // } + // defer f.Close() + // _, err = f.Write([]byte("[DEBUG] Hello from the tool launcher!\n")) + // if err != nil { + // fmt.Fprintf(os.Stderr, "Failed to write to output.txt: %v\n", err) + // os.Exit(1) + // } +} From 4190947b34ebc2e1a5b2fd5cd012ae8cf2576f4b Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Sun, 16 Nov 2025 15:40:05 -0800 Subject: [PATCH 2/2] use wasmzero runtime, seems easier --- MODULE.bazel | 2 +- go.mod | 2 +- go.sum | 12 ++------- path/to/package/tool.go | 4 +-- tools/run_binary/BUILD.bazel | 5 +++- tools/run_binary/main.go | 52 +++++++++++++++--------------------- 6 files changed, 31 insertions(+), 46 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 937ea2bcc..f0b42e293 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -62,7 +62,7 @@ go_deps.from_file(go_mod = "//:go.mod") use_repo( go_deps, "com_github_bmatcuk_doublestar_v4", - "com_github_bytecodealliance_wasmtime_go", + "com_github_tetratelabs_wazero", "org_golang_x_exp", "org_golang_x_sys", ) diff --git a/go.mod b/go.mod index 5a09afbbe..ac40dac61 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.24.6 require ( github.com/bazelbuild/rules_go v0.55.0 github.com/bmatcuk/doublestar/v4 v4.7.1 - github.com/bytecodealliance/wasmtime-go v1.0.0 + github.com/tetratelabs/wazero v1.10.1 golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 golang.org/x/sys v0.30.0 ) diff --git a/go.sum b/go.sum index b1100cc93..dc20c27c8 100644 --- a/go.sum +++ b/go.sum @@ -2,17 +2,9 @@ github.com/bazelbuild/rules_go v0.55.0 h1:S8X/b/Oygw/Dtv7NuyW7ht0QwdynMEdXQqYigX github.com/bazelbuild/rules_go v0.55.0/go.mod h1:T90Gpyq4HDFlsrvtQa2CBdHNJ2P4rAu/uUTmQbanzf0= github.com/bmatcuk/doublestar/v4 v4.7.1 h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0CXv75Q= github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/bytecodealliance/wasmtime-go v1.0.0 h1:9u9gqaUiaJeN5IoD1L7egD8atOnTGyJcNp8BhkL9cUU= -github.com/bytecodealliance/wasmtime-go v1.0.0/go.mod h1:jjlqQbWUfVSbehpErw3UoWFndBXRRMvfikYH6KsCwOg= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8= +github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/path/to/package/tool.go b/path/to/package/tool.go index e9951b363..5a4c6d504 100644 --- a/path/to/package/tool.go +++ b/path/to/package/tool.go @@ -8,8 +8,8 @@ import ( ) //go:wasmexport spawn -func spawn(path string) { - f, err := os.Create(path) +func spawn() { + f, err := os.Create("bazel-out/darwin_arm64-fastbuild/bin/path/to/package/output.txt") if err != nil { fmt.Fprintf(os.Stderr, "Failed to create file: %v\n", err) os.Exit(1) diff --git a/tools/run_binary/BUILD.bazel b/tools/run_binary/BUILD.bazel index c12d7e7ba..2ffa6ea4c 100644 --- a/tools/run_binary/BUILD.bazel +++ b/tools/run_binary/BUILD.bazel @@ -5,7 +5,10 @@ go_library( srcs = ["main.go"], importpath = "github.com/bazel-contrib/bazel-lib/tools/run_binary", visibility = ["//visibility:private"], - deps = ["@com_github_bytecodealliance_wasmtime_go//:wasmtime"], + deps = [ + "@com_github_tetratelabs_wazero//:wazero", + "@com_github_tetratelabs_wazero//imports/wasi_snapshot_preview1", + ], ) go_binary( diff --git a/tools/run_binary/main.go b/tools/run_binary/main.go index e6ad78ff0..8f5755927 100644 --- a/tools/run_binary/main.go +++ b/tools/run_binary/main.go @@ -16,54 +16,44 @@ package main import ( + "context" "fmt" "os" - // add a high-performance WASM runtime - "github.com/bytecodealliance/wasmtime-go" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" ) func main() { - - engine := wasmtime.NewEngine() - store := wasmtime.NewStore(engine) - // Load wasm code by reading /Users/alexeagle/Projects/bazel-lib/bazel-bin/path/to/package/tool_/tool.wasm - wasmCode, err := os.ReadFile("/Users/alexeagle/Projects/bazel-lib/bazel-bin/path/to/package/tool_/tool.wasm") + wasmCode, err := os.ReadFile("/Users/alexeagle/Projects/bazel-lib/bazel-bin/path/to/package/wasm_tool_/wasm_tool.wasm") if err != nil { fmt.Fprintf(os.Stderr, "Failed to read wasm code: %v\n", err) os.Exit(1) } - module, err := wasmtime.NewModule(engine, []byte(wasmCode)) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to create module: %v\n", err) - os.Exit(1) - } - instance, err := wasmtime.NewInstance(store, module, []wasmtime.AsExtern{}) + + // Choose the context to use for function calls. + ctx := context.Background() + r := wazero.NewRuntime(ctx) + defer r.Close(ctx) + + wasi_snapshot_preview1.MustInstantiate(ctx, r) + + module, err := r.Instantiate(ctx, wasmCode) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to create instance: %v\n", err) + fmt.Fprintf(os.Stderr, "Failed to instantiate module: %v\n", err) os.Exit(1) } - spawnFunc := instance.GetFunc(store, "spawn") - if spawnFunc == nil { + + // Call the exported spawn function with the output path + // Go's wasmexport handles string parameter marshalling automatically + spawn := module.ExportedFunction("spawn") + if spawn == nil { fmt.Fprintf(os.Stderr, "Failed to get spawn function\n") os.Exit(1) } - _, err = spawnFunc.Call(store, "bazel-out/darwin_arm64-fastbuild/bin/path/to/package/output.txt") + _, err = spawn.Call(ctx) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to call spawn function: %v\n", err) + fmt.Fprintf(os.Stderr, "Failed to call spawn: %v\n", err) os.Exit(1) } - - // // writes to bazel-bin/path/to/package/output.txt - // f, err := os.Create("bazel-out/darwin_arm64-fastbuild/bin/path/to/package/output.txt") - // if err != nil { - // fmt.Fprintf(os.Stderr, "Failed to create output.txt: %v\n", err) - // os.Exit(1) - // } - // defer f.Close() - // _, err = f.Write([]byte("[DEBUG] Hello from the tool launcher!\n")) - // if err != nil { - // fmt.Fprintf(os.Stderr, "Failed to write to output.txt: %v\n", err) - // os.Exit(1) - // } }