rust: compile alloc from the in-tree sources

Includes:

  - Cross-building a new sysroot with our custom `alloc` in order
    to support `alloc` tests.
  - Documentation generation for in-tree `alloc`.
  - Split `RUSTDOC` command in two for clarity, since now we have
    to also handle `sysroot` differently.
  - Proper handling of both silent and verbose mode for tests
    and `cargo`.
  - Deny missing docs globally `alloc`.
  - Allow broken intra docs for `alloc`.
  - Do not format `rust/alloc` nor `rust/test`, including handling
    the case where `srctree` is a prent folder.
  - `make clean` now cleans `rust/doc` and `rust/test`.
  - Move allocs out of the second expansion section since it is
    in-tree now.
  - Other small cleanups.

I want to clean this up further and factor a few things,
but let's move ahead.

Signed-off-by: Miguel Ojeda <ojeda@kernel.org>

Author: ojeda

Makefile

@@ -461,9 +461,11 @@ STRIP		= $(CROSS_COMPILE)strip
 endif
 RUSTC		= rustc
 RUSTC_BOOTSTRAP = 1
+RUSTDOC		= rustdoc
 RUSTFMT		= rustfmt
 CLIPPY_DRIVER	= clippy-driver
 BINDGEN		= bindgen
+CARGO		= cargo
 PAHOLE		= pahole
 RESOLVE_BTFIDS	= $(objtree)/tools/bpf/resolve_btfids/resolve_btfids
 LEX		= flex
@@ -525,7 +527,7 @@ KBUILD_RUSTCFLAGS := --emit=dep-info,obj,metadata --edition=2018 \
 		     -Cpanic=abort -Cembed-bitcode=n -Clto=n -Crpath=n \
 		     -Cforce-unwind-tables=n -Ccodegen-units=1 \
 		     -Zbinary_dep_depinfo=y -Zsymbol-mangling-version=v0 \
-		     -D unsafe_op_in_unsafe_fn
+		     -Dunsafe_op_in_unsafe_fn -Wmissing_docs
 KBUILD_AFLAGS_KERNEL :=
 KBUILD_CFLAGS_KERNEL :=
 KBUILD_RUSTCFLAGS_KERNEL :=
@@ -543,14 +545,13 @@ else
 	RUSTC_OR_CLIPPY_QUIET := RUSTC
 	RUSTC_OR_CLIPPY = $(RUSTC)
 endif
-export RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY
 
 ifdef RUST_LIB_SRC
 	export RUST_LIB_SRC
 endif
 
 export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
-export RUSTC RUSTC_BOOTSTRAP BINDGEN
+export RUSTC RUSTC_BOOTSTRAP RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN CARGO
 export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL
 export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
 export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD
@@ -1598,7 +1599,7 @@ endif # CONFIG_MODULES
 # Directories & files removed with 'make clean'
 CLEAN_FILES += include/ksym vmlinux.symvers modules-only.symvers \
 	       modules.builtin modules.builtin.modinfo modules.nsdeps \
-	       compile_commands.json .thinlto-cache
+	       compile_commands.json .thinlto-cache rust/test rust/doc
 
 # Directories & files removed with 'make mrproper'
 MRPROPER_FILES += include/config include/generated          \
@@ -1728,7 +1729,7 @@ help:
 	@echo  '  rustdoc	  - Generate Rust documentation'
 	@echo  '		    (requires kernel .config)'
 	@echo  '  rusttest        - Runs the Rust tests'
-	@echo  '                    (requires kernel .config)'
+	@echo  '                    (requires kernel .config; downloads external repos)'
 	@echo  '  rust-analyzer	  - Generate rust-project.json rust-analyzer support file'
 	@echo  '		    (requires kernel .config)'
 	@echo  ''
@@ -1822,11 +1823,21 @@ rusttest: prepare0
 # Formatting targets
 PHONY += rustfmt rustfmtcheck
 
+# We skip `rust/alloc` since we want to minimize the diff w.r.t. upstream.
+#
+# We match using absolute paths since `find` does not resolve them
+# when matching, which is a problem when e.g. `srctree` is `..`.
+# We `grep` afterwards in order to remove the directory entry itself.
 rustfmt:
-	find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT)
-
-rustfmtcheck:
-	find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT) --check
+	$(Q)find $(abs_srctree) -type f -name '*.rs' \
+		-o -path $(abs_srctree)/rust/alloc -prune \
+		-o -path $(abs_objtree)/rust/test -prune \
+		| grep -Fv $(abs_srctree)/rust/alloc \
+		| grep -Fv $(abs_objtree)/rust/test \
+		| xargs $(RUSTFMT) $(rustfmt_flags)
+
+rustfmtcheck: rustfmt_flags = --check
+rustfmtcheck: rustfmt
 
 # IDE support targets
 PHONY += rust-analyzer

rust/Makefile

@@ -17,18 +17,27 @@ endif
 
 obj-$(CONFIG_RUST) += exports.o
 
-RUSTDOC = rustdoc
+ifeq ($(quiet),silent_)
+cargo_quiet=-q
+rust_test_quiet=-q
+rustdoc_test_quiet=--test-args -q
+else ifeq ($(quiet),quiet_)
+rust_test_quiet=-q
+rustdoc_test_quiet=--test-args -q
+else
+cargo_quiet=--verbose
+endif
 
-quiet_cmd_rustdoc = RUSTDOC $(if $(filter --test,$(rustdoc_target_flags)),T, ) $(if $(rustdoc_host),H, ) $<
+quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $<
       cmd_rustdoc = \
 	RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
 	$(RUSTDOC) $(if $(rustdoc_host),,$(rustc_cross_flags)) \
-		$(filter-out --emit=%, $(rustc_flags)) $(rustc_target_flags) $(rustdoc_target_flags) \
-		-L $(objtree)/rust/$(if $(filter --test,$(rustdoc_target_flags)),test/) \
-		--output $(objtree)/rust/doc --crate-name $(subst rusttest-,,$(subst rustdoc-,,$@)) \
-		-Fmissing-docs @$(objtree)/include/generated/rustc_cfg $<
+		$(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags))) \
+		$(rustc_target_flags) -L $(objtree)/rust \
+		--output $(objtree)/rust/doc --crate-name $(subst rustdoc-,,$@) \
+		@$(objtree)/include/generated/rustc_cfg $<
 
-rustdoc: rustdoc-macros rustdoc-compiler_builtins rustdoc-kernel
+rustdoc: rustdoc-macros rustdoc-compiler_builtins rustdoc-alloc rustdoc-kernel
 
 rustdoc-macros: private rustdoc_host = yes
 rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \
@@ -39,6 +48,15 @@ rustdoc-macros: $(srctree)/rust/macros/lib.rs FORCE
 rustdoc-compiler_builtins: $(srctree)/rust/compiler_builtins.rs FORCE
 	$(call if_changed,rustdoc)
 
+# We need to allow `broken_intra_doc_links` because some
+# `no_global_oom_handling` functions refer to non-`no_global_oom_handling`
+# functions. Ideally `rustdoc` would have a way to distinguish broken links
+# due to things that are "configured out" vs. entirely non-existing ones.
+rustdoc-alloc: private rustc_target_flags = \
+	-Abroken_intra_doc_links
+rustdoc-alloc: $(srctree)/rust/alloc/lib.rs FORCE
+	$(call if_changed,rustdoc)
+
 rustdoc-kernel: private rustc_target_flags = --extern alloc \
     --extern build_error \
     --extern macros=$(objtree)/rust/libmacros.so
@@ -49,50 +67,108 @@ rustdoc-kernel: $(srctree)/rust/kernel/lib.rs rustdoc-macros \
 quiet_cmd_rustc_test_library = RUSTC TL $<
       cmd_rustc_test_library = \
 	RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
-	$(RUSTC) $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags))) \
+	$(RUSTC) $(filter-out --sysroot=%, $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags)))) \
 		$(rustc_target_flags) --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \
 		--out-dir $(objtree)/rust/test/ --cfg testlib \
+		--sysroot $(objtree)/rust/test/sysroot \
 		-L $(objtree)/rust/test/ --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $<
 
-rusttestlib-build_error: $(srctree)/rust/build_error.rs FORCE
+rusttestlib-build_error: $(srctree)/rust/build_error.rs rusttest-prepare FORCE
 	$(call if_changed,rustc_test_library)
 
 rusttestlib-macros: private rustc_target_flags = --extern proc_macro
 rusttestlib-macros: private rustc_test_library_proc = yes
-rusttestlib-macros: $(srctree)/rust/macros/lib.rs FORCE
+rusttestlib-macros: $(srctree)/rust/macros/lib.rs rusttest-prepare FORCE
 	$(call if_changed,rustc_test_library)
 
+quiet_cmd_rustdoc_test = RUSTDOC T $<
+      cmd_rustdoc_test = \
+	RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
+	$(RUSTDOC) --test $(filter-out --sysroot=%, $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags)))) \
+		$(rustc_target_flags) $(rustdoc_test_target_flags) \
+		--sysroot $(objtree)/rust/test/sysroot $(rustdoc_test_quiet) \
+		-L $(objtree)/rust/test \
+		--output $(objtree)/rust/doc --crate-name $(subst rusttest-,,$@) \
+		@$(objtree)/include/generated/rustc_cfg $<
+
 # We cannot use `-Zpanic-abort-tests` because some tests are dynamic,
 # so for the moment we skip `-Cpanic=abort`.
 quiet_cmd_rustc_test = RUSTC T  $<
       cmd_rustc_test = \
 	RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
-	$(RUSTC) --test $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags))) \
+	$(RUSTC) --test $(filter-out --sysroot=%, $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags)))) \
 		$(rustc_target_flags) --out-dir $(objtree)/rust/test \
+		--sysroot $(objtree)/rust/test/sysroot \
 		-L $(objtree)/rust/test/ --crate-name $(subst rusttest-,,$@) $<; \
-		$(objtree)/rust/test/$(subst rusttest-,,$@) $(rustc_test_run_flags)
+	$(objtree)/rust/test/$(subst rusttest-,,$@) $(rust_test_quiet) \
+		$(rustc_test_run_flags)
 
 rusttest: rusttest-macros rusttest-kernel
 
+# This prepares a custom sysroot with our custom `alloc` instead of
+# the standard one.
+#
+# This requires several hacks:
+#   - Unlike `core` and `alloc`, `std` depends on more than a dozen crates,
+#     including third-party crates that need to be downloaded, plus custom
+#     `build.rs` steps. Thus hardcoding things here is not maintainable.
+#   - `cargo` knows how to build the standard library, but it is an unstable
+#     feature so far (`-Zbuild-std`).
+#   - `cargo` only considers the use case of building the standard library
+#     to use it in a given package. Thus we need to create a dummy package
+#     and pick the generated libraries from there.
+#   - Since we only keep a subset of upstream `alloc` in-tree, we need
+#     to recreate it on the fly by putting our sources on top.
+#   - The usual ways of modifying the dependency graph in `cargo` do not seem
+#     to apply for the `-Zbuild-std` steps, thus we have to mislead it
+#     by modifying the sources in the sysroot.
+#   - To avoid messing with the user's Rust installation, we create a clone
+#     of the sysroot. However, `cargo` ignores `RUSTFLAGS` in the `-Zbuild-std`
+#     steps, thus we use a wrapper binary passed via `RUSTC` to pass the flag.
+#
+# In the future, we hope to avoid the whole ordeal by either:
+#   - Making the `test` crate not depend on `std` (either improving upstream
+#     or having our own custom crate).
+#   - Making the tests run in kernel space (requires the previous point).
+#   - Making `std` and friends be more like a "normal" crate, so that
+#     `-Zbuild-std` and related hacks are not needed.
+quiet_cmd_rustsysroot = RUSTSYSROOT
+      cmd_rustsysroot = \
+	rm -rf $(objtree)/rust/test; \
+	mkdir -p $(objtree)/rust/test; \
+	cp -a $(rustc_sysroot) $(objtree)/rust/test/sysroot; \
+	cp -r $(srctree)/rust/alloc/* \
+		$(objtree)/rust/test/sysroot/lib/rustlib/src/rust/library/alloc/src; \
+	echo '\#!/bin/sh' > $(objtree)/rust/test/rustc_sysroot; \
+	echo "$(RUSTC) --sysroot=$(abspath $(objtree)/rust/test/sysroot) \"\$$@\"" \
+		>> $(objtree)/rust/test/rustc_sysroot; \
+	chmod u+x $(objtree)/rust/test/rustc_sysroot; \
+	$(CARGO) -q new $(objtree)/rust/test/dummy; \
+	RUSTC=$(objtree)/rust/test/rustc_sysroot $(CARGO) $(cargo_quiet) \
+		test -Zbuild-std --target $(rustc_host_target) \
+		--manifest-path $(objtree)/rust/test/dummy/Cargo.toml; \
+	rm $(objtree)/rust/test/sysroot/lib/rustlib/$(rustc_host_target)/lib/*; \
+	cp $(objtree)/rust/test/dummy/target/$(rustc_host_target)/debug/deps/* \
+		$(objtree)/rust/test/sysroot/lib/rustlib/$(rustc_host_target)/lib
+
+rusttest-prepare: FORCE
+	$(call if_changed,rustsysroot)
+
 rusttest-macros: private rustc_target_flags = --extern proc_macro
-rusttest-macros: private rustdoc_host = yes
-rusttest-macros: private rustdoc_target_flags = --test --crate-type proc-macro
-rusttest-macros: $(srctree)/rust/macros/lib.rs FORCE
+rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro
+rusttest-macros: $(srctree)/rust/macros/lib.rs rusttest-prepare FORCE
 	$(call if_changed,rustc_test)
-	$(call if_changed,rustdoc)
+	$(call if_changed,rustdoc_test)
 
 rusttest-kernel: private rustc_target_flags = --extern alloc \
-    --extern build_error \
-    --extern macros=$(objtree)/rust/test/libmacros.so
+    --extern build_error --extern macros
 rusttest-kernel: private rustc_test_run_flags = \
     --skip bindgen_test_layout_
-rusttest-kernel: private rustdoc_host = yes
-rusttest-kernel: private rustdoc_target_flags = --test
-rusttest-kernel: $(srctree)/rust/kernel/lib.rs rusttestlib-build_error \
-    rusttestlib-macros FORCE
+rusttest-kernel: $(srctree)/rust/kernel/lib.rs rusttest-prepare \
+    rusttestlib-build_error rusttestlib-macros FORCE
 	$(call if_changed,rustc_test)
 	$(call if_changed,rustc_test_library)
-	$(call if_changed,rustdoc)
+	$(call if_changed,rustdoc_test)
 
 ifdef CONFIG_CC_IS_CLANG
 bindgen_c_flags = $(c_flags)
@@ -188,6 +264,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 
 # `$(rustc_flags)` is passed in case the user added `--sysroot`.
 rustc_sysroot = $(shell $(RUSTC) $(rustc_flags) --print sysroot)
+rustc_host_target = $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2)
 RUST_LIB_SRC ?= $(rustc_sysroot)/lib/rustlib/src/rust/library
 
 rust-analyzer:
@@ -198,6 +275,11 @@ $(objtree)/rust/compiler_builtins.o: $(srctree)/rust/compiler_builtins.rs \
     $(objtree)/rust/core.o FORCE
 	$(call if_changed_dep,rustc_library)
 
+$(objtree)/rust/alloc.o: private skip_clippy = 1
+$(objtree)/rust/alloc.o: $(srctree)/rust/alloc/lib.rs \
+    $(objtree)/rust/compiler_builtins.o FORCE
+	$(call if_changed_dep,rustc_library)
+
 $(objtree)/rust/build_error.o: $(srctree)/rust/build_error.rs \
     $(objtree)/rust/compiler_builtins.o FORCE
 	$(call if_changed_dep,rustc_library)
@@ -216,8 +298,3 @@ $(objtree)/rust/kernel.o: $(srctree)/rust/kernel/lib.rs $(objtree)/rust/alloc.o
 $(objtree)/rust/core.o: private skip_clippy = 1
 $(objtree)/rust/core.o: $$(RUST_LIB_SRC)/core/src/lib.rs FORCE
 	$(call if_changed_dep,rustc_library)
-
-$(objtree)/rust/alloc.o: private skip_clippy = 1
-$(objtree)/rust/alloc.o: $$(RUST_LIB_SRC)/alloc/src/lib.rs \
-    $(objtree)/rust/compiler_builtins.o FORCE
-	$(call if_changed_dep,rustc_library)