Auto merge of #140064 - EnzymeAD:enable-autodiff-in-ci, r=<try>

[DO NOT MERGE] start building Enzyme in CI

My goal is to put this in CI on April 26, to have a week to land some of the outstanding PRs (removed # in front of it to avoid spamming them every time I do a try build in this PR):
139700 [Landed]
139308 [Landed]
140104 [Landed]
140030 [Landed]
140049 [Almost ready]
140244 [Closed due to design changes]
140697[WIP]
The autodiff flags PR should land first, but otherwise they don't overlap and are mostly ready, so it shouldn't be too hard to land them. In the meantime, I'll experiment here with some builders. There are some related CMake PRs which I also made in the Enzyme repo. I can temporarily merge them into the r-l/Enzyme fork, so we are not blocked on them.

A fix for 140137 (Apple CI) would be nice, but I'm also happy to start with the working linux (x86-64 + aarch) as the most common target, and expand from there.

r? `@oli-obk`

Tracking:

- 124509

**Linux**
succeeded try with plugins and without plugins & with llvm-enzyme enabled:  dist-x86_64-linux
succeeded try with plugins and without plugins & with llvm-enzyme enabled:  dist-aarch64-linux

**MacOS**
failed try with plugins & llvm-enzyme enabled: dist-x86_64-apple (#140064 (comment))
failed try with plugins & llvm-enzyme enabled: dist-apple-various (#140064 (comment))
failed try with plugins & llvm-enzyme enabled: dist-aarch64-apple (#140064 (comment))

**Windows**
failed try with plugins & llvm-enzyme enabled: dist-x86_64-mingw (#140064 (comment))

try-job: dist-aarch64-apple

Author: bors

src/bootstrap/src/core/build_steps/compile.rs

@@ -1187,8 +1187,8 @@ pub fn rustc_cargo(
     // functionality from Enzyme core. For that we need to link against Enzyme.
     if builder.config.llvm_enzyme {
         let arch = builder.build.build;
-        let enzyme_dir = builder.build.out.join(arch).join("enzyme").join("lib");
-        cargo.rustflag("-L").rustflag(enzyme_dir.to_str().expect("Invalid path"));
+        let _enzyme_dir = builder.build.out.join(arch).join("enzyme").join("lib");
+        //cargo.rustflag("-L").rustflag(enzyme_dir.to_str().expect("Invalid path"));
 
         if let Some(llvm_config) = builder.llvm_config(builder.config.build) {
             let llvm_version_major = llvm::get_llvm_version_major(builder, &llvm_config);
@@ -2068,7 +2068,7 @@ impl Step for Assemble {
             let enzyme_install = builder.ensure(llvm::Enzyme { target: build_compiler.host });
             let llvm_config = builder.llvm_config(builder.config.build).unwrap();
             let llvm_version_major = llvm::get_llvm_version_major(builder, &llvm_config);
-            let lib_ext = std::env::consts::DLL_EXTENSION;
+            let lib_ext = "so";
             let libenzyme = format!("libEnzyme-{llvm_version_major}");
             let src_lib =
                 enzyme_install.join("build/Enzyme").join(&libenzyme).with_extension(lib_ext);
@@ -2079,6 +2079,12 @@ impl Step for Assemble {
             let target_dst_lib = target_libdir.join(&libenzyme).with_extension(lib_ext);
             builder.copy_link(&src_lib, &dst_lib, FileType::NativeLibrary);
             builder.copy_link(&src_lib, &target_dst_lib, FileType::NativeLibrary);
+            //add_to_sysroot(
+            //    builder,
+            //    &src_lib,
+            //    &target_dst_lib,
+            //    &build_stamp::librustc_stamp(builder, build_compiler, target_compiler.host),
+            //);
         }
 
         // Build the libraries for this compiler to link to (i.e., the libraries

src/bootstrap/src/core/build_steps/llvm.rs

@@ -970,7 +970,8 @@ impl Step for Enzyme {
             .env("LLVM_CONFIG_REAL", &llvm_config)
             .define("LLVM_ENABLE_ASSERTIONS", "ON")
             .define("ENZYME_EXTERNAL_SHARED_LIB", "ON")
-            .define("LLVM_DIR", builder.llvm_out(target));
+            .define("ENZYME_STATIC_LIB", "ON")
+            .define("LLVM_DIR", builder.llvm_out(target).join("build"));
 
         cfg.build();
 

src/bootstrap/src/core/builder/cargo.rs

@@ -993,9 +993,15 @@ impl Builder<'_> {
                 let llvm_libdir =
                     command(llvm_config).arg("--libdir").run_capture_stdout(self).stdout();
                 if target.is_msvc() {
+                    // FIXME(autodiff): Enzyme does currently not support MSVC.
                     rustflags.arg(&format!("-Clink-arg=-LIBPATH:{llvm_libdir}"));
                 } else {
                     rustflags.arg(&format!("-Clink-arg=-L{llvm_libdir}"));
+                    if self.config.llvm_enzyme {
+                        let arch = self.build.build;
+                        let enzyme_dir = self.build.out.join(arch).join("enzyme").join("lib");
+                        rustflags.arg(&format!("-Clink-arg=-L{}", enzyme_dir.display()));
+                    }
                 }
             }
         }

src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile

@@ -89,6 +89,7 @@ ENV RUST_CONFIGURE_ARGS \
       --set llvm.thin-lto=true \
       --set llvm.libzstd=true \
       --set llvm.ninja=false \
+      --set llvm.enzyme=true \
       --set rust.debug-assertions=false \
       --set rust.jemalloc \
       --set rust.use-lld=true \

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -90,6 +90,7 @@ ENV RUST_CONFIGURE_ARGS \
       --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
       --set llvm.thin-lto=true \
       --set llvm.ninja=false \
+      --set llvm.enzyme=true \
       --set llvm.libzstd=true \
       --set rust.jemalloc \
       --set rust.use-lld=true \

src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile

@@ -30,4 +30,5 @@ ENV RUST_CONFIGURE_ARGS \
  --enable-profiler \
  --enable-compiler-docs \
  --set llvm.libzstd=true
+ --set llvm.enzyme=true \
 ENV SCRIPT python3 ../x.py --stage 2 test

src/ci/github-actions/jobs.yml

@@ -391,7 +391,7 @@ auto:
   - name: dist-x86_64-apple
     env:
       SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
-      RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1
+      RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set llvm.enzyme=true --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
       # Ensure that host tooling is built to support our minimum support macOS version.
       MACOSX_DEPLOYMENT_TARGET: 10.12
@@ -409,7 +409,7 @@ auto:
       SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim,aarch64-apple-ios-macabi,x86_64-apple-ios-macabi
       # Mac Catalyst cannot currently compile the sanitizer:
       # https://github.com/rust-lang/rust/issues/129069
-      RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set target.aarch64-apple-ios-macabi.sanitizers=false --set target.x86_64-apple-ios-macabi.sanitizers=false
+      RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.enzyme=true --set target.aarch64-apple-ios-macabi.sanitizers=false --set target.x86_64-apple-ios-macabi.sanitizers=false
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
       # Ensure that host tooling is built to support our minimum support macOS version.
       # FIXME(madsmtm): This might be redundant, as we're not building host tooling here (?)
@@ -441,6 +441,7 @@ auto:
         --enable-profiler
         --set rust.jemalloc
         --set llvm.ninja=false
+        --set llvm.enzyme=true
         --set rust.lto=thin
         --set rust.codegen-units=1
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
@@ -627,6 +628,7 @@ auto:
       SCRIPT: python x.py dist bootstrap --include-default-paths
       RUST_CONFIGURE_ARGS: >-
         --build=x86_64-pc-windows-gnu
+        --set llvm.enzyme=true
         --enable-full-tools
       DIST_REQUIRE_ALL_TOOLS: 1
       CODEGEN_BACKENDS: llvm,cranelift