feat: Rework CMake search path settings (#880)

- [x] Gate the install path's site-packages by an option
- [x] Add logic for `entry_points(group="cmake.root")`:
- [x] Documentation
    - Add a new page on all the search logics
    - Encourage the usage of `cmake.root` over `cmake.prefix`
- [x] Tests

Closes #831 
Closes #885
Relates to #860 (still keeping it open because it doesn't address the
specific issue)

---------

Signed-off-by: Cristian Le <cristian.le@mpsd.mpg.de>
Signed-off-by: Cristian Le <git@lecris.dev>

Author: LecrisUT

README.md

@@ -314,6 +314,10 @@ messages.after-failure = ""
 # A message to print after a successful build.
 messages.after-success = ""
 
+# Add the python build environment site_packages folder to the CMake prefix
+# paths.
+search.site-packages = true
+
 # List dynamic metadata fields and hook locations in this table.
 metadata = {}
 

docs/configuration/search_paths.md

@@ -0,0 +1,184 @@
+# Search paths
+
+Scikit-build-core populates CMake search paths to take into account any other
+CMake project installed in the same environment. In order to take advantage of
+this the dependent project must populate a `cmake.*` entry-point.
+
+## `<PackageName>_ROOT`
+
+This is the recommended interface to be used for importing dependent packages
+using `find_package`. This variable is populated by the dependent project's
+entry-point `cmake.root`.
+
+To configure the `cmake.root` entry-point to export to other projects, you can
+use the CMake standard install paths in you `CMakeLists.txt` if you use
+`wheel.install-dir` option, e.g.
+
+```{code-block} cmake
+:caption: CMakeLists.txt
+:emphasize-lines: 14-16
+
+include(CMakePackageConfigHelpers)
+include(GNUInstallDirs)
+write_basic_package_version_file(
+    MyProjectConfigVersion.cmake
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY SameMajorVersion
+)
+configure_package_config_file(
+    cmake/MyProjectConfig.cmake.in
+    MyProjectConfig.cmake
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/MyProject
+)
+install(FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/MyProjectConfigVersion.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/MyProjectConfig.cmake
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/MyProject
+)
+```
+
+```{code-block} toml
+:caption: pyproject.toml
+:emphasize-lines: 2,5
+
+[tool.scikit-build]
+wheel.install-dir = "myproject"
+
+[project.entry-points."cmake.root"]
+MyProject = "myproject"
+```
+
+With this any consuming project that depends on this would automatically work
+with `find_package(MyProject)` as long as it is in the `build-system.requires`
+list.
+
+````{tab} pyproject.toml
+
+```toml
+[tool.scikit-build.search]
+ignore_entry_point = ["MyProject"]
+[tool.scikit-build.search.roots]
+OtherProject = "/path/to/other_project"
+```
+
+````
+
+`````{tab} config-settings
+
+
+````{tab} pip
+
+```console
+$ pip install . -v --config-settings=search.ignore_entry_point="MyProject" --config-settings=search.roots.OtherProject="/path/to/other_project"
+```
+
+````
+
+````{tab} build
+
+```console
+$ pipx run build --wheel -Csearch.ignore_entry_point="MyProject" -Csearch.roots.OtherProject="/path/to/other_project"
+```
+
+````
+
+````{tab} cibuildwheel
+
+```toml
+[tool.cibuildwheel.config-settings]
+"search.ignore_entry_point" = ["MyProject"]
+"search.roots.OtherProject" = "/path/to/other_project"
+```
+
+````
+
+`````
+
+````{tab} Environment
+
+
+```yaml
+SKBUILD_SEARCH_IGNORE_ENTRY_POINT: "MyProject"
+SKBUILD_SEARCH_ROOTS_OtherProject: "/path/to/other_project"
+```
+
+````
+
+## `CMAKE_PREFIX_PATH`
+
+Another common search path that scikit-build-core populates is the
+`CMAKE_PREFIX_PATH` which is a common catch-all for all CMake search paths, e.g.
+`find_package`, `find_program`, `find_path`. This is populated by default with
+the `site-packages` folder where the project will be installed or the build
+isolation's `site-packages` folder. This default can be disabled by setting
+
+```toml
+[tool.scikit-build.search]
+search.use-site-packages = false
+```
+
+Additionally, scikit-build-core reads the entry-point `cmake.prefix` of the
+dependent projects, which is similarly export as
+
+```toml
+[project.entry-points."cmake.prefix"]
+MyProject = "myproject"
+```
+
+````{tab} pyproject.toml
+
+```toml
+[tool.scikit-build.search]
+ignore_entry_point = ["MyProject"]
+prefixes = ["/path/to/prefixA", "/path/to/prefixB"]
+```
+
+````
+
+`````{tab} config-settings
+
+
+````{tab} pip
+
+```console
+$ pip install . -v --config-settings=search.ignore_entry_point="MyProject" --config-settings=search.prefixes="/path/to/prefixA;/path/to/prefixB"
+```
+
+````
+
+````{tab} build
+
+```console
+$ pipx run build --wheel -Csearch.ignore_entry_point="MyProject" -Csearch.prefixes="/path/to/prefixA;/path/to/prefixB"
+```
+
+````
+
+````{tab} cibuildwheel
+
+```toml
+[tool.cibuildwheel.config-settings]
+"search.ignore_entry_point" = ["MyProject"]
+"search.prefixes" = ["/path/to/prefixA", "/path/to/prefixB"]
+```
+
+````
+
+`````
+
+````{tab} Environment
+
+
+```yaml
+SKBUILD_SEARCH_IGNORE_ENTRY_POINT: "MyProject"
+SKBUILD_SEARCH_PREFIXES: "/path/to/prefixA;/path/to/prefixB"
+```
+
+````
+
+## `CMAKE_MODULE_PATH`
+
+Scikit-build-core also populates `CMAKE_MODULE_PATH` variable used to search for
+CMake modules using the `include()` command (if the `.cmake` suffix is omitted).
+
+[`CMAKE_PREFIX_PATH`]: #cmake-prefix-path

docs/guide/cmakelists.md

@@ -77,15 +77,13 @@ succeed.
 
 ## Finding other packages
 
-Scikit-build-core includes the site-packages directory in CMake's search path,
-so packages can provide a find package config with a name matching the package
-name - such as the `pybind11` package.
-
-Third party packages can declare entry-points `cmake.module` and `cmake.prefix`,
-and the specified module will be added to `CMAKE_MODULE_PATH` and
-`CMAKE_PREFIX_PATH`, respectively. Currently, the key is not used, but
-eventually there might be a way to request or exclude certain entry-points by
-key.
+Scikit-build-core includes various pythonic paths to the CMake search paths by
+default so that usually you only need to include the dependent project inside
+the `build-system.requires` section. Note that `cmake` and `ninja` should not be
+included in that section.
+
+See [search paths section](../configuration/search_paths.md) for more details on
+how the search paths are constructed.
 
 ## Install directories
 

docs/index.md

@@ -44,6 +44,7 @@ configuration/index
 configuration/overrides
 configuration/dynamic
 configuration/formatted
+configuration/search_paths
 ```
 
 ```{toctree}

src/scikit_build_core/builder/builder.py

@@ -120,6 +120,21 @@ def get_cmake_args(self) -> list[str]:
     def get_generator(self, *args: str) -> str | None:
         return self.config.get_generator(*self.get_cmake_args(), *args)
 
+    def _get_entry_point_search_path(self, entry_point: str) -> dict[str, list[Path]]:
+        """Get the search path dict from the entry points"""
+        search_paths = {}
+        eps = metadata.entry_points(group=entry_point)
+        if eps:
+            logger.debug(
+                "Loading search paths {} from entry-points: {}", entry_point, len(eps)
+            )
+            for ep in eps:
+                ep_value = _sanitize_path(resources.files(ep.load()))
+                logger.debug("{}: {} -> {}", ep.name, ep.value, ep_value)
+                if ep_value:
+                    search_paths[ep.name] = ep_value
+        return search_paths
+
     def configure(
         self,
         *,
@@ -136,25 +151,35 @@ def configure(
         }
 
         # Add any extra CMake modules
-        eps = metadata.entry_points(group="cmake.module")
         self.config.module_dirs.extend(
-            p for ep in eps for p in _sanitize_path(resources.files(ep.load()))
+            p
+            for ep_paths in self._get_entry_point_search_path("cmake.module").values()
+            for p in ep_paths
         )
+        logger.debug("cmake.modules: {}", self.config.module_dirs)
 
         # Add any extra CMake prefixes
-        eps = metadata.entry_points(group="cmake.prefix")
         self.config.prefix_dirs.extend(
-            p for ep in eps for p in _sanitize_path(resources.files(ep.load()))
+            p
+            for ep_paths in self._get_entry_point_search_path("cmake.prefix").values()
+            for p in ep_paths
         )
+        logger.debug("cmake.prefix: {}", self.config.prefix_dirs)
+
+        # Add all CMake roots
+        # TODO: Check for unique uppercase names
+        self.config.prefix_roots.update(self._get_entry_point_search_path("cmake.root"))
+        logger.debug("cmake.root: {}", self.config.prefix_roots)
 
         # Add site-packages to the prefix path for CMake
         site_packages = Path(sysconfig.get_path("purelib"))
-        self.config.prefix_dirs.append(site_packages)
-        logger.debug("SITE_PACKAGES: {}", site_packages)
-        if site_packages != DIR.parent.parent:
-            self.config.prefix_dirs.append(DIR.parent.parent)
-            logger.debug("Extra SITE_PACKAGES: {}", DIR.parent.parent)
-            logger.debug("PATH: {}", sys.path)
+        if self.settings.search.site_packages:
+            self.config.prefix_dirs.append(site_packages)
+            logger.debug("SITE_PACKAGES: {}", site_packages)
+            if site_packages != DIR.parent.parent:
+                self.config.prefix_dirs.append(DIR.parent.parent)
+                logger.debug("Extra SITE_PACKAGES: {}", DIR.parent.parent)
+                logger.debug("PATH: {}", sys.path)
 
         # Add the FindPython backport if needed
         if self.config.cmake.version < self.settings.backport.find_python:

src/scikit_build_core/cmake.py

@@ -79,6 +79,7 @@ class CMaker:
     build_type: str
     module_dirs: list[Path] = dataclasses.field(default_factory=list)
     prefix_dirs: list[Path] = dataclasses.field(default_factory=list)
+    prefix_roots: dict[str, list[Path]] = dataclasses.field(default_factory=dict)
     init_cache_file: Path = dataclasses.field(init=False, default=Path())
     env: dict[str, str] = dataclasses.field(init=False, default_factory=os.environ.copy)
     single_config: bool = not sysconfig.get_platform().startswith("win")
@@ -183,6 +184,17 @@ def init_cache(
                 )
                 f.write('set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE "BOTH" CACHE PATH "")\n')
 
+            if self.prefix_roots:
+                for pkg, path_list in self.prefix_roots.items():
+                    paths_str = ";".join(map(str, path_list)).replace("\\", "/")
+                    f.write(
+                        f'set({pkg}_ROOT [===[{paths_str}]===] CACHE PATH "" FORCE)\n'
+                    )
+                    # Available since CMake 3.27 with CMP0144
+                    f.write(
+                        f'set({pkg.upper()}_ROOT [===[{paths_str}]===] CACHE PATH "" FORCE)\n'
+                    )
+
         contents = self.init_cache_file.read_text(encoding="utf-8").strip()
         logger.debug(
             "{}:\n{}",

src/scikit_build_core/resources/scikit-build.schema.json

@@ -407,6 +407,17 @@
         }
       }
     },
+    "search": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "site-packages": {
+          "type": "boolean",
+          "default": true,
+          "description": "Add the python build environment site_packages folder to the CMake prefix paths."
+        }
+      }
+    },
     "metadata": {
       "type": "object",
       "description": "List dynamic metadata fields and hook locations in this table.",
@@ -618,6 +629,9 @@
           "messages": {
             "$ref": "#/properties/messages"
           },
+          "search": {
+            "$ref": "#/properties/search"
+          },
           "metadata": {
             "$ref": "#/properties/metadata"
           },

src/scikit_build_core/settings/skbuild_model.py

@@ -20,6 +20,7 @@
     "NinjaSettings",
     "SDistSettings",
     "ScikitBuildSettings",
+    "SearchSettings",
     "WheelSettings",
 ]
 
@@ -103,6 +104,14 @@ class CMakeSettings:
     """
 
 
+@dataclasses.dataclass
+class SearchSettings:
+    site_packages: bool = True
+    """
+    Add the python build environment site_packages folder to the CMake prefix paths.
+    """
+
+
 @dataclasses.dataclass
 class NinjaSettings:
     minimum_version: Optional[Version] = None
@@ -355,6 +364,7 @@ class ScikitBuildSettings:
     install: InstallSettings = dataclasses.field(default_factory=InstallSettings)
     generate: List[GenerateSettings] = dataclasses.field(default_factory=list)
     messages: MessagesSettings = dataclasses.field(default_factory=MessagesSettings)
+    search: SearchSettings = dataclasses.field(default_factory=SearchSettings)
 
     metadata: Dict[str, Dict[str, Any]] = dataclasses.field(default_factory=dict)
     """

tests/packages/custom_cmake/CMakeLists.txt

@@ -6,7 +6,7 @@ project(
   VERSION 2.3.4)
 
 find_package(ExamplePkg REQUIRED)
-
+find_package(ExampleRoot REQUIRED)
 include(ExampleInclude)
 
 if(NOT EXAMPLE_INCLUDE_FOUND)

tests/packages/custom_cmake/extern/custom_cmake_testing_stuff/cmake_root/ExampleRootConfig.cmake

@@ -0,0 +1,3 @@
+set(ExampleRoot_FOUND
+    TRUE
+    CACHE BOOL "ExampleRoot found" FORCE)