feat: poly commands should respect any existing project-specific exclude pattern (#387)

* refactor(hatch): move exclude pattern collector to the toml brick

* feat(toml): collect exclude patterns from Hatch, PDM and Poetry

* wip: use exclude list

* wip: collext imports to exclude

* wip: create report with bricks and libs to exclude

* feat(uv): get uv build backend configured exclude patterns

* test: check - report

* test: check - report

* test: toml, collect exclude patterns

* bump hatch brick hook to 1.5.3

* bump PDM build hook to 1.3.4

* bump PDM workspace hook to 1.3.4

* bump Poetry plugin to 1.44.0

* bump CLI to 1.37.0

* feat: libs, excluded third-party imports

Author: DavidVujic

components/polylith/check/report.py

@@ -42,7 +42,22 @@ def print_missing_deps(diff: Set[str], project_name: str) -> None:
 
     missing = ", ".join(sorted(diff))
 
-    console.print(f":thinking_face: Cannot locate [data]{missing}[/] in [proj]{project_name}[/]")
+    console.print(
+        f":thinking_face: Cannot locate [data]{missing}[/] in [proj]{project_name}[/]"
+    )
+
+
+def print_excluded_deps(excluded: Set[str], project_name: str) -> None:
+    if not excluded:
+        return
+
+    console = Console(theme=theme.poly_theme)
+
+    skipped = ", ".join(sorted(excluded))
+
+    console.print(
+        f":information: Marked as excluded in [proj]{project_name}[/]: [data]{skipped}[/]"
+    )
 
 
 def print_unused_bricks(bricks: Set[str], project_name: str) -> None:
@@ -54,7 +69,25 @@ def print_unused_bricks(bricks: Set[str], project_name: str) -> None:
     unused = ", ".join(sorted(bricks))
     verb = "Are" if len(bricks) > 1 else "Is"
 
-    console.print(f":mag_right: {verb} [comp]{unused}[/] needed in [proj]{project_name}[/]?")
+    console.print(
+        f":mag_right: {verb} [comp]{unused}[/] needed in [proj]{project_name}[/]?"
+    )
+
+
+def extract_collected_imports(
+    ns: str, imports_in_bases: dict, imports_in_components: dict
+) -> dict:
+    brick_imports = {
+        "bases": grouping.extract_brick_imports(imports_in_bases, ns),
+        "components": grouping.extract_brick_imports(imports_in_components, ns),
+    }
+
+    third_party_imports = {
+        "bases": libs.extract_third_party_imports(imports_in_bases, ns),
+        "components": libs.extract_third_party_imports(imports_in_components, ns),
+    }
+
+    return {"brick_imports": brick_imports, "third_party_imports": third_party_imports}
 
 
 def collect_all_imports(root: Path, ns: str, project_data: dict) -> dict:
@@ -67,17 +100,39 @@ def collect_all_imports(root: Path, ns: str, project_data: dict) -> dict:
     all_imports_in_bases = imports.fetch_all_imports(bases_paths)
     all_imports_in_components = imports.fetch_all_imports(components_paths)
 
-    brick_imports = {
-        "bases": grouping.extract_brick_imports(all_imports_in_bases, ns),
-        "components": grouping.extract_brick_imports(all_imports_in_components, ns),
-    }
+    return extract_collected_imports(
+        ns, all_imports_in_bases, all_imports_in_components
+    )
 
-    third_party_imports = {
-        "bases": libs.extract_third_party_imports(all_imports_in_bases, ns),
-        "components": libs.extract_third_party_imports(all_imports_in_components, ns),
-    }
 
-    return {"brick_imports": brick_imports, "third_party_imports": third_party_imports}
+def collect_imports_to_exclude(root: Path, ns: str, project_data: dict) -> dict:
+    exclude = project_data["exclude"]
+
+    if not exclude:
+        return {}
+
+    bases = set(project_data.get("bases", []))
+    components = set(project_data.get("components", []))
+
+    bases_paths = workspace.paths.collect_bases_paths(root, ns, bases)
+    components_paths = workspace.paths.collect_components_paths(root, ns, components)
+
+    excludes_in_bases = imports.fetch_excluded_imports(bases_paths, exclude)
+    excludes_in_components = imports.fetch_excluded_imports(components_paths, exclude)
+
+    return extract_collected_imports(ns, excludes_in_bases, excludes_in_components)
+
+
+def create_exclude_report(collected_excludes: dict) -> dict:
+    fallback: dict = {"bases": {}, "components": {}}
+
+    collected_bricks = collected_excludes.get("brick_imports", fallback)
+    collected_third_party = collected_excludes.get("third_party_imports", fallback)
+
+    bricks = collect.to_flattened_imports(collected_bricks)
+    third_party = collect.to_flattened_imports(collected_third_party)
+
+    return {"brick_exclude": bricks, "libs_exclude": third_party}
 
 
 def create_report(

components/polylith/commands/check.py

@@ -78,6 +78,7 @@ def run_each(
 ) -> Tuple[bool, dict]:
     is_quiet = options["quiet"]
     is_strict = options["strict"]
+    is_verbose = options["verbose"]
 
     name = project_data["name"]
     deps = project_data["deps"]
@@ -86,6 +87,7 @@ def run_each(
     from_lock_file = libs.is_from_lock_file(deps)
 
     collected_imports = check.report.collect_all_imports(root, ns, project_data)
+    collected_excludes = check.report.collect_imports_to_exclude(root, ns, project_data)
     collected_libs = distributions.known_aliases_and_sub_dependencies(
         deps,
         alias,
@@ -100,15 +102,29 @@ def run_each(
         is_strict or from_lock_file,
     )
 
-    res = all([not details["brick_diff"], not details["libs_diff"]])
+    exclude_details = check.report.create_exclude_report(collected_excludes)
+
+    brick_diff = details["brick_diff"]
+    libs_diff = details["libs_diff"]
+    brick_exclude = exclude_details["brick_exclude"]
+    libs_exclude = exclude_details["libs_exclude"]
+
+    missing_bricks = brick_diff.difference(brick_exclude)
+    missing_libs = libs_diff.difference(libs_exclude)
+
+    res = all([not missing_bricks, not missing_libs])
 
     if not is_quiet:
-        check.report.print_missing_deps(details["brick_diff"], name)
-        check.report.print_missing_deps(details["libs_diff"], name)
+        check.report.print_missing_deps(missing_bricks, name)
+        check.report.print_missing_deps(missing_libs, name)
 
     if is_strict and not is_quiet:
         check.report.print_unused_bricks(details["unused_bricks"], name)
 
+    if is_verbose:
+        check.report.print_excluded_deps(brick_exclude, name)
+        check.report.print_excluded_deps(libs_exclude, name)
+
     return res, details
 
 

components/polylith/hatch/hooks/bricks.py

@@ -34,15 +34,6 @@ def filtered_bricks(data: dict, version: str) -> dict:
     return bricks
 
 
-def collect_configured_exclude_patterns(data: dict, target_name: str) -> set:
-    entry = data.get("tool", {}).get("hatch", {}).get("build", {})
-    target = entry.get("targets", {}).get(target_name, {})
-
-    exclude = target.get("exclude", [])
-
-    return set(exclude)
-
-
 def copy_bricks(bricks: dict, work_dir: Path, exclude_patterns: Set[str]) -> List[Path]:
     return [
         parsing.copy_brick(source, brick, work_dir, exclude_patterns)
@@ -76,7 +67,7 @@ def initialize(self, version: str, build_data: Dict[str, Any]) -> None:
         ns = parsing.parse_brick_namespace_from_path(bricks)
         top_ns = core.get_top_namespace(data, self.config)
         work_dir = core.get_work_dir(self.config)
-        exclude_patterns = collect_configured_exclude_patterns(data, self.target_name)
+        exclude_patterns = toml.collect_configured_exclude_patterns(data, self.target_name)
 
         if not top_ns and not exclude_patterns:
             build_data[include_key] = bricks

components/polylith/imports/__init__.py

@@ -1,3 +1,13 @@
-from polylith.imports.parser import extract_top_ns, fetch_all_imports, list_imports
+from polylith.imports.parser import (
+    extract_top_ns,
+    fetch_all_imports,
+    fetch_excluded_imports,
+    list_imports,
+)
 
-__all__ = ["extract_top_ns", "fetch_all_imports", "list_imports"]
+__all__ = [
+    "extract_top_ns",
+    "fetch_all_imports",
+    "fetch_excluded_imports",
+    "list_imports",
+]

components/polylith/imports/parser.py

@@ -1,4 +1,5 @@
 import ast
+from collections.abc import Iterable
 from functools import lru_cache
 from pathlib import Path
 from typing import List, Set
@@ -43,18 +44,26 @@ def extract_imports(path: Path) -> List[str]:
     return [i for node in ast.walk(tree) for i in parse_imports(node) if i is not None]
 
 
+def extract_and_flatten(py_modules: Iterable) -> Set[str]:
+    extracted = (extract_imports(m) for m in py_modules)
+    flattened = (i for imports in extracted for i in imports)
+
+    return set(flattened)
+
+
 def is_python_file(path: Path) -> bool:
     return path.is_file() and path.suffix == ".py"
 
 
+def find_files(path: Path) -> Iterable:
+    return [path] if is_python_file(path) else path.rglob("*.py")
+
+
 @lru_cache(maxsize=None)
 def list_imports(path: Path) -> Set[str]:
-    py_modules = [path] if is_python_file(path) else path.rglob("*.py")
-
-    extracted = (extract_imports(m) for m in py_modules)
-    flattened = (i for imports in extracted for i in imports)
+    py_modules = find_files(path)
 
-    return set(flattened)
+    return extract_and_flatten(py_modules)
 
 
 def fetch_all_imports(paths: Set[Path]) -> dict:
@@ -63,6 +72,24 @@ def fetch_all_imports(paths: Set[Path]) -> dict:
     return {k: v for row in rows for k, v in row.items()}
 
 
+def should_exclude(path: Path, excludes: Set[str]):
+    return any(path.match(pattern) for pattern in excludes)
+
+
+def list_excluded_imports(path: Path, excludes: Set[str]) -> Set[str]:
+    py_modules = find_files(path)
+
+    filtered = [p for p in py_modules if should_exclude(p, excludes)]
+
+    return extract_and_flatten(filtered)
+
+
+def fetch_excluded_imports(paths: Set[Path], excludes: Set[str]) -> dict:
+    rows = [{p.name: list_excluded_imports(p, excludes)} for p in paths]
+
+    return {k: v for row in rows for k, v in row.items()}
+
+
 def extract_top_ns_from_imports(imports: Set[str]) -> Set:
     return {imp.split(".")[0] for imp in imports}
 

components/polylith/libs/grouping.py

@@ -3,7 +3,7 @@
 from typing import Set
 
 from polylith import configuration
-from polylith.imports import extract_top_ns, fetch_all_imports
+from polylith.imports import extract_top_ns, fetch_all_imports, fetch_excluded_imports
 from polylith.libs.stdlib import standard_libs
 
 
@@ -41,9 +41,21 @@ def extract_third_party_imports(all_imports: dict, top_ns: str) -> dict:
     return exclude_empty(with_third_party)
 
 
-def get_third_party_imports(root: Path, paths: Set[Path]) -> dict:
+def get_third_party_imports(
+    root: Path, paths: Set[Path], project_data: dict
+) -> dict:
     top_ns = configuration.get_namespace_from_config(root)
 
     all_imports = fetch_all_imports(paths)
 
-    return extract_third_party_imports(all_imports, top_ns)
+    third_party = extract_third_party_imports(all_imports, top_ns)
+
+    exclude = project_data["exclude"]
+
+    if not exclude:
+        return third_party
+
+    excluded = fetch_excluded_imports(paths, exclude)
+    excluded_third_party = extract_third_party_imports(excluded, top_ns)
+
+    return {k: v for k, v in third_party.items() if k not in excluded_third_party}

components/polylith/libs/report.py

@@ -19,8 +19,10 @@ def get_third_party_imports(root: Path, ns: str, project_data: dict) -> dict:
     bases_paths = workspace.paths.collect_bases_paths(root, ns, bases)
     components_paths = workspace.paths.collect_components_paths(root, ns, components)
 
-    bases_imports = grouping.get_third_party_imports(root, bases_paths)
-    components_imports = grouping.get_third_party_imports(root, components_paths)
+    bases_imports = grouping.get_third_party_imports(root, bases_paths, project_data)
+    components_imports = grouping.get_third_party_imports(
+        root, components_paths, project_data
+    )
 
     return {"bases": bases_imports, "components": components_imports}
 

components/polylith/project/get.py

@@ -139,6 +139,7 @@ def get_packages_for_projects(root: Path) -> List[dict]:
             "path": d["path"],
             "type": d["type"],
             "deps": toml.get_project_dependencies(d["toml"]),
+            "exclude": toml.collect_configured_exclude_patterns(d["toml"]),
         }
         for d in toml_files
     ]

components/polylith/repo/__init__.py

@@ -9,6 +9,7 @@
     is_pdm,
     is_pep_621_ready,
     is_poetry,
+    is_uv,
     load_workspace_config,
     projects_dir,
     readme_file,
@@ -27,6 +28,7 @@
     "is_pdm",
     "is_pep_621_ready",
     "is_poetry",
+    "is_uv",
     "load_workspace_config",
     "projects_dir",
     "readme_file",

components/polylith/repo/repo.py

@@ -130,5 +130,9 @@ def is_pdm(pyproject: dict) -> bool:
     return has_build_requires(pyproject, "pdm")
 
 
+def is_uv(pyproject: dict) -> bool:
+    return has_build_requires(pyproject, "uv_build")
+
+
 def is_pep_621_ready(pyproject: dict) -> bool:
     return pyproject.get("project", {}).get("name") is not None