Merge branch 'main' into performance-optimization

Author: davidkopp

CLAUDE.md

@@ -15,7 +15,7 @@ pip install -r requirements.txt
 
 # Run specific detectors for testing
 python3 -m dependency_resolver docker <container_name>
-python3 -m dependency_resolver host --skip-system-scope --skip-hash-generation
+python3 -m dependency_resolver host --skip-os-packages --skip-hash-generation
 
 # Execute linting and formatting
 pre-commit run --files $(git diff --name-only --diff-filter=ACMR HEAD)

SPECIFICATION.md

@@ -60,35 +60,23 @@ For complete architecture documentation, see [docs/technical/architecture/overvi
 
 ### JSON Output Schema
 
-The tool outputs structured JSON with packages aggregated by scope (project/system):
+The tool outputs structured JSON with detected package managers and their dependencies:
 
 ```json
 {
-  "source": { "type": "container", "name": "nginx-container", "image": "nginx:latest", "hash": "sha256:..." },
-  "project": {
-    "packages": [
-      { "name": "numpy", "version": "1.3.3", "type": "pip" }
-    ],
-    "package-management": {
-      "pip": { "location": "/app/venv/lib/python3.12/site-packages", "hash": "sha256:..." }
-    }
-  },
-  "system": {
-    "packages": [
-      { "name": "curl", "version": "7.81.0-1ubuntu1.18 amd64", "type": "dpkg", "hash": "abc123..." }
-    ]
-  }
+  "source": { "name": "nginx-container", "image": "nginx:latest", "hash": "sha256:..." },
+  "dpkg": { "scope": "system", "dependencies": { "curl": { "version": "7.81.0-1ubuntu1.18 amd64" } } },
+  "pip": { "scope": "project", "location": "/app/venv/lib/python3.12/site-packages", "dependencies": { "numpy": { "version": "1.3.3" } } }
 }
 ```
 
 For complete JSON schema documentation, field definitions, and examples, see [docs/usage/output-format.md](docs/usage/output-format.md).
 
 ### Key Schema Concepts
 
-- **Project Packages**: All project-scoped packages aggregated in `project.packages[]` array
-- **System Packages**: All system-scoped packages aggregated in `system.packages[]` array
-- **Package Type**: Each package includes `type` field (`pip`, `npm`, `dpkg`, `apk`, `maven`)
-- **Metadata**: Package manager metadata stored in `project.{manager}` sections (location, hash)
+- **Scope**: `"system"` (system-wide packages) or `"project"` (project-specific packages)
+- **Location**: Path to project-local installations (project scope only)
+- **Hash**: Dependency integrity verification when available
 - **Source Info**: Environment metadata included as `source` when applicable (includes `type` field for container/host)
 
 ## Implementation Constraints

dependency_resolver/__init__.py

@@ -11,7 +11,7 @@
 def resolve_host_dependencies(
     working_dir: Optional[str] = None,
     debug: bool = False,
-    skip_system_scope: bool = False,
+    skip_os_packages: bool = False,
     venv_path: Optional[str] = None,
     pretty_print: bool = False,
 ) -> str:
@@ -21,15 +21,15 @@ def resolve_host_dependencies(
     Args:
         working_dir: Working directory to analyze (defaults to current directory)
         debug: Enable debug output
-        skip_system_scope: Skip system-scope package managers
+        skip_os_packages: Skip OS package managers (dpkg, apk)
         venv_path: Explicit virtual environment path for pip detector
         pretty_print: Format JSON output with indentation
 
     Returns:
         JSON string containing all discovered dependencies
     """
     executor = HostExecutor(debug=debug)
-    orchestrator = Orchestrator(debug=debug, skip_system_scope=skip_system_scope, venv_path=venv_path)
+    orchestrator = Orchestrator(debug=debug, skip_os_packages=skip_os_packages, venv_path=venv_path)
     dependencies = orchestrator.resolve_dependencies(executor, working_dir)
     formatter = OutputFormatter(debug=debug)
     return formatter.format_json(dependencies, pretty_print=pretty_print)
@@ -39,7 +39,7 @@ def resolve_docker_dependencies(
     container_identifier: str,
     working_dir: Optional[str] = None,
     debug: bool = False,
-    skip_system_scope: bool = False,
+    skip_os_packages: bool = False,
     venv_path: Optional[str] = None,
     only_container_info: bool = False,
     pretty_print: bool = False,
@@ -51,7 +51,7 @@ def resolve_docker_dependencies(
         container_identifier: Container ID or name
         working_dir: Working directory to analyze within the container
         debug: Enable debug output
-        skip_system_scope: Skip system-scope package managers
+        skip_os_packages: Skip OS package managers (dpkg, apk)
         venv_path: Explicit virtual environment path for pip detector
         only_container_info: Only analyze container metadata (skip dependency detection)
         pretty_print: Format JSON output with indentation
@@ -60,7 +60,7 @@ def resolve_docker_dependencies(
         JSON string containing all discovered dependencies
     """
     executor = DockerExecutor(container_identifier, debug=debug)
-    orchestrator = Orchestrator(debug=debug, skip_system_scope=skip_system_scope, venv_path=venv_path)
+    orchestrator = Orchestrator(debug=debug, skip_os_packages=skip_os_packages, venv_path=venv_path)
     dependencies = orchestrator.resolve_dependencies(executor, working_dir, only_container_info)
     formatter = OutputFormatter(debug=debug)
     return formatter.format_json(dependencies, pretty_print=pretty_print)
@@ -70,7 +70,7 @@ def resolve_docker_dependencies_as_dict(
     container_identifier: str,
     working_dir: Optional[str] = None,
     debug: bool = False,
-    skip_system_scope: bool = False,
+    skip_os_packages: bool = False,
     venv_path: Optional[str] = None,
     only_container_info: bool = False,
 ) -> dict[str, Any]:
@@ -84,7 +84,7 @@ def resolve_docker_dependencies_as_dict(
         container_identifier: Container ID or name
         working_dir: Working directory to analyze within the container
         debug: Enable debug output
-        skip_system_scope: Skip system-scope package managers
+        skip_os_packages: Skip OS package managers (dpkg, apk)
         venv_path: Explicit virtual environment path for pip detector
         only_container_info: Only analyze container metadata (skip dependency detection)
 
@@ -99,7 +99,7 @@ def resolve_docker_dependencies_as_dict(
         raise ValueError("Container identifier is required")
 
     executor = DockerExecutor(container_identifier, debug=debug)
-    orchestrator = Orchestrator(debug=debug, skip_system_scope=skip_system_scope, venv_path=venv_path)
+    orchestrator = Orchestrator(debug=debug, skip_os_packages=skip_os_packages, venv_path=venv_path)
     return orchestrator.resolve_dependencies(executor, working_dir, only_container_info)
 
 
@@ -108,7 +108,7 @@ def resolve_dependencies_as_dict(
     environment_identifier: Optional[str] = None,
     working_dir: Optional[str] = None,
     debug: bool = False,
-    skip_system_scope: bool = False,
+    skip_os_packages: bool = False,
     venv_path: Optional[str] = None,
     only_container_info: bool = False,
 ) -> dict[str, Any]:
@@ -120,7 +120,7 @@ def resolve_dependencies_as_dict(
         environment_identifier: Environment identifier (required for docker)
         working_dir: Working directory to analyze
         debug: Enable debug output
-        skip_system_scope: Skip system-scope package managers
+        skip_os_packages: Skip OS package managers (dpkg, apk)
         venv_path: Explicit virtual environment path for pip detector
         only_container_info: Only analyze container metadata (for docker environments)
 
@@ -137,7 +137,7 @@ def resolve_dependencies_as_dict(
     else:
         raise ValueError(f"Unsupported environment type: {environment_type}")
 
-    orchestrator = Orchestrator(debug=debug, skip_system_scope=skip_system_scope, venv_path=venv_path)
+    orchestrator = Orchestrator(debug=debug, skip_os_packages=skip_os_packages, venv_path=venv_path)
     return orchestrator.resolve_dependencies(executor, working_dir, only_container_info)
 
 

dependency_resolver/__main__.py

@@ -35,7 +35,7 @@ def parse_arguments() -> argparse.Namespace:
   %(prog)s --working-dir /tmp/repo                # Set working directory on target environment
   %(prog)s --venv-path ~/.virtualenvs/myproject   # Use specific virtual environment for pip
   %(prog)s --debug                                # Enable debug output
-  %(prog)s --skip-system-scope                    # Skip system scope package managers
+  %(prog)s --skip-os-packages                     # Skip OS package managers (dpkg, apk)
   %(prog)s --skip-hash-collection                 # Skip hash collection for improved performance
   %(prog)s --select-detectors "pip,dpkg"          # Use only pip and dpkg detectors
         """,
@@ -63,9 +63,9 @@ def parse_arguments() -> argparse.Namespace:
     parser.add_argument("--debug", action="store_true", help="Print debug statements")
 
     parser.add_argument(
-        "--skip-system-scope",
+        "--skip-os-packages",
         action="store_true",
-        help="Skip system scope package managers (system packages or globally installed Python packages)",
+        help="Skip OS package managers (dpkg, apk) - language package managers like pip/npm will still run",
     )
 
     parser.add_argument(
@@ -137,7 +137,7 @@ def main() -> None:
         executor = create_executor(args.environment_type, args.environment_identifier, debug=args.debug)
         orchestrator = Orchestrator(
             debug=args.debug,
-            skip_system_scope=args.skip_system_scope,
+            skip_os_packages=args.skip_os_packages,
             venv_path=args.venv_path,
             skip_hash_collection=args.skip_hash_collection,
             selected_detectors=args.select_detectors,

dependency_resolver/core/interfaces.py

@@ -33,7 +33,7 @@ def is_usable(self, executor: EnvironmentExecutor, working_dir: Optional[str] =
     @abstractmethod
     def get_dependencies(
         self, executor: EnvironmentExecutor, working_dir: Optional[str] = None, skip_hash_collection: bool = False
-    ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    ) -> dict[str, Any]:
         """Extract dependencies with versions and hashes.
 
         Args:
@@ -49,10 +49,10 @@ def get_dependencies(
         raise NotImplementedError
 
     @abstractmethod
-    def has_system_scope(self, executor: EnvironmentExecutor, working_dir: Optional[str] = None) -> bool:
-        """Check if this detector operates at system scope for the given environment.
+    def is_os_package_manager(self) -> bool:
+        """Check if this detector manages OS-level packages (like dpkg, apk).
 
-        Returns True if the detector would return scope: "system" in get_dependencies().
-        This allows efficient scope checking without the overhead of dependency extraction.
+        Returns True for OS package managers that install system packages.
+        Returns False for language package managers (pip, npm, maven) and other detectors.
         """
         raise NotImplementedError

dependency_resolver/core/orchestrator.py

@@ -14,13 +14,13 @@ class Orchestrator:
     def __init__(
         self,
         debug: bool = False,
-        skip_system_scope: bool = False,
+        skip_os_packages: bool = False,
         venv_path: str | None = None,
         skip_hash_collection: bool = False,
         selected_detectors: str | None = None,
     ):
         self.debug = debug
-        self.skip_system_scope = skip_system_scope
+        self.skip_os_packages = skip_os_packages
         self.skip_hash_collection = skip_hash_collection
 
         # Create all detector instances
@@ -62,10 +62,6 @@ def resolve_dependencies(
             raise ValueError(f"Working directory does not exist: {working_dir}")
 
         result: dict[str, Any] = {}
-        project_packages: list[dict[str, Any]] = []
-        project_metadata: dict[str, dict[str, Any]] = {}
-        system_packages: list[dict[str, Any]] = []
-        system_metadata: dict[str, dict[str, Any]] = {}
 
         if only_container_info:
             # Only run docker-info detector when only container info is requested
@@ -81,45 +77,33 @@ def resolve_dependencies(
 
             try:
                 if detector.is_usable(executor, working_dir):
-                    # Check if detector has system scope and skip if requested
-                    if self.skip_system_scope and detector.has_system_scope(executor, working_dir):
+                    # Check if detector is OS package manager and skip if requested
+                    if self.skip_os_packages and detector.is_os_package_manager():
                         if self.debug:
-                            print(f"Skipping {detector_name} (system scope, --skip-system-scope enabled)")
+                            print(f"Skipping {detector_name} (OS package manager, --skip-os-packages enabled)")
                         continue
 
                     if self.debug:
                         print(f"{detector_name} is usable, extracting dependencies...")
 
-                    # Special handling for docker-info detector
+                    dependencies = detector.get_dependencies(
+                        executor, working_dir, skip_hash_collection=self.skip_hash_collection
+                    )
+
+                    # Special handling for docker-info detector (simplified format)
                     if detector_name == "docker-info":
-                        packages, metadata = detector.get_dependencies(
-                            executor, working_dir, skip_hash_collection=self.skip_hash_collection
-                        )
-                        if metadata:  # Only include if we got container info
-                            metadata["type"] = "container"
-                            result["source"] = metadata
+                        result["source"] = dependencies
+                        result["source"]["type"] = "container"
                         if self.debug:
                             print(f"Found container info for {detector_name}")
                     else:
-                        # Standard handling for package detectors
-                        packages, metadata = detector.get_dependencies(
-                            executor, working_dir, skip_hash_collection=self.skip_hash_collection
-                        )
-
-                        if detector.has_system_scope(executor, working_dir):
-                            # System scope packages
-                            system_packages.extend(packages)
-                            if metadata:
-                                system_metadata[detector_name] = metadata
-                            if self.debug:
-                                print(f"Found {len(packages)} system packages for {detector_name}")
-                        else:
-                            # Project scope packages
-                            project_packages.extend(packages)
-                            if metadata:
-                                project_metadata[detector_name] = metadata
-                            if self.debug:
-                                print(f"Found {len(packages)} project packages for {detector_name}")
+                        # Standard handling for other detectors
+                        if dependencies.get("dependencies") or self.debug:
+                            result[detector_name] = dependencies
+
+                        if self.debug:
+                            dep_count = len(dependencies.get("dependencies", {}))
+                            print(f"Found {dep_count} dependencies for {detector_name}")
                 else:
                     if self.debug:
                         print(f"{detector_name} is not available")
@@ -129,13 +113,4 @@ def resolve_dependencies(
                     print(f"Error checking {detector_name}: {str(e)}")
                 continue
 
-        # Build final result structure matching proposal
-        if project_packages or project_metadata:
-            project_section: dict[str, Any] = {"packages": project_packages, "package-management": project_metadata}
-            result["project"] = project_section
-
-        if system_packages or system_metadata:
-            system_section: dict[str, Any] = {"packages": system_packages, "package-management": system_metadata}
-            result["system"] = system_section
-
         return result

dependency_resolver/core/output_formatter.py

@@ -22,39 +22,30 @@ def format_json(self, dependencies: dict[str, Any], pretty_print: bool = True) -
             else:
                 return json.dumps(dependencies)
 
-    def create_excerpt(self, dependencies: dict[str, Any], max_deps_per_section: int = 3) -> dict[str, Any]:
+    def create_excerpt(self, dependencies: dict[str, Any], max_deps_per_manager: int = 3) -> dict[str, Any]:
         """Create an excerpt of dependencies for debug mode."""
         excerpt: dict[str, Any] = {}
 
-        for section_name, section_data in dependencies.items():
-            if section_name.startswith("_"):
-                # Copy container info and other metadata as-is
-                excerpt[section_name] = section_data
-            elif section_name in ("project", "system"):
-                # Handle new unified structure with packages array
-                excerpt[section_name] = {}
-
-                for key, value in section_data.items():
-                    if key != "packages":
-                        # Copy metadata (package-management and other metadata)
-                        excerpt[section_name][key] = value
-
-                if "packages" in section_data:
-                    packages = section_data["packages"]
-                    total_packages = len(packages)
-
-                    if total_packages <= max_deps_per_section:
-                        excerpt[section_name]["packages"] = packages
-                    else:
-                        limited_packages = packages[:max_deps_per_section]
-                        excerpt[section_name]["packages"] = limited_packages
-                        excerpt[section_name]["_excerpt_info"] = {
-                            "total_packages": total_packages,
-                            "shown": max_deps_per_section,
-                            "note": f"Showing {max_deps_per_section} of {total_packages} packages (debug mode excerpt)",
-                        }
-            else:
-                # Fallback for any legacy structure
-                excerpt[section_name] = section_data
+        for manager_name, manager_data in dependencies.items():
+            excerpt[manager_name] = {}
+
+            for key, value in manager_data.items():
+                if key != "dependencies":
+                    excerpt[manager_name][key] = value
+
+            if "dependencies" in manager_data:
+                deps = manager_data["dependencies"]
+                total_deps = len(deps)
+
+                if total_deps <= max_deps_per_manager:
+                    excerpt[manager_name]["dependencies"] = deps
+                else:
+                    limited_deps = dict(list(deps.items())[:max_deps_per_manager])
+                    excerpt[manager_name]["dependencies"] = limited_deps
+                    excerpt[manager_name]["_excerpt_info"] = {
+                        "total_dependencies": total_deps,
+                        "shown": max_deps_per_manager,
+                        "note": f"Showing {max_deps_per_manager} of {total_deps} dependencies (debug mode excerpt)",
+                    }
 
         return excerpt