Change to new output format separating packages by project/system scope instead of package managers

Author: davidkopp

SPECIFICATION.md

@@ -60,24 +60,34 @@ For complete architecture documentation, see [docs/technical/architecture/overvi
 
 ### JSON Output Schema
 
-The tool outputs structured JSON with detected package managers and their dependencies:
+The tool outputs structured JSON with packages aggregated by scope (project/system):
 
 ```json
 {
   "_container-info": { "name": "nginx-container", "image": "nginx:latest", "hash": "sha256:..." },
-  "dpkg": { "scope": "system", "dependencies": { "curl": { "version": "7.81.0-1ubuntu1.18 amd64" } } },
-  "pip": { "scope": "project", "location": "/app/venv/lib/python3.12/site-packages", "dependencies": { "numpy": { "version": "1.3.3" } } }
+  "project": {
+    "packages": [
+      { "name": "numpy", "version": "1.3.3", "type": "pip" }
+    ],
+    "pip": { "location": "/app/venv/lib/python3.12/site-packages", "hash": "sha256:..." }
+  },
+  "system": {
+    "packages": [
+      { "name": "curl", "version": "7.81.0-1ubuntu1.18 amd64", "type": "dpkg", "hash": "abc123..." }
+    ]
+  }
 }
 ```
 
 For complete JSON schema documentation, field definitions, and examples, see [docs/usage/output-format.md](docs/usage/output-format.md).
 
 ### Key Schema Concepts
 
-- **Scope**: `"system"` (system-wide packages) or `"project"` (project-specific packages)
-- **Location**: Path to project-local installations (project scope only)
-- **Hash**: Dependency integrity verification when available
-- **Container Info**: Docker metadata included as `_container-info`
+- **Project Packages**: All project-scoped packages aggregated in `project.packages[]` array
+- **System Packages**: All system-scoped packages aggregated in `system.packages[]` array
+- **Package Type**: Each package includes `type` field (`pip`, `npm`, `dpkg`, `apk`, `maven`)
+- **Metadata**: Package manager metadata stored in `project.{manager}` sections (location, hash)
+- **Container Info**: Docker metadata included as `_container-info` when applicable
 
 ## Implementation Constraints
 

dependency_resolver/core/interfaces.py

@@ -31,8 +31,16 @@ def is_usable(self, executor: EnvironmentExecutor, working_dir: Optional[str] =
         raise NotImplementedError
 
     @abstractmethod
-    def get_dependencies(self, executor: EnvironmentExecutor, working_dir: Optional[str] = None) -> dict[str, Any]:
-        """Extract dependencies with versions and hashes."""
+    def get_dependencies(
+        self, executor: EnvironmentExecutor, working_dir: Optional[str] = None
+    ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+        """Extract dependencies with versions and hashes.
+
+        Returns:
+            tuple: (packages, metadata)
+            - packages: List of package dicts with name, version, type, and optional hash
+            - metadata: Dict with location and hash for project scope, empty for system scope
+        """
         raise NotImplementedError
 
     @abstractmethod

dependency_resolver/core/orchestrator.py

@@ -38,7 +38,11 @@ def resolve_dependencies(
         if working_dir is not None and not executor.path_exists(working_dir):
             raise ValueError(f"Working directory does not exist: {working_dir}")
 
-        result = {}
+        result: dict[str, Any] = {}
+        project_packages: list[dict[str, Any]] = []
+        project_metadata: dict[str, dict[str, Any]] = {}
+        system_packages: list[dict[str, Any]] = []
+        system_metadata: dict[str, dict[str, Any]] = {}
 
         if only_container_info:
             # Only run docker-info detector when only container info is requested
@@ -63,22 +67,31 @@ def resolve_dependencies(
                     if self.debug:
                         print(f"{detector_name} is usable, extracting dependencies...")
 
-                    dependencies = detector.get_dependencies(executor, working_dir)
-
-                    # Special handling for docker-info detector (simplified format)
+                    # Special handling for docker-info detector
                     if detector_name == "docker-info":
-                        if dependencies:  # Only include if we got container info
-                            result["_container-info"] = dependencies
+                        packages, metadata = detector.get_dependencies(executor, working_dir)
+                        if metadata:  # Only include if we got container info
+                            result["_container-info"] = metadata
                         if self.debug:
                             print(f"Found container info for {detector_name}")
                     else:
-                        # Standard handling for other detectors
-                        if dependencies.get("dependencies") or self.debug:
-                            result[detector_name] = dependencies
-
-                        if self.debug:
-                            dep_count = len(dependencies.get("dependencies", {}))
-                            print(f"Found {dep_count} dependencies for {detector_name}")
+                        # Standard handling for package detectors
+                        packages, metadata = detector.get_dependencies(executor, working_dir)
+
+                        if detector.has_system_scope(executor, working_dir):
+                            # System scope packages
+                            system_packages.extend(packages)
+                            if metadata:
+                                system_metadata[detector_name] = metadata
+                            if self.debug:
+                                print(f"Found {len(packages)} system packages for {detector_name}")
+                        else:
+                            # Project scope packages
+                            project_packages.extend(packages)
+                            if metadata:
+                                project_metadata[detector_name] = metadata
+                            if self.debug:
+                                print(f"Found {len(packages)} project packages for {detector_name}")
                 else:
                     if self.debug:
                         print(f"{detector_name} is not available")
@@ -88,4 +101,15 @@ def resolve_dependencies(
                     print(f"Error checking {detector_name}: {str(e)}")
                 continue
 
+        # Build final result structure matching proposal
+        if project_packages or project_metadata:
+            project_section: dict[str, Any] = {"packages": project_packages}
+            project_section.update(project_metadata)
+            result["project"] = project_section
+
+        if system_packages or system_metadata:
+            system_section: dict[str, Any] = {"packages": system_packages}
+            system_section.update(system_metadata)
+            result["system"] = system_section
+
         return result

dependency_resolver/core/output_formatter.py

@@ -22,30 +22,39 @@ def format_json(self, dependencies: dict[str, Any], pretty_print: bool = True) -
             else:
                 return json.dumps(dependencies)
 
-    def create_excerpt(self, dependencies: dict[str, Any], max_deps_per_manager: int = 3) -> dict[str, Any]:
+    def create_excerpt(self, dependencies: dict[str, Any], max_deps_per_section: int = 3) -> dict[str, Any]:
         """Create an excerpt of dependencies for debug mode."""
         excerpt: dict[str, Any] = {}
 
-        for manager_name, manager_data in dependencies.items():
-            excerpt[manager_name] = {}
-
-            for key, value in manager_data.items():
-                if key != "dependencies":
-                    excerpt[manager_name][key] = value
-
-            if "dependencies" in manager_data:
-                deps = manager_data["dependencies"]
-                total_deps = len(deps)
-
-                if total_deps <= max_deps_per_manager:
-                    excerpt[manager_name]["dependencies"] = deps
-                else:
-                    limited_deps = dict(list(deps.items())[:max_deps_per_manager])
-                    excerpt[manager_name]["dependencies"] = limited_deps
-                    excerpt[manager_name]["_excerpt_info"] = {
-                        "total_dependencies": total_deps,
-                        "shown": max_deps_per_manager,
-                        "note": f"Showing {max_deps_per_manager} of {total_deps} dependencies (debug mode excerpt)",
-                    }
+        for section_name, section_data in dependencies.items():
+            if section_name.startswith("_"):
+                # Copy container info and other metadata as-is
+                excerpt[section_name] = section_data
+            elif section_name in ("project", "system"):
+                # Handle new unified structure with packages array
+                excerpt[section_name] = {}
+
+                for key, value in section_data.items():
+                    if key != "packages":
+                        # Copy metadata (location, hash for project managers)
+                        excerpt[section_name][key] = value
+
+                if "packages" in section_data:
+                    packages = section_data["packages"]
+                    total_packages = len(packages)
+
+                    if total_packages <= max_deps_per_section:
+                        excerpt[section_name]["packages"] = packages
+                    else:
+                        limited_packages = packages[:max_deps_per_section]
+                        excerpt[section_name]["packages"] = limited_packages
+                        excerpt[section_name]["_excerpt_info"] = {
+                            "total_packages": total_packages,
+                            "shown": max_deps_per_section,
+                            "note": f"Showing {max_deps_per_section} of {total_packages} packages (debug mode excerpt)",
+                        }
+            else:
+                # Fallback for any legacy structure
+                excerpt[section_name] = section_data
 
         return excerpt

dependency_resolver/detectors/apk_detector.py

@@ -1,6 +1,9 @@
 from typing import Optional, Any
 from ..core.interfaces import EnvironmentExecutor, PackageManagerDetector
 
+# Package type constant
+PACKAGE_TYPE_APK = "apk"
+
 
 class ApkDetector(PackageManagerDetector):
     """Detector for system packages managed by apk (Alpine Linux)."""
@@ -21,19 +24,26 @@ def is_usable(self, executor: EnvironmentExecutor, working_dir: Optional[str] =
         _, _, apk_exit_code = executor.execute_command("apk --version")
         return apk_exit_code == 0
 
-    def get_dependencies(self, executor: EnvironmentExecutor, working_dir: Optional[str] = None) -> dict[str, Any]:
+    def get_dependencies(
+        self, executor: EnvironmentExecutor, working_dir: Optional[str] = None
+    ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
         """Extract system packages with versions and architecture using apk list.
 
         Uses 'apk list --installed' for comprehensive package information including architecture.
         See docs/technical/detectors/apk_detector.md
+
+        Returns:
+            tuple: (packages, metadata)
+            - packages: List of package dicts with name, version, type
+            - metadata: Empty dict (system scope has no metadata)
         """
         command = "apk list --installed"
         stdout, _, exit_code = executor.execute_command(command, working_dir)
 
         if exit_code != 0:
-            return {"scope": "system", "dependencies": {}}
+            return [], {}
 
-        dependencies = {}
+        packages = []
         for line in stdout.strip().split("\n"):
             line = line.strip()
 
@@ -56,13 +66,9 @@ def get_dependencies(self, executor: EnvironmentExecutor, working_dir: Optional[
 
                         full_version = f"{version} {architecture}" if architecture else version
 
-                        package_data = {
-                            "version": full_version,
-                        }
-
-                        dependencies[package_name] = package_data
+                        packages.append({"name": package_name, "version": full_version, "type": PACKAGE_TYPE_APK})
 
-        return {"scope": "system", "dependencies": dependencies}
+        return packages, {}
 
     def has_system_scope(self, executor: EnvironmentExecutor, working_dir: Optional[str] = None) -> bool:
         """APK always has system scope (system packages)."""

dependency_resolver/detectors/docker_info_detector.py

@@ -12,14 +12,21 @@ def is_usable(self, executor: EnvironmentExecutor, working_dir: Optional[str] =
         """Check if this is a Docker environment."""
         return isinstance(executor, DockerExecutor)
 
-    def get_dependencies(self, executor: EnvironmentExecutor, working_dir: Optional[str] = None) -> dict[str, Any]:
-        """Extract Docker container metadata."""
+    def get_dependencies(
+        self, executor: EnvironmentExecutor, working_dir: Optional[str] = None
+    ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+        """Extract Docker container metadata.
+
+        Note: This detector returns metadata rather than packages, but conforms to the interface.
+        The orchestrator handles this specially.
+        """
         if not isinstance(executor, DockerExecutor):
-            return {}
+            return [], {}
 
         container_info = executor.get_container_info()
 
-        # Return simplified container info structure
+        # Return simplified container info structure as metadata
+        # The orchestrator will handle this specially for _container-info section
         result = {
             "name": container_info["name"],
             "image": container_info["image"],
@@ -30,7 +37,8 @@ def get_dependencies(self, executor: EnvironmentExecutor, working_dir: Optional[
         if "error" in container_info:
             result["error"] = container_info["error"]
 
-        return result
+        # Return empty packages list and the container info as metadata
+        return [], result
 
     def has_system_scope(self, executor: EnvironmentExecutor, working_dir: Optional[str] = None) -> bool:
         """Docker container info has container scope, not system scope."""

dependency_resolver/detectors/dpkg_detector.py

@@ -3,6 +3,9 @@
 
 from ..core.interfaces import EnvironmentExecutor, PackageManagerDetector
 
+# Package type constant
+PACKAGE_TYPE_DPKG = "dpkg"
+
 
 class DpkgDetector(PackageManagerDetector):
     """Detector for system packages managed by dpkg (Debian/Ubuntu)."""
@@ -27,22 +30,29 @@ def is_usable(self, executor: EnvironmentExecutor, working_dir: Optional[str] =
         _, _, dpkg_exit_code = executor.execute_command("dpkg-query --version")
         return dpkg_exit_code == 0
 
-    def get_dependencies(self, executor: EnvironmentExecutor, working_dir: Optional[str] = None) -> dict[str, Any]:
+    def get_dependencies(
+        self, executor: EnvironmentExecutor, working_dir: Optional[str] = None
+    ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
         """Extract system packages with versions using dpkg-query.
 
         Uses dpkg-query -W -f for reliable package information extraction.
         See docs/technical/detectors/dpkg_detector.md
+
+        Returns:
+            tuple: (packages, metadata)
+            - packages: List of package dicts with name, version, type, and hash
+            - metadata: Empty dict (system scope has no metadata)
         """
         command = "dpkg-query -W -f='${Package}\t${Version}\t${Architecture}\n'"
         stdout, _, exit_code = executor.execute_command(command, working_dir)
 
         if exit_code != 0:
-            return {"scope": "system", "dependencies": {}}
+            return [], {}
 
         # Collect all package hashes in a single batch operation
         batch_hashes = self._collect_all_package_hashes(executor)
 
-        dependencies = {}
+        packages = []
         for line in stdout.strip().split("\n"):
             if line and "\t" in line:
                 parts = line.split("\t")
@@ -53,9 +63,7 @@ def get_dependencies(self, executor: EnvironmentExecutor, working_dir: Optional[
 
                     full_version = f"{version} {architecture}" if architecture else version
 
-                    package_data = {
-                        "version": full_version,
-                    }
+                    package_data = {"name": package_name, "version": full_version, "type": PACKAGE_TYPE_DPKG}
 
                     # Use batch-collected hash or fallback to individual lookup
                     package_hash = batch_hashes.get(package_name)
@@ -65,9 +73,9 @@ def get_dependencies(self, executor: EnvironmentExecutor, working_dir: Optional[
                     if package_hash:
                         package_data["hash"] = package_hash
 
-                    dependencies[package_name] = package_data
+                    packages.append(package_data)
 
-        return {"scope": "system", "dependencies": dependencies}
+        return packages, {}
 
     def _get_package_hash(self, executor: EnvironmentExecutor, package_name: str, architecture: str = "") -> str | None:
         """Get package hash from dpkg md5sums file if available.