Merge branch 'main' into idrissrio/preprocessor-multiline

Author: IdrissRio

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -109,7 +109,7 @@ public HashSet<AssemblyLookupLocation> Restore()
                 if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds))
                 {
                     // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
-                    var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds);
+                    var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds([], explicitFeeds);
                     return unresponsiveMissingPackageLocation is null
                         ? []
                         : [unresponsiveMissingPackageLocation];
@@ -166,11 +166,11 @@ public HashSet<AssemblyLookupLocation> Restore()
                 .ToList();
             assemblyLookupLocations.UnionWith(paths.Select(p => new AssemblyLookupLocation(p)));
 
-            LogAllUnusedPackages(dependencies);
+            var usedPackageNames = GetAllUsedPackageDirNames(dependencies);
 
             var missingPackageLocation = checkNugetFeedResponsiveness
-                ? DownloadMissingPackagesFromSpecificFeeds(explicitFeeds)
-                : DownloadMissingPackages();
+                ? DownloadMissingPackagesFromSpecificFeeds(usedPackageNames, explicitFeeds)
+                : DownloadMissingPackages(usedPackageNames);
 
             if (missingPackageLocation is not null)
             {
@@ -297,21 +297,21 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
             compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<string>? feedsFromNugetConfigs)
+        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(IEnumerable<string> usedPackageNames, HashSet<string>? feedsFromNugetConfigs)
         {
             var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
             if (reachableFallbackFeeds.Count > 0)
             {
-                return DownloadMissingPackages(fallbackNugetFeeds: reachableFallbackFeeds);
+                return DownloadMissingPackages(usedPackageNames, fallbackNugetFeeds: reachableFallbackFeeds);
             }
 
             logger.LogWarning("Skipping download of missing packages from specific feeds as no fallback Nuget feeds are reachable.");
             return null;
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable<string>? fallbackNugetFeeds = null)
+        private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable<string> usedPackageNames, IEnumerable<string>? fallbackNugetFeeds = null)
         {
-            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(PackageDirectory.DirInfo);
+            var alreadyDownloadedPackages = usedPackageNames.Select(p => p.ToLowerInvariant());
             var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
 
             var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
@@ -418,17 +418,23 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
             return nugetConfig;
         }
 
-        private void LogAllUnusedPackages(DependencyContainer dependencies)
+        private IEnumerable<string> GetAllUsedPackageDirNames(DependencyContainer dependencies)
         {
             var allPackageDirectories = GetAllPackageDirectories();
 
             logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
             logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
 
-            allPackageDirectories
-                .Where(package => !dependencies.Packages.Contains(package))
+            var usage = allPackageDirectories.Select(package => (package, isUsed: dependencies.Packages.Contains(package)));
+
+            usage
+                .Where(package => !package.isUsed)
                 .Order()
-                .ForEach(package => logger.LogDebug($"Unused package: {package}"));
+                .ForEach(package => logger.LogDebug($"Unused package: {package.package}"));
+
+            return usage
+                .Where(package => package.isUsed)
+                .Select(package => package.package);
         }
 
         private ICollection<string> GetAllPackageDirectories()

csharp/ql/src/change-notes/2025-03-21-dependency-fetching.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Improved dependency resolution in `build-mode: none` extraction to handle failing `dotnet restore` processes that managed to download a subset of the dependencies before the failure.

rust/ql/integration-tests/hello-workspace/exe/src/main.rs

@@ -1,7 +1,7 @@
-use lib::a_module::hello;
+use lib::a_module::hello; // $ item=HELLO
 
 mod a_module;
 
 fn main() {
-    hello();
+    hello(); // $ item=HELLO
 }

rust/ql/integration-tests/hello-workspace/lib/src/a_module/mod.rs

@@ -1,3 +1,3 @@
 pub fn hello() {
     println!("Hello, world!");
-}
+} // HELLO

rust/ql/integration-tests/hello-workspace/path-resolution.expected

@@ -0,0 +1 @@
+import utils.test.PathResolutionInlineExpectationsTest

rust/ql/integration-tests/hello-workspace/path-resolution.ql

@@ -2,14 +2,23 @@
   "sysroot_src": "filled by the rust_project fixture",
   "crates": [
     {
+      "display_name": "exe",
+      "version": "0.1.0",
       "root_module": "exe/src/main.rs",
       "edition": "2021",
-      "deps": [{"crate": 1, "name": "lib"}]
+      "deps": [
+        {
+          "crate": 1,
+          "name": "lib"
+        }
+      ]
     },
     {
+      "display_name": "lib",
+      "version": "0.1.0",
       "root_module": "lib/src/lib.rs",
       "edition": "2021",
       "deps": []
     }
   ]
-}
+}

rust/ql/integration-tests/hello-workspace/rust-project.json

@@ -1,4 +1,4 @@
-| Elements extracted | 87 |
+| Elements extracted | 90 |
 | Elements unextracted | 0 |
 | Extraction errors | 0 |
 | Extraction warnings | 0 |

rust/ql/integration-tests/hello-workspace/summary.cargo.expected

@@ -1,4 +1,4 @@
-| Elements extracted | 87 |
+| Elements extracted | 90 |
 | Elements unextracted | 0 |
 | Extraction errors | 0 |
 | Extraction warnings | 0 |

rust/ql/integration-tests/hello-workspace/summary.rust-project.expected

@@ -13,6 +13,7 @@ private import codeql.rust.elements.internal.generated.Crate
 module Impl {
   private import rust
   private import codeql.rust.elements.internal.NamedCrate
+  private import codeql.rust.internal.PathResolution
 
   class Crate extends Generated::Crate {
     override string toStringImpl() {
@@ -58,6 +59,14 @@ module Impl {
      */
     Crate getADependency() { result = this.getDependency(_) }
 
+    /** Gets the source file that defines this crate, if any. */
+    SourceFile getSourceFile() { result.getFile() = this.getModule().getFile() }
+
+    /**
+     * Gets a source file that belongs to this crate, if any.
+     */
+    SourceFile getASourceFile() { result = this.(CrateItemNode).getASourceFile() }
+
     override Location getLocation() { result = this.getModule().getLocation() }
   }
 }