Merge pull request #16248 from michaelnebel/csharp/groupsprojectbeforerestore

C#: Restore projects and collect dependencies for projects in the same folder sequentially.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs

@@ -16,6 +16,11 @@ internal class Assets
     {
         private readonly ILogger logger;
 
+        /// <summary>
+        /// Contains the dependencies found in the parsed assets files.
+        /// </summary>
+        public DependencyContainer Dependencies { get; } = new();
+
         internal Assets(ILogger logger)
         {
             this.logger = logger;
@@ -72,7 +77,7 @@ private record class ReferenceInfo(string? Type, Dictionary<string, object>? Com
         ///     "json.net"
         ///   }
         /// </summary>
-        private void AddPackageDependencies(JObject json, DependencyContainer dependencies)
+        private void AddPackageDependencies(JObject json)
         {
             // If there is more than one framework we need to pick just one.
             // To ensure stability we pick one based on the lexicographic order of
@@ -107,13 +112,13 @@ private void AddPackageDependencies(JObject json, DependencyContainer dependenci
                         // If this is a framework reference then include everything.
                         if (FrameworkPackageNames.AllFrameworks.Any(framework => name.StartsWith(framework)))
                         {
-                            dependencies.AddFramework(name);
+                            Dependencies.AddFramework(name);
                         }
                         return;
                     }
 
                     info.Compile
-                        .ForEach(r => dependencies.Add(name, r.Key));
+                        .ForEach(r => Dependencies.Add(name, r.Key));
                 });
 
             return;
@@ -149,7 +154,7 @@ private void AddPackageDependencies(JObject json, DependencyContainer dependenci
         ///     "microsoft.netcore.app.ref"
         ///   }
         /// </summary>
-        private void AddFrameworkDependencies(JObject json, DependencyContainer dependencies)
+        private void AddFrameworkDependencies(JObject json)
         {
 
             var frameworks = json
@@ -178,21 +183,21 @@ private void AddFrameworkDependencies(JObject json, DependencyContainer dependen
 
             references
                 .Properties()
-                .ForEach(f => dependencies.AddFramework($"{f.Name}.Ref".ToLowerInvariant()));
+                .ForEach(f => Dependencies.AddFramework($"{f.Name}.Ref".ToLowerInvariant()));
         }
 
         /// <summary>
         /// Parse `json` as project.assets.json content and add relative paths to the dependencies
         /// (together with used package information) required for compilation.
         /// </summary>
         /// <returns>True if parsing succeeds, otherwise false.</returns>
-        public bool TryParse(string json, DependencyContainer dependencies)
+        public bool TryParse(string json)
         {
             try
             {
                 var obj = JObject.Parse(json);
-                AddPackageDependencies(obj, dependencies);
-                AddFrameworkDependencies(obj, dependencies);
+                AddPackageDependencies(obj);
+                AddFrameworkDependencies(obj);
                 return true;
             }
             catch (Exception e)
@@ -217,19 +222,24 @@ private static bool TryReadAllText(string path, ILogger logger, [NotNullWhen(ret
             }
         }
 
-        public static DependencyContainer GetCompilationDependencies(ILogger logger, IEnumerable<string> assets)
+        /// <summary>
+        /// Add the dependencies from the assets file to the dependencies.
+        /// </summary>
+        /// <param name="asset">Path to an assets file.</param>
+        public void AddDependencies(string asset)
         {
-            var parser = new Assets(logger);
-            var dependencies = new DependencyContainer();
-            assets.ForEach(asset =>
+            if (TryReadAllText(asset, logger, out var json))
             {
-                if (TryReadAllText(asset, logger, out var json))
-                {
-                    parser.TryParse(json, dependencies);
-                }
-            });
-            return dependencies;
+                TryParse(json);
+            }
         }
+
+        /// <summary>
+        /// Add the dependencies from the assets files to the dependencies.
+        /// </summary>
+        /// <param name="assets">Collection of paths to assets files.</param>
+        public void AddDependenciesRange(IEnumerable<string> assets) =>
+            assets.ForEach(AddDependencies);
     }
 
     internal static class JsonExtensions

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyContainer.cs

@@ -12,7 +12,7 @@ internal class DependencyContainer
         /// <summary>
         /// Paths to dependencies required for compilation.
         /// </summary>
-        public List<string> Paths { get; } = new();
+        public HashSet<string> Paths { get; } = new();
 
         /// <summary>
         /// Packages that are used as a part of the required dependencies.
@@ -45,7 +45,7 @@ public void Add(string package, string dependency)
             var p = package.Replace('/', Path.DirectorySeparatorChar);
             var d = dependency.Replace('/', Path.DirectorySeparatorChar);
 
-            // In most cases paths in asset files point to dll's or the empty _._ file.
+            // In most cases paths in assets files point to dll's or the empty _._ file.
             // That is, for _._ we don't need to add anything.
             if (Path.GetFileName(d) == "_._")
             {
@@ -68,4 +68,18 @@ public void AddFramework(string framework)
             Packages.Add(GetPackageName(p));
         }
     }
-}
+
+    internal static class DependencyContainerExtensions
+    {
+        /// <summary>
+        /// Flatten a list of containers into a single container.
+        /// </summary>
+        public static DependencyContainer Flatten(this IEnumerable<DependencyContainer> containers, DependencyContainer init) =>
+            containers.Aggregate(init, (acc, container) =>
+            {
+                acc.Paths.UnionWith(container.Paths);
+                acc.Packages.UnionWith(container.Packages);
+                return acc;
+            });
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -3,8 +3,6 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
 using System.Threading.Tasks;
 
 using Semmle.Util;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -144,11 +145,11 @@ public HashSet<AssemblyLookupLocation> Restore()
                 logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
             }
 
-            var restoredProjects = RestoreSolutions(out var assets1);
+            var restoredProjects = RestoreSolutions(out var container);
             var projects = fileProvider.Projects.Except(restoredProjects);
-            RestoreProjects(projects, out var assets2);
+            RestoreProjects(projects, out var containers);
 
-            var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
+            var dependencies = containers.Flatten(container);
 
             var paths = dependencies
                 .Paths
@@ -198,14 +199,14 @@ private List<string> GetReachableFallbackNugetFeeds()
         /// As opposed to RestoreProjects this is not run in parallel using PLINQ
         /// as `dotnet restore` on a solution already uses multiple threads for restoring
         /// the projects (this can be disabled with the `--disable-parallel` flag).
-        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// Populates dependencies with the relevant dependencies from the assets files generated by the restore.
         /// Returns a list of projects that are up to date with respect to restore.
         /// </summary>
-        private IEnumerable<string> RestoreSolutions(out IEnumerable<string> assets)
+        private IEnumerable<string> RestoreSolutions(out DependencyContainer dependencies)
         {
             var successCount = 0;
             var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
+            var assets = new Assets(logger);
             var projects = fileProvider.Solutions.SelectMany(solution =>
                 {
                     logger.LogInfo($"Restoring solution {solution}...");
@@ -218,10 +219,10 @@ private IEnumerable<string> RestoreSolutions(out IEnumerable<string> assets)
                     {
                         nugetSourceFailures++;
                     }
-                    assetFiles.AddRange(res.AssetsFilePaths);
+                    assets.AddDependenciesRange(res.AssetsFilePaths);
                     return res.RestoredProjects;
                 }).ToList();
-            assets = assetFiles;
+            dependencies = assets.Dependencies;
             compilationInfoContainer.CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
             compilationInfoContainer.CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
             compilationInfoContainer.CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
@@ -231,33 +232,39 @@ private IEnumerable<string> RestoreSolutions(out IEnumerable<string> assets)
         /// <summary>
         /// Executes `dotnet restore` on all projects in projects.
         /// This is done in parallel for performance reasons.
-        /// Populates assets with the relative paths to the assets files generated by the restore.
+        /// Populates dependencies with the relative paths to the assets files generated by the restore.
         /// </summary>
         /// <param name="projects">A list of paths to project files.</param>
-        private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
+        private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<DependencyContainer> dependencies)
         {
             var successCount = 0;
             var nugetSourceFailures = 0;
-            var assetFiles = new List<string>();
+            ConcurrentBag<DependencyContainer> collectedDependencies = [];
             var sync = new object();
-            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = DependencyManager.Threads }, project =>
+            var projectGroups = projects.GroupBy(Path.GetDirectoryName);
+            Parallel.ForEach(projectGroups, new ParallelOptions { MaxDegreeOfParallelism = DependencyManager.Threads }, projectGroup =>
             {
-                logger.LogInfo($"Restoring project {project}...");
-                var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                lock (sync)
+                var assets = new Assets(logger);
+                foreach (var project in projectGroup)
                 {
-                    if (res.Success)
+                    logger.LogInfo($"Restoring project {project}...");
+                    var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                    assets.AddDependenciesRange(res.AssetsFilePaths);
+                    lock (sync)
                     {
-                        successCount++;
-                    }
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        nugetSourceFailures++;
+                        if (res.Success)
+                        {
+                            successCount++;
+                        }
+                        if (res.HasNugetPackageSourceError)
+                        {
+                            nugetSourceFailures++;
+                        }
                     }
-                    assetFiles.AddRange(res.AssetsFilePaths);
                 }
+                collectedDependencies.Add(assets.Dependencies);
             });
-            assets = assetFiles;
+            dependencies = collectedDependencies;
             compilationInfoContainer.CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
             compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
         }