Merge pull request #14111 from michaelnebel/csharp/reduceprojectrestore

C#: Avoid explicitly restoring projects in solution files.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -103,8 +103,9 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                     progressMonitor.MissingNuGet();
                 }
 
-                Restore(solutions);
-                Restore(allProjects);
+                var restoredProjects = RestoreSolutions(solutions);
+                var projects = allProjects.Except(restoredProjects);
+                RestoreProjects(projects);
                 DownloadMissingPackages(allFiles);
             }
 
@@ -351,14 +352,48 @@ private void AnalyseProject(FileInfo project)
 
         }
 
-        private bool Restore(string target, string? pathToNugetConfig = null) =>
-            dotnet.RestoreToDirectory(target, packageDirectory.DirInfo.FullName, pathToNugetConfig);
+        private bool RestoreProject(string project, out string stdout, string? pathToNugetConfig = null) =>
+            dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, out stdout, pathToNugetConfig);
 
-        private void Restore(IEnumerable<string> targets, string? pathToNugetConfig = null)
+        private bool RestoreSolution(string solution, out IEnumerable<string> projects) =>
+            dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, out projects);
+
+        /// <summary>
+        /// Executes `dotnet restore` on all solution files in solutions.
+        /// As opposed to RestoreProjects this is not run in parallel using PLINQ
+        /// as `dotnet restore` on a solution already uses multiple threads for restoring
+        /// the projects (this can be disabled with the `--disable-parallel` flag).
+        /// Returns a list of projects that are up to date with respect to restore.
+        /// </summary>
+        /// <param name="solutions">A list of paths to solution files.</param>
+        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions) =>
+            solutions.SelectMany(solution =>
+                {
+                    RestoreSolution(solution, out var restoredProjects);
+                    return restoredProjects;
+                });
+
+        /// <summary>
+        /// Executes `dotnet restore` on all projects in projects.
+        /// This is done in parallel for performance reasons.
+        /// To ensure that output is not interleaved, the output of each
+        /// restore is collected and printed.
+        /// </summary>
+        /// <param name="projects">A list of paths to project files.</param>
+        private void RestoreProjects(IEnumerable<string> projects)
         {
-            foreach (var target in targets)
+            var stdoutLines = projects
+                .AsParallel()
+                .WithDegreeOfParallelism(options.Threads)
+                .Select(project =>
+                    {
+                        RestoreProject(project, out var stdout);
+                        return stdout;
+                    })
+                .ToList();
+            foreach (var line in stdoutLines)
             {
-                Restore(target, pathToNugetConfig);
+                Console.WriteLine(line);
             }
         }
 
@@ -401,10 +436,10 @@ private void DownloadMissingPackages(List<FileInfo> allFiles)
                     continue;
                 }
 
-                success = Restore(tempDir.DirInfo.FullName, nugetConfig);
+                success = RestoreProject(tempDir.DirInfo.FullName, out var stdout, nugetConfig);
+                Console.WriteLine(stdout);
 
                 // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
-
                 if (!success)
                 {
                     progressMonitor.FailedToRestoreNugetPackage(package);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -2,14 +2,16 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
 using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     /// <summary>
     /// Utilities to run the "dotnet" command.
     /// </summary>
-    internal class DotNet : IDotNet
+    internal partial class DotNet : IDotNet
     {
         private readonly ProgressMonitor progressMonitor;
         private readonly string dotnet;
@@ -31,17 +33,22 @@ private void Info()
             }
         }
 
-        private ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput) =>
-            new ProcessStartInfo(dotnet, args)
+        private ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput)
+        {
+            var startInfo = new ProcessStartInfo(dotnet, args)
             {
                 UseShellExecute = false,
                 RedirectStandardOutput = redirectStandardOutput
             };
+            // Set the .NET CLI language to English to avoid localized output.
+            startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
+            return startInfo;
+        }
 
         private bool RunCommand(string args)
         {
             progressMonitor.RunningProcess($"{dotnet} {args}");
-            using var proc = Process.Start(this.MakeDotnetStartInfo(args, redirectStandardOutput: false));
+            using var proc = Process.Start(MakeDotnetStartInfo(args, redirectStandardOutput: false));
             proc?.WaitForExit();
             var exitCode = proc?.ExitCode ?? -1;
             if (exitCode != 0)
@@ -52,12 +59,50 @@ private bool RunCommand(string args)
             return true;
         }
 
-        public bool RestoreToDirectory(string projectOrSolutionFile, string packageDirectory, string? pathToNugetConfig = null)
+        private bool RunCommand(string args, out IList<string> output)
+        {
+            progressMonitor.RunningProcess($"{dotnet} {args}");
+            var pi = MakeDotnetStartInfo(args, redirectStandardOutput: true);
+            var exitCode = pi.ReadOutput(out output);
+            if (exitCode != 0)
+            {
+                progressMonitor.CommandFailed(dotnet, args, exitCode);
+                return false;
+            }
+            return true;
+        }
+
+        private static string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory) =>
+            $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
+
+        public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, out string stdout, string? pathToNugetConfig = null)
         {
-            var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
+            var args = GetRestoreArgs(projectFile, packageDirectory);
             if (pathToNugetConfig != null)
+            {
                 args += $" --configfile \"{pathToNugetConfig}\"";
-            return RunCommand(args);
+            }
+            var success = RunCommand(args, out var output);
+            stdout = string.Join("\n", output);
+            return success;
+        }
+
+        public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects)
+        {
+            var args = GetRestoreArgs(solutionFile, packageDirectory);
+            args += " --verbosity normal";
+            if (RunCommand(args, out var output))
+            {
+                var regex = RestoreProjectRegex();
+                projects = output
+                    .Select(line => regex.Match(line))
+                    .Where(match => match.Success)
+                    .Select(match => match.Groups[1].Value);
+                return true;
+            }
+
+            projects = Array.Empty<string>();
+            return false;
         }
 
         public bool New(string folder)
@@ -78,22 +123,21 @@ public bool AddPackage(string folder, string package)
 
         private IList<string> GetListed(string args, string artifact)
         {
-            progressMonitor.RunningProcess($"{dotnet} {args}");
-            var pi = this.MakeDotnetStartInfo(args, redirectStandardOutput: true);
-            var exitCode = pi.ReadOutput(out var artifacts);
-            if (exitCode != 0)
+            if (RunCommand(args, out var artifacts))
             {
-                progressMonitor.CommandFailed(dotnet, args, exitCode);
-                return new List<string>();
+                progressMonitor.LogInfo($"Found {artifact}s: {string.Join("\n", artifacts)}");
+                return artifacts;
             }
-            progressMonitor.LogInfo($"Found {artifact}s: {string.Join("\n", artifacts)}");
-            return artifacts;
+            return new List<string>();
         }
 
         public bool Exec(string execArgs)
         {
             var args = $"exec {execArgs}";
             return RunCommand(args);
         }
+
+        [GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
+        private static partial Regex RestoreProjectRegex();
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -4,7 +4,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal interface IDotNet
     {
-        bool RestoreToDirectory(string project, string directory, string? pathToNugetConfig = null);
+        bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null);
+        bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects);
         bool New(string folder);
         bool AddPackage(string folder, string package);
         IList<string> GetListedRuntimes();

csharp/extractor/Semmle.Extraction.Tests/FileContent.cs

@@ -1,6 +1,5 @@
 using Xunit;
 using System.Collections.Generic;
-using System.Linq;
 using Semmle.Util.Logging;
 using Semmle.Extraction.CSharp.DependencyFetching;
 

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -1,4 +1,5 @@
 using Xunit;
+using System;
 using System.Collections.Generic;
 using Semmle.Extraction.CSharp.DependencyFetching;
 
@@ -18,7 +19,17 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
 
         public bool New(string folder) => true;
 
-        public bool RestoreToDirectory(string project, string directory, string? pathToNugetConfig = null) => true;
+        public bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null)
+        {
+            stdout = "";
+            return true;
+        }
+
+        public bool RestoreSolutionToDirectory(string solution, string directory, out IEnumerable<string> projects)
+        {
+            projects = Array.Empty<string>();
+            return true;
+        }
 
         public IList<string> GetListedRuntimes() => runtimes;