Merge pull request #15395 from tamasvajk/feature/standalone-nuget-restore-retry

tamasvajk · web-flow · commit df8d453058b4 · 2024-01-23T14:45:00.000+01:00
C#: Try fallback `dotnet restore` without nuget.config
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
@@ -650,12 +650,6 @@ private void AnalyseProject(FileInfo project)
 
         }
 
-        private bool RestoreProject(string project, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null) =>
-            dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching, out assets, pathToNugetConfig);
-
-        private bool RestoreSolution(string solution, out IEnumerable<string> projects, out IEnumerable<string> assets) =>
-            dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out projects, out assets);
-
         /// <summary>
         /// Executes `dotnet restore` on all solution files in solutions.
         /// As opposed to RestoreProjects this is not run in parallel using PLINQ
@@ -670,7 +664,7 @@ private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out
             var assetFiles = new List<string>();
             var projects = solutions.SelectMany(solution =>
                 {
-                    RestoreSolution(solution, out var restoredProjects, out var a);
+                    dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a);
                     assetFiles.AddRange(a);
                     return restoredProjects;
                 });
@@ -689,7 +683,7 @@ private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<strin
             var assetFiles = new List<string>();
             Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
             {
-                RestoreProject(project, forceDotnetRefAssemblyFetching: true, out var a);
+                dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _);
                 assetFiles.AddRange(a);
             });
             assets = assetFiles;
@@ -736,11 +730,21 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
                     return;
                 }
 
-                dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, pathToNugetConfig: nugetConfig);
-                // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
+                success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var outputLines, pathToNugetConfig: nugetConfig);
                 if (!success)
                 {
-                    progressMonitor.FailedToRestoreNugetPackage(package);
+                    if (outputLines?.Any(s => s.Contains("NU1301")) == true)
+                    {
+                        // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
+                        success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var _, pathToNugetConfig: null, force: true);
+                    }
+
+                    // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
+
+                    if (!success)
+                    {
+                        progressMonitor.FailedToRestoreNugetPackage(package);
+                    }
                 }
             });
 
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
@@ -72,16 +72,21 @@ private static IEnumerable<string> GetAssetsFilePaths(IEnumerable<string> lines)
         private static IEnumerable<string> GetRestoredProjects(IEnumerable<string> lines) =>
             GetFirstGroupOnMatch(RestoredProjectRegex(), lines);
 
-        public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null)
+        public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)
         {
             var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching);
             if (pathToNugetConfig != null)
             {
                 args += $" --configfile \"{pathToNugetConfig}\"";
             }
 
-            var success = dotnetCliInvoker.RunCommand(args, out var output);
-            assets = success ? GetAssetsFilePaths(output) : Array.Empty<string>();
+            if (force)
+            {
+                args += " --force";
+            }
+
+            var success = dotnetCliInvoker.RunCommand(args, out outputLines);
+            assets = success ? GetAssetsFilePaths(outputLines) : Array.Empty<string>();
             return success;
         }
 
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
@@ -4,7 +4,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal interface IDotNet
     {
-        bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null);
+        bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false);
         bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets);
         bool New(string folder);
         bool AddPackage(string folder, string package);
diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
@@ -101,7 +101,7 @@ public void TestDotnetRestoreProjectToDirectory1()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets);
+            dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _);
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -116,7 +116,7 @@ public void TestDotnetRestoreProjectToDirectory2()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, "myconfig.config");
+            dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config");
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -126,6 +126,24 @@ public void TestDotnetRestoreProjectToDirectory2()
             Assert.Contains("/path/to/project2.assets.json", assets);
         }
 
+        [Fact]
+        public void TestDotnetRestoreProjectToDirectory3()
+        {
+            // Setup
+            var dotnetCliInvoker = new DotNetCliInvokerStub(MakeDotnetRestoreOutput());
+            var dotnet = MakeDotnet(dotnetCliInvoker);
+
+            // Execute
+            dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config", force: true);
+
+            // Verify
+            var lastArgs = dotnetCliInvoker.GetLastArgs();
+            Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\" --force", lastArgs);
+            Assert.Equal(2, assets.Count());
+            Assert.Contains("/path/to/project.assets.json", assets);
+            Assert.Contains("/path/to/project2.assets.json", assets);
+        }
+
         [Fact]
         public void TestDotnetRestoreSolutionToDirectory1()
         {
diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
@@ -19,9 +19,10 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
 
         public bool New(string folder) => true;
 
-        public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null)
+        public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)
         {
             assets = Array.Empty<string>();
+            outputLines = Array.Empty<string>();
             return true;
         }
 
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/Assemblies.expected
@@ -0,0 +1 @@
+| newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/Assemblies.ql
@@ -0,0 +1,11 @@
+import csharp
+
+private string getPath(Assembly a) {
+  not a.getCompilation().getOutputAssembly() = a and
+  exists(string s | s = a.getFile().getAbsolutePath() |
+    result = s.substring(s.indexOf("newtonsoft.json"), s.length())
+  )
+}
+
+from Assembly a
+select getPath(a)
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/proj/Program.cs b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/proj/Program.cs
@@ -0,0 +1,6 @@
+class Program
+{
+    static void Main(string[] args)
+    {
+    }
+}
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/proj/nuget.config b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/proj/nuget.config
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="x" value="https://abc.abc/packages/" />
+  </packageSources>
+</configuration>
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/proj/proj.csproj b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/proj/proj.csproj
@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net8.0</TargetFrameworks>
+  </PropertyGroup>
+
+  <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+    <RemoveDir Directories=".\bin" />
+    <RemoveDir Directories=".\obj" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+</Project>
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/standalone.sln b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/standalone.sln
@@ -0,0 +1,19 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.002.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "proj", "proj\proj.csproj", "{6ED00460-7666-4AE9-A405-4B6C8B02279A}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {4ED55A1C-066C-43DF-B32E-7EAA035985EE}
+	EndGlobalSection
+EndGlobal
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/test.py
@@ -0,0 +1,3 @@
+from create_database_utils import *
+
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])

Original file line number	Diff line number	Diff line change
`@@ -72,16 +72,21 @@ private static IEnumerable<string> GetAssetsFilePaths(IEnumerable<string> lines)`
`72`	`72`	`private static IEnumerable<string> GetRestoredProjects(IEnumerable<string> lines) =>`
`73`	`73`	`GetFirstGroupOnMatch(RestoredProjectRegex(), lines);`
`74`	`74`
`75`		`- public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null)`
	`75`	`+ public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)`
`76`	`76`	`{`
`77`	`77`	`var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching);`
`78`	`78`	`if (pathToNugetConfig != null)`
`79`	`79`	`{`
`80`	`80`	`args += $" --configfile \"{pathToNugetConfig}\"";`
`81`	`81`	`}`
`82`	`82`
`83`		`- var success = dotnetCliInvoker.RunCommand(args, out var output);`
`84`		`- assets = success ? GetAssetsFilePaths(output) : Array.Empty<string>();`
	`83`	`+ if (force)`
	`84`	`+ {`
	`85`	`+ args += " --force";`
	`86`	`+ }`
	`87`	`+`
	`88`	`+ var success = dotnetCliInvoker.RunCommand(args, out outputLines);`
	`89`	`+ assets = success ? GetAssetsFilePaths(outputLines) : Array.Empty<string>();`
`85`	`90`	`return success;`
`86`	`91`	`}`
`87`	`92`
Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching`
`4`	`4`	`{`
`5`	`5`	`internal interface IDotNet`
`6`	`6`	`{`
`7`		`- bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null);`
	`7`	`+ bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false);`
`8`	`8`	`bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets);`
`9`	`9`	`bool New(string folder);`
`10`	`10`	`bool AddPackage(string folder, string package);`
Original file line number	Diff line number	Diff line change
`@@ -19,9 +19,10 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)`
`19`	`19`
`20`	`20`	`public bool New(string folder) => true;`
`21`	`21`
`22`		`- public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null)`
	`22`	`+ public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)`
`23`	`23`	`{`
`24`	`24`	`assets = Array.Empty<string>();`
	`25`	`+ outputLines = Array.Empty<string>();`
`25`	`26`	`return true;`
`26`	`27`	`}`
`27`	`28`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+\| newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll \|`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,6 @@ @@
 +class Program
 +{
 +    static void Main(string[] args)
 +    {
 +    }
 +}