Merge pull request #16164 from tamasvajk/buildless/nuget-feed-fallback-feed-check

tamasvajk · web-flow · commit 91f2ea572c9d · 2024-04-12T13:17:06.000+02:00
C#: Check fallback nuget feeds before trying to use them in the fallb…
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
@@ -339,7 +339,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
                 {
                     logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
 
-                    if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
+                    if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
                     {
                         runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
                     }
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
@@ -95,9 +95,9 @@ public bool AddPackage(string folder, string package)
 
         public IList<string> GetListedSdks() => GetResultList("--list-sdks");
 
-        private IList<string> GetResultList(string args)
+        private IList<string> GetResultList(string args, string? workingDirectory = null)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var results))
+            if (dotnetCliInvoker.RunCommand(args, workingDirectory, out var results))
             {
                 return results;
             }
@@ -111,7 +111,19 @@ public bool Exec(string execArgs)
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
+        private const string nugetListSourceCommand = "nuget list source --format Short";
+
+        public IList<string> GetNugetFeeds(string nugetConfig)
+        {
+            logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
+            return GetResultList($"{nugetListSourceCommand} --configfile \"{nugetConfig}\"");
+        }
+
+        public IList<string> GetNugetFeedsFromFolder(string folderPath)
+        {
+            logger.LogInfo($"Getting Nuget feeds in folder '{folderPath}'...");
+            return GetResultList(nugetListSourceCommand, folderPath);
+        }
 
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "8.0.101";
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
@@ -21,41 +21,49 @@ public DotNetCliInvoker(ILogger logger, string exec)
             this.Exec = exec;
         }
 
-        private ProcessStartInfo MakeDotnetStartInfo(string args)
+        private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory)
         {
             var startInfo = new ProcessStartInfo(Exec, args)
             {
                 UseShellExecute = false,
                 RedirectStandardOutput = true,
                 RedirectStandardError = true
             };
+            if (!string.IsNullOrWhiteSpace(workingDirectory))
+            {
+                startInfo.WorkingDirectory = workingDirectory;
+            }
             // Set the .NET CLI language to English to avoid localized output.
             startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
             startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1";
             startInfo.EnvironmentVariables["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "true";
             return startInfo;
         }
 
-        private bool RunCommandAux(string args, out IList<string> output)
+        private bool RunCommandAux(string args, string? workingDirectory, out IList<string> output)
         {
-            logger.LogInfo($"Running {Exec} {args}");
-            var pi = MakeDotnetStartInfo(args);
+            var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}";
+            logger.LogInfo($"Running {Exec} {args}{dirLog}");
+            var pi = MakeDotnetStartInfo(args, workingDirectory);
             var threadId = Environment.CurrentManagedThreadId;
             void onOut(string s) => logger.LogInfo(s, threadId);
             void onError(string s) => logger.LogError(s, threadId);
             var exitCode = pi.ReadOutput(out output, onOut, onError);
             if (exitCode != 0)
             {
-                logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
+                logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}");
                 return false;
             }
             return true;
         }
 
         public bool RunCommand(string args) =>
-            RunCommandAux(args, out _);
+            RunCommandAux(args, null, out _);
 
         public bool RunCommand(string args, out IList<string> output) =>
-            RunCommandAux(args, out output);
+            RunCommandAux(args, null, out output);
+
+        public bool RunCommand(string args, string? workingDirectory, out IList<string> output) =>
+            RunCommandAux(args, workingDirectory, out output);
     }
 }
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
@@ -33,10 +33,28 @@ internal class EnvironmentVariableNames
         public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
 
         /// <summary>
-        /// Specifies how many requests to make to the NuGet feed to check its responsiveness.
+        /// Specifies the timeout (as an integer) in milliseconds for the initial check of fallback NuGet feeds responsiveness. The value is then doubled for each subsequent check.
+        /// This is primarily used in testing.
+        /// </summary>
+        internal const string NugetFeedResponsivenessInitialTimeoutForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT";
+
+        /// <summary>
+        /// Specifies how many requests to make to the NuGet feeds to check their responsiveness.
         /// </summary>
         public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
 
+        /// <summary>
+        /// Specifies how many requests to make to the fallback NuGet feeds to check their responsiveness.
+        /// This is primarily used in testing.
+        /// </summary>
+        internal const string NugetFeedResponsivenessRequestCountForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT";
+
+        /// <summary>
+        /// Specifies the NuGet feeds to use for fallback Nuget dependency fetching. The value is a space-separated list of feed URLs.
+        /// The default value is `https://api.nuget.org/v3/index.json`.
+        /// </summary>
+        public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
+
         /// <summary>
         /// Specifies the location of the diagnostic directory.
         /// </summary>
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
@@ -14,6 +14,7 @@ public interface IDotNet
         IList<string> GetListedSdks();
         bool Exec(string execArgs);
         IList<string> GetNugetFeeds(string nugetConfig);
+        IList<string> GetNugetFeedsFromFolder(string folderPath);
     }
 
     public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs
@@ -19,5 +19,11 @@ internal interface IDotNetCliInvoker
         /// The output of the command is returned in `output`.
         /// </summary>
         bool RunCommand(string args, out IList<string> output);
+
+        /// <summary>
+        /// Execute `dotnet <args>` in `<workingDirectory>` and return true if the command succeeded, otherwise false.
+        /// The output of the command is returned in `output`.
+        /// </summary>
+        bool RunCommand(string args, string? workingDirectory, out IList<string> output);
     }
 }
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
@@ -243,7 +243,7 @@ private void RunMonoNugetCommand(string command, out IList<string> stdout)
         private void AddDefaultPackageSource(string nugetConfig)
         {
             logger.LogInfo("Adding default package source...");
-            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source https://api.nuget.org/v3/index.json -ConfigFile \"{nugetConfig}\"", out var _);
+            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {DependencyManager.PublicNugetFeed} -ConfigFile \"{nugetConfig}\"", out var _);
         }
 
         public void Dispose()
diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
@@ -10,6 +10,7 @@ internal class DotNetCliInvokerStub : IDotNetCliInvoker
     {
         private readonly IList<string> output;
         private string lastArgs = "";
+        public string WorkingDirectory { get; private set; } = "";
         public bool Success { get; set; } = true;
 
         public DotNetCliInvokerStub(IList<string> output)
@@ -32,6 +33,12 @@ public bool RunCommand(string args, out IList<string> output)
             return Success;
         }
 
+        public bool RunCommand(string args, string? workingDirectory, out IList<string> output)
+        {
+            WorkingDirectory = workingDirectory ?? "";
+            return RunCommand(args, out output);
+        }
+
         public string GetLastArgs() => lastArgs;
     }
 
@@ -262,5 +269,36 @@ public void TestDotnetExec()
             var lastArgs = dotnetCliInvoker.GetLastArgs();
             Assert.Equal("exec myarg1 myarg2", lastArgs);
         }
+
+        [Fact]
+        public void TestNugetFeeds()
+        {
+            // Setup
+            var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
+            var dotnet = MakeDotnet(dotnetCliInvoker);
+
+            // Execute
+            dotnet.GetNugetFeeds("abc");
+
+            // Verify
+            var lastArgs = dotnetCliInvoker.GetLastArgs();
+            Assert.Equal("nuget list source --format Short --configfile \"abc\"", lastArgs);
+        }
+
+        [Fact]
+        public void TestNugetFeedsFromFolder()
+        {
+            // Setup
+            var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
+            var dotnet = MakeDotnet(dotnetCliInvoker);
+
+            // Execute
+            dotnet.GetNugetFeedsFromFolder("abc");
+
+            // Verify
+            var lastArgs = dotnetCliInvoker.GetLastArgs();
+            Assert.Equal("nuget list source --format Short", lastArgs);
+            Assert.Equal("abc", dotnetCliInvoker.WorkingDirectory);
+        }
     }
 }
diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
@@ -28,6 +28,8 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
         public bool Exec(string execArgs) => true;
 
         public IList<string> GetNugetFeeds(string nugetConfig) => [];
+
+        public IList<string> GetNugetFeedsFromFolder(string folderPath) => [];
     }
 
     public class RuntimeTests
diff --git a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Globalization;
 using System.Numerics;
 
@@ -34,5 +35,10 @@ public static bool GetBoolean(string name)
             var _ = bool.TryParse(env, out var value);
             return value;
         }
+
+        public static IEnumerable<string> GetURLs(string name)
+        {
+            return Environment.GetEnvironmentVariable(name)?.Split(" ", StringSplitOptions.RemoveEmptyEntries) ?? [];
+        }
     }
 }
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected
@@ -1,5 +1,6 @@
 | All Nuget feeds reachable | 0.0 |
 | Fallback nuget restore | 1.0 |
+| Inherited Nuget feed count | 1.0 |
 | Project files on filesystem | 1.0 |
 | Resolved assembly conflicts | 7.0 |
 | Restored .NET framework variants | 0.0 |
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/nuget.config b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/nuget.config
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
-    <clear />
+    <!-- <clear /> -->
     <add key="x" value="https://localhost:53/packages/" />
     <add key="y" value="https://localhost:80/packages/" />
   </packageSources>
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py
@@ -6,5 +6,12 @@
 os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1"     # 1ms, the GET request should fail with such short timeout
 os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1"       # Limit the count of checks to 1
 os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"] = "https://abc.de:8000/packages/"   # Exclude this feed from check
+
+# Making sure the reachability test of `nuget.org` succeeds:
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1000"
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "5"
+# The second feed is ignored in the fallback restore, because of network issues:
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK"] = "https://api.nuget.org/v3/index.json https://abc.def:8000/packages/"
+
 run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
 check_diagnostics()

Original file line number	Diff line number	Diff line change
`@@ -339,7 +339,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet`
`339`	`339`	`{`
`340`	`340`	`logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");`
`341`	`341`
`342`		`- if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))`
	`342`	`+ if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))`
`343`	`343`	`{`
`344`	`344`	`runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);`
`345`	`345`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,41 +21,49 @@ public DotNetCliInvoker(ILogger logger, string exec)`
`21`	`21`	`this.Exec = exec;`
`22`	`22`	`}`
`23`	`23`
`24`		`- private ProcessStartInfo MakeDotnetStartInfo(string args)`
	`24`	`+ private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory)`
`25`	`25`	`{`
`26`	`26`	`var startInfo = new ProcessStartInfo(Exec, args)`
`27`	`27`	`{`
`28`	`28`	`UseShellExecute = false,`
`29`	`29`	`RedirectStandardOutput = true,`
`30`	`30`	`RedirectStandardError = true`
`31`	`31`	`};`
	`32`	`+ if (!string.IsNullOrWhiteSpace(workingDirectory))`
	`33`	`+ {`
	`34`	`+ startInfo.WorkingDirectory = workingDirectory;`
	`35`	`+ }`
`32`	`36`	`// Set the .NET CLI language to English to avoid localized output.`
`33`	`37`	`startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";`
`34`	`38`	`startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1";`
`35`	`39`	`startInfo.EnvironmentVariables["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "true";`
`36`	`40`	`return startInfo;`
`37`	`41`	`}`
`38`	`42`
`39`		`- private bool RunCommandAux(string args, out IList<string> output)`
	`43`	`+ private bool RunCommandAux(string args, string? workingDirectory, out IList<string> output)`
`40`	`44`	`{`
`41`		`- logger.LogInfo($"Running {Exec} {args}");`
`42`		`- var pi = MakeDotnetStartInfo(args);`
	`45`	`+ var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}";`
	`46`	`+ logger.LogInfo($"Running {Exec} {args}{dirLog}");`
	`47`	`+ var pi = MakeDotnetStartInfo(args, workingDirectory);`
`43`	`48`	`var threadId = Environment.CurrentManagedThreadId;`
`44`	`49`	`void onOut(string s) => logger.LogInfo(s, threadId);`
`45`	`50`	`void onError(string s) => logger.LogError(s, threadId);`
`46`	`51`	`var exitCode = pi.ReadOutput(out output, onOut, onError);`
`47`	`52`	`if (exitCode != 0)`
`48`	`53`	`{`
`49`		`- logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");`
	`54`	`+ logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}");`
`50`	`55`	`return false;`
`51`	`56`	`}`
`52`	`57`	`return true;`
`53`	`58`	`}`
`54`	`59`
`55`	`60`	`public bool RunCommand(string args) =>`
`56`		`- RunCommandAux(args, out _);`
	`61`	`+ RunCommandAux(args, null, out _);`
`57`	`62`
`58`	`63`	`public bool RunCommand(string args, out IList<string> output) =>`
`59`		`- RunCommandAux(args, out output);`
	`64`	`+ RunCommandAux(args, null, out output);`
	`65`	`+`
	`66`	`+ public bool RunCommand(string args, string? workingDirectory, out IList<string> output) =>`
	`67`	`+ RunCommandAux(args, workingDirectory, out output);`
`60`	`68`	`}`
`61`	`69`	`}`
Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,7 @@ public interface IDotNet`
`14`	`14`	`IList<string> GetListedSdks();`
`15`	`15`	`bool Exec(string execArgs);`
`16`	`16`	`IList<string> GetNugetFeeds(string nugetConfig);`
	`17`	`+ IList<string> GetNugetFeedsFromFolder(string folderPath);`
`17`	`18`	`}`
`18`	`19`
`19`	`20`	`public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);`
Original file line number	Diff line number	Diff line change
`@@ -19,5 +19,11 @@ internal interface IDotNetCliInvoker`
`19`	`19`	/// The output of the command is returned in `output`.
`20`	`20`	`/// </summary>`
`21`	`21`	`bool RunCommand(string args, out IList<string> output);`
	`22`	`+`
	`23`	`+ /// <summary>`
	`24`	+ /// Execute `dotnet <args>` in `<workingDirectory>` and return true if the command succeeded, otherwise false.
	`25`	+ /// The output of the command is returned in `output`.
	`26`	`+ /// </summary>`
	`27`	`+ bool RunCommand(string args, string? workingDirectory, out IList<string> output);`
`22`	`28`	`}`
`23`	`29`	`}`
Original file line number	Diff line number	Diff line change
`@@ -243,7 +243,7 @@ private void RunMonoNugetCommand(string command, out IList<string> stdout)`
`243`	`243`	`private void AddDefaultPackageSource(string nugetConfig)`
`244`	`244`	`{`
`245`	`245`	`logger.LogInfo("Adding default package source...");`
`246`		`- RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source https://api.nuget.org/v3/index.json -ConfigFile \"{nugetConfig}\"", out var _);`
	`246`	`+ RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {DependencyManager.PublicNugetFeed} -ConfigFile \"{nugetConfig}\"", out var _);`
`247`	`247`	`}`
`248`	`248`
`249`	`249`	`public void Dispose()`
Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,8 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)`
`28`	`28`	`public bool Exec(string execArgs) => true;`
`29`	`29`
`30`	`30`	`public IList<string> GetNugetFeeds(string nugetConfig) => [];`
	`31`	`+`
	`32`	`+ public IList<string> GetNugetFeedsFromFolder(string folderPath) => [];`
`31`	`33`	`}`
`32`	`34`
`33`	`35`	`public class RuntimeTests`