C#: Check fallback nuget feeds before trying to use them in the fallback restore process

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Net.Http;
+using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -14,12 +15,13 @@ public sealed partial class DependencyManager
     {
         private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<AssemblyLookupLocation> dllLocations)
         {
+            var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
             try
             {
-                var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
                 if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
                 {
-                    DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false);
+                    // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
+                    DownloadMissingPackagesFromSpecificFeeds(allNonBinaryFiles, dllLocations);
                     return;
                 }
 
@@ -75,7 +77,35 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<
             dllLocations.UnionWith(paths.Select(p => new AssemblyLookupLocation(p)));
 
             LogAllUnusedPackages(dependencies);
-            DownloadMissingPackages(allNonBinaryFiles, dllLocations);
+
+            if (checkNugetFeedResponsiveness)
+            {
+                DownloadMissingPackagesFromSpecificFeeds(allNonBinaryFiles, dllLocations);
+            }
+            else
+            {
+                DownloadMissingPackages(allNonBinaryFiles, dllLocations);
+            }
+        }
+
+        internal const string PublicNugetFeed = "https://api.nuget.org/v3/index.json";
+
+        private List<string> GetReachableFallbackNugetFeeds()
+        {
+            var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
+            if (fallbackFeeds.Count == 0)
+            {
+                fallbackFeeds.Add(PublicNugetFeed);
+            }
+
+            logger.LogInfo("Checking fallback Nuget feed reachability");
+            var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed)).ToList();
+            if (reachableFallbackFeeds.Count == 0)
+            {
+                logger.LogWarning("No fallback Nuget feeds are reachable. Skipping fallback Nuget package restoration.");
+            }
+
+            return reachableFallbackFeeds;
         }
 
         /// <summary>
@@ -148,7 +178,16 @@ private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<strin
             CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
         }
 
-        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<AssemblyLookupLocation> dllLocations, bool withNugetConfig = true)
+        private void DownloadMissingPackagesFromSpecificFeeds(List<FileInfo> allNonBinaryFiles, HashSet<AssemblyLookupLocation> dllLocations)
+        {
+            var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
+            if (reachableFallbackFeeds.Count > 0)
+            {
+                DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false, fallbackNugetFeeds: reachableFallbackFeeds);
+            }
+        }
+
+        private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLookupLocation> dllLocations, bool withNugetConfig = true, IEnumerable<string>? fallbackNugetFeeds = null)
         {
             var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
             var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -181,9 +220,10 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<AssemblyLooku
             }
 
             logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
+            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "nugetconfig"));
             var nugetConfig = withNugetConfig
                 ? GetNugetConfig(allFiles)
-                : null;
+                : CreateFallbackNugetConfig(fallbackNugetFeeds, tempDir.DirInfo.FullName);
 
             CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
 
@@ -209,6 +249,33 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<AssemblyLooku
             dllLocations.Add(missingPackageDirectory.DirInfo.FullName);
         }
 
+        private string? CreateFallbackNugetConfig(IEnumerable<string>? fallbackNugetFeeds, string folderPath)
+        {
+            if (fallbackNugetFeeds is null)
+            {
+                // We're not overriding the inherited Nuget feeds
+                return null;
+            }
+
+            var sb = new StringBuilder();
+            fallbackNugetFeeds.ForEach((feed, index) => sb.AppendLine($"<add key=\"feed{index}\" value=\"{feed}\" />"));
+
+            var nugetConfigPath = Path.Combine(folderPath, "nuget.config");
+            logger.LogInfo($"Creating fallback nuget.config file {nugetConfigPath}.");
+            File.WriteAllText(nugetConfigPath,
+                $"""
+                <?xml version="1.0" encoding="utf-8"?>
+                <configuration>
+                    <packageSources>
+                        <clear />
+                {sb}
+                    </packageSources>
+                </configuration>
+                """);
+
+            return nugetConfigPath;
+        }
+
         private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();
 
         private string? GetNugetConfig(List<FileInfo> allFiles)
@@ -429,18 +496,18 @@ private bool IsFeedReachable(string feed)
         private bool CheckFeeds(List<FileInfo> allFiles)
         {
             logger.LogInfo("Checking Nuget feeds...");
-            var feeds = GetAllFeeds(allFiles);
+            var (explicitFeeds, allFeeds) = GetAllFeeds(allFiles);
+            var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet();
 
-            var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
-                ?.Split(" ", StringSplitOptions.RemoveEmptyEntries)
+            var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
                 .ToHashSet() ?? [];
 
             if (excludedFeeds.Count > 0)
             {
                 logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
             }
 
-            var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
+            var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
             if (!allFeedsReachable)
             {
                 logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
@@ -454,13 +521,19 @@ private bool CheckFeeds(List<FileInfo> allFiles)
                 ));
             }
             CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
+
+            if (inheritedFeeds.Count > 0)
+            {
+                logger.LogInfo($"Inherited Nuget feeds: {string.Join(", ", inheritedFeeds.OrderBy(f => f))}");
+                CompilationInfos.Add(("Inherited Nuget feed count", inheritedFeeds.Count.ToString()));
+            }
+
             return allFeedsReachable;
         }
 
-        private IEnumerable<string> GetFeeds(string nugetConfig)
+        private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
         {
-            logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
-            var results = dotnet.GetNugetFeeds(nugetConfig);
+            var results = getNugetFeeds();
             var regex = EnabledNugetFeed();
             foreach (var result in results)
             {
@@ -479,27 +552,63 @@ private IEnumerable<string> GetFeeds(string nugetConfig)
                     continue;
                 }
 
-                yield return url;
+                if (!string.IsNullOrWhiteSpace(url))
+                {
+                    yield return url;
+                }
             }
         }
 
-        private HashSet<string> GetAllFeeds(List<FileInfo> allFiles)
+        private (HashSet<string>, HashSet<string>) GetAllFeeds(List<FileInfo> allFiles)
         {
+            IList<string> GetNugetFeeds(string nugetConfig)
+            {
+                logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
+                return dotnet.GetNugetFeeds(nugetConfig);
+            }
+
+            IList<string> GetNugetFeedsFromFolder(string folderPath)
+            {
+                logger.LogInfo($"Getting Nuget feeds in folder '{folderPath}'...");
+                return dotnet.GetNugetFeedsFromFolder(folderPath);
+            }
+
             var nugetConfigs = GetAllNugetConfigs(allFiles);
-            var feeds = nugetConfigs
-                .SelectMany(GetFeeds)
-                .Where(str => !string.IsNullOrWhiteSpace(str))
+            var explicitFeeds = nugetConfigs
+                .SelectMany(config => GetFeeds(() => GetNugetFeeds(config)))
                 .ToHashSet();
 
-            if (feeds.Count > 0)
+            if (explicitFeeds.Count > 0)
             {
-                logger.LogInfo($"Found {feeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}");
+                logger.LogInfo($"Found {explicitFeeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", explicitFeeds.OrderBy(f => f))}");
             }
             else
             {
                 logger.LogDebug("No Nuget feeds found in nuget.config files.");
             }
-            return feeds;
+
+            // todo: this could be improved.
+            // We don't have to get the feeds from each of the folders from below, it would be enought to check the folders that recursively contain the others.
+            var allFeeds = nugetConfigs
+                .Select(config =>
+                {
+                    try
+                    {
+                        return new FileInfo(config).Directory?.FullName;
+                    }
+                    catch (Exception exc)
+                    {
+                        logger.LogWarning($"Failed to get directory of '{config}': {exc}");
+                    }
+                    return null;
+                })
+                .Where(folder => folder != null)
+                .SelectMany(folder => GetFeeds(() => GetNugetFeedsFromFolder(folder!)))
+                .ToHashSet();
+
+            logger.LogInfo($"Found {allFeeds.Count} Nuget feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}");
+
+            return (explicitFeeds, allFeeds);
         }
 
         [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -95,9 +95,9 @@ public bool AddPackage(string folder, string package)
 
         public IList<string> GetListedSdks() => GetResultList("--list-sdks");
 
-        private IList<string> GetResultList(string args)
+        private IList<string> GetResultList(string args, string? workingDirectory = null)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var results))
+            if (dotnetCliInvoker.RunCommand(args, workingDirectory, out var results))
             {
                 return results;
             }
@@ -111,7 +111,11 @@ public bool Exec(string execArgs)
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
+        private const string nugetListSourceCommand = "nuget list source --format Short";
+
+        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"{nugetListSourceCommand} --configfile \"{nugetConfig}\"");
+
+        public IList<string> GetNugetFeedsFromFolder(string folderPath) => GetResultList(nugetListSourceCommand, folderPath);
 
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "8.0.101";

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs

@@ -21,41 +21,49 @@ public DotNetCliInvoker(ILogger logger, string exec)
             this.Exec = exec;
         }
 
-        private ProcessStartInfo MakeDotnetStartInfo(string args)
+        private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory)
         {
             var startInfo = new ProcessStartInfo(Exec, args)
             {
                 UseShellExecute = false,
                 RedirectStandardOutput = true,
                 RedirectStandardError = true
             };
+            if (!string.IsNullOrWhiteSpace(workingDirectory))
+            {
+                startInfo.WorkingDirectory = workingDirectory;
+            }
             // Set the .NET CLI language to English to avoid localized output.
             startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
             startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1";
             startInfo.EnvironmentVariables["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "true";
             return startInfo;
         }
 
-        private bool RunCommandAux(string args, out IList<string> output)
+        private bool RunCommandAux(string args, string? workingDirectory, out IList<string> output)
         {
-            logger.LogInfo($"Running {Exec} {args}");
-            var pi = MakeDotnetStartInfo(args);
+            var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}";
+            logger.LogInfo($"Running {Exec} {args}{dirLog}");
+            var pi = MakeDotnetStartInfo(args, workingDirectory);
             var threadId = Environment.CurrentManagedThreadId;
             void onOut(string s) => logger.LogInfo(s, threadId);
             void onError(string s) => logger.LogError(s, threadId);
             var exitCode = pi.ReadOutput(out output, onOut, onError);
             if (exitCode != 0)
             {
-                logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
+                logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}");
                 return false;
             }
             return true;
         }
 
         public bool RunCommand(string args) =>
-            RunCommandAux(args, out _);
+            RunCommandAux(args, null, out _);
 
         public bool RunCommand(string args, out IList<string> output) =>
-            RunCommandAux(args, out output);
+            RunCommandAux(args, null, out output);
+
+        public bool RunCommand(string args, string? workingDirectory, out IList<string> output) =>
+            RunCommandAux(args, workingDirectory, out output);
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -37,6 +37,12 @@ internal class EnvironmentVariableNames
         /// </summary>
         public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
 
+        /// <summary>
+        /// Specifies the NuGet feeds to use for fallback Nuget dependency fetching. The value is a space-separated list of feed URLs.
+        /// The default value is `https://api.nuget.org/v3/index.json`.
+        /// </summary>
+        public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
+
         /// <summary>
         /// Specifies the location of the diagnostic directory.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -14,6 +14,7 @@ public interface IDotNet
         IList<string> GetListedSdks();
         bool Exec(string execArgs);
         IList<string> GetNugetFeeds(string nugetConfig);
+        IList<string> GetNugetFeedsFromFolder(string folderPath);
     }
 
     public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs

@@ -19,5 +19,11 @@ internal interface IDotNetCliInvoker
         /// The output of the command is returned in `output`.
         /// </summary>
         bool RunCommand(string args, out IList<string> output);
+
+        /// <summary>
+        /// Execute `dotnet <args>` in `<workingDirectory>` and return true if the command succeeded, otherwise false.
+        /// The output of the command is returned in `output`.
+        /// </summary>
+        bool RunCommand(string args, string? workingDirectory, out IList<string> output);
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs

@@ -243,7 +243,7 @@ private void RunMonoNugetCommand(string command, out IList<string> stdout)
         private void AddDefaultPackageSource(string nugetConfig)
         {
             logger.LogInfo("Adding default package source...");
-            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source https://api.nuget.org/v3/index.json -ConfigFile \"{nugetConfig}\"", out var _);
+            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {DependencyManager.PublicNugetFeed} -ConfigFile \"{nugetConfig}\"", out var _);
         }
 
         public void Dispose()

csharp/extractor/Semmle.Extraction.Tests/DotNet.cs

@@ -10,6 +10,7 @@ internal class DotNetCliInvokerStub : IDotNetCliInvoker
     {
         private readonly IList<string> output;
         private string lastArgs = "";
+        public string WorkingDirectory { get; private set; } = "";
         public bool Success { get; set; } = true;
 
         public DotNetCliInvokerStub(IList<string> output)
@@ -32,6 +33,12 @@ public bool RunCommand(string args, out IList<string> output)
             return Success;
         }
 
+        public bool RunCommand(string args, string? workingDirectory, out IList<string> output)
+        {
+            WorkingDirectory = workingDirectory ?? "";
+            return RunCommand(args, out output);
+        }
+
         public string GetLastArgs() => lastArgs;
     }
 

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -28,6 +28,8 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
         public bool Exec(string execArgs) => true;
 
         public IList<string> GetNugetFeeds(string nugetConfig) => [];
+
+        public IList<string> GetNugetFeedsFromFolder(string folderPath) => [];
     }
 
     public class RuntimeTests