Merge pull request #15600 from tamasvajk/buildless/no-mono-dlls

C# Change desktop dotnet assembly lookup to fall back to nuget reference assemblies

Author: tamasvajk

csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs

@@ -75,7 +75,7 @@ public static string[] AsListWithExpandedEnvVars(this string? value, IBuildActio
                 return defaultValue;
 
             return value.
-                Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).
+                Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries).
                 Select(s => AsStringWithExpandedEnvVars(s, actions)).ToArray();
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -62,7 +62,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             try
             {
                 this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
-                runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
+                runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet, logger));
             }
             catch
             {
@@ -112,7 +112,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 logger.LogInfo($"Unresolved reference {r.Key} in project {r.Value}");
             }
 
-            var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS");
+            var webViewExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.WebViewGeneration);
             if (webViewExtractionOption == null ||
                 bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
                 shouldExtractWebViews)
@@ -159,6 +159,53 @@ private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
         {
             var frameworkLocations = new HashSet<string>();
 
+            var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);
+            var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
+            _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders);
+            if (!string.IsNullOrWhiteSpace(frameworkReferences))
+            {
+                RemoveFrameworkNugetPackages(dllPaths);
+                RemoveNugetPackageReference(FrameworkPackageNames.AspNetCoreFramework, dllPaths);
+                RemoveNugetPackageReference(FrameworkPackageNames.WindowsDesktopFramework, dllPaths);
+
+                var frameworkPaths = frameworkReferences.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries);
+
+                foreach (var path in frameworkPaths)
+                {
+                    if (!Directory.Exists(path))
+                    {
+                        logger.LogError($"Specified framework reference path '{path}' does not exist.");
+                        continue;
+                    }
+
+                    if (useSubfolders)
+                    {
+                        dllPaths.Add(path);
+                        frameworkLocations.Add(path);
+                        continue;
+                    }
+
+                    try
+                    {
+                        var dlls = Directory.GetFiles(path, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                        if (dlls.Length == 0)
+                        {
+                            logger.LogError($"No DLLs found in specified framework reference path '{path}'.");
+                            continue;
+                        }
+
+                        dllPaths.UnionWith(dlls);
+                        frameworkLocations.UnionWith(dlls);
+                    }
+                    catch (Exception e)
+                    {
+                        logger.LogError($"Error while searching for DLLs in '{path}': {e.Message}");
+                    }
+                }
+
+                return frameworkLocations;
+            }
+
             AddNetFrameworkDlls(dllPaths, frameworkLocations);
             AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations);
             AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations);
@@ -204,9 +251,9 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<
                 nugetPackageDllPaths.ExceptWith(excludedPaths);
                 dllPaths.UnionWith(nugetPackageDllPaths);
             }
-            catch (Exception)
+            catch (Exception exc)
             {
-                logger.LogError("Failed to restore Nuget packages with nuget.exe");
+                logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
             }
 
             var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
@@ -297,14 +344,23 @@ private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePat
                 .ToArray();
         }
 
+        private void RemoveFrameworkNugetPackages(ISet<string> dllPaths, int fromIndex = 0)
+        {
+            var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
+            for (var i = fromIndex; i < packagesInPrioOrder.Length; i++)
+            {
+                RemoveNugetPackageReference(packagesInPrioOrder[i], dllPaths);
+            }
+        }
+
         private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
         {
             // Multiple dotnet framework packages could be present.
             // The order of the packages is important, we're adding the first one that is present in the nuget cache.
             var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
 
             var frameworkPaths = packagesInPrioOrder
-                .Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
+                .Select((s, index) => (Index: index, Path: GetPackageDirectory(s, packageDirectory)))
                 .Where(pair => pair.Path is not null)
                 .ToArray();
 
@@ -318,12 +374,7 @@ private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLo
                 }
 
                 SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations);
-
-                for (var i = frameworkPath.Index + 1; i < packagesInPrioOrder.Length; i++)
-                {
-                    RemoveNugetPackageReference(packagesInPrioOrder[i], dllPaths);
-                }
-
+                RemoveFrameworkNugetPackages(dllPaths, frameworkPath.Index + 1);
                 return;
             }
 
@@ -336,6 +387,16 @@ private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLo
             else if (fileContent.IsLegacyProjectStructureUsed)
             {
                 runtimeLocation = Runtime.DesktopRuntime;
+
+                if (runtimeLocation is null)
+                {
+                    logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
+
+                    if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
+                    {
+                        runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
+                    }
+                }
             }
 
             runtimeLocation ??= Runtime.ExecutingRuntime;
@@ -375,7 +436,7 @@ private void AddAspNetCoreFrameworkDlls(ISet<string> dllPaths, ISet<string> fram
             }
 
             // First try to find ASP.NET Core assemblies in the NuGet packages
-            if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
+            if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage)
             {
                 SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllPaths, frameworkLocations);
                 return;
@@ -391,15 +452,15 @@ private void AddAspNetCoreFrameworkDlls(ISet<string> dllPaths, ISet<string> fram
 
         private void AddMicrosoftWindowsDesktopDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
         {
-            if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
+            if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp)
             {
                 SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllPaths, frameworkLocations);
             }
         }
 
-        private string? GetPackageDirectory(string packagePrefix)
+        private string? GetPackageDirectory(string packagePrefix, TemporaryDirectory root)
         {
-            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
+            return new DirectoryInfo(root.DirInfo.FullName)
                 .EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
                 .FirstOrDefault()?
                 .FullName;
@@ -435,19 +496,19 @@ private void GenerateSourceFileFromImplicitUsings()
             }
 
             // Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives
-            usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
-                "System.Threading.Tasks" });
+            usings.UnionWith([ "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
+                "System.Threading.Tasks" ]);
 
             if (fileContent.UseAspNetCoreDlls)
             {
-                usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
+                usings.UnionWith([ "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
                     "Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
-                    "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" });
+                    "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" ]);
             }
 
             if (fileContent.UseWindowsForms)
             {
-                usings.UnionWith(new[] { "System.Drawing", "System.Windows.Forms" });
+                usings.UnionWith(["System.Drawing", "System.Windows.Forms"]);
             }
 
             usings.UnionWith(fileContent.CustomImplicitUsings);
@@ -869,47 +930,58 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
 
             Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
             {
-                logger.LogInfo($"Restoring package {package}...");
-                using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
-                var success = dotnet.New(tempDir.DirInfo.FullName);
+                var success = TryRestorePackageManually(package, nugetConfig);
                 if (!success)
                 {
                     return;
                 }
 
-                success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
-                if (!success)
+                lock (sync)
                 {
-                    return;
+                    successCount++;
                 }
+            });
 
-                var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
-                if (!res.Success)
-                {
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
-                        res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
-                    }
+            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
 
-                    // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
+            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
+        }
 
-                    if (!res.Success)
-                    {
-                        logger.LogInfo($"Failed to restore nuget package {package}");
-                        return;
-                    }
-                }
+        private bool TryRestorePackageManually(string package, string? nugetConfig)
+        {
+            logger.LogInfo($"Restoring package {package}...");
+            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
+            var success = dotnet.New(tempDir.DirInfo.FullName);
+            if (!success)
+            {
+                return false;
+            }
 
-                lock (sync)
+            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
+            if (!success)
+            {
+                return false;
+            }
+
+            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
+            if (!res.Success)
+            {
+                if (res.HasNugetPackageSourceError && nugetConfig is not null)
                 {
-                    successCount++;
+                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
+                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
                 }
-            });
 
-            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
+                // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
 
-            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
+                if (!res.Success)
+                {
+                    logger.LogInfo($"Failed to restore nuget package {package}");
+                    return false;
+                }
+            }
+
+            return true;
         }
 
         public void Dispose(TemporaryDirectory? dir, string name)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -0,0 +1,20 @@
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    internal class EnvironmentVariableNames
+    {
+        /// <summary>
+        /// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`).
+        /// </summary>
+        public const string WebViewGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS";
+
+        /// <summary>
+        /// Specifies the location of .Net framework references added to the compilation.
+        /// </summary>
+        public const string DotnetFrameworkReferences = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES";
+
+        /// <summary>
+        /// Controls whether to use framework dependencies from subfolders.
+        /// </summary>
+        public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs

@@ -31,7 +31,7 @@ private record class PathFilter(Regex Regex, bool Include);
 
         public IEnumerable<FileInfo> Filter(IEnumerable<FileInfo> files)
         {
-            var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
+            var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries);
             if (filters.Length == 0)
             {
                 return files;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs

@@ -1,25 +1,25 @@
 using System.Collections.Generic;
-using System.Linq;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal static class FrameworkPackageNames
     {
+        public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481";
+
         public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";
 
         public static string WindowsDesktopFramework { get; } = "microsoft.windowsdesktop.app.ref";
 
         // The order of the packages is important.
-        public static string[] NetFrameworks { get; } = new string[]
-            {
+        public static string[] NetFrameworks { get; } =
+            [
                 "microsoft.netcore.app.ref", // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0
                 "microsoft.netframework.referenceassemblies.", // net48, ..., net20
                 "netstandard.library.ref", // netstandard2.1
                 "netstandard.library" // netstandard2.0
-            };
+            ];
 
         public static IEnumerable<string> AllFrameworks { get; } =
-            NetFrameworks
-                .Union(new string[] { AspNetCoreFramework, WindowsDesktopFramework });
+            [.. NetFrameworks, AspNetCoreFramework, WindowsDesktopFramework];
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs

@@ -71,7 +71,7 @@ public IEnumerable<string> GenerateFiles(IEnumerable<string> cshtmls, IEnumerabl
                 var args = new StringBuilder();
                 args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");
 
-                foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll"))
+                foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }))
                 {
                     args.Append($"/analyzer:\"{f}\" ");
                 }