Introduce environment variable to specify framework assembly locations

tamasvajk · tamasvajk · commit f8b29ad70e05 · 2024-02-16T11:54:19.000+01:00
diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs
@@ -75,7 +75,7 @@ public static string[] AsListWithExpandedEnvVars(this string? value, IBuildActio
                 return defaultValue;
 
             return value.
-                Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).
+                Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries).
                 Select(s => AsStringWithExpandedEnvVars(s, actions)).ToArray();
         }
 
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
@@ -158,6 +158,49 @@ private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
         {
             var frameworkLocations = new HashSet<string>();
 
+            var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);
+            var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
+            _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders);
+            if (!string.IsNullOrWhiteSpace(frameworkReferences))
+            {
+                var frameworkPaths = frameworkReferences.Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries);
+
+                foreach (var path in frameworkPaths)
+                {
+                    if (!Directory.Exists(path))
+                    {
+                        logger.LogError($"Specified framework reference path '{path}' does not exist.");
+                        continue;
+                    }
+
+                    if (useSubfolders)
+                    {
+                        dllPaths.Add(path);
+                        frameworkLocations.Add(path);
+                        continue;
+                    }
+
+                    try
+                    {
+                        var dlls = Directory.GetFiles(path, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                        if (dlls.Length == 0)
+                        {
+                            logger.LogError($"No DLLs found in specified framework reference path '{path}'.");
+                            continue;
+                        }
+
+                        dllPaths.UnionWith(dlls);
+                        frameworkLocations.UnionWith(dlls);
+                    }
+                    catch (Exception e)
+                    {
+                        logger.LogError($"Error while searching for DLLs in '{path}': {e.Message}");
+                    }
+                }
+
+                return frameworkLocations;
+            }
+
             AddNetFrameworkDlls(dllPaths, frameworkLocations);
             AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations);
             AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations);
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
@@ -2,7 +2,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal class EnvironmentVariableNames
     {
+        /// <summary>
+        /// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`).
+        /// </summary>
         public const string WebViewGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS";
-        public const string MonoPath = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH";
+
+        /// <summary>
+        /// Specifies the location of .Net framework references added to the compilation.
+        /// </summary>
+        public const string DotnetFrameworkReferences = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES";
+
+        /// <summary>
+        /// Controls whether to use framework dependencies from subfolders.
+        /// </summary>
+        public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
     }
 }
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs
@@ -31,7 +31,7 @@ private record class PathFilter(Regex Regex, bool Include);
 
         public IEnumerable<FileInfo> Filter(IEnumerable<FileInfo> files)
         {
-            var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
+            var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries);
             if (filters.Length == 0)
             {
                 return files;
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs
@@ -71,7 +71,7 @@ public IEnumerable<string> GenerateFiles(IEnumerable<string> cshtmls, IEnumerabl
                 var args = new StringBuilder();
                 args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");
 
-                foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll"))
+                foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }))
                 {
                     args.Append($"/analyzer:\"{f}\" ");
                 }
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs
@@ -78,30 +78,12 @@ private IEnumerable<string> DesktopRuntimes
                         .OrderByDescending(Path.GetFileName);
                 }
 
-                string? monoDir = null;
-
-                var monoPathEnv = Environment.GetEnvironmentVariable(EnvironmentVariableNames.MonoPath);
-                if (monoPathEnv is not null)
-                {
-                    if (Directory.Exists(monoPathEnv))
-                    {
-                        monoDir = monoPathEnv;
-                    }
-                    else
-                    {
-                        logger.LogError($"The directory specified in {EnvironmentVariableNames.MonoPath} does not exist: {monoPathEnv}");
-                    }
-                }
-                else
-                {
-                    var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono");
-                    string[] monoDirs = monoPath is not null
-                        ? [Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath]
-                        : ["/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono"];
-
-                    monoDir = monoDirs.FirstOrDefault(Directory.Exists);
-                }
+                var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono");
+                string[] monoDirs = monoPath is not null
+                    ? [Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath]
+                    : ["/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono"];
 
+                var monoDir = monoDirs.FirstOrDefault(Directory.Exists);
                 if (monoDir is not null)
                 {
                     return Directory.EnumerateDirectories(monoDir)
diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs
@@ -386,7 +386,7 @@ private static ExitCode AnalyseTracing(
 
                     if (compilerArguments.GeneratedFilesOutputDirectory is not null)
                     {
-                        paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", SearchOption.AllDirectories));
+                        paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive }));
                     }
 
                     return ReadSyntaxTrees(
diff --git a/csharp/extractor/Semmle.Extraction/CsProjFile.cs b/csharp/extractor/Semmle.Extraction/CsProjFile.cs
@@ -112,7 +112,7 @@ private static (string[] csFiles, string[] references, string[] projectReference
                     .Where(s => s is not null)
                     ?? Enumerable.Empty<string>();
 
-                var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", SearchOption.AllDirectories);
+                var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive });
 
                 var projectReferences = root
                     .SelectNodes("/Project/ItemGroup/ProjectReference/@Include", mgr)
diff --git a/csharp/extractor/Semmle.Util/FileUtils.cs b/csharp/extractor/Semmle.Util/FileUtils.cs
@@ -13,6 +13,8 @@ public static class FileUtils
     {
         public const string NugetExeUrl = "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe";
 
+        public static readonly char[] NewLineCharacters = ['\r', '\n'];
+
         public static string ConvertToWindows(string path)
         {
             return path.Replace('/', '\\');
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_mono/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_mono/test.py
@@ -1,5 +1,5 @@
 from create_database_utils import *
 import os
 
-os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH"] = "/non-existent-path"
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES"] = "/non-existent-path"
 run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])

Original file line number	Diff line number	Diff line change
`@@ -75,7 +75,7 @@ public static string[] AsListWithExpandedEnvVars(this string? value, IBuildActio`
`75`	`75`	`return defaultValue;`
`76`	`76`
`77`	`77`	`return value.`
`78`		`- Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).`
	`78`	`+ Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries).`
`79`	`79`	`Select(s => AsStringWithExpandedEnvVars(s, actions)).ToArray();`
`80`	`80`	`}`
`81`	`81`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching`
`2`	`2`	`{`
`3`	`3`	`internal class EnvironmentVariableNames`
`4`	`4`	`{`
	`5`	`+ /// <summary>`
	`6`	+ /// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`).
	`7`	`+ /// </summary>`
`5`	`8`	`public const string WebViewGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS";`
`6`		`- public const string MonoPath = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH";`
	`9`	`+`
	`10`	`+ /// <summary>`
	`11`	`+ /// Specifies the location of .Net framework references added to the compilation.`
	`12`	`+ /// </summary>`
	`13`	`+ public const string DotnetFrameworkReferences = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES";`
	`14`	`+`
	`15`	`+ /// <summary>`
	`16`	`+ /// Controls whether to use framework dependencies from subfolders.`
	`17`	`+ /// </summary>`
	`18`	`+ public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";`
`7`	`19`	`}`
`8`	`20`	`}`
Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ private record class PathFilter(Regex Regex, bool Include);`
`31`	`31`
`32`	`32`	`public IEnumerable<FileInfo> Filter(IEnumerable<FileInfo> files)`
`33`	`33`	`{`
`34`		`- var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);`
	`34`	`+ var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries);`
`35`	`35`	`if (filters.Length == 0)`
`36`	`36`	`{`
`37`	`37`	`return files;`
Original file line number	Diff line number	Diff line change
`@@ -71,7 +71,7 @@ public IEnumerable<string> GenerateFiles(IEnumerable<string> cshtmls, IEnumerabl`
`71`	`71`	`var args = new StringBuilder();`
`72`	`72`	`args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");`
`73`	`73`
`74`		`- foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll"))`
	`74`	`+ foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }))`
`75`	`75`	`{`
`76`	`76`	`args.Append($"/analyzer:\"{f}\" ");`
`77`	`77`	`}`
Original file line number	Diff line number	Diff line change
`@@ -386,7 +386,7 @@ private static ExitCode AnalyseTracing(`
`386`	`386`
`387`	`387`	`if (compilerArguments.GeneratedFilesOutputDirectory is not null)`
`388`	`388`	`{`
`389`		`- paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", SearchOption.AllDirectories));`
	`389`	`+ paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive }));`
`390`	`390`	`}`
`391`	`391`
`392`	`392`	`return ReadSyntaxTrees(`
Original file line number	Diff line number	Diff line change
`@@ -13,6 +13,8 @@ public static class FileUtils`
`13`	`13`	`{`
`14`	`14`	`public const string NugetExeUrl = "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe";`
`15`	`15`
	`16`	`+ public static readonly char[] NewLineCharacters = ['\r', '\n'];`
	`17`	`+`
`16`	`18`	`public static string ConvertToWindows(string path)`
`17`	`19`	`{`
`18`	`20`	`return path.Replace('/', '\\');`