Merge pull request #16225 from tamasvajk/buildless/resx

C#: Add resource generator

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -152,7 +152,8 @@ void exitCallback(int ret, string msg, bool silent)
             var sourceGenerators = new ISourceGenerator[]
             {
                 new ImplicitUsingsGenerator(fileContent, logger, tempWorkingDirectory),
-                new WebViewGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys)
+                new RazorGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
+                new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, nugetPackageRestorer, tempWorkingDirectory, usedReferences.Keys),
             };
 
             foreach (var sourceGenerator in sourceGenerators)
@@ -255,35 +256,6 @@ private void RemoveNugetAnalyzerReferences()
             }
         }
 
-        private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet<AssemblyLookupLocation> dllLocations, ISet<string> frameworkLocations)
-        {
-            var versionFolders = GetPackageVersionSubDirectories(frameworkPath);
-            if (versionFolders.Length > 1)
-            {
-                var versions = string.Join(", ", versionFolders.Select(d => d.Name));
-                logger.LogDebug($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
-            }
-
-            var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
-            if (selectedFrameworkFolder is null)
-            {
-                logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
-                selectedFrameworkFolder = frameworkPath;
-            }
-
-            dllLocations.Add(selectedFrameworkFolder);
-            frameworkLocations.Add(selectedFrameworkFolder);
-            logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
-        }
-
-        private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath)
-        {
-            return new DirectoryInfo(packagePath)
-                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
-                .OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
-                .ToArray();
-        }
-
         private void RemoveFrameworkNugetPackages(ISet<AssemblyLookupLocation> dllLocations, int fromIndex = 0)
         {
             var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
@@ -310,10 +282,12 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
             {
                 foreach (var fp in frameworkPaths)
                 {
-                    dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length;
+                    dotnetFrameworkVersionVariantCount += NugetPackageRestorer.GetOrderedPackageVersionSubDirectories(fp.Path!).Length;
                 }
 
-                SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllLocations, frameworkLocations);
+                var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(frameworkPath.Path, ".NET Framework");
+                dllLocations.Add(folder);
+                frameworkLocations.Add(folder);
                 RemoveFrameworkNugetPackages(dllLocations, frameworkPath.Index + 1);
                 return;
             }
@@ -331,7 +305,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
                 if (runtimeLocation is null)
                 {
                     logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
-                    runtimeLocation = nugetPackageRestorer.TryRestoreLatestNetFrameworkReferenceAssemblies();
+                    runtimeLocation = nugetPackageRestorer.TryRestore(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies);
                 }
             }
 
@@ -371,7 +345,9 @@ private void AddAspNetCoreFrameworkDlls(ISet<AssemblyLookupLocation> dllLocation
             // First try to find ASP.NET Core assemblies in the NuGet packages
             if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
             {
-                SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations);
+                var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(aspNetCorePackage, "ASP.NET Core");
+                dllLocations.Add(folder);
+                frameworkLocations.Add(folder);
                 return;
             }
 
@@ -387,7 +363,9 @@ private void AddMicrosoftWindowsDesktopDlls(ISet<AssemblyLookupLocation> dllLoca
         {
             if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
             {
-                SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations);
+                var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(windowsDesktopApp, "Windows Desktop App");
+                dllLocations.Add(folder);
+                frameworkLocations.Add(folder);
             }
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -2,6 +2,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal class EnvironmentVariableNames
     {
+        /// <summary>
+        /// Controls whether to generate source files from resources (`.resx`).
+        /// </summary>
+        public const string ResourceGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_RESOURCES";
+
         /// <summary>
         /// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`).
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs

@@ -22,6 +22,7 @@ public class FileProvider
         private readonly Lazy<string[]> nugetConfigs;
         private readonly Lazy<string[]> globalJsons;
         private readonly Lazy<string[]> razorViews;
+        private readonly Lazy<string[]> resources;
         private readonly Lazy<string?> rootNugetConfig;
 
         public FileProvider(DirectoryInfo sourceDir, ILogger logger)
@@ -44,6 +45,7 @@ public FileProvider(DirectoryInfo sourceDir, ILogger logger)
             nugetConfigs = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("nuget.config").ToArray());
             globalJsons = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("global.json").ToArray());
             razorViews = new Lazy<string[]>(() => SelectTextFileNamesByExtension("razor view", ".cshtml", ".razor"));
+            resources = new Lazy<string[]>(() => SelectTextFileNamesByExtension("resource", ".resx"));
 
             rootNugetConfig = new Lazy<string?>(() => all.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault());
         }
@@ -116,5 +118,6 @@ private FileInfo[] GetAllFiles()
         public string? RootNugetConfig => rootNugetConfig.Value;
         public IEnumerable<string> GlobalJsons => globalJsons.Value;
         public ICollection<string> RazorViews => razorViews.Value;
+        public ICollection<string> Resources => resources.Value;
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -30,6 +30,9 @@ public partial record class RestoreResult(bool Success, IList<string> Output)
         private readonly Lazy<bool> hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
         public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
 
+        private readonly Lazy<bool> hasNugetNoStablePackageVersionError = new(() => Output.Any(s => s.Contains("NU1103")));
+        public bool HasNugetNoStablePackageVersionError => hasNugetNoStablePackageVersionError.Value;
+
         private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
             lines
                 .Select(line => regex.Match(line))

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -48,16 +48,48 @@ public NugetPackageRestorer(
             missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "missingpackages"), "missing package", logger);
         }
 
-        public string? TryRestoreLatestNetFrameworkReferenceAssemblies()
+        public string? TryRestore(string package)
         {
-            if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
+            if (TryRestorePackageManually(package))
             {
-                return DependencyManager.GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory.DirInfo);
+                var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo);
+                if (packageDir is not null)
+                {
+                    return GetNewestNugetPackageVersionFolder(packageDir, package);
+                }
             }
 
             return null;
         }
 
+        public string GetNewestNugetPackageVersionFolder(string packagePath, string packageFriendlyName)
+        {
+            var versionFolders = GetOrderedPackageVersionSubDirectories(packagePath);
+            if (versionFolders.Length > 1)
+            {
+                var versions = string.Join(", ", versionFolders.Select(d => d.Name));
+                logger.LogDebug($"Found multiple {packageFriendlyName} DLLs in NuGet packages at {packagePath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
+            }
+
+            var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
+            if (selectedFrameworkFolder is null)
+            {
+                logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {packagePath}, but no version folder was found.");
+                selectedFrameworkFolder = packagePath;
+            }
+
+            logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {selectedFrameworkFolder}.");
+            return selectedFrameworkFolder;
+        }
+
+        public static DirectoryInfo[] GetOrderedPackageVersionSubDirectories(string packagePath)
+        {
+            return new DirectoryInfo(packagePath)
+                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
+                .OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
+                .ToArray();
+        }
+
         public HashSet<AssemblyLookupLocation> Restore()
         {
             var assemblyLookupLocations = new HashSet<AssemblyLookupLocation>();
@@ -408,7 +440,8 @@ private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInf
                 .Select(d => Path.GetFileName(d).ToLowerInvariant());
         }
 
-        private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryWithoutNugetConfig = true)
+        private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj,
+            bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true)
         {
             logger.LogInfo($"Restoring package {package}...");
             using var tempDir = new TemporaryDirectory(
@@ -430,59 +463,87 @@ private bool TryRestorePackageManually(string package, string? nugetConfig = nul
                 return false;
             }
 
-            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
-            if (!res.Success)
+            var res = TryRestorePackageManually(package, nugetConfig, tempDir, tryPrereleaseVersion);
+            if (res.Success)
+            {
+                return true;
+            }
+
+            if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
             {
-                if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
+                logger.LogDebug($"Trying to restore '{package}' without nuget.config.");
+                // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
+                res = TryRestorePackageManually(package, nugetConfig: null, tempDir, tryPrereleaseVersion);
+                if (res.Success)
                 {
-                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
-                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
+                    return true;
                 }
+            }
 
-                // TODO: the restore might fail, we could retry with
-                // - a prerelease (*-* instead of *) version of the package,
-                // - a different target framework moniker.
+            logger.LogInfo($"Failed to restore nuget package {package}");
+            return false;
+        }
+
+        private RestoreResult TryRestorePackageManually(string package, string? nugetConfig, TemporaryDirectory tempDir, bool tryPrereleaseVersion)
+        {
+            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
+
+            if (!res.Success && tryPrereleaseVersion && res.HasNugetNoStablePackageVersionError)
+            {
+                logger.LogDebug($"Failed to restore nuget package {package} because no stable version was found.");
+                TryChangePackageVersion(tempDir.DirInfo, "*-*");
 
+                res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
                 if (!res.Success)
                 {
-                    logger.LogInfo($"Failed to restore nuget package {package}");
-                    return false;
+                    TryChangePackageVersion(tempDir.DirInfo, "*");
                 }
             }
 
-            return true;
+            return res;
         }
 
         private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
+        {
+            TryChangeProjectFile(tempDir, TargetFramework(), $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", "target framework moniker");
+        }
+
+        private void TryChangePackageVersion(DirectoryInfo tempDir, string newVersion)
+        {
+            TryChangeProjectFile(tempDir, PackageReferenceVersion(), $"Version=\"{newVersion}\"", "package reference version");
+        }
+
+        private bool TryChangeProjectFile(DirectoryInfo projectDir, Regex pattern, string replacement, string patternName)
         {
             try
             {
-                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
+                logger.LogDebug($"Changing the {patternName} in {projectDir.FullName}...");
 
-                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                var csprojs = projectDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
                 if (csprojs.Length != 1)
                 {
-                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
-                    return;
+                    logger.LogError($"Could not find the .csproj file in {projectDir.FullName}, count = {csprojs.Length}");
+                    return false;
                 }
 
                 var csproj = csprojs[0];
                 var content = File.ReadAllText(csproj.FullName);
-                var matches = TargetFramework().Matches(content);
+                var matches = pattern.Matches(content);
                 if (matches.Count == 0)
                 {
-                    logger.LogError($"Could not find target framework in {csproj.FullName}");
-                }
-                else
-                {
-                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
-                    File.WriteAllText(csproj.FullName, content);
+                    logger.LogError($"Could not find the {patternName} in {csproj.FullName}");
+                    return false;
                 }
+
+                content = pattern.Replace(content, replacement, 1);
+                File.WriteAllText(csproj.FullName, content);
+                return true;
             }
             catch (Exception exc)
             {
-                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
+                logger.LogError($"Failed to change the {patternName} in {projectDir.FullName}: {exc}");
             }
+            return false;
         }
 
         private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
@@ -664,6 +725,9 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
         [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
         private static partial Regex TargetFramework();
 
+        [GeneratedRegex(@"Version=""(\*|\*-\*)""", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex PackageReferenceVersion();
+
         [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
         private static partial Regex LegacyNugetPackage();