Restore and use Microsoft.CodeAnalysis.ResxSourceGenerator

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -153,7 +153,7 @@ void exitCallback(int ret, string msg, bool silent)
             {
                 new ImplicitUsingsGenerator(fileContent, logger, tempWorkingDirectory),
                 new RazorGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
-                new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
+                new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, nugetPackageRestorer, tempWorkingDirectory, usedReferences.Keys),
             };
 
             foreach (var sourceGenerator in sourceGenerators)
@@ -256,35 +256,6 @@ private void RemoveNugetAnalyzerReferences()
             }
         }
 
-        private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet<AssemblyLookupLocation> dllLocations, ISet<string> frameworkLocations)
-        {
-            var versionFolders = GetPackageVersionSubDirectories(frameworkPath);
-            if (versionFolders.Length > 1)
-            {
-                var versions = string.Join(", ", versionFolders.Select(d => d.Name));
-                logger.LogDebug($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
-            }
-
-            var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
-            if (selectedFrameworkFolder is null)
-            {
-                logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
-                selectedFrameworkFolder = frameworkPath;
-            }
-
-            dllLocations.Add(selectedFrameworkFolder);
-            frameworkLocations.Add(selectedFrameworkFolder);
-            logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
-        }
-
-        private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath)
-        {
-            return new DirectoryInfo(packagePath)
-                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
-                .OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
-                .ToArray();
-        }
-
         private void RemoveFrameworkNugetPackages(ISet<AssemblyLookupLocation> dllLocations, int fromIndex = 0)
         {
             var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
@@ -311,10 +282,12 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
             {
                 foreach (var fp in frameworkPaths)
                 {
-                    dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length;
+                    dotnetFrameworkVersionVariantCount += NugetPackageRestorer.GetOrderedPackageVersionSubDirectories(fp.Path!).Length;
                 }
 
-                SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllLocations, frameworkLocations);
+                var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(frameworkPath.Path, ".NET Framework");
+                dllLocations.Add(folder);
+                frameworkLocations.Add(folder);
                 RemoveFrameworkNugetPackages(dllLocations, frameworkPath.Index + 1);
                 return;
             }
@@ -332,7 +305,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
                 if (runtimeLocation is null)
                 {
                     logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
-                    runtimeLocation = nugetPackageRestorer.TryRestoreLatestNetFrameworkReferenceAssemblies();
+                    runtimeLocation = nugetPackageRestorer.TryRestore(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies);
                 }
             }
 
@@ -372,7 +345,9 @@ private void AddAspNetCoreFrameworkDlls(ISet<AssemblyLookupLocation> dllLocation
             // First try to find ASP.NET Core assemblies in the NuGet packages
             if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
             {
-                SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations);
+                var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(aspNetCorePackage, "ASP.NET Core");
+                dllLocations.Add(folder);
+                frameworkLocations.Add(folder);
                 return;
             }
 
@@ -388,7 +363,9 @@ private void AddMicrosoftWindowsDesktopDlls(ISet<AssemblyLookupLocation> dllLoca
         {
             if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
             {
-                SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations);
+                var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(windowsDesktopApp, "Windows Desktop App");
+                dllLocations.Add(folder);
+                frameworkLocations.Add(folder);
             }
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -30,6 +30,9 @@ public partial record class RestoreResult(bool Success, IList<string> Output)
         private readonly Lazy<bool> hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
         public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
 
+        private readonly Lazy<bool> hasNugetNoStablePackageVersionError = new(() => Output.Any(s => s.Contains("NU1103")));
+        public bool HasNugetNoStablePackageVersionError => hasNugetNoStablePackageVersionError.Value;
+
         private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
             lines
                 .Select(line => regex.Match(line))

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -48,16 +48,48 @@ public NugetPackageRestorer(
             missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "missingpackages"), "missing package", logger);
         }
 
-        public string? TryRestoreLatestNetFrameworkReferenceAssemblies()
+        public string? TryRestore(string package)
         {
-            if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
+            if (TryRestorePackageManually(package))
             {
-                return DependencyManager.GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory.DirInfo);
+                var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo);
+                if (packageDir is not null)
+                {
+                    return GetNewestNugetPackageVersionFolder(packageDir, package);
+                }
             }
 
             return null;
         }
 
+        public string GetNewestNugetPackageVersionFolder(string packagePath, string packageFriendlyName)
+        {
+            var versionFolders = GetOrderedPackageVersionSubDirectories(packagePath);
+            if (versionFolders.Length > 1)
+            {
+                var versions = string.Join(", ", versionFolders.Select(d => d.Name));
+                logger.LogDebug($"Found multiple {packageFriendlyName} DLLs in NuGet packages at {packagePath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
+            }
+
+            var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
+            if (selectedFrameworkFolder is null)
+            {
+                logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {packagePath}, but no version folder was found.");
+                selectedFrameworkFolder = packagePath;
+            }
+
+            logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {selectedFrameworkFolder}.");
+            return selectedFrameworkFolder;
+        }
+
+        public static DirectoryInfo[] GetOrderedPackageVersionSubDirectories(string packagePath)
+        {
+            return new DirectoryInfo(packagePath)
+                .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
+                .OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
+                .ToArray();
+        }
+
         public HashSet<AssemblyLookupLocation> Restore()
         {
             var assemblyLookupLocations = new HashSet<AssemblyLookupLocation>();
@@ -408,7 +440,8 @@ private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInf
                 .Select(d => Path.GetFileName(d).ToLowerInvariant());
         }
 
-        private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryWithoutNugetConfig = true)
+        private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj,
+            bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true)
         {
             logger.LogInfo($"Restoring package {package}...");
             using var tempDir = new TemporaryDirectory(
@@ -430,59 +463,91 @@ private bool TryRestorePackageManually(string package, string? nugetConfig = nul
                 return false;
             }
 
-            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
-            if (!res.Success)
+            var res = TryRestorePackageManually(package, nugetConfig, tempDir, tryPrereleaseVersion);
+            if (res.Success)
             {
-                if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
+                return true;
+            }
+
+            if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
+            {
+                logger.LogDebug($"Trying to restore '{package}' without nuget.config.");
+                // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
+                res = TryRestorePackageManually(package, nugetConfig: null, tempDir, tryPrereleaseVersion);
+                if (res.Success)
                 {
-                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
-                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
+                    return true;
                 }
+            }
+
+            logger.LogInfo($"Failed to restore nuget package {package}");
+            return false;
+        }
 
-                // TODO: the restore might fail, we could retry with
-                // - a prerelease (*-* instead of *) version of the package,
-                // - a different target framework moniker.
+        private RestoreResult TryRestorePackageManually(string package, string? nugetConfig, TemporaryDirectory tempDir, bool tryPrereleaseVersion)
+        {
+            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
 
-                if (!res.Success)
+            if (!res.Success && tryPrereleaseVersion && res.HasNugetNoStablePackageVersionError)
+            {
+                logger.LogDebug($"Failed to restore nuget package {package} because no stable version was found.");
+                try
                 {
-                    logger.LogInfo($"Failed to restore nuget package {package}");
-                    return false;
+                    TryChangePackageVersion(tempDir.DirInfo, "*-*");
+
+                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
+                    return res;
+                }
+                finally
+                {
+                    TryChangePackageVersion(tempDir.DirInfo, "*");
                 }
             }
 
-            return true;
+            return res;
         }
 
         private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
+        {
+            TryChangeProjectFile(tempDir, TargetFramework(), $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", "target framework moniker");
+        }
+
+        private void TryChangePackageVersion(DirectoryInfo tempDir, string newVersion)
+        {
+            TryChangeProjectFile(tempDir, PackageReferenceVersion(), $"Version=\"{newVersion}\"", "package reference version");
+        }
+
+        private bool TryChangeProjectFile(DirectoryInfo projectDir, Regex pattern, string replacement, string patternName)
         {
             try
             {
-                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
+                logger.LogDebug($"Changing the {patternName} in {projectDir.FullName}...");
 
-                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                var csprojs = projectDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
                 if (csprojs.Length != 1)
                 {
-                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
-                    return;
+                    logger.LogError($"Could not find the .csproj file in {projectDir.FullName}, count = {csprojs.Length}");
+                    return false;
                 }
 
                 var csproj = csprojs[0];
                 var content = File.ReadAllText(csproj.FullName);
-                var matches = TargetFramework().Matches(content);
+                var matches = pattern.Matches(content);
                 if (matches.Count == 0)
                 {
-                    logger.LogError($"Could not find target framework in {csproj.FullName}");
-                }
-                else
-                {
-                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
-                    File.WriteAllText(csproj.FullName, content);
+                    logger.LogError($"Could not find the {patternName} in {csproj.FullName}");
+                    return false;
                 }
+
+                content = pattern.Replace(content, replacement, 1);
+                File.WriteAllText(csproj.FullName, content);
+                return true;
             }
             catch (Exception exc)
             {
-                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
+                logger.LogError($"Failed to change the {patternName} in {projectDir.FullName}: {exc}");
             }
+            return false;
         }
 
         private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
@@ -664,6 +729,9 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
         [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
         private static partial Regex TargetFramework();
 
+        [GeneratedRegex(@"Version=""[*|*-*]""", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex PackageReferenceVersion();
+
         [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
         private static partial Regex LegacyNugetPackage();
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs

@@ -15,12 +15,18 @@ public ResxGenerator(
             IDotNet dotnet,
             ICompilationInfoContainer compilationInfoContainer,
             ILogger logger,
+            NugetPackageRestorer nugetPackageRestorer,
             TemporaryDirectory tempWorkingDirectory,
             IEnumerable<string> references) : base(fileProvider, fileContent, dotnet, compilationInfoContainer, logger, tempWorkingDirectory, references)
         {
             try
             {
-                // todo: download latest `Microsoft.CodeAnalysis.ResxSourceGenerator` and set `sourceGeneratorFolder`
+                // The package is downloaded to `missingpackages`, which is okay, we're already after the DLL collection phase.
+                var nugetFolder = nugetPackageRestorer.TryRestore("Microsoft.CodeAnalysis.ResxSourceGenerator");
+                if (nugetFolder is not null)
+                {
+                    sourceGeneratorFolder = System.IO.Path.Combine(nugetFolder, "analyzers", "dotnet", "cs");
+                }
             }
             catch (Exception e)
             {

csharp/ql/integration-tests/all-platforms/standalone_resx/Files.expected

@@ -0,0 +1,3 @@
+| Program.cs |
+| test-db/working/implicitUsings/GlobalUsings.g.cs |
+| test-db/working/resx/[...]/Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp/Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp.CSharpResxGenerator/test.Designer.cs |

csharp/ql/integration-tests/all-platforms/standalone_resx/Files.ql

@@ -0,0 +1,22 @@
+import csharp
+
+private string getPath(File f) {
+  result = f.getRelativePath() and
+  not exists(
+    result
+        .indexOf("Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp/Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp.CSharpResxGenerator")
+  )
+  or
+  exists(int index |
+    index =
+      f.getRelativePath()
+          .indexOf("Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp/Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp.CSharpResxGenerator") and
+    result =
+      f.getRelativePath().substring(0, index - 32 - 2) + "/[...]/" +
+        f.getRelativePath().substring(index, f.getRelativePath().length())
+  )
+}
+
+from File f
+where f.fromSource()
+select getPath(f)

csharp/ql/integration-tests/all-platforms/standalone_resx/Program.cs

@@ -0,0 +1 @@
+var dummy = "dummy";

csharp/ql/integration-tests/all-platforms/standalone_resx/global.json

@@ -0,0 +1,5 @@
+{
+    "sdk": {
+        "version": "8.0.101"
+    }
+}

csharp/ql/integration-tests/all-platforms/standalone_resx/resx.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+ <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+   <RemoveDir Directories=".\bin" />
+   <RemoveDir Directories=".\obj" />
+ </Target>
+</Project>

csharp/ql/integration-tests/all-platforms/standalone_resx/test.py

@@ -0,0 +1,4 @@
+import os
+from create_database_utils import *
+
+run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])