Merge pull request #15680 from tamasvajk/buildless/no-mono-fallback-impr

tamasvajk · web-flow · commit 9ecac0470958 · 2024-02-22T08:40:16.000+01:00
C#: Improve fallback nuget package restore in buildless
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
@@ -882,14 +882,14 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
             var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
             var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
 
-            var notYetDownloadedPackages = new HashSet<string>(fileContent.AllPackages);
+            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
             foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
             {
-                notYetDownloadedPackages.Remove(alreadyDownloadedPackage);
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
             }
             foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
             {
-                notYetDownloadedPackages.Remove(alreadyDownloadedLegacyPackage);
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
             }
 
             if (notYetDownloadedPackages.Count == 0)
@@ -930,7 +930,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
 
             Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
             {
-                var success = TryRestorePackageManually(package, nugetConfig);
+                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
                 if (!success)
                 {
                     return;
@@ -947,7 +947,10 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
             dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
         }
 
-        private bool TryRestorePackageManually(string package, string? nugetConfig)
+        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex TargetFramework();
+
+        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
         {
             logger.LogInfo($"Restoring package {package}...");
             using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
@@ -957,6 +960,11 @@ private bool TryRestorePackageManually(string package, string? nugetConfig)
                 return false;
             }
 
+            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
+            {
+                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
+            }
+
             success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
             if (!success)
             {
@@ -972,7 +980,9 @@ private bool TryRestorePackageManually(string package, string? nugetConfig)
                     res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
                 }
 
-                // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
+                // TODO: the restore might fail, we could retry with
+                // - a prerelease (*-* instead of *) version of the package,
+                // - a different target framework moniker.
 
                 if (!res.Success)
                 {
@@ -984,6 +994,38 @@ private bool TryRestorePackageManually(string package, string? nugetConfig)
             return true;
         }
 
+        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
+        {
+            try
+            {
+                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
+
+                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                if (csprojs.Length != 1)
+                {
+                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
+                    return;
+                }
+
+                var csproj = csprojs[0];
+                var content = File.ReadAllText(csproj.FullName);
+                var matches = TargetFramework().Matches(content);
+                if (matches.Count == 0)
+                {
+                    logger.LogError($"Could not find target framework in {csproj.FullName}");
+                }
+                else
+                {
+                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
+                    File.WriteAllText(csproj.FullName, content);
+                }
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
+            }
+        }
+
         public void Dispose(TemporaryDirectory? dir, string name)
         {
             try
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
@@ -19,11 +19,11 @@ internal partial class FileContent
         private readonly ILogger logger;
         private readonly IUnsafeFileReader unsafeFileReader;
         private readonly IEnumerable<string> files;
-        private readonly HashSet<string> allPackages = new HashSet<string>();
+        private readonly HashSet<PackageReference> allPackages = new HashSet<PackageReference>();
         private readonly HashSet<string> implicitUsingNamespaces = new HashSet<string>();
         private readonly Initializer initialize;
 
-        public HashSet<string> AllPackages
+        public HashSet<PackageReference> AllPackages
         {
             get
             {
@@ -157,15 +157,15 @@ private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string gr
             return false;
         }
 
-        private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex)
+        private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex, PackageReferenceSource source)
         {
             foreach (var valueMatch in regex().EnumerateMatches(line))
             {
                 // We can't get the group from the ValueMatch, so doing it manually:
                 var packageName = GetGroup(line, valueMatch, groupName).ToLowerInvariant();
                 if (!string.IsNullOrEmpty(packageName))
                 {
-                    allPackages.Add(packageName);
+                    allPackages.Add(new PackageReference(packageName, source));
                 }
             }
         }
@@ -181,11 +181,13 @@ private void DoInitialize()
                     foreach (ReadOnlySpan<char> line in unsafeFileReader.ReadLines(file))
                     {
                         // Find all the packages.
-                        AddPackageReference(line, "Include", PackageReference);
-
                         if (isPackagesConfig)
                         {
-                            AddPackageReference(line, "id", LegacyPackageReference);
+                            AddPackageReference(line, "id", LegacyPackageReference, PackageReferenceSource.PackagesConfig);
+                        }
+                        else
+                        {
+                            AddPackageReference(line, "Include", PackageReference, PackageReferenceSource.SdkCsProj);
                         }
 
                         // Determine if ASP.NET is used.
@@ -268,4 +270,12 @@ public IEnumerable<string> ReadLines(string file)
             }
         }
     }
+
+    public enum PackageReferenceSource
+    {
+        SdkCsProj,
+        PackagesConfig
+    }
+
+    public record PackageReference(string Name, PackageReferenceSource PackageReferenceSource);
 }
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs
@@ -4,7 +4,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal static class FrameworkPackageNames
     {
-        public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481";
+        public const string LatestNetFrameworkMoniker = "net481";
+
+        public static string LatestNetFrameworkReferenceAssemblies { get; } = $"microsoft.netframework.referenceassemblies.{LatestNetFrameworkMoniker}";
 
         public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";
 
diff --git a/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs b/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
@@ -55,9 +55,9 @@ public void TestFileContent1()
             // Verify
             Assert.False(useAspNetDlls);
             Assert.Equal(3, allPackages.Count);
-            Assert.Contains("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), allPackages);
-            Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
-            Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
+            Assert.Contains(new PackageReference("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
+            Assert.Contains(new PackageReference("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
+            Assert.Contains(new PackageReference("StyleCop.Analyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
         }
 
         [Fact]
@@ -80,8 +80,8 @@ public void TestFileContent2()
             // Verify
             Assert.True(useAspNetDlls);
             Assert.Equal(2, allPackages.Count);
-            Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
-            Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
+            Assert.Contains(new PackageReference("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
+            Assert.Contains(new PackageReference("StyleCop.Analyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
         }
 
         private static void CsProjSettingsTest(string line, bool expected, Func<FileContent, bool> func)

Original file line number	Diff line number	Diff line change
`@@ -19,11 +19,11 @@ internal partial class FileContent`
`19`	`19`	`private readonly ILogger logger;`
`20`	`20`	`private readonly IUnsafeFileReader unsafeFileReader;`
`21`	`21`	`private readonly IEnumerable<string> files;`
`22`		`- private readonly HashSet<string> allPackages = new HashSet<string>();`
	`22`	`+ private readonly HashSet<PackageReference> allPackages = new HashSet<PackageReference>();`
`23`	`23`	`private readonly HashSet<string> implicitUsingNamespaces = new HashSet<string>();`
`24`	`24`	`private readonly Initializer initialize;`
`25`	`25`
`26`		`- public HashSet<string> AllPackages`
	`26`	`+ public HashSet<PackageReference> AllPackages`
`27`	`27`	`{`
`28`	`28`	`get`
`29`	`29`	`{`
`@@ -157,15 +157,15 @@ private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string gr`
`157`	`157`	`return false;`
`158`	`158`	`}`
`159`	`159`
`160`		`- private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex)`
	`160`	`+ private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex, PackageReferenceSource source)`
`161`	`161`	`{`
`162`	`162`	`foreach (var valueMatch in regex().EnumerateMatches(line))`
`163`	`163`	`{`
`164`	`164`	`// We can't get the group from the ValueMatch, so doing it manually:`
`165`	`165`	`var packageName = GetGroup(line, valueMatch, groupName).ToLowerInvariant();`
`166`	`166`	`if (!string.IsNullOrEmpty(packageName))`
`167`	`167`	`{`
`168`		`- allPackages.Add(packageName);`
	`168`	`+ allPackages.Add(new PackageReference(packageName, source));`
`169`	`169`	`}`
`170`	`170`	`}`
`171`	`171`	`}`
`@@ -181,11 +181,13 @@ private void DoInitialize()`
`181`	`181`	`foreach (ReadOnlySpan<char> line in unsafeFileReader.ReadLines(file))`
`182`	`182`	`{`
`183`	`183`	`// Find all the packages.`
`184`		`- AddPackageReference(line, "Include", PackageReference);`
`185`		`-`
`186`	`184`	`if (isPackagesConfig)`
`187`	`185`	`{`
`188`		`- AddPackageReference(line, "id", LegacyPackageReference);`
	`186`	`+ AddPackageReference(line, "id", LegacyPackageReference, PackageReferenceSource.PackagesConfig);`
	`187`	`+ }`
	`188`	`+ else`
	`189`	`+ {`
	`190`	`+ AddPackageReference(line, "Include", PackageReference, PackageReferenceSource.SdkCsProj);`
`189`	`191`	`}`
`190`	`192`
`191`	`193`	`// Determine if ASP.NET is used.`
`@@ -268,4 +270,12 @@ public IEnumerable<string> ReadLines(string file)`
`268`	`270`	`}`
`269`	`271`	`}`
`270`	`272`	`}`
	`273`	`+`
	`274`	`+ public enum PackageReferenceSource`
	`275`	`+ {`
	`276`	`+ SdkCsProj,`
	`277`	`+ PackagesConfig`
	`278`	`+ }`
	`279`	`+`
	`280`	`+ public record PackageReference(string Name, PackageReferenceSource PackageReferenceSource);`
`271`	`281`	`}`
Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching`
`4`	`4`	`{`
`5`	`5`	`internal static class FrameworkPackageNames`
`6`	`6`	`{`
`7`		`- public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481";`
	`7`	`+ public const string LatestNetFrameworkMoniker = "net481";`
	`8`	`+`
	`9`	`+ public static string LatestNetFrameworkReferenceAssemblies { get; } = $"microsoft.netframework.referenceassemblies.{LatestNetFrameworkMoniker}";`
`8`	`10`
`9`	`11`	`public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";`
`10`	`12`