Merge pull request #15650 from tamasvajk/buildless/fallback-nuget-install

C#: Add fallback logic to `nuget install`

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -5,6 +5,7 @@
 using System.Linq;
 using System.Security.Cryptography;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Semmle.Util;
 using Semmle.Util.Logging;
@@ -14,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// <summary>
     /// Main implementation of the build analysis.
     /// </summary>
-    public sealed class DependencyManager : IDisposable
+    public sealed partial class DependencyManager : IDisposable
     {
         private readonly AssemblyCache assemblyCache;
         private readonly ILogger logger;
@@ -783,13 +784,53 @@ private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<strin
             CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
         }
 
-        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
+        [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex LegacyNugetPackage();
+
+
+        private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
         {
-            var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
+            return Directory.GetDirectories(root.FullName)
                 .Select(d => Path.GetFileName(d).ToLowerInvariant());
-            var notYetDownloadedPackages = fileContent.AllPackages
-                .Except(alreadyDownloadedPackages)
-                .ToList();
+        }
+
+        private IEnumerable<string> GetRestoredLegacyPackageNames()
+        {
+            var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
+            foreach (var oldPackageDirectory in oldPackageDirectories)
+            {
+                // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
+                // typical folder names look like:
+                //   newtonsoft.json.13.0.3
+                // there are more complex ones too, such as:
+                //   runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
+
+                var match = LegacyNugetPackage().Match(oldPackageDirectory);
+                if (!match.Success)
+                {
+                    logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
+                    continue;
+                }
+
+                yield return match.Groups[1].Value.ToLowerInvariant();
+            }
+        }
+
+        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
+        {
+            var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
+            var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
+
+            var notYetDownloadedPackages = new HashSet<string>(fileContent.AllPackages);
+            foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
+            {
+                notYetDownloadedPackages.Remove(alreadyDownloadedPackage);
+            }
+            foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
+            {
+                notYetDownloadedPackages.Remove(alreadyDownloadedLegacyPackage);
+            }
+
             if (notYetDownloadedPackages.Count == 0)
             {
                 return;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -157,23 +157,35 @@ private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string gr
             return false;
         }
 
+        private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex)
+        {
+            foreach (var valueMatch in regex().EnumerateMatches(line))
+            {
+                // We can't get the group from the ValueMatch, so doing it manually:
+                var packageName = GetGroup(line, valueMatch, groupName).ToLowerInvariant();
+                if (!string.IsNullOrEmpty(packageName))
+                {
+                    allPackages.Add(packageName);
+                }
+            }
+        }
+
         private void DoInitialize()
         {
             foreach (var file in files)
             {
                 try
                 {
+                    var isPackagesConfig = file.EndsWith("packages.config", StringComparison.OrdinalIgnoreCase);
+
                     foreach (ReadOnlySpan<char> line in unsafeFileReader.ReadLines(file))
                     {
                         // Find all the packages.
-                        foreach (var valueMatch in PackageReference().EnumerateMatches(line))
+                        AddPackageReference(line, "Include", PackageReference);
+
+                        if (isPackagesConfig)
                         {
-                            // We can't get the group from the ValueMatch, so doing it manually:
-                            var packageName = GetGroup(line, valueMatch, "Include").ToLowerInvariant();
-                            if (!string.IsNullOrEmpty(packageName))
-                            {
-                                allPackages.Add(packageName);
-                            }
+                            AddPackageReference(line, "id", LegacyPackageReference);
                         }
 
                         // Determine if ASP.NET is used.
@@ -223,6 +235,9 @@ private void DoInitialize()
         [GeneratedRegex("(?<!<!--.*)<PackageReference.*\\sInclude=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
         private static partial Regex PackageReference();
 
+        [GeneratedRegex("(?<!<!--.*)<package.*\\sid=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex LegacyPackageReference();
+
         [GeneratedRegex("(?<!<!--.*)<FrameworkReference.*\\sInclude=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
         private static partial Regex FrameworkReference();