C#: Move package filtering logic from FileContent to DependencyManager.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -60,7 +60,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
 
-            this.fileContent = new FileContent(packageDirectory, progressMonitor, () => GetFiles("*.*"));
+            this.fileContent = new FileContent(progressMonitor, () => GetFiles("*.*"));
             this.allSources = GetFiles("*.cs").ToList();
             var allProjects = GetFiles("*.csproj");
             var solutions = options.SolutionFile is not null
@@ -388,7 +388,11 @@ private void DownloadMissingPackages()
                 nugetConfig = nugetConfigs.FirstOrDefault();
             }
 
-            foreach (var package in fileContent.NotYetDownloadedPackages)
+            var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
+                .Select(d => Path.GetFileName(d)
+                .ToLowerInvariant());
+            var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
+            foreach (var package in notYetDownloadedPackages)
             {
                 progressMonitor.NugetInstall(package);
                 using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -19,17 +19,15 @@ internal partial class FileContent
         private readonly ProgressMonitor progressMonitor;
         private readonly IUnsafeFileReader unsafeFileReader;
         private readonly Func<IEnumerable<string>> getFiles;
-        private readonly Func<HashSet<string>> getAlreadyDownloadedPackages;
         private readonly HashSet<string> allPackages = new HashSet<string>();
         private readonly Initializer initialize;
 
-        public IEnumerable<string> NotYetDownloadedPackages
+        public HashSet<string> AllPackages
         {
             get
             {
                 initialize.Run();
-                var alreadyDownloadedPackages = getAlreadyDownloadedPackages();
-                return allPackages.Except(alreadyDownloadedPackages);
+                return allPackages;
             }
         }
 
@@ -51,23 +49,18 @@ public bool UseAspNetDlls
             }
         }
 
-        internal FileContent(Func<HashSet<string>> getAlreadyDownloadedPackages,
-            ProgressMonitor progressMonitor,
+        internal FileContent(ProgressMonitor progressMonitor,
             Func<IEnumerable<string>> getFiles,
             IUnsafeFileReader unsafeFileReader)
         {
-            this.getAlreadyDownloadedPackages = getAlreadyDownloadedPackages;
             this.progressMonitor = progressMonitor;
             this.getFiles = getFiles;
             this.unsafeFileReader = unsafeFileReader;
             this.initialize = new Initializer(DoInitialize);
         }
 
 
-        public FileContent(TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor, Func<IEnumerable<string>> getFiles) : this(() => Directory.GetDirectories(packageDirectory.DirInfo.FullName)
-                .Select(d => Path.GetFileName(d)
-                .ToLowerInvariant())
-                .ToHashSet(), progressMonitor, getFiles, new UnsafeFileReader())
+        public FileContent(ProgressMonitor progressMonitor, Func<IEnumerable<string>> getFiles) : this(progressMonitor, getFiles, new UnsafeFileReader())
         { }
 
         private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix)
@@ -114,7 +107,7 @@ private void DoInitialize()
                         {
                             // We can't get the group from the ValueMatch, so doing it manually:
                             var packageName = GetGroup(line, valueMatch, "Include");
-                            if (!string.IsNullOrEmpty(packageName) && !allPackages.Contains(packageName))
+                            if (!string.IsNullOrEmpty(packageName))
                             {
                                 allPackages.Add(packageName);
                             }

csharp/extractor/Semmle.Extraction.Tests/FileContent.cs

@@ -34,8 +34,7 @@ public IEnumerable<string> ReadLines(string file)
 
     internal class TestFileContent : FileContent
     {
-        public TestFileContent(List<string> lines) : base(() => new HashSet<string>(),
-            new ProgressMonitor(new LoggerStub()),
+        public TestFileContent(List<string> lines) : base(new ProgressMonitor(new LoggerStub()),
             () => new List<string>() { "test1.cs" },
             new UnsafeFileReaderStub(lines))
         { }
@@ -58,15 +57,15 @@ public void TestFileContent1()
             var fileContent = new TestFileContent(lines);
 
             // Execute
-            var notYetDownloadedPackages = fileContent.NotYetDownloadedPackages;
+            var allPackages = fileContent.AllPackages;
             var useAspNetDlls = fileContent.UseAspNetDlls;
 
             // Verify
             Assert.False(useAspNetDlls);
-            Assert.Equal(3, notYetDownloadedPackages.Count());
-            Assert.Contains("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), notYetDownloadedPackages);
-            Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), notYetDownloadedPackages);
-            Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), notYetDownloadedPackages);
+            Assert.Equal(3, allPackages.Count);
+            Assert.Contains("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), allPackages);
+            Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
+            Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
         }
 
         [Fact]
@@ -84,13 +83,13 @@ public void TestFileContent2()
 
             // Execute
             var useAspNetDlls = fileContent.UseAspNetDlls;
-            var notYetDownloadedPackages = fileContent.NotYetDownloadedPackages;
+            var allPackages = fileContent.AllPackages;
 
             // Verify
             Assert.True(useAspNetDlls);
-            Assert.Equal(2, notYetDownloadedPackages.Count());
-            Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), notYetDownloadedPackages);
-            Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), notYetDownloadedPackages);
+            Assert.Equal(2, allPackages.Count);
+            Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
+            Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
         }
     }
 }