C#: Fix lazy evaluation of not yet downloaded packages

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -12,23 +12,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     // This class is used to read a set of files and decide different properties about the
     // content (by reading the content of the files only once).
     // The implementation is lazy, so the properties are only calculated when
-    // the first property is accessed. 
+    // the first property is accessed.
     // </summary>
     internal partial class FileContent
     {
         private readonly ProgressMonitor progressMonitor;
         private readonly IUnsafeFileReader unsafeFileReader;
         private readonly Func<IEnumerable<string>> getFiles;
         private readonly Func<HashSet<string>> getAlreadyDownloadedPackages;
-        private readonly HashSet<string> notYetDownloadedPackages = new HashSet<string>();
+        private readonly HashSet<string> allPackages = new HashSet<string>();
         private readonly Initializer initialize;
 
-        public HashSet<string> NotYetDownloadedPackages
+        public IEnumerable<string> NotYetDownloadedPackages
         {
             get
             {
                 initialize.Run();
-                return notYetDownloadedPackages;
+                var alreadyDownloadedPackages = getAlreadyDownloadedPackages();
+                return allPackages.Except(alreadyDownloadedPackages);
             }
         }
 
@@ -101,22 +102,21 @@ private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string gr
 
         private void DoInitialize()
         {
-            var alreadyDownloadedPackages = getAlreadyDownloadedPackages();
             foreach (var file in getFiles())
             {
                 try
                 {
                     foreach (ReadOnlySpan<char> line in unsafeFileReader.ReadLines(file))
                     {
 
-                        // Find the not yet downloaded packages.
+                        // Find all the packages.
                         foreach (var valueMatch in PackageReference().EnumerateMatches(line))
                         {
                             // We can't get the group from the ValueMatch, so doing it manually:
                             var packageName = GetGroup(line, valueMatch, "Include");
-                            if (!string.IsNullOrEmpty(packageName) && !alreadyDownloadedPackages.Contains(packageName))
+                            if (!string.IsNullOrEmpty(packageName) && !allPackages.Contains(packageName))
                             {
-                                notYetDownloadedPackages.Add(packageName);
+                                allPackages.Add(packageName);
                             }
                         }
 

csharp/extractor/Semmle.Extraction.Tests/FileContent.cs

@@ -1,5 +1,6 @@
 using Xunit;
 using System.Collections.Generic;
+using System.Linq;
 using Semmle.Util.Logging;
 using Semmle.Extraction.CSharp.DependencyFetching;
 
@@ -62,7 +63,7 @@ public void TestFileContent1()
 
             // Verify
             Assert.False(useAspNetDlls);
-            Assert.Equal(3, notYetDownloadedPackages.Count);
+            Assert.Equal(3, notYetDownloadedPackages.Count());
             Assert.Contains("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), notYetDownloadedPackages);
             Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), notYetDownloadedPackages);
             Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), notYetDownloadedPackages);
@@ -87,7 +88,7 @@ public void TestFileContent2()
 
             // Verify
             Assert.True(useAspNetDlls);
-            Assert.Equal(2, notYetDownloadedPackages.Count);
+            Assert.Equal(2, notYetDownloadedPackages.Count());
             Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), notYetDownloadedPackages);
             Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), notYetDownloadedPackages);
         }