Extract total number of diagnostic per ID and compilation

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.IO;
 using System.Linq;
 using Microsoft.CodeAnalysis;
@@ -8,6 +9,8 @@ namespace Semmle.Extraction.CSharp.Entities
 {
     internal class Compilation : CachedEntity<object>
     {
+        internal readonly ConcurrentDictionary<string, int> messageCounts = new();
+
         private static (string Cwd, string[] Args) settings;
         private static int hashCode;
 
@@ -78,9 +81,11 @@ public override void Populate(TextWriter trapFile)
                 .ForEach((file, index) => trapFile.compilation_referencing_files(this, index, file));
 
             // Diagnostics
-            Context.Compilation
-                .GetDiagnostics()
-                .ForEach((diag, index) => new CompilerDiagnostic(Context, diag, this, index));
+            var diags = Context.Compilation.GetDiagnostics();
+            diags.ForEach((diag, index) => new CompilerDiagnostic(Context, diag, this, index));
+
+            var diagCounts = diags.GroupBy(diag => diag.Id).ToDictionary(group => group.Key, group => group.Count());
+            diagCounts.ForEach(pair => trapFile.compilation_info(this, $"Compiler diagnostic count for {pair.Key}", pair.Value.ToString()));
         }
 
         public void PopulatePerformance(PerformanceMetrics p)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs

@@ -1,4 +1,3 @@
-using System.Collections.Concurrent;
 using System.IO;
 using Semmle.Util;
 
@@ -7,7 +6,6 @@ namespace Semmle.Extraction.CSharp.Entities
     internal class CompilerDiagnostic : FreshEntity
     {
         private static readonly int limit = EnvironmentVariables.TryGetExtractorNumberOption<int>("COMPILER_DIAGNOSTIC_LIMIT") ?? 1000;
-        private static readonly ConcurrentDictionary<string, int> messageCounts = new();
 
         private readonly Microsoft.CodeAnalysis.Diagnostic diagnostic;
         private readonly Compilation compilation;
@@ -25,12 +23,12 @@ protected override void Populate(TextWriter trapFile)
         {
             // The below doesn't limit the extractor messages to the exact limit, but it's good enough.
             var key = diagnostic.Id;
-            var messageCount = messageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
+            var messageCount = compilation.messageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
             if (messageCount > limit)
             {
                 if (messageCount == limit + 1)
                 {
-                    Context.Extractor.Logger.LogWarning($"Stopped logging {key} compiler diagnostics after reaching {limit}");
+                    Context.Extractor.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}");
                 }
 
                 return;

csharp/ql/integration-tests/all-platforms/standalone/Diag.expected

@@ -2,3 +2,6 @@ extractorMessages
 | 5 |
 compilerDiagnostics
 | 4 |
+compilationInfo
+| Compiler diagnostic count for CS0103 | 3.0 |
+| Compiler diagnostic count for CS8019 | 7.0 |

csharp/ql/integration-tests/all-platforms/standalone/Diag.ql

@@ -4,3 +4,11 @@ import semmle.code.csharp.commons.Diagnostics
 query predicate extractorMessages(int c) { c = count(ExtractorMessage msg) }
 
 query predicate compilerDiagnostics(int c) { c = count(Diagnostic diag) }
+
+query predicate compilationInfo(string key, float value) {
+  exists(Compilation c, string infoValue |
+    infoValue = c.getInfo(key) and key.matches("Compiler diagnostic count for%")
+  |
+    value = infoValue.toFloat()
+  )
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/CompilationInfo.ql

@@ -3,6 +3,7 @@ import semmle.code.csharp.commons.Diagnostics
 
 query predicate compilationInfo(string key, float value) {
   key != "Resolved references" and
+  not key.matches("Compiler diagnostic count for%") and
   exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
     key = infoKey and
     value = infoValue.toFloat()

csharp/ql/src/Telemetry/ExtractorInformation.ql

@@ -10,6 +10,7 @@ import csharp
 import semmle.code.csharp.commons.Diagnostics
 
 predicate compilationInfo(string key, float value) {
+  not key.matches("Compiler diagnostic count for%") and
   exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
     key = infoKey and
     value = infoValue.toFloat()