Merge pull request #16021 from tamasvajk/feature/add-buildless-telemetry

C#: Add high level diagnostic messages for buildless extraction (star…

Author: tamasvajk

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -51,7 +51,9 @@ public override BuildScript GetBuildScript()
                     break;
                 case CSharpBuildStrategy.Buildless:
                     // No need to check that the extractor has been executed in buildless mode
-                    attempt = new StandaloneBuildRule().Analyse(this, false);
+                    attempt = BuildScript.Bind(
+                        AddBuildlessStartedDiagnostic() & new StandaloneBuildRule().Analyse(this, false),
+                        AddBuildlessEndedDiagnostic);
                     break;
                 case CSharpBuildStrategy.MSBuild:
                     attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
@@ -86,6 +88,52 @@ public BuildScript CheckExtractorRun(bool warnOnFailure) =>
                 return 1;
             });
 
+        private BuildScript AddBuildlessStartedDiagnostic()
+        {
+            return BuildScript.Create(actions =>
+            {
+                AddDiagnostic(new DiagnosticMessage(
+                    Options.Language,
+                    "buildless/mode-active",
+                    "C# with build-mode set to 'none'",
+                    visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
+                    markdownMessage: "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
+                    severity: DiagnosticMessage.TspSeverity.Note
+                ));
+                return 0;
+            });
+        }
+
+        private BuildScript AddBuildlessEndedDiagnostic(int buildResult)
+        {
+            return BuildScript.Create(actions =>
+            {
+                if (buildResult == 0)
+                {
+                    AddDiagnostic(new DiagnosticMessage(
+                        Options.Language,
+                        "buildless/complete",
+                        "C# analysis with build-mode 'none' completed",
+                        visibility: new DiagnosticMessage.TspVisibility(statusPage: false, cliSummaryTable: true, telemetry: true),
+                        markdownMessage: "C# analysis with build-mode 'none' completed.",
+                        severity: DiagnosticMessage.TspSeverity.Unknown
+                    ));
+                }
+                else
+                {
+                    AddDiagnostic(new DiagnosticMessage(
+                        Options.Language,
+                        "buildless/failed",
+                        "C# analysis with build-mode 'none' failed",
+                        visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
+                        markdownMessage: "C# analysis with build-mode 'none' failed.",
+                        severity: DiagnosticMessage.TspSeverity.Error
+                    ));
+                }
+                return buildResult;
+            });
+        }
+
         protected override void AutobuildFailureDiagnostic()
         {
             // if `ScriptPath` is not null here, the `BuildCommandAuto` rule was

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -728,6 +728,11 @@ private void ResolveConflicts(IEnumerable<string> frameworkPaths)
         /// </summary>
         public IEnumerable<string> GeneratedSourceFiles => generatedSources;
 
+        /// <summary>
+        /// All of the non-generated source files in the source directory.
+        /// </summary>
+        public IEnumerable<string> NonGeneratedSourcesFiles => nonGeneratedSources;
+
         /// <summary>
         /// All of the source files in the source directory.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -146,7 +146,7 @@ public static ExitCode Run(Options options)
             logger.Log(Severity.Info, "Extracting C# in buildless mode");
             using var dependencyManager = new DependencyManager(options.SrcDir, logger);
 
-            if (!dependencyManager.AllSourceFiles.Any())
+            if (!dependencyManager.NonGeneratedSourcesFiles.Any())
             {
                 logger.Log(Severity.Error, "No source files found");
                 return ExitCode.Errors;

csharp/extractor/Semmle.Util/ToolStatusPage.cs

@@ -46,6 +46,7 @@ public TspSource(string id, string name, string? extractorName = null)
         [JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
         public enum TspSeverity
         {
+            Unknown,
             Note,
             Warning,
             Error

csharp/ql/integration-tests/all-platforms/standalone/diagnostics.expected

@@ -0,0 +1,28 @@
+{
+  "markdownMessage": "C# analysis with build-mode 'none' completed.",
+  "severity": "unknown",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/complete",
+    "name": "C# analysis with build-mode 'none' completed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
+  "severity": "note",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/mode-active",
+    "name": "C# with build-mode set to 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/all-platforms/standalone/test.py

@@ -1,6 +1,9 @@
 import os
 from create_database_utils import *
+from diagnostics_test_utils import *
 
 os.environ['CODEQL_EXTRACTOR_CSHARP_OPTION_COMPILER_DIAGNOSTIC_LIMIT'] = '2'
 os.environ['CODEQL_EXTRACTOR_CSHARP_OPTION_MESSAGE_LIMIT'] = '5'
 run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+
+check_diagnostics()

csharp/ql/integration-tests/all-platforms/standalone_failed/diagnostics.expected

@@ -0,0 +1,28 @@
+{
+  "markdownMessage": "C# analysis with build-mode 'none' failed.",
+  "severity": "error",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/failed",
+    "name": "C# analysis with build-mode 'none' failed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
+  "severity": "note",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/mode-active",
+    "name": "C# with build-mode set to 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/all-platforms/standalone_failed/standalone.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+ <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+   <RemoveDir Directories=".\bin" />
+   <RemoveDir Directories=".\obj" />
+ </Target>
+</Project>

csharp/ql/integration-tests/all-platforms/standalone_failed/test.py

@@ -0,0 +1,6 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], db=None, lang="csharp", extra_args=["--build-mode=none"], runFunction=runUnsuccessfully)
+
+check_diagnostics()