fix: Ensure all providers's recollect works agains the new gidhub page format (react-app.embeddedData)

davidkallesen · davidkallesen · commit 3b7a876508d9 · 2024-04-02T21:52:26.000+02:00
diff --git a/src/Atc.CodingRules.AnalyzerProviders/GlobalUsings.cs b/src/Atc.CodingRules.AnalyzerProviders/GlobalUsings.cs
@@ -1,20 +1,15 @@
-global using System;
-global using System.Collections.Generic;
 global using System.Collections.ObjectModel;
 global using System.Diagnostics;
 global using System.Diagnostics.CodeAnalysis;
-global using System.IO;
-global using System.Linq;
 global using System.Net;
 global using System.Text.Encodings.Web;
 global using System.Text.Json;
 global using System.Text.Json.Serialization;
-global using System.Threading.Tasks;
 global using System.Xml;
 
 global using Atc.CodingRules.AnalyzerProviders.Models;
 global using Atc.CodingRules.AnalyzerProviders.Providers;
-
+global using Atc.Serialization;
 global using HtmlAgilityPack;
 
 global using Microsoft.Extensions.Logging;
diff --git a/src/Atc.CodingRules.AnalyzerProviders/Providers/AsyncFixerProvider.cs b/src/Atc.CodingRules.AnalyzerProviders/Providers/AsyncFixerProvider.cs
@@ -18,6 +18,7 @@ public AsyncFixerProvider(
     protected override AnalyzerProviderBaseRuleData CreateData()
         => new(Name);
 
+    [SuppressMessage("Design", "MA0051:Method is too long", Justification = "OK.")]
     protected override async Task ReCollect(
         AnalyzerProviderBaseRuleData data)
     {
@@ -26,6 +27,43 @@ protected override async Task ReCollect(
         var web = new HtmlWeb();
         var htmlDoc = await web.LoadFromWebAsync(DocumentationLink!.AbsoluteUri);
 
+        var embeddedNode = htmlDoc.DocumentNode.SelectSingleNode("//script[@data-target='react-app.embeddedData']");
+        if (embeddedNode is not null)
+        {
+            var dynamicJson = new DynamicJson(embeddedNode.InnerText);
+            if (dynamicJson.GetValue("payload.blob.headerInfo.toc") is List<object> tocObjects)
+            {
+                foreach (var tocObject in tocObjects)
+                {
+                    if (tocObject is not Dictionary<string, object> tocItem)
+                    {
+                        continue;
+                    }
+
+                    if (!tocItem.ContainsKey("text"))
+                    {
+                        continue;
+                    }
+
+                    var sa = tocItem["text"]
+                        .ToString()!
+                        .Split(':', StringSplitOptions.RemoveEmptyEntries);
+
+                    var code = sa[0].Trim();
+                    var title = sa[1].Trim();
+
+                    data.Rules.Add(
+                        new Rule(
+                            code,
+                            title,
+                            link: string.Empty,
+                            description: title));
+                }
+
+                return;
+            }
+        }
+
         var headers3 = htmlDoc.DocumentNode.SelectNodes("//h3").ToList();
 
         foreach (var item in headers3)
diff --git a/src/Atc.CodingRules.AnalyzerProviders/Providers/MeziantouProvider.cs b/src/Atc.CodingRules.AnalyzerProviders/Providers/MeziantouProvider.cs
@@ -29,6 +29,16 @@ protected override async Task ReCollect(
 
         var web = new HtmlWeb();
         var htmlDoc = await web.LoadFromWebAsync(DocumentationLink!.AbsoluteUri).ConfigureAwait(false);
+
+        var embeddedNode = htmlDoc.DocumentNode.SelectSingleNode("//script[@data-target='react-app.embeddedData']");
+        if (embeddedNode is not null)
+        {
+            var dynamicJson = new DynamicJson(embeddedNode.InnerText);
+            var html = dynamicJson.GetValue("payload.tree.readme.richText")?.ToString();
+
+            htmlDoc.LoadHtml(html);
+        }
+
         var articleNode = htmlDoc.DocumentNode.SelectNodes("//article[@class='markdown-body entry-content container-lg']").First();
         var articleTableRows = articleNode.SelectNodes("//*//table[1]//tr").ToList();
 
diff --git a/src/Atc.CodingRules.AnalyzerProviders/Providers/MicrosoftCompilerErrorsProvider.cs b/src/Atc.CodingRules.AnalyzerProviders/Providers/MicrosoftCompilerErrorsProvider.cs
@@ -13,7 +13,7 @@ public MicrosoftCompilerErrorsProvider(
 
     public static string Name => "Microsoft.CompilerErrors";
 
-    public override Uri? DocumentationLink { get; set; } = new("https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/compiler-messages", UriKind.Absolute);
+    public override Uri? DocumentationLink { get; set; } = new("https://learn.microsoft.com/en-us/dotnet/csharp/language-reference", UriKind.Absolute);
 
     protected override AnalyzerProviderBaseRuleData CreateData()
         => new(Name);
@@ -37,26 +37,42 @@ protected override async Task ReCollect(
         while (jsonDocItems.MoveNext())
         {
             var jsonElement = jsonDocItems.Current;
+
             if (!jsonElement.GetRawText().Contains("children", StringComparison.Ordinal))
             {
                 continue;
             }
 
+            var tocTitle = jsonElement.GetProperty("toc_title").ToString();
+            if (!tocTitle.Equals("C# compiler messages", StringComparison.Ordinal))
+            {
+                continue;
+            }
+
             var jsonChildItems = jsonElement.GetProperty("children").EnumerateArray();
             while (jsonChildItems.MoveNext())
             {
                 var jsonChildElement = jsonChildItems.Current;
-                var code = jsonChildElement.GetProperty("toc_title").ToString()!;
-                var hrefPart = jsonChildElement.GetProperty("href").ToString()!;
-
-                var link = hrefPart.StartsWith("../../misc/", StringComparison.Ordinal)
-                    ? "https://docs.microsoft.com/en-us/dotnet/csharp/" + hrefPart.Replace("../../", string.Empty, StringComparison.Ordinal)
-                    : DocumentationLink.AbsoluteUri + "/" + hrefPart;
+                if (jsonChildElement.ValueKind != JsonValueKind.Object ||
+                    !jsonChildElement.TryGetProperty("children", out var jsonChildElement2))
+                {
+                    continue;
+                }
 
-                var rule = await GetRuleByCode(code, link);
-                if (rule is not null)
+                foreach (var element in jsonChildElement2.EnumerateArray())
                 {
-                    data.Rules.Add(rule);
+                    var hrefPart = element.GetProperty("href").ToString();
+                    var code = element.GetProperty("toc_title").ToString();
+
+                    var link = hrefPart.StartsWith("../misc/", StringComparison.Ordinal)
+                        ? "https://docs.microsoft.com/en-us/dotnet/csharp/" + hrefPart.Replace("../", string.Empty, StringComparison.Ordinal)
+                        : DocumentationLink.AbsoluteUri + "/" + hrefPart;
+
+                    var rule = await GetRuleByCode(code, link);
+                    if (rule is not null)
+                    {
+                        data.Rules.Add(rule);
+                    }
                 }
             }
         }
@@ -79,18 +95,16 @@ protected override async Task ReCollect(
             return null;
         }
 
+        var header = mainNode.SelectSingleNode(".//h1");
+
         var paragraphs = mainNode.SelectNodes(".//p").ToList();
-        if (paragraphs.Count < 3)
+        if (paragraphs.Count < 2)
         {
             return null;
         }
 
-        var title = paragraphs[2].InnerText;
-        var description = string.Empty;
-        if (paragraphs.Count > 3)
-        {
-            description = paragraphs[3].InnerText;
-        }
+        var title = header.InnerText;
+        var description = paragraphs[1].InnerText;
 
         return new Rule(
             code,
diff --git a/src/Atc.CodingRules.AnalyzerProviders/Providers/MicrosoftVisualStudioThreadingAnalyzersProvider.cs b/src/Atc.CodingRules.AnalyzerProviders/Providers/MicrosoftVisualStudioThreadingAnalyzersProvider.cs
@@ -29,6 +29,16 @@ protected override async Task ReCollect(
 
         var web = new HtmlWeb();
         var htmlDoc = await web.LoadFromWebAsync(DocumentationLink!.AbsoluteUri).ConfigureAwait(false);
+
+        var embeddedNode = htmlDoc.DocumentNode.SelectSingleNode("//script[@data-target='react-app.embeddedData']");
+        if (embeddedNode is not null)
+        {
+            var dynamicJson = new DynamicJson(embeddedNode.InnerText);
+            var html = dynamicJson.GetValue("payload.blob.richText")?.ToString();
+
+            htmlDoc.LoadHtml(html);
+        }
+
         var articleNode = htmlDoc.DocumentNode.SelectNodes("//article[@class='markdown-body entry-content container-lg']").First();
         var articleTableRows = articleNode.SelectNodes("//*//table[1]//tr").ToList();
 
diff --git a/src/Atc.CodingRules.AnalyzerProviders/Providers/NSubstituteAnalyzersProvider.cs b/src/Atc.CodingRules.AnalyzerProviders/Providers/NSubstituteAnalyzersProvider.cs
@@ -29,6 +29,16 @@ protected override async Task ReCollect(
 
         var web = new HtmlWeb();
         var htmlDoc = await web.LoadFromWebAsync(DocumentationLink!.AbsoluteUri).ConfigureAwait(false);
+
+        var embeddedNode = htmlDoc.DocumentNode.SelectSingleNode("//script[@data-target='react-app.embeddedData']");
+        if (embeddedNode is not null)
+        {
+            var dynamicJson = new DynamicJson(embeddedNode.InnerText);
+            var html = dynamicJson.GetValue("payload.tree.readme.richText")?.ToString();
+
+            htmlDoc.LoadHtml(html);
+        }
+
         var articleNode = htmlDoc.DocumentNode.SelectNodes("//article[@class='markdown-body entry-content container-lg']").First();
         var articleTableRows = articleNode.SelectNodes("//*//table[1]//tr").ToList();
 
diff --git a/src/Atc.CodingRules.AnalyzerProviders/Providers/SonarAnalyzerCSharpProvider.cs b/src/Atc.CodingRules.AnalyzerProviders/Providers/SonarAnalyzerCSharpProvider.cs
@@ -27,16 +27,39 @@ protected override async Task ReCollect(
 
         var web = new HtmlWeb();
         var htmlDoc = await web.LoadFromWebAsync(DocumentationLink!.AbsoluteUri).ConfigureAwait(false);
-        var jsonDoc = JsonDocument.Parse(htmlDoc.DocumentNode.InnerText);
-        var jsonDocItems = jsonDoc.RootElement.GetProperty("result").GetProperty("pageContext").GetProperty("rules").EnumerateArray();
 
-        while (jsonDocItems.MoveNext())
+        var dynamicJson = new DynamicJson(htmlDoc.DocumentNode.InnerText);
+        if (dynamicJson.GetValue("result.data.allFile.nodes") is not List<object> nodes)
         {
-            var jsonElement = jsonDocItems.Current;
-            var ruleKey = jsonElement.GetProperty("ruleKey").GetString();
-            var summary = jsonElement.GetProperty("summary").GetString() ?? string.Empty;
+            return;
+        }
+
+        if (nodes[0] is not Dictionary<string, object> node)
+        {
+            return;
+        }
+
+        if (node["childLanguageJson"] is not Dictionary<string, object> childLanguageJson)
+        {
+            return;
+        }
+
+        if (childLanguageJson["rules"] is not List<object> rules)
+        {
+            return;
+        }
+
+        foreach (var ruleObj in rules)
+        {
+            if (ruleObj is not Dictionary<string, object> ruleDict)
+            {
+                continue;
+            }
+
+            var ruleKey = ruleDict["ruleKey"].ToString();
+            var summary = ruleDict["summary"].ToString() ?? string.Empty;
             var link = RuleLinkBase + ruleKey;
-            var description = jsonElement.GetProperty("description").GetString();
+            var description = ruleDict["description"].ToString();
 
             if (ruleKey is null)
             {
diff --git a/src/Atc.CodingRules.AnalyzerProviders/Providers/StyleCopAnalyzersProvider.cs b/src/Atc.CodingRules.AnalyzerProviders/Providers/StyleCopAnalyzersProvider.cs
@@ -30,6 +30,16 @@ protected override async Task ReCollect(
 
         var web = new HtmlWeb();
         var htmlDoc = await web.LoadFromWebAsync(DocumentationLink!.AbsoluteUri).ConfigureAwait(false);
+
+        var embeddedNode = htmlDoc.DocumentNode.SelectSingleNode("//script[@data-target='react-app.embeddedData']");
+        if (embeddedNode is not null)
+        {
+            var dynamicJson = new DynamicJson(embeddedNode.InnerText);
+            var html = dynamicJson.GetValue("payload.blob.richText")?.ToString();
+
+            htmlDoc.LoadHtml(html);
+        }
+
         var articleNode = htmlDoc.DocumentNode.SelectNodes("//article[@class='markdown-body entry-content container-lg']").First();
         var articleRuleLinks = articleNode.SelectNodes("//*//strong//a").ToList();
 
@@ -49,6 +59,16 @@ private static async Task<List<Rule>> GetRules(
         var link = $"https://github.com{item.Attributes["href"].Value}";
         var web = new HtmlWeb();
         var htmlDoc = await web.LoadFromWebAsync(link).ConfigureAwait(false);
+
+        var embeddedNode = htmlDoc.DocumentNode.SelectSingleNode("//script[@data-target='react-app.embeddedData']");
+        if (embeddedNode is not null)
+        {
+            var dynamicJson = new DynamicJson(embeddedNode.InnerText);
+            var html = dynamicJson.GetValue("payload.blob.richText")?.ToString();
+
+            htmlDoc.LoadHtml(html);
+        }
+
         var articleNode = htmlDoc.DocumentNode.SelectNodes("//article[@class='markdown-body entry-content container-lg']").First();
         var articleTableRows = articleNode.SelectNodes("//*//table[1]//tr").ToList();
         var category = articleNode.Descendants("h3").First().InnerText;
diff --git a/src/Atc.CodingRules.AnalyzerProviders/Providers/XunitProvider.cs b/src/Atc.CodingRules.AnalyzerProviders/Providers/XunitProvider.cs
@@ -3,7 +3,6 @@ namespace Atc.CodingRules.AnalyzerProviders.Providers;
 public class XunitProvider : AnalyzerProviderBase
 {
     private const int TableThColumnId = 0;
-    private const int TableTdColumnCategory = 2;
     private const int TableTdColumnTitle = 0;
 
     public XunitProvider(
@@ -56,14 +55,13 @@ protected override async Task ReCollect(
             var code = aHrefNode.InnerText.RemoveNewLines().Trim();
             var title = HtmlEntity.DeEntitize(cellsTd[TableTdColumnTitle].InnerText);
             var link = $"{DocumentationLink}/{code}";
-            var category = cellsTd[TableTdColumnCategory].InnerText;
 
             data.Rules.Add(
                 new Rule(
                     code,
                     title,
                     link,
-                    category: category));
+                    category: null));
         }
     }
 }
diff --git a/src/Atc.CodingRules.Updater.CLI/ProjectHelper.cs b/src/Atc.CodingRules.Updater.CLI/ProjectHelper.cs
@@ -391,7 +391,7 @@ private static async Task CreateSuppressionsFileInTempPath(
         ILogger logger,
         DirectoryInfo temporarySuppressionsPath,
         bool temporarySuppressionAsExcel,
-        IEnumerable<Tuple<string, List<string>>> suppressionLinesPrAnalyzer)
+        IList<Tuple<string, List<string>>> suppressionLinesPrAnalyzer)
     {
         var temporarySuppressionsFile = Path.Join(
             temporarySuppressionsPath.FullName,
@@ -610,5 +610,5 @@ private static string Pluralize(
         string word,
         string pluralSuffix = "s",
         string singularSuffix = "")
-        => $@"{number} {word}{(number != 1 ? pluralSuffix : singularSuffix)}";
+        => $"{number} {word}{(number != 1 ? pluralSuffix : singularSuffix)}";
 }
diff --git a/test/Atc.CodingRules.AnalyzerProviders.Tests/Providers/MicrosoftCompilerErrorsProviderTests.cs b/test/Atc.CodingRules.AnalyzerProviders.Tests/Providers/MicrosoftCompilerErrorsProviderTests.cs
@@ -21,6 +21,6 @@ public async Task CollectBaseRules(ProviderCollectingMode providerCollectingMode
         Assert.NotNull(actual.Name);
         Assert.Equal(MicrosoftCompilerErrorsProvider.Name, actual.Name);
         Assert.NotNull(actual.Rules);
-        Assert.True(actual.Rules.Count >= 935);
+        Assert.True(actual.Rules.Count >= 910);
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,6 @@ namespace Atc.CodingRules.AnalyzerProviders.Providers;`
`3`	`3`	`public class XunitProvider : AnalyzerProviderBase`
`4`	`4`	`{`
`5`	`5`	`private const int TableThColumnId = 0;`
`6`		`- private const int TableTdColumnCategory = 2;`
`7`	`6`	`private const int TableTdColumnTitle = 0;`
`8`	`7`
`9`	`8`	`public XunitProvider(`
`@@ -56,14 +55,13 @@ protected override async Task ReCollect(`
`56`	`55`	`var code = aHrefNode.InnerText.RemoveNewLines().Trim();`
`57`	`56`	`var title = HtmlEntity.DeEntitize(cellsTd[TableTdColumnTitle].InnerText);`
`58`	`57`	`var link = $"{DocumentationLink}/{code}";`
`59`		`- var category = cellsTd[TableTdColumnCategory].InnerText;`
`60`	`58`
`61`	`59`	`data.Rules.Add(`
`62`	`60`	`new Rule(`
`63`	`61`	`code,`
`64`	`62`	`title,`
`65`	`63`	`link,`
`66`		`- category: category));`
	`64`	`+ category: null));`
`67`	`65`	`}`
`68`	`66`	`}`
`69`	`67`	`}`