Skip to content

Commit fdf9916

Browse files
JamieMageeJamieMagee
authored
Migrate vcpkg and Go detectors from Newtonsoft.Json to System.TextJson (#1570)
* Migrate vcpkg and Go detectors from `Newtonsoft.Json` to `System.Text.Json` - `VcpkgComponentDetector`: Replace `JsonConvert.DeserializeObject` with `JsonSerializer.Deserialize` - `GoCLIParser`: Replace `JsonTextReader`/`JsonSerializer` with `Utf8JsonReader` for parsing multi-object JSON output from `go list` - .NET 9 added `AllowMultipleValues`, but unfortunately we are still on .NET 8 - Add explicit `[JsonPropertyName]` attributes to vcpkg contract classes using SPDX 2.2.1 schema field names * Optimize Go CLI output parsing by converting to utf-8 only once * Add unit test coverage for Go CLI parser * PR comments * Pass `CancellationToken` to `ParseAsync`
1 parent b88d00b commit fdf9916

File tree

12 files changed

+547
-47
lines changed

12 files changed

+547
-47
lines changed

‎src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
115115
case ".MOD":
116116
{
117117
this.Logger.LogDebug("Found Go.mod: {Location}", file.Location);
118-
var wasModParsedSuccessfully = await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
118+
var wasModParsedSuccessfully = await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record, cancellationToken);
119119

120120
// Check if go.mod was parsed successfully and Go version is >= 1.17 in go.mod
121121
if (wasModParsedSuccessfully &&
@@ -155,7 +155,7 @@ await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync(
155155
{
156156
if (!wasGoCliDisabled)
157157
{
158-
wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
158+
wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record, cancellationToken);
159159
}
160160
else
161161
{
@@ -176,7 +176,7 @@ await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync(
176176
{
177177
record.WasGoFallbackStrategyUsed = true;
178178
this.Logger.LogDebug("Go CLI scan when considering {GoSumLocation} was not successful. Falling back to scanning go.sum", file.Location);
179-
await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
179+
await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record, cancellationToken);
180180
}
181181
else
182182
{

‎src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoCLIParser.cs‎

Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,15 @@ namespace Microsoft.ComponentDetection.Detectors.Go;
33

44
using System;
55
using System.Collections.Generic;
6-
using System.Diagnostics.CodeAnalysis;
76
using System.IO;
7+
using System.Text;
8+
using System.Text.Json;
9+
using System.Threading;
810
using System.Threading.Tasks;
911
using Microsoft.ComponentDetection.Common.Telemetry.Records;
1012
using Microsoft.ComponentDetection.Contracts;
1113
using Microsoft.ComponentDetection.Contracts.TypedComponent;
1214
using Microsoft.Extensions.Logging;
13-
using Newtonsoft.Json;
1415

1516
public class GoCLIParser : IGoParser
1617
{
@@ -25,11 +26,11 @@ public GoCLIParser(ILogger logger, IFileUtilityService fileUtilityService, IComm
2526
this.commandLineInvocationService = commandLineInvocationService;
2627
}
2728

28-
[SuppressMessage("Maintainability", "CA1508:Avoid dead conditional code", Justification = "False positive")]
2929
public async Task<bool> ParseAsync(
3030
ISingleFileComponentRecorder singleFileComponentRecorder,
3131
IComponentStream file,
32-
GoGraphTelemetryRecord record)
32+
GoGraphTelemetryRecord record,
33+
CancellationToken cancellationToken = default)
3334
{
3435
record.WasGraphSuccessful = false;
3536
record.DidGoCliCommandFail = false;
@@ -49,7 +50,7 @@ public async Task<bool> ParseAsync(
4950
"Detection time may be improved by activating fallback strategy (https://github.com/microsoft/component-detection/blob/main/docs/detectors/go.md#fallback-detection-strategy). " +
5051
"But, it will introduce noise into the detected components.");
5152

52-
var goDependenciesProcess = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, workingDirectory: projectRootDirectory, ["list", "-mod=readonly", "-m", "-json", "all"]);
53+
var goDependenciesProcess = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, projectRootDirectory, cancellationToken, "list", "-mod=readonly", "-m", "-json", "all");
5354
if (goDependenciesProcess.ExitCode != 0)
5455
{
5556
this.logger.LogError("Go CLI command \"go list -m -json all\" failed with error: {GoDependenciesProcessStdErr}", goDependenciesProcess.StdErr);
@@ -66,27 +67,41 @@ public async Task<bool> ParseAsync(
6667
this.logger,
6768
singleFileComponentRecorder,
6869
projectRootDirectory.FullName,
69-
record);
70+
record,
71+
cancellationToken);
7072

7173
return true;
7274
}
7375

7476
private void RecordBuildDependencies(string goListOutput, ISingleFileComponentRecorder singleFileComponentRecorder, string projectRootDirectoryFullName)
7577
{
7678
var goBuildModules = new List<GoBuildModule>();
77-
var reader = new JsonTextReader(new StringReader(goListOutput))
78-
{
79-
SupportMultipleContent = true,
80-
};
8179

8280
using var record = new GoReplaceTelemetryRecord();
8381

84-
while (reader.Read())
82+
// Go CLI outputs multiple JSON objects (not an array), so we need to parse them one by one
83+
var utf8Bytes = Encoding.UTF8.GetBytes(goListOutput);
84+
var remaining = utf8Bytes.AsSpan();
85+
while (!remaining.IsEmpty)
8586
{
86-
var serializer = new JsonSerializer();
87-
var buildModule = serializer.Deserialize<GoBuildModule>(reader);
87+
var reader = new Utf8JsonReader(remaining);
88+
try
89+
{
90+
var buildModule = JsonSerializer.Deserialize<GoBuildModule>(ref reader);
91+
if (buildModule != null)
92+
{
93+
goBuildModules.Add(buildModule);
94+
}
8895

89-
goBuildModules.Add(buildModule);
96+
// Move past the consumed bytes plus any whitespace
97+
var consumed = (int)reader.BytesConsumed;
98+
remaining = remaining[consumed..];
99+
}
100+
catch (JsonException ex)
101+
{
102+
this.logger.LogWarning("Failed to parse Go build module JSON: {ErrorMessage}", ex.Message);
103+
break;
104+
}
90105
}
91106

92107
foreach (var dependency in goBuildModules)

‎src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ namespace Microsoft.ComponentDetection.Detectors.Go;
55
using System.Collections.Generic;
66
using System.IO;
77
using System.Text.RegularExpressions;
8+
using System.Threading;
89
using System.Threading.Tasks;
910
using Microsoft.ComponentDetection.Common.Telemetry.Records;
1011
using Microsoft.ComponentDetection.Contracts;
@@ -71,7 +72,8 @@ private static void HandleReplaceDirective(
7172
public async Task<bool> ParseAsync(
7273
ISingleFileComponentRecorder singleFileComponentRecorder,
7374
IComponentStream file,
74-
GoGraphTelemetryRecord record)
75+
GoGraphTelemetryRecord record,
76+
CancellationToken cancellationToken = default)
7577
{
7678
// Collect replace directives
7779
var (replacePathDirectives, moduleReplacements) = await this.GetAllReplaceDirectivesAsync(file);
@@ -84,7 +86,7 @@ public async Task<bool> ParseAsync(
8486
// There can be multiple require( ) sections in go 1.17+. loop over all of them.
8587
while (!reader.EndOfStream)
8688
{
87-
var line = await reader.ReadLineAsync();
89+
var line = await reader.ReadLineAsync(cancellationToken);
8890

8991
while (line != null && !line.StartsWith("require ("))
9092
{
@@ -100,11 +102,11 @@ public async Task<bool> ParseAsync(
100102
this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replacePathDirectives, moduleReplacements);
101103
}
102104

103-
line = await reader.ReadLineAsync();
105+
line = await reader.ReadLineAsync(cancellationToken);
104106
}
105107

106108
// Stopping at the first ) restrict the detection to only the require section.
107-
while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(')'))
109+
while ((line = await reader.ReadLineAsync(cancellationToken)) != null && !line.EndsWith(')'))
108110
{
109111
this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replacePathDirectives, moduleReplacements);
110112
}

‎src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoSumParser.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Go;
33

44
using System.IO;
55
using System.Text.RegularExpressions;
6+
using System.Threading;
67
using System.Threading.Tasks;
78
using Microsoft.ComponentDetection.Common.Telemetry.Records;
89
using Microsoft.ComponentDetection.Contracts;
@@ -24,12 +25,13 @@ public class GoSumParser : IGoParser
2425
public async Task<bool> ParseAsync(
2526
ISingleFileComponentRecorder singleFileComponentRecorder,
2627
IComponentStream file,
27-
GoGraphTelemetryRecord record)
28+
GoGraphTelemetryRecord record,
29+
CancellationToken cancellationToken = default)
2830
{
2931
using var reader = new StreamReader(file.Stream);
3032

3133
string line;
32-
while ((line = await reader.ReadLineAsync()) != null)
34+
while ((line = await reader.ReadLineAsync(cancellationToken)) != null)
3335
{
3436
if (this.TryToCreateGoComponentFromSumLine(line, out var goComponent))
3537
{

‎src/Microsoft.ComponentDetection.Detectors/go/Parsers/IGoParser.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
#nullable disable
22
namespace Microsoft.ComponentDetection.Detectors.Go;
33

4+
using System.Threading;
45
using System.Threading.Tasks;
56
using Microsoft.ComponentDetection.Common.Telemetry.Records;
67
using Microsoft.ComponentDetection.Contracts;
78

89
public interface IGoParser
910
{
10-
Task<bool> ParseAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file, GoGraphTelemetryRecord record);
11+
Task<bool> ParseAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file, GoGraphTelemetryRecord record, CancellationToken cancellationToken = default);
1112
}

‎src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,19 @@
22
namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts;
33

44
using System;
5+
using System.Text.Json.Serialization;
56

67
public class Annotation
78
{
9+
[JsonPropertyName("annotationDate")]
810
public DateTime Date { get; set; }
911

12+
[JsonPropertyName("comment")]
1013
public string Comment { get; set; }
1114

15+
[JsonPropertyName("annotationType")]
1216
public string Type { get; set; }
1317

18+
[JsonPropertyName("annotator")]
1419
public string Annotator { get; set; }
1520
}

‎src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/ManifestInfo.cs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@
22
namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts;
33

44
using System.Text.Json.Serialization;
5-
using Newtonsoft.Json;
65

76
public class ManifestInfo
87
{
9-
[JsonProperty("manifest-path")]
108
[JsonPropertyName("manifest-path")]
119
public string ManifestPath { get; set; }
1210
}

‎src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,31 @@
11
#nullable disable
22
namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts;
33

4+
using System.Text.Json.Serialization;
5+
46
public class Package
57
{
8+
[JsonPropertyName("SPDXID")]
69
public string SPDXID { get; set; }
710

11+
[JsonPropertyName("versionInfo")]
812
public string VersionInfo { get; set; }
913

14+
[JsonPropertyName("downloadLocation")]
1015
public string DownloadLocation { get; set; }
1116

17+
[JsonPropertyName("packageFileName")]
1218
public string Filename { get; set; }
1319

20+
[JsonPropertyName("homepage")]
1421
public string Homepage { get; set; }
1522

23+
[JsonPropertyName("description")]
1624
public string Description { get; set; }
1725

26+
[JsonPropertyName("name")]
1827
public string Name { get; set; }
1928

29+
[JsonPropertyName("annotations")]
2030
public Annotation[] Annotations { get; set; }
2131
}

‎src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
#nullable disable
22
namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts;
33

4+
using System.Text.Json.Serialization;
5+
46
/// <summary>
57
/// Matches a subset of https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json.
68
/// </summary>
79
public class VcpkgSBOM
810
{
11+
[JsonPropertyName("packages")]
912
public Package[] Packages { get; set; }
1013

14+
[JsonPropertyName("name")]
1115
public string Name { get; set; }
1216
}

‎src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ namespace Microsoft.ComponentDetection.Detectors.Vcpkg;
77
using System.IO;
88
using System.Linq;
99
using System.Reactive.Linq;
10+
using System.Text.Json;
1011
using System.Threading;
1112
using System.Threading.Tasks;
1213
using Microsoft.ComponentDetection.Contracts;
1314
using Microsoft.ComponentDetection.Contracts.Internal;
1415
using Microsoft.ComponentDetection.Contracts.TypedComponent;
1516
using Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts;
1617
using Microsoft.Extensions.Logging;
17-
using Newtonsoft.Json;
1818

1919
public class VcpkgComponentDetector : FileComponentDetector
2020
{
@@ -83,7 +83,7 @@ await processRequests.ForEachAsync(async pr =>
8383
using (var reader = new StreamReader(pr.ComponentStream.Stream))
8484
{
8585
var contents = await reader.ReadToEndAsync().ConfigureAwait(false);
86-
var manifestData = JsonConvert.DeserializeObject<ManifestInfo>(contents);
86+
var manifestData = JsonSerializer.Deserialize<ManifestInfo>(contents);
8787

8888
if (manifestData == null || string.IsNullOrWhiteSpace(manifestData.ManifestPath))
8989
{
@@ -112,7 +112,7 @@ private async Task ParseSpdxFileAsync(
112112
VcpkgSBOM sbom;
113113
try
114114
{
115-
sbom = JsonConvert.DeserializeObject<VcpkgSBOM>(await reader.ReadToEndAsync());
115+
sbom = JsonSerializer.Deserialize<VcpkgSBOM>(await reader.ReadToEndAsync());
116116
}
117117
catch (Exception)
118118
{

0 commit comments

Comments
 (0)