diff --git a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs index 620c309c3..2625d2f56 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs @@ -115,7 +115,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID case ".MOD": { this.Logger.LogDebug("Found Go.mod: {Location}", file.Location); - var wasModParsedSuccessfully = await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + var wasModParsedSuccessfully = await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record, cancellationToken); // Check if go.mod was parsed successfully and Go version is >= 1.17 in go.mod if (wasModParsedSuccessfully && @@ -155,7 +155,7 @@ await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync( { if (!wasGoCliDisabled) { - wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record, cancellationToken); } else { @@ -176,7 +176,7 @@ await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync( { record.WasGoFallbackStrategyUsed = true; this.Logger.LogDebug("Go CLI scan when considering {GoSumLocation} was not successful. Falling back to scanning go.sum", file.Location); - await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record, cancellationToken); } else { diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoCLIParser.cs b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoCLIParser.cs index a088ced7b..aea3c4355 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoCLIParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoCLIParser.cs @@ -3,14 +3,15 @@ namespace Microsoft.ComponentDetection.Detectors.Go; using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Text; +using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; public class GoCLIParser : IGoParser { @@ -25,11 +26,11 @@ public GoCLIParser(ILogger logger, IFileUtilityService fileUtilityService, IComm this.commandLineInvocationService = commandLineInvocationService; } - [SuppressMessage("Maintainability", "CA1508:Avoid dead conditional code", Justification = "False positive")] public async Task ParseAsync( ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file, - GoGraphTelemetryRecord record) + GoGraphTelemetryRecord record, + CancellationToken cancellationToken = default) { record.WasGraphSuccessful = false; record.DidGoCliCommandFail = false; @@ -49,7 +50,7 @@ public async Task ParseAsync( "Detection time may be improved by activating fallback strategy (https://github.com/microsoft/component-detection/blob/main/docs/detectors/go.md#fallback-detection-strategy). " + "But, it will introduce noise into the detected components."); - var goDependenciesProcess = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, workingDirectory: projectRootDirectory, ["list", "-mod=readonly", "-m", "-json", "all"]); + var goDependenciesProcess = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, projectRootDirectory, cancellationToken, "list", "-mod=readonly", "-m", "-json", "all"); if (goDependenciesProcess.ExitCode != 0) { this.logger.LogError("Go CLI command \"go list -m -json all\" failed with error: {GoDependenciesProcessStdErr}", goDependenciesProcess.StdErr); @@ -66,7 +67,8 @@ public async Task ParseAsync( this.logger, singleFileComponentRecorder, projectRootDirectory.FullName, - record); + record, + cancellationToken); return true; } @@ -74,19 +76,32 @@ public async Task ParseAsync( private void RecordBuildDependencies(string goListOutput, ISingleFileComponentRecorder singleFileComponentRecorder, string projectRootDirectoryFullName) { var goBuildModules = new List(); - var reader = new JsonTextReader(new StringReader(goListOutput)) - { - SupportMultipleContent = true, - }; using var record = new GoReplaceTelemetryRecord(); - while (reader.Read()) + // Go CLI outputs multiple JSON objects (not an array), so we need to parse them one by one + var utf8Bytes = Encoding.UTF8.GetBytes(goListOutput); + var remaining = utf8Bytes.AsSpan(); + while (!remaining.IsEmpty) { - var serializer = new JsonSerializer(); - var buildModule = serializer.Deserialize(reader); + var reader = new Utf8JsonReader(remaining); + try + { + var buildModule = JsonSerializer.Deserialize(ref reader); + if (buildModule != null) + { + goBuildModules.Add(buildModule); + } - goBuildModules.Add(buildModule); + // Move past the consumed bytes plus any whitespace + var consumed = (int)reader.BytesConsumed; + remaining = remaining[consumed..]; + } + catch (JsonException ex) + { + this.logger.LogWarning("Failed to parse Go build module JSON: {ErrorMessage}", ex.Message); + break; + } } foreach (var dependency in goBuildModules) diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs index 2c3afee0b..2d3f98e45 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs @@ -5,6 +5,7 @@ namespace Microsoft.ComponentDetection.Detectors.Go; using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; @@ -71,7 +72,8 @@ private static void HandleReplaceDirective( public async Task ParseAsync( ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file, - GoGraphTelemetryRecord record) + GoGraphTelemetryRecord record, + CancellationToken cancellationToken = default) { // Collect replace directives var (replacePathDirectives, moduleReplacements) = await this.GetAllReplaceDirectivesAsync(file); @@ -84,7 +86,7 @@ public async Task ParseAsync( // There can be multiple require( ) sections in go 1.17+. loop over all of them. while (!reader.EndOfStream) { - var line = await reader.ReadLineAsync(); + var line = await reader.ReadLineAsync(cancellationToken); while (line != null && !line.StartsWith("require (")) { @@ -100,11 +102,11 @@ public async Task ParseAsync( this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replacePathDirectives, moduleReplacements); } - line = await reader.ReadLineAsync(); + line = await reader.ReadLineAsync(cancellationToken); } // Stopping at the first ) restrict the detection to only the require section. - while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(')')) + while ((line = await reader.ReadLineAsync(cancellationToken)) != null && !line.EndsWith(')')) { this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replacePathDirectives, moduleReplacements); } diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoSumParser.cs b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoSumParser.cs index 0b7229f27..c612731a0 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoSumParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoSumParser.cs @@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Go; using System.IO; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; @@ -24,12 +25,13 @@ public class GoSumParser : IGoParser public async Task ParseAsync( ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file, - GoGraphTelemetryRecord record) + GoGraphTelemetryRecord record, + CancellationToken cancellationToken = default) { using var reader = new StreamReader(file.Stream); string line; - while ((line = await reader.ReadLineAsync()) != null) + while ((line = await reader.ReadLineAsync(cancellationToken)) != null) { if (this.TryToCreateGoComponentFromSumLine(line, out var goComponent)) { diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/IGoParser.cs b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/IGoParser.cs index 36ce27164..851528e45 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/IGoParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/IGoParser.cs @@ -1,11 +1,12 @@ #nullable disable namespace Microsoft.ComponentDetection.Detectors.Go; +using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; public interface IGoParser { - Task ParseAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file, GoGraphTelemetryRecord record); + Task ParseAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file, GoGraphTelemetryRecord record, CancellationToken cancellationToken = default); } diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs index 073e2a6ab..df17d80d4 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs @@ -2,14 +2,19 @@ namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts; using System; +using System.Text.Json.Serialization; public class Annotation { + [JsonPropertyName("annotationDate")] public DateTime Date { get; set; } + [JsonPropertyName("comment")] public string Comment { get; set; } + [JsonPropertyName("annotationType")] public string Type { get; set; } + [JsonPropertyName("annotator")] public string Annotator { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/ManifestInfo.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/ManifestInfo.cs index 1e2be2e9f..4fd150f98 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/ManifestInfo.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/ManifestInfo.cs @@ -2,11 +2,9 @@ namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts; using System.Text.Json.Serialization; -using Newtonsoft.Json; public class ManifestInfo { - [JsonProperty("manifest-path")] [JsonPropertyName("manifest-path")] public string ManifestPath { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs index 1c33c5148..bec371da8 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs @@ -1,21 +1,31 @@ #nullable disable namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts; +using System.Text.Json.Serialization; + public class Package { + [JsonPropertyName("SPDXID")] public string SPDXID { get; set; } + [JsonPropertyName("versionInfo")] public string VersionInfo { get; set; } + [JsonPropertyName("downloadLocation")] public string DownloadLocation { get; set; } + [JsonPropertyName("packageFileName")] public string Filename { get; set; } + [JsonPropertyName("homepage")] public string Homepage { get; set; } + [JsonPropertyName("description")] public string Description { get; set; } + [JsonPropertyName("name")] public string Name { get; set; } + [JsonPropertyName("annotations")] public Annotation[] Annotations { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs index 1eef00945..34bf19fd6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs @@ -1,12 +1,16 @@ #nullable disable namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts; +using System.Text.Json.Serialization; + /// /// Matches a subset of https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json. /// public class VcpkgSBOM { + [JsonPropertyName("packages")] public Package[] Packages { get; set; } + [JsonPropertyName("name")] public string Name { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index 59bc7f07a..ddc30ad78 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -7,6 +7,7 @@ namespace Microsoft.ComponentDetection.Detectors.Vcpkg; using System.IO; using System.Linq; using System.Reactive.Linq; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; @@ -14,7 +15,6 @@ namespace Microsoft.ComponentDetection.Detectors.Vcpkg; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; public class VcpkgComponentDetector : FileComponentDetector { @@ -83,7 +83,7 @@ await processRequests.ForEachAsync(async pr => using (var reader = new StreamReader(pr.ComponentStream.Stream)) { var contents = await reader.ReadToEndAsync().ConfigureAwait(false); - var manifestData = JsonConvert.DeserializeObject(contents); + var manifestData = JsonSerializer.Deserialize(contents); if (manifestData == null || string.IsNullOrWhiteSpace(manifestData.ManifestPath)) { @@ -112,7 +112,7 @@ private async Task ParseSpdxFileAsync( VcpkgSBOM sbom; try { - sbom = JsonConvert.DeserializeObject(await reader.ReadToEndAsync()); + sbom = JsonSerializer.Deserialize(await reader.ReadToEndAsync()); } catch (Exception) { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/GoCLIParserTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/GoCLIParserTests.cs new file mode 100644 index 000000000..316102737 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/GoCLIParserTests.cs @@ -0,0 +1,463 @@ +#nullable disable +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using AwesomeAssertions; +using Microsoft.ComponentDetection.Common.Telemetry.Records; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Go; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class GoCLIParserTests +{ + private GoCLIParser parser; + private Mock commandLineMock; + private Mock fileUtilityMock; + private Mock loggerMock; + + [TestInitialize] + public void TestInitialize() + { + this.commandLineMock = new Mock(); + this.fileUtilityMock = new Mock(); + this.loggerMock = new Mock(); + + this.parser = new GoCLIParser( + this.loggerMock.Object, + this.fileUtilityMock.Object, + this.commandLineMock.Object); + } + + [TestMethod] + public async Task ParseAsync_GoCliNotAvailable_ReturnsFalse() + { + // Arrange + this.SetupGoAvailable(false); + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, _) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeFalse(); + record.IsGoAvailable.Should().BeFalse(); + } + + [TestMethod] + public async Task ParseAsync_GoListCommandFails_ReturnsFalse() + { + // Arrange + this.SetupGoAvailable(true); + this.SetupGoListCommand(stdOut: string.Empty, exitCode: 1, stdErr: "go: error loading module"); + + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, _) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeFalse(); + record.DidGoCliCommandFail.Should().BeTrue(); + record.GoCliCommandError.Should().Be("go: error loading module"); + } + + [TestMethod] + public async Task ParseAsync_SingleDirectDependency_RegistersWithExplicitFlag() + { + // Arrange + this.SetupGoAvailable(true); + var goListOutput = BuildGoListJsonOutput( + new GoBuildModuleTestData { Path = "example.com/main", Version = "v1.0.0", Main = true }, + new GoBuildModuleTestData { Path = "github.com/dep/one", Version = "v1.2.3", Indirect = false }); + this.SetupGoListCommand(goListOutput); + this.SetupGoModGraphCommand(string.Empty); + + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, captured) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeTrue(); + captured.Should().ContainSingle(); + + var (component, isExplicit) = captured[0]; + var goComponent = component.Component as GoComponent; + goComponent.Name.Should().Be("github.com/dep/one"); + goComponent.Version.Should().Be("v1.2.3"); + isExplicit.Should().BeTrue(); + } + + [TestMethod] + public async Task ParseAsync_SingleIndirectDependency_RegistersWithoutExplicitFlag() + { + // Arrange + this.SetupGoAvailable(true); + var goListOutput = BuildGoListJsonOutput( + new GoBuildModuleTestData { Path = "example.com/main", Version = "v1.0.0", Main = true }, + new GoBuildModuleTestData { Path = "github.com/dep/indirect", Version = "v2.0.0", Indirect = true }); + this.SetupGoListCommand(goListOutput); + this.SetupGoModGraphCommand(string.Empty); + + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, captured) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeTrue(); + captured.Should().ContainSingle(); + + var (component, isExplicit) = captured[0]; + var goComponent = component.Component as GoComponent; + goComponent.Name.Should().Be("github.com/dep/indirect"); + isExplicit.Should().BeFalse(); + } + + [TestMethod] + public async Task ParseAsync_MainModuleSkipped_NotRegistered() + { + // Arrange + this.SetupGoAvailable(true); + var goListOutput = BuildGoListJsonOutput( + new GoBuildModuleTestData { Path = "example.com/main", Version = "v1.0.0", Main = true }); + this.SetupGoListCommand(goListOutput); + this.SetupGoModGraphCommand(string.Empty); + + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, captured) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeTrue(); + captured.Should().BeEmpty(); + } + + [TestMethod] + public async Task ParseAsync_ReplaceWithVersion_UsesReplacementModule() + { + // Arrange + this.SetupGoAvailable(true); + var goListOutput = BuildGoListJsonOutput( + new GoBuildModuleTestData { Path = "example.com/main", Version = "v1.0.0", Main = true }, + new GoBuildModuleTestData + { + Path = "github.com/original/pkg", + Version = "v1.0.0", + Replace = new GoBuildModuleTestData { Path = "github.com/fork/pkg", Version = "v1.1.0" }, + }); + this.SetupGoListCommand(goListOutput); + this.SetupGoModGraphCommand(string.Empty); + + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, captured) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeTrue(); + captured.Should().ContainSingle(); + + var goComponent = captured[0].Component.Component as GoComponent; + goComponent.Name.Should().Be("github.com/fork/pkg"); + goComponent.Version.Should().Be("v1.1.0"); + } + + [TestMethod] + public async Task ParseAsync_ReplaceWithLocalPath_FileExists_SkipsModule() + { + // Arrange + this.SetupGoAvailable(true); + var goListOutput = BuildGoListJsonOutput( + new GoBuildModuleTestData { Path = "example.com/main", Version = "v1.0.0", Main = true }, + new GoBuildModuleTestData + { + Path = "github.com/replaced/pkg", + Version = "v1.0.0", + Replace = new GoBuildModuleTestData { Path = "../local/module" }, + }); + this.SetupGoListCommand(goListOutput); + this.SetupGoModGraphCommand(string.Empty); + + // Local go.mod file exists + this.fileUtilityMock.Setup(f => f.Exists(It.IsAny())).Returns(true); + + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, captured) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeTrue(); + captured.Should().BeEmpty(); // Local replacement should be skipped + } + + [TestMethod] + public async Task ParseAsync_ReplaceWithLocalPath_FileNotExists_RegistersOriginalModule() + { + // Arrange + this.SetupGoAvailable(true); + var goListOutput = BuildGoListJsonOutput( + new GoBuildModuleTestData { Path = "example.com/main", Version = "v1.0.0", Main = true }, + new GoBuildModuleTestData + { + Path = "github.com/replaced/pkg", + Version = "v1.0.0", + Replace = new GoBuildModuleTestData { Path = "../missing/module" }, + }); + this.SetupGoListCommand(goListOutput); + this.SetupGoModGraphCommand(string.Empty); + + // Local go.mod file does NOT exist + this.fileUtilityMock.Setup(f => f.Exists(It.IsAny())).Returns(false); + + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, captured) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeTrue(); + + // Should still register the original module since local replacement is invalid + captured.Should().ContainSingle(); + var goComponent = captured[0].Component.Component as GoComponent; + goComponent.Name.Should().Be("github.com/replaced/pkg"); + goComponent.Version.Should().Be("v1.0.0"); + } + + [TestMethod] + public async Task ParseAsync_MultipleDependencies_RegistersAll() + { + // Arrange + this.SetupGoAvailable(true); + var goListOutput = BuildGoListJsonOutput( + new GoBuildModuleTestData { Path = "example.com/main", Version = "v1.0.0", Main = true }, + new GoBuildModuleTestData { Path = "github.com/dep/one", Version = "v1.0.0", Indirect = false }, + new GoBuildModuleTestData { Path = "github.com/dep/two", Version = "v2.0.0", Indirect = true }, + new GoBuildModuleTestData { Path = "github.com/dep/three", Version = "v3.0.0", Indirect = false }); + this.SetupGoListCommand(goListOutput); + this.SetupGoModGraphCommand(string.Empty); + + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, captured) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeTrue(); + captured.Should().HaveCount(3); + + var names = captured.Select(c => (c.Component.Component as GoComponent).Name).ToList(); + names.Should().Contain("github.com/dep/one"); + names.Should().Contain("github.com/dep/two"); + names.Should().Contain("github.com/dep/three"); + } + + [TestMethod] + public async Task ParseAsync_Success_PopulatesTelemetryRecord() + { + // Arrange + this.SetupGoAvailable(true); + var goListOutput = BuildGoListJsonOutput( + new GoBuildModuleTestData { Path = "example.com/main", Version = "v1.0.0", Main = true }); + this.SetupGoListCommand(goListOutput); + this.SetupGoModGraphCommand(string.Empty, exitCode: 0); + + var stream = CreateComponentStream("/project/go.mod"); + var record = new GoGraphTelemetryRecord(); + var (recorderMock, _) = CreateCapturingRecorder(); + + // Act + var result = await this.parser.ParseAsync(recorderMock.Object, stream, record); + + // Assert + result.Should().BeTrue(); + record.IsGoAvailable.Should().BeTrue(); + record.DidGoCliCommandFail.Should().BeFalse(); + record.ProjectRoot.Should().EndWith("project"); + record.WasGraphSuccessful.Should().BeTrue(); + } + + /// + /// Creates a mock IComponentStream with the specified location. + /// + private static IComponentStream CreateComponentStream(string location) + { + var mock = new Mock(); + mock.Setup(s => s.Location).Returns(location); + return mock.Object; + } + + /// + /// Creates a mock ISingleFileComponentRecorder that captures registered components. + /// + private static (Mock Mock, List<(DetectedComponent Component, bool IsExplicit)> Captured) CreateCapturingRecorder() + { + var captured = new List<(DetectedComponent Component, bool IsExplicit)>(); + var mock = new Mock(); + + mock.Setup(r => r.RegisterUsage( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback( + (component, isExplicit, parentId, isDev, scope, framework) => + { + captured.Add((component, isExplicit)); + }); + + return (mock, captured); + } + + /// + /// Builds a JSON output string for go list -m -json all command. + /// Each module is a separate JSON object (not an array). + /// + private static string BuildGoListJsonOutput(params GoBuildModuleTestData[] modules) + { + var sb = new StringBuilder(); + foreach (var module in modules) + { + sb.AppendLine(module.ToJson()); + } + + return sb.ToString(); + } + + /// + /// Sets up the command line mock to indicate Go CLI is available. + /// + private void SetupGoAvailable(bool isAvailable = true) + { + this.commandLineMock + .Setup(c => c.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.Is(args => args.Contains("version")))) + .ReturnsAsync(isAvailable); + } + + /// + /// Sets up the go list command to return the specified output. + /// + private void SetupGoListCommand(string stdOut, int exitCode = 0, string stdErr = "") + { + this.commandLineMock + .Setup(c => c.ExecuteCommandAsync( + "go", + null, + It.IsAny(), + It.IsAny(), + It.Is(args => args.Contains("list") && args.Contains("-m") && args.Contains("-json") && args.Contains("all")))) + .ReturnsAsync(new CommandLineExecutionResult + { + ExitCode = exitCode, + StdOut = stdOut, + StdErr = stdErr, + }); + } + + /// + /// Sets up the go mod graph command to return the specified output. + /// + private void SetupGoModGraphCommand(string stdOut, int exitCode = 0) + { + this.commandLineMock + .Setup(c => c.ExecuteCommandAsync( + "go", + null, + It.IsAny(), + It.IsAny(), + It.Is(args => args.Contains("mod") && args.Contains("graph")))) + .ReturnsAsync(new CommandLineExecutionResult + { + ExitCode = exitCode, + StdOut = stdOut, + }); + } + + /// + /// Test data class for building Go module JSON. + /// + private class GoBuildModuleTestData + { + public string Path { get; set; } + + public string Version { get; set; } + + public bool Main { get; set; } + + public bool Indirect { get; set; } + + public GoBuildModuleTestData Replace { get; set; } + + public string ToJson() + { + var parts = new List + { + $"\"Path\": \"{this.Path}\"", + }; + + if (this.Version != null) + { + parts.Add($"\"Version\": \"{this.Version}\""); + } + + if (this.Main) + { + parts.Add("\"Main\": true"); + } + + if (this.Indirect) + { + parts.Add("\"Indirect\": true"); + } + + if (this.Replace != null) + { + var replaceParts = new List { $"\"Path\": \"{this.Replace.Path}\"" }; + if (this.Replace.Version != null) + { + replaceParts.Add($"\"Version\": \"{this.Replace.Version}\""); + } + + parts.Add($"\"Replace\": {{ {string.Join(", ", replaceParts)} }}"); + } + + return $"{{ {string.Join(", ", parts)} }}"; + } + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs index 97d06c3e3..7f159d323 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs @@ -869,7 +869,7 @@ public async Task GoModDetector_GoModFileFound_GoModParserIsExecuted() scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - goModParserMock.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + goModParserMock.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } /// @@ -888,10 +888,10 @@ public async Task GoDetector_GoSum_GoSumParserExecuted(bool goCliSucceeds) this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); // Setup go cli parser to succeed/fail - this.mockGoCliParser.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(goCliSucceeds); + this.mockGoCliParser.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(goCliSucceeds); // Setup go sum parser to succeed - this.mockGoSumParser.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); + this.mockGoSumParser.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.sum", string.Empty) .ExecuteDetectorAsync(); @@ -913,8 +913,8 @@ public async Task GoDetector_GoSum_GoSumParserExecutedIfCliDisabled() // Setup environment variable to disable CLI scan this.envVarService.Setup(s => s.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(true); - // Setup go sum parser to succed - goSumParserMock.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); + // Setup go sum parser to succeed + goSumParserMock.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.sum", string.Empty) @@ -941,7 +941,7 @@ public async Task GoModDetector_ExecutingGoVersionFails_DetectorDoesNotFail() scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - this.mockGoModParser.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + this.mockGoModParser.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } [TestMethod] @@ -996,9 +996,9 @@ public async Task GoDetector_GoMod_VerifyNestedRootsUnderGTE117_AreSkipped() var processedFiles = new List(); this.SetupMockGoModParser(); this.mockGoModParser - .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(true) - .Callback((_, file, record) => + .Callback((_, file, record, _) => { processedFiles.Add(file.Location); record.GoModVersion = "1.18"; @@ -1031,9 +1031,9 @@ public async Task GoDetector_GoMod_VerifyNestedRootsUnderLT117AreNotSkipped() var processedFiles = new List(); this.SetupMockGoModParser(); this.mockGoModParser - .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(true) - .Callback((_, file, record) => + .Callback((_, file, record, _) => { processedFiles.Add(file.Location); var rootMod = Path.Combine(root, "go.mod"); @@ -1080,8 +1080,8 @@ public async Task GoDetector_GoMod_VerifyNestedRootsAreNotSkippedIfParentParseFa this.SetupMockGoModParser(); this.mockGoModParser - .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record) => + .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record, CancellationToken cancellationToken) => { processedFiles.Add(file.Location); var aMod = Path.Combine(root, "a", "go.mod"); @@ -1134,9 +1134,9 @@ public async Task GoDetector_GoSum_VerifyNestedRootsUnderGoSum_AreSkipped() this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); this.SetupMockGoCLIParser(); this.mockGoCliParser - .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(true) - .Callback((_, file, record) => + .Callback((_, file, record, _) => { processedFiles.Add(file.Location); }); @@ -1167,8 +1167,8 @@ public async Task GoDetector_GoSum_VerifyNestedRootsAreNotSkippedIfParentParseFa this.SetupMockGoSumParser(); this.mockGoModParser - .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record) => + .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record, CancellationToken cancellationToken) => { processedFiles.Add(file.Location); var bMod = Path.Combine(root, "b", "go.mod"); @@ -1182,8 +1182,8 @@ public async Task GoDetector_GoSum_VerifyNestedRootsAreNotSkippedIfParentParseFa }); this.mockGoCliParser - .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record) => + .Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((ISingleFileComponentRecorder recorder, IComponentStream file, GoGraphTelemetryRecord record, CancellationToken cancellationToken) => { processedFiles.Add(file.Location); return file.Location != Path.Combine(root, "a", "go.sum");