Skip to content

Commit a82a9aa

Browse files
pauld-msftpauld-msft
authored
Improve detection graph traversal for large result sets (#1401)
* test * update comments * initial VERY rough attempt to cache roots/ancestors * pre-compute typed components * remove * use loop * clarity and testing * manually use typed component fill isntead of type logic * update comments * styling * linq * confirm compoennt nodes exist
1 parent 3ebea36 commit a82a9aa

File tree

5 files changed

+222
-101
lines changed

5 files changed

+222
-101
lines changed

‎src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using System.Text;
88
using Microsoft.ComponentDetection.Contracts;
99
using Microsoft.ComponentDetection.Contracts.BcdeModels;
10+
using Microsoft.ComponentDetection.Contracts.TypedComponent;
1011

1112
[assembly: InternalsVisibleTo("Microsoft.ComponentDetection.Common.Tests")]
1213

@@ -141,6 +142,34 @@ public ICollection<string> GetAncestors(string componentId)
141142
.ToList();
142143
}
143144

145+
public HashSet<TypedComponent> GetAncestorsAsTypedComponents(string componentId, Func<string, TypedComponent> toTypedComponent)
146+
{
147+
ArgumentNullException.ThrowIfNull(componentId);
148+
return this.GetAncestors(componentId)
149+
.Select(a => this.componentNodes.TryGetValue(a, out var component) ? component : null)
150+
.Where(a => a != null)
151+
.Select(a => a.TypedComponent ?? toTypedComponent(a.Id))
152+
.ToHashSet(new ComponentComparer());
153+
}
154+
155+
public HashSet<TypedComponent> GetRootsAsTypedComponents(string componentId, Func<string, TypedComponent> toTypedComponent)
156+
{
157+
ArgumentNullException.ThrowIfNull(componentId);
158+
return this.GetExplicitReferencedDependencyIds(componentId)
159+
.Select(r => this.componentNodes.TryGetValue(r, out var component) ? component : null)
160+
.Where(r => r != null)
161+
.Select(r => r.TypedComponent ?? toTypedComponent(r.Id))
162+
.ToHashSet(new ComponentComparer());
163+
}
164+
165+
public void FillTypedComponents(Func<string, TypedComponent> toTypedComponent)
166+
{
167+
foreach (var componentId in this.componentNodes.Values)
168+
{
169+
componentId.TypedComponent = toTypedComponent(componentId.Id);
170+
}
171+
}
172+
144173
IEnumerable<string> IDependencyGraph.GetDependenciesForComponent(string componentId)
145174
{
146175
return this.GetDependenciesForComponent(componentId).ToImmutableList();
@@ -229,5 +258,7 @@ internal ComponentRefNode()
229258
internal bool? IsDevelopmentDependency { get; set; }
230259

231260
internal DependencyScope? DependencyScope { get; set; }
261+
262+
internal TypedComponent TypedComponent { get; set; }
232263
}
233264
}

‎src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphTranslationRecord.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
namespace Microsoft.ComponentDetection.Common.Telemetry.Records;
2+
3+
using System;
4+
5+
public class DependencyGraphTranslationRecord : BaseDetectionTelemetryRecord
6+
{
7+
public override string RecordName => "DependencyGraphTranslationRecord";
8+
9+
public string DetectorId { get; set; }
10+
11+
public TimeSpan? TimeToAddRoots { get; set; }
12+
13+
public TimeSpan? TimeToAddAncestors { get; set; }
14+
}

‎src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
namespace Microsoft.ComponentDetection.Contracts;
22

3+
using System;
34
using System.Collections.Generic;
45
using Microsoft.ComponentDetection.Contracts.BcdeModels;
56

@@ -124,4 +125,28 @@ public interface IDependencyGraph
124125
/// <param name="componentId">The component id to look up ancestors for.</param>
125126
/// <returns>The componentIds that are ancestors for a given componentId.</returns>
126127
ICollection<string> GetAncestors(string componentId);
128+
129+
/// <summary>
130+
/// Gets the component IDs of all explicitly referenced components, and converts them to a set of typed components.
131+
/// WARNING: Using this method without calling <see cref="FillTypedComponents"/> first will result in a performance hit.
132+
/// </summary>
133+
/// <param name="componentId">The component to find all roots for.</param>
134+
/// <param name="toTypedComponent">Function that converts the component id to the typed component object.</param>
135+
/// <returns>Set of TypedComponents containing the roots.</returns>
136+
public HashSet<TypedComponent.TypedComponent> GetRootsAsTypedComponents(string componentId, Func<string, TypedComponent.TypedComponent> toTypedComponent);
137+
138+
/// <summary>
139+
/// Gets the component IDs of all ancestors for a given component id, and converts them to a set of typed components.
140+
/// WARNING: Using this method without calling <see cref="FillTypedComponents"/> first will result in a performance hit.
141+
/// </summary>
142+
/// <param name="componentId">The component to find all roots for.</param>
143+
/// <param name="toTypedComponent">Function that converts the component id to the typed component object.</param>
144+
/// <returns>Set of TypedComponents containing the ancestors.</returns>
145+
public HashSet<TypedComponent.TypedComponent> GetAncestorsAsTypedComponents(string componentId, Func<string, TypedComponent.TypedComponent> toTypedComponent);
146+
147+
/// <summary>
148+
/// This operation pre-fills all nodes with the specified typed component, which improves performance for subsequent runs
149+
/// of <see cref="GetRootsAsTypedComponents"/> and <see cref="GetAncestorsAsTypedComponents"/>.
150+
/// </summary>
151+
public void FillTypedComponents(Func<string, TypedComponent.TypedComponent> toTypedComponent);
127152
}

‎src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -90,11 +90,24 @@ private IEnumerable<DetectedComponent> GatherSetOfDetectedComponentsUnmerged(IEn
9090
.Where(recorderDetectorPair => recorderDetectorPair.Recorder != null)
9191
.SelectMany(recorderDetectorPair =>
9292
{
93+
using var record = new DependencyGraphTranslationRecord()
94+
{
95+
DetectorId = recorderDetectorPair.Detector.Id,
96+
};
97+
9398
var detector = recorderDetectorPair.Detector;
9499
var componentRecorder = recorderDetectorPair.Recorder;
95100
var detectedComponents = componentRecorder.GetDetectedComponents();
96101
var dependencyGraphsByLocation = componentRecorder.GetDependencyGraphsByLocation();
97102

103+
foreach (var graph in dependencyGraphsByLocation.Values)
104+
{
105+
graph.FillTypedComponents(componentRecorder.GetComponent);
106+
}
107+
108+
var totalTimeToAddRoots = TimeSpan.Zero;
109+
var totalTimeToAddAncestors = TimeSpan.Zero;
110+
98111
// Note that it looks like we are building up detected components functionally, but they are not immutable -- the code is just written
99112
// to look like a pipeline.
100113
foreach (var component in detectedComponents)
@@ -118,10 +131,17 @@ private IEnumerable<DetectedComponent> GatherSetOfDetectedComponentsUnmerged(IEn
118131
var dependencyGraph = graphKvp.Value;
119132

120133
// Calculate roots of the component
134+
var rootStartTime = DateTime.UtcNow;
121135
this.AddRootsToDetectedComponent(component, dependencyGraph, componentRecorder);
136+
var rootEndTime = DateTime.UtcNow;
137+
totalTimeToAddRoots += rootEndTime - rootStartTime;
122138

123139
// Calculate Ancestors of the component
140+
var ancestorStartTime = DateTime.UtcNow;
124141
this.AddAncestorsToDetectedComponent(component, dependencyGraph, componentRecorder);
142+
var ancestorEndTime = DateTime.UtcNow;
143+
totalTimeToAddAncestors += ancestorEndTime - ancestorStartTime;
144+
125145
component.DevelopmentDependency = this.MergeDevDependency(component.DevelopmentDependency, dependencyGraph.IsDevelopmentDependency(component.Component.Id));
126146
component.DependencyScope = DependencyScopeComparer.GetMergedDependencyScope(component.DependencyScope, dependencyGraph.GetDependencyScope(component.Component.Id));
127147
component.DetectedBy = detector;
@@ -151,6 +171,9 @@ private IEnumerable<DetectedComponent> GatherSetOfDetectedComponentsUnmerged(IEn
151171
}
152172
}
153173

174+
record.TimeToAddRoots = totalTimeToAddRoots;
175+
record.TimeToAddAncestors = totalTimeToAddAncestors;
176+
154177
return detectedComponents;
155178
}).ToList();
156179
}
@@ -223,35 +246,23 @@ private DetectedComponent MergeComponents(IEnumerable<DetectedComponent> enumera
223246
private void AddRootsToDetectedComponent(DetectedComponent detectedComponent, IDependencyGraph dependencyGraph, IComponentRecorder componentRecorder)
224247
{
225248
detectedComponent.DependencyRoots ??= new HashSet<TypedComponent>(new ComponentComparer());
226-
227249
if (dependencyGraph == null)
228250
{
229251
return;
230252
}
231253

232-
var roots = dependencyGraph.GetExplicitReferencedDependencyIds(detectedComponent.Component.Id);
233-
234-
foreach (var rootId in roots)
235-
{
236-
detectedComponent.DependencyRoots.Add(componentRecorder.GetComponent(rootId));
237-
}
254+
detectedComponent.DependencyRoots.UnionWith(dependencyGraph.GetRootsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent));
238255
}
239256

240257
private void AddAncestorsToDetectedComponent(DetectedComponent detectedComponent, IDependencyGraph dependencyGraph, IComponentRecorder componentRecorder)
241258
{
242259
detectedComponent.AncestralDependencyRoots ??= new HashSet<TypedComponent>(new ComponentComparer());
243-
244260
if (dependencyGraph == null)
245261
{
246262
return;
247263
}
248264

249-
var roots = dependencyGraph.GetAncestors(detectedComponent.Component.Id);
250-
251-
foreach (var rootId in roots)
252-
{
253-
detectedComponent.AncestralDependencyRoots.Add(componentRecorder.GetComponent(rootId));
254-
}
265+
detectedComponent.AncestralDependencyRoots.UnionWith(dependencyGraph.GetAncestorsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent));
255266
}
256267

257268
private HashSet<string> MakeFilePathsRelative(ILogger logger, DirectoryInfo rootDirectory, HashSet<string> filePaths)

0 commit comments

Comments
 (0)