Skip to content

Commit e2d3359

Browse files
hazefullyhazefully
committed
Group Debugger events by PlannerPhase in stats
This commit modifies the EventState class and the StatsMaps that it produces to group Debugger events by the planner phase it was emitted during to enable the ability of getting stats for each planner phase separately. Given that some Debugger events (e.g. InsertIntoMemoEvent) do not store the planner phase they were emitted in (as it not visible to the code that emits them), these events are stored separately and can be retrieved from the StatsMaps (in addition to the ability of retrieving all events, regardless of the planner phase that they were emitted in).
1 parent 4f60160 commit e2d3359

File tree

4 files changed

+153
-44
lines changed

4 files changed

+153
-44
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/debug/EventState.java‎

Lines changed: 46 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@
2323
import com.apple.foundationdb.record.RecordCoreException;
2424
import com.apple.foundationdb.record.logging.KeyValueLogMessage;
2525
import com.apple.foundationdb.record.query.plan.cascades.CascadesRule;
26+
import com.apple.foundationdb.record.query.plan.cascades.PlannerPhase;
2627
import com.apple.foundationdb.record.query.plan.cascades.debug.eventprotos.PEvent;
2728
import com.apple.foundationdb.record.util.pair.Pair;
2829
import com.google.common.base.Verify;
29-
import com.google.common.collect.ImmutableMap;
3030
import com.google.common.collect.Lists;
3131
import com.google.common.collect.Maps;
3232
import org.slf4j.Logger;
@@ -35,29 +35,30 @@
3535
import javax.annotation.Nonnull;
3636
import javax.annotation.Nullable;
3737
import java.util.ArrayDeque;
38+
import java.util.Collections;
3839
import java.util.Deque;
3940
import java.util.Iterator;
41+
import java.util.LinkedHashMap;
4042
import java.util.List;
41-
import java.util.Locale;
4243
import java.util.Map;
4344
import java.util.Objects;
44-
import java.util.concurrent.TimeUnit;
4545

4646
@SuppressWarnings("PMD.SystemPrintln")
4747
public class EventState {
4848
@Nonnull
4949
private static final Logger logger = LoggerFactory.getLogger(EventState.class);
5050

51-
@Nullable private final List<Debugger.Event> events;
52-
@Nullable private final List<PEvent> eventProtos;
53-
@Nullable private final Iterable<PEvent> prerecordedEventProtoIterable;
54-
@Nullable private Iterator<PEvent> prerecordedEventProtoIterator;
51+
@Nullable protected final List<Debugger.Event> events;
52+
@Nullable protected final List<PEvent> eventProtos;
53+
@Nullable protected final Iterable<PEvent> prerecordedEventProtoIterable;
54+
@Nullable protected Iterator<PEvent> prerecordedEventProtoIterator;
5555

56-
@Nonnull private final Map<Class<? extends Debugger.Event>, MutableStats> eventClassStatsMap;
56+
@Nonnull protected final Map<Class<? extends Debugger.Event>, MutableStats> eventWithoutStateClassStatsMap;
57+
@Nonnull protected final Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, MutableStats>> eventWithStateClassStatsMapByPlannerPhase;
5758

58-
@Nonnull private final Map<Class<? extends CascadesRule<?>>, MutableStats> plannerRuleClassStatsMap;
59+
@Nonnull protected final Map<Class<? extends CascadesRule<?>>, MutableStats> plannerRuleClassStatsMap;
5960

60-
@Nonnull private final Deque<Pair<Class<? extends Debugger.Event>, EventDurations>> eventProfilingStack;
61+
@Nonnull protected final Deque<Pair<Class<? extends Debugger.Event>, EventDurations>> eventProfilingStack;
6162

6263
protected int currentTick;
6364
protected final long startTs;
@@ -72,7 +73,8 @@ public static EventState copyOf(final EventState source) {
7273
source.events == null ? null : Lists.newArrayList(source.events),
7374
source.eventProtos == null ? null : Lists.newArrayList(source.eventProtos),
7475
source.prerecordedEventProtoIterable,
75-
Maps.newLinkedHashMap(source.eventClassStatsMap),
76+
Maps.newLinkedHashMap(source.eventWithoutStateClassStatsMap),
77+
Maps.newEnumMap(source.eventWithStateClassStatsMapByPlannerPhase),
7678
Maps.newLinkedHashMap(source.plannerRuleClassStatsMap),
7779
new ArrayDeque<>(source.eventProfilingStack),
7880
source.getCurrentTick(),
@@ -86,28 +88,31 @@ protected EventState(final boolean isRecordEvents, final boolean isRecordEventPr
8688
isRecordEvents ? Lists.newArrayList() : null,
8789
prerecordedEventProtoIterable,
8890
Maps.newLinkedHashMap(),
91+
Maps.newEnumMap(PlannerPhase.class),
8992
Maps.newLinkedHashMap(),
9093
new ArrayDeque<>(),
9194
-1,
9295
System.nanoTime());
9396
}
9497

9598
protected EventState(
96-
@Nullable final List<Debugger.Event> events,
97-
@Nullable final List<PEvent> eventProtos,
98-
@Nullable final Iterable<PEvent> prerecordedEventProtoIterable,
99-
@Nonnull final Map<Class<? extends Debugger.Event>, MutableStats> eventClassStatsMap,
100-
@Nonnull final Map<Class<? extends CascadesRule<?>>, MutableStats> plannerRuleClassStatsMap,
101-
@Nonnull final Deque<Pair<Class<? extends Debugger.Event>, EventDurations>> eventProfilingStack,
102-
final int currentTick,
103-
final long startTs) {
99+
@Nullable final List<Debugger.Event> events,
100+
@Nullable final List<PEvent> eventProtos,
101+
@Nullable final Iterable<PEvent> prerecordedEventProtoIterable,
102+
@Nonnull final Map<Class<? extends Debugger.Event>, MutableStats> eventWithoutStateClassStatsMap,
103+
@Nonnull final Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, MutableStats>> eventWithStateClassStatsMapByPlannerPhase,
104+
@Nonnull final Map<Class<? extends CascadesRule<?>>, MutableStats> plannerRuleClassStatsMap,
105+
@Nonnull final Deque<Pair<Class<? extends Debugger.Event>, EventDurations>> eventProfilingStack,
106+
final int currentTick,
107+
final long startTs) {
104108

105109
this.events = events;
106110
this.eventProtos = eventProtos;
107111
this.prerecordedEventProtoIterable = prerecordedEventProtoIterable;
108112
this.prerecordedEventProtoIterator = prerecordedEventProtoIterable == null
109113
? null : prerecordedEventProtoIterable.iterator();
110-
this.eventClassStatsMap = eventClassStatsMap;
114+
this.eventWithoutStateClassStatsMap = eventWithoutStateClassStatsMap;
115+
this.eventWithStateClassStatsMapByPlannerPhase = eventWithStateClassStatsMapByPlannerPhase;
111116
this.plannerRuleClassStatsMap = plannerRuleClassStatsMap;
112117
this.eventProfilingStack = eventProfilingStack;
113118
this.currentTick = currentTick;
@@ -179,7 +184,7 @@ public void addCurrentEvent(@Nonnull final Debugger.Event event) {
179184
final long totalTime = currentTsInNs - eventDurations.getStartTsInNs();
180185
final long ownTime = totalTime - eventDurations.getAdjustmentForOwnTimeInNs();
181186

182-
final MutableStats forEventClass = getEventStatsForEventClass(currentEventClass);
187+
final MutableStats forEventClass = getEventStatsForEvent(event);
183188
forEventClass.increaseTotalTimeInNs(totalTime);
184189
forEventClass.increaseOwnTimeInNs(ownTime);
185190
if (event instanceof Debugger.TransformRuleCallEvent) {
@@ -229,8 +234,9 @@ private void verifyCurrentEventProto(final PEvent currentEventProto) {
229234

230235
@SuppressWarnings("unchecked")
231236
private void updateCounts(@Nonnull final Debugger.Event event) {
232-
final MutableStats forEventClass = getEventStatsForEventClass(event.getClass());
237+
final MutableStats forEventClass = getEventStatsForEvent(event);
233238
forEventClass.increaseCount(event.getLocation(), 1L);
239+
234240
if (event instanceof Debugger.EventWithRule) {
235241
final CascadesRule<?> rule = ((Debugger.EventWithRule)event).getRule();
236242
final Class<? extends CascadesRule<?>> ruleClass = (Class<? extends CascadesRule<?>>)rule.getClass();
@@ -239,17 +245,31 @@ private void updateCounts(@Nonnull final Debugger.Event event) {
239245
}
240246
}
241247

242-
private MutableStats getEventStatsForEventClass(@Nonnull Class<? extends Debugger.Event> eventClass) {
243-
return eventClassStatsMap.compute(eventClass, (eC, mutableStats) -> mutableStats != null ? mutableStats : new MutableStats());
248+
private MutableStats getEventStatsForEvent(@Nonnull Debugger.Event event) {
249+
return (event instanceof Debugger.EventWithState) ?
250+
getEventStatsForEventWithStateClassByPlannerPhase((Debugger.EventWithState)event) :
251+
getEventStatsForEventWithoutStateClass(event.getClass());
252+
}
253+
254+
private MutableStats getEventStatsForEventWithoutStateClass(@Nonnull Class<? extends Debugger.Event> eventClass) {
255+
return eventWithoutStateClassStatsMap.compute(eventClass, (eC, mutableStats) -> mutableStats != null ? mutableStats : new MutableStats());
256+
}
257+
258+
private MutableStats getEventStatsForEventWithStateClassByPlannerPhase(@Nonnull Debugger.EventWithState event) {
259+
return eventWithStateClassStatsMapByPlannerPhase.computeIfAbsent(event.getPlannerPhase(), pP -> new LinkedHashMap<>())
260+
.computeIfAbsent(event.getClass(), (eC) -> new MutableStats());
244261
}
245262

246263
private MutableStats getEventStatsForPlannerRuleClass(@Nonnull Class<? extends CascadesRule<?>> plannerRuleClass) {
247-
return plannerRuleClassStatsMap.compute(plannerRuleClass, (eC, mutableStats) -> mutableStats != null ? mutableStats : new MutableStats());
264+
return plannerRuleClassStatsMap.computeIfAbsent(plannerRuleClass, (eC) -> new MutableStats());
248265
}
249266

250267
@Nonnull
251268
StatsMaps getStatsMaps() {
252-
return new StatsMaps(eventClassStatsMap, plannerRuleClassStatsMap);
269+
return new StatsMaps(
270+
Collections.unmodifiableMap(eventWithoutStateClassStatsMap),
271+
Collections.unmodifiableMap(eventWithStateClassStatsMapByPlannerPhase),
272+
Collections.unmodifiableMap(plannerRuleClassStatsMap));
253273
}
254274

255275
private static class MutableStats extends Stats {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/debug/Stats.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424

2525
import javax.annotation.Nonnull;
2626
import java.util.Map;
27+
import java.util.Set;
28+
import java.util.stream.Stream;
2729

2830
public class Stats {
2931
@Nonnull
@@ -61,4 +63,14 @@ public long getOwnTimeInNs() {
6163
public Stats toImmutable() {
6264
return new Stats(ImmutableMap.copyOf(locationCountMap), totalTimeInNs, ownTimeInNs);
6365
}
66+
67+
public static Stats merge(final Stats first, final Stats second) {
68+
return new Stats(
69+
Stream.of(first.getLocationCountMap(), second.getLocationCountMap())
70+
.map(Map::entrySet)
71+
.flatMap(Set::stream)
72+
.collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue, Long::sum)),
73+
first.getTotalTimeInNs() + second.getTotalTimeInNs(),
74+
first.getOwnTimeInNs() + second.getOwnTimeInNs());
75+
}
6476
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/debug/StatsMaps.java‎

Lines changed: 67 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,29 +21,45 @@
2121
package com.apple.foundationdb.record.query.plan.cascades.debug;
2222

2323
import com.apple.foundationdb.record.query.plan.cascades.CascadesRule;
24+
import com.apple.foundationdb.record.query.plan.cascades.PlannerPhase;
2425
import com.google.common.base.Suppliers;
2526
import com.google.common.collect.ImmutableMap;
2627

2728
import javax.annotation.Nonnull;
29+
import java.util.LinkedHashMap;
2830
import java.util.Map;
31+
import java.util.Optional;
2932
import java.util.function.Supplier;
3033

3134
public class StatsMaps {
3235
@Nonnull
33-
private final Map<Class<? extends Debugger.Event>, ? extends Stats> eventClassStatsMap;
36+
private final Map<Class<? extends Debugger.Event>, ? extends Stats> eventWithoutStateClassStatsMap;
37+
@Nonnull
38+
private final Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, ? extends Stats>> eventWithStateClassStatsByPlannerPhaseMap;
3439
@Nonnull
3540
private final Map<Class<? extends CascadesRule<?>>, ? extends Stats> plannerRuleClassStatsMap;
3641

3742
@Nonnull
3843
private final Supplier<Map<Class<? extends Debugger.Event>, Stats>> immutableEventClassStatsMapSupplier;
3944
@Nonnull
45+
private final Supplier<Map<Class<? extends Debugger.Event>, Stats>> immutableEventWithoutStateClassStatsMapSupplier;
46+
@Nonnull
47+
private final Supplier<Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, Stats>>> immutableEventWithStateClassStatsByPlannerPhaseMapSupplier;
48+
@Nonnull
4049
private final Supplier<Map<Class<? extends CascadesRule<?>>, Stats>> immutablePlannerRuleClassStatsMapSupplier;
4150

42-
public StatsMaps(@Nonnull final Map<Class<? extends Debugger.Event>, ? extends Stats> eventClassStatsMap,
51+
52+
public StatsMaps(@Nonnull final Map<Class<? extends Debugger.Event>, ? extends Stats> eventWithoutStateClassStatsMap,
53+
@Nonnull final Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, ? extends Stats>> eventWithStateClassStatsByPlannerPhaseMap,
4354
@Nonnull final Map<Class<? extends CascadesRule<?>>, ? extends Stats> plannerRuleClassStatsMap) {
44-
this.eventClassStatsMap = eventClassStatsMap;
55+
56+
this.eventWithoutStateClassStatsMap = eventWithoutStateClassStatsMap;
57+
this.eventWithStateClassStatsByPlannerPhaseMap = eventWithStateClassStatsByPlannerPhaseMap;
4558
this.plannerRuleClassStatsMap = plannerRuleClassStatsMap;
59+
4660
this.immutableEventClassStatsMapSupplier = Suppliers.memoize(this::computeImmutableEventClassStatsMap);
61+
this.immutableEventWithoutStateClassStatsMapSupplier = Suppliers.memoize(this::computeImmutableEventWithoutStateClassStatsMap);
62+
this.immutableEventWithStateClassStatsByPlannerPhaseMapSupplier = Suppliers.memoize(this::computeImmutableEventWithStateClassStatsByPlannerPhaseMap);
4763
this.immutablePlannerRuleClassStatsMapSupplier = Suppliers.memoize(this::computeImmutablePlannerRuleClassStatsMap);
4864
}
4965

@@ -58,13 +74,56 @@ public Map<Class<? extends CascadesRule<?>>, Stats> getPlannerRuleClassStatsMap(
5874
}
5975

6076
@Nonnull
61-
private Map<Class<? extends Debugger.Event>, Stats> computeImmutableEventClassStatsMap() {
62-
final var eventClassStatsMapBuilder =
77+
public Map<Class<? extends Debugger.Event>, Stats> getEventWithoutStateClassStatsMap() {
78+
return immutableEventClassStatsMapSupplier.get();
79+
}
80+
81+
@Nonnull
82+
public Optional<Map<Class<? extends Debugger.EventWithState>, Stats>> getEventWithStateClassStatsMapByPlannerPhase(@Nonnull PlannerPhase plannerPhase) {
83+
return Optional.ofNullable(immutableEventWithStateClassStatsByPlannerPhaseMapSupplier.get().get(plannerPhase));
84+
}
85+
86+
private Map<Class<? extends Debugger.Event>, Stats> computeImmutableEventWithoutStateClassStatsMap() {
87+
final var eventWithoutStateClassStatsMapBuilder =
6388
ImmutableMap.<Class<? extends Debugger.Event>, Stats>builder();
64-
for (final var entry : eventClassStatsMap.entrySet()) {
65-
eventClassStatsMapBuilder.put(entry.getKey(), entry.getValue().toImmutable());
89+
90+
eventWithoutStateClassStatsMap.forEach(
91+
(eventClass, stats) -> eventWithoutStateClassStatsMapBuilder.put(eventClass, stats.toImmutable())
92+
);
93+
94+
return eventWithoutStateClassStatsMapBuilder.build();
95+
}
96+
97+
@Nonnull
98+
private Map<Class<? extends Debugger.Event>, Stats> computeImmutableEventClassStatsMap() {
99+
// Add all events not tied to a specific planner phase first
100+
Map<Class<? extends Debugger.Event>, Stats> result = new LinkedHashMap<>(this.immutableEventWithoutStateClassStatsMapSupplier.get());
101+
102+
// Merge the stats all events tied to a specific planner phase with other events
103+
for (final var eventWithStateClassStats : eventWithStateClassStatsByPlannerPhaseMap.values()) {
104+
for (final var eventWithStateClassStatsEntry : eventWithStateClassStats.entrySet()) {
105+
result.merge(
106+
eventWithStateClassStatsEntry.getKey(),
107+
eventWithStateClassStatsEntry.getValue().toImmutable(),
108+
(s1, s2) -> Stats.merge(s1, s2).toImmutable());
109+
}
110+
}
111+
112+
return ImmutableMap.copyOf(result);
113+
}
114+
115+
@Nonnull
116+
private Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, Stats>> computeImmutableEventWithStateClassStatsByPlannerPhaseMap() {
117+
final var eventClassStatsByPlannerPhaseMapBuilder =
118+
ImmutableMap.<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, Stats>>builder();
119+
for (final var eventClassStatsByPlannerPhaseEntry : eventWithStateClassStatsByPlannerPhaseMap.entrySet()) {
120+
final Map<Class<? extends Debugger.EventWithState>, Stats> eventClassImmutableStats =
121+
eventClassStatsByPlannerPhaseEntry.getValue().entrySet().stream().collect(
122+
ImmutableMap.toImmutableMap(Map.Entry::getKey, e -> e.getValue().toImmutable())
123+
);
124+
eventClassStatsByPlannerPhaseMapBuilder.put(eventClassStatsByPlannerPhaseEntry.getKey(), eventClassImmutableStats);
66125
}
67-
return eventClassStatsMapBuilder.build();
126+
return eventClassStatsByPlannerPhaseMapBuilder.build();
68127
}
69128

70129
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/debug/StatsViewer.java‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,14 @@
2020

2121
package com.apple.foundationdb.record.query.plan.cascades.debug;
2222

23+
import com.apple.foundationdb.record.query.plan.cascades.PlannerPhase;
2324
import com.apple.foundationdb.record.util.pair.Pair;
2425
import com.google.common.collect.ImmutableMap;
2526

2627
import javax.annotation.Nonnull;
2728
import java.util.Locale;
2829
import java.util.Map;
30+
import java.util.Optional;
2931
import java.util.concurrent.TimeUnit;
3032

3133
/** Only used by debuggers to display stats.*/
@@ -39,16 +41,32 @@ public static String showStats(@Nonnull final StatsDebugger statsDebugger) {
3941
}
4042

4143
StringBuilder tableBuilder = new StringBuilder();
42-
tableBuilder.append("<table class=\"table\">");
43-
tableHeader(tableBuilder, "Event");
44-
final ImmutableMap<String, Stats> eventStatsMap =
45-
statsMaps.get().getEventClassStatsMap().entrySet()
46-
.stream()
47-
.map(entry -> Pair.of(entry.getKey().getSimpleName(), entry.getValue()))
48-
.sorted(Map.Entry.comparingByKey())
49-
.collect(ImmutableMap.toImmutableMap(Pair::getKey, Pair::getValue));
50-
tableBody(tableBuilder, eventStatsMap);
51-
tableBuilder.append("</table>");
44+
45+
final var phaseNameToStatsMap = ImmutableMap.of(
46+
"Rewriting", statsMaps.get().getEventWithStateClassStatsMapByPlannerPhase(PlannerPhase.REWRITING),
47+
"Planning", statsMaps.get().getEventWithStateClassStatsMapByPlannerPhase(PlannerPhase.PLANNING),
48+
"Unspecified", Optional.of(statsMaps.get().getEventWithoutStateClassStatsMap())
49+
);
50+
51+
for (final var phaseNameToStatsMapEntry : phaseNameToStatsMap.entrySet()) {
52+
if (phaseNameToStatsMapEntry.getValue().map(Map::isEmpty).orElse(false)) {
53+
continue;
54+
}
55+
56+
tableBuilder.append("<h4>").append(phaseNameToStatsMapEntry.getKey()).append(" Phase:</h4>");
57+
tableBuilder.append("<table class=\"table\">");
58+
tableHeader(tableBuilder, "Event");
59+
60+
final ImmutableMap<String, Stats> eventStatsMap =
61+
phaseNameToStatsMapEntry.getValue().get().entrySet()
62+
.stream()
63+
.map(entry -> Pair.of(entry.getKey().getSimpleName(), entry.getValue()))
64+
.sorted(Map.Entry.comparingByKey())
65+
.collect(ImmutableMap.toImmutableMap(Pair::getKey, Pair::getValue));
66+
67+
tableBody(tableBuilder, eventStatsMap);
68+
tableBuilder.append("</table>");
69+
}
5270

5371
final String eventProfilingString = tableBuilder.toString();
5472

0 commit comments

Comments
 (0)