Skip to content

Commit b629272

Browse files
hazefullyhazefully
committed
Group Debugger events by PlannerPhase in stats
This commit modifies the EventState class and the StatsMaps that it produces to group Debugger events by the planner phase it was emitted during to enable the ability of getting stats for each planner phase separately. Given that some Debugger events (e.g. InsertIntoMemoEvent) do not store the planner phase they were emitted in (as it not visible to the code that emits them), these events are stored separately and can be retrieved from the StatsMaps (in addition to the ability of retrieving all events, regardless of the planner phase that they were emitted in).
1 parent 2a9c324 commit b629272

File tree

4 files changed

+153
-44
lines changed

4 files changed

+153
-44
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/debug/EventState.java‎

Lines changed: 46 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@
2323
import com.apple.foundationdb.record.RecordCoreException;
2424
import com.apple.foundationdb.record.logging.KeyValueLogMessage;
2525
import com.apple.foundationdb.record.query.plan.cascades.CascadesRule;
26+
import com.apple.foundationdb.record.query.plan.cascades.PlannerPhase;
2627
import com.apple.foundationdb.record.query.plan.cascades.debug.eventprotos.PEvent;
2728
import com.apple.foundationdb.record.util.pair.Pair;
2829
import com.google.common.base.Verify;
29-
import com.google.common.collect.ImmutableMap;
3030
import com.google.common.collect.Lists;
3131
import com.google.common.collect.Maps;
3232
import org.slf4j.Logger;
@@ -35,28 +35,29 @@
3535
import javax.annotation.Nonnull;
3636
import javax.annotation.Nullable;
3737
import java.util.ArrayDeque;
38+
import java.util.Collections;
3839
import java.util.Deque;
3940
import java.util.Iterator;
41+
import java.util.LinkedHashMap;
4042
import java.util.List;
41-
import java.util.Locale;
4243
import java.util.Map;
4344
import java.util.Objects;
44-
import java.util.concurrent.TimeUnit;
4545

4646
public class EventState {
4747
@Nonnull
4848
private static final Logger logger = LoggerFactory.getLogger(EventState.class);
4949

50-
@Nullable private final List<Debugger.Event> events;
51-
@Nullable private final List<PEvent> eventProtos;
52-
@Nullable private final Iterable<PEvent> prerecordedEventProtoIterable;
53-
@Nullable private Iterator<PEvent> prerecordedEventProtoIterator;
50+
@Nullable protected final List<Debugger.Event> events;
51+
@Nullable protected final List<PEvent> eventProtos;
52+
@Nullable protected final Iterable<PEvent> prerecordedEventProtoIterable;
53+
@Nullable protected Iterator<PEvent> prerecordedEventProtoIterator;
5454

55-
@Nonnull private final Map<Class<? extends Debugger.Event>, MutableStats> eventClassStatsMap;
55+
@Nonnull protected final Map<Class<? extends Debugger.Event>, MutableStats> eventWithoutStateClassStatsMap;
56+
@Nonnull protected final Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, MutableStats>> eventWithStateClassStatsMapByPlannerPhase;
5657

57-
@Nonnull private final Map<Class<? extends CascadesRule<?>>, MutableStats> plannerRuleClassStatsMap;
58+
@Nonnull protected final Map<Class<? extends CascadesRule<?>>, MutableStats> plannerRuleClassStatsMap;
5859

59-
@Nonnull private final Deque<Pair<Class<? extends Debugger.Event>, EventDurations>> eventProfilingStack;
60+
@Nonnull protected final Deque<Pair<Class<? extends Debugger.Event>, EventDurations>> eventProfilingStack;
6061

6162
protected int currentTick;
6263
protected final long startTs;
@@ -71,7 +72,8 @@ public static EventState copyOf(final EventState source) {
7172
source.events == null ? null : Lists.newArrayList(source.events),
7273
source.eventProtos == null ? null : Lists.newArrayList(source.eventProtos),
7374
source.prerecordedEventProtoIterable,
74-
Maps.newLinkedHashMap(source.eventClassStatsMap),
75+
Maps.newLinkedHashMap(source.eventWithoutStateClassStatsMap),
76+
Maps.newEnumMap(source.eventWithStateClassStatsMapByPlannerPhase),
7577
Maps.newLinkedHashMap(source.plannerRuleClassStatsMap),
7678
new ArrayDeque<>(source.eventProfilingStack),
7779
source.getCurrentTick(),
@@ -85,28 +87,31 @@ protected EventState(final boolean isRecordEvents, final boolean isRecordEventPr
8587
isRecordEvents ? Lists.newArrayList() : null,
8688
prerecordedEventProtoIterable,
8789
Maps.newLinkedHashMap(),
90+
Maps.newEnumMap(PlannerPhase.class),
8891
Maps.newLinkedHashMap(),
8992
new ArrayDeque<>(),
9093
-1,
9194
System.nanoTime());
9295
}
9396

9497
protected EventState(
95-
@Nullable final List<Debugger.Event> events,
96-
@Nullable final List<PEvent> eventProtos,
97-
@Nullable final Iterable<PEvent> prerecordedEventProtoIterable,
98-
@Nonnull final Map<Class<? extends Debugger.Event>, MutableStats> eventClassStatsMap,
99-
@Nonnull final Map<Class<? extends CascadesRule<?>>, MutableStats> plannerRuleClassStatsMap,
100-
@Nonnull final Deque<Pair<Class<? extends Debugger.Event>, EventDurations>> eventProfilingStack,
101-
final int currentTick,
102-
final long startTs) {
98+
@Nullable final List<Debugger.Event> events,
99+
@Nullable final List<PEvent> eventProtos,
100+
@Nullable final Iterable<PEvent> prerecordedEventProtoIterable,
101+
@Nonnull final Map<Class<? extends Debugger.Event>, MutableStats> eventWithoutStateClassStatsMap,
102+
@Nonnull final Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, MutableStats>> eventWithStateClassStatsMapByPlannerPhase,
103+
@Nonnull final Map<Class<? extends CascadesRule<?>>, MutableStats> plannerRuleClassStatsMap,
104+
@Nonnull final Deque<Pair<Class<? extends Debugger.Event>, EventDurations>> eventProfilingStack,
105+
final int currentTick,
106+
final long startTs) {
103107

104108
this.events = events;
105109
this.eventProtos = eventProtos;
106110
this.prerecordedEventProtoIterable = prerecordedEventProtoIterable;
107111
this.prerecordedEventProtoIterator = prerecordedEventProtoIterable == null
108112
? null : prerecordedEventProtoIterable.iterator();
109-
this.eventClassStatsMap = eventClassStatsMap;
113+
this.eventWithoutStateClassStatsMap = eventWithoutStateClassStatsMap;
114+
this.eventWithStateClassStatsMapByPlannerPhase = eventWithStateClassStatsMapByPlannerPhase;
110115
this.plannerRuleClassStatsMap = plannerRuleClassStatsMap;
111116
this.eventProfilingStack = eventProfilingStack;
112117
this.currentTick = currentTick;
@@ -178,7 +183,7 @@ public void addCurrentEvent(@Nonnull final Debugger.Event event) {
178183
final long totalTime = currentTsInNs - eventDurations.getStartTsInNs();
179184
final long ownTime = totalTime - eventDurations.getAdjustmentForOwnTimeInNs();
180185

181-
final MutableStats forEventClass = getEventStatsForEventClass(currentEventClass);
186+
final MutableStats forEventClass = getEventStatsForEvent(event);
182187
forEventClass.increaseTotalTimeInNs(totalTime);
183188
forEventClass.increaseOwnTimeInNs(ownTime);
184189
if (event instanceof Debugger.TransformRuleCallEvent) {
@@ -228,8 +233,9 @@ private void verifyCurrentEventProto(final PEvent currentEventProto) {
228233

229234
@SuppressWarnings("unchecked")
230235
private void updateCounts(@Nonnull final Debugger.Event event) {
231-
final MutableStats forEventClass = getEventStatsForEventClass(event.getClass());
236+
final MutableStats forEventClass = getEventStatsForEvent(event);
232237
forEventClass.increaseCount(event.getLocation(), 1L);
238+
233239
if (event instanceof Debugger.EventWithRule) {
234240
final CascadesRule<?> rule = ((Debugger.EventWithRule)event).getRule();
235241
final Class<? extends CascadesRule<?>> ruleClass = (Class<? extends CascadesRule<?>>)rule.getClass();
@@ -238,17 +244,31 @@ private void updateCounts(@Nonnull final Debugger.Event event) {
238244
}
239245
}
240246

241-
private MutableStats getEventStatsForEventClass(@Nonnull Class<? extends Debugger.Event> eventClass) {
242-
return eventClassStatsMap.compute(eventClass, (eC, mutableStats) -> mutableStats != null ? mutableStats : new MutableStats());
247+
private MutableStats getEventStatsForEvent(@Nonnull Debugger.Event event) {
248+
return (event instanceof Debugger.EventWithState) ?
249+
getEventStatsForEventWithStateClassByPlannerPhase((Debugger.EventWithState)event) :
250+
getEventStatsForEventWithoutStateClass(event.getClass());
251+
}
252+
253+
private MutableStats getEventStatsForEventWithoutStateClass(@Nonnull Class<? extends Debugger.Event> eventClass) {
254+
return eventWithoutStateClassStatsMap.compute(eventClass, (eC, mutableStats) -> mutableStats != null ? mutableStats : new MutableStats());
255+
}
256+
257+
private MutableStats getEventStatsForEventWithStateClassByPlannerPhase(@Nonnull Debugger.EventWithState event) {
258+
return eventWithStateClassStatsMapByPlannerPhase.computeIfAbsent(event.getPlannerPhase(), pP -> new LinkedHashMap<>())
259+
.computeIfAbsent(event.getClass(), (eC) -> new MutableStats());
243260
}
244261

245262
private MutableStats getEventStatsForPlannerRuleClass(@Nonnull Class<? extends CascadesRule<?>> plannerRuleClass) {
246-
return plannerRuleClassStatsMap.compute(plannerRuleClass, (eC, mutableStats) -> mutableStats != null ? mutableStats : new MutableStats());
263+
return plannerRuleClassStatsMap.computeIfAbsent(plannerRuleClass, (eC) -> new MutableStats());
247264
}
248265

249266
@Nonnull
250267
StatsMaps getStatsMaps() {
251-
return new StatsMaps(eventClassStatsMap, plannerRuleClassStatsMap);
268+
return new StatsMaps(
269+
Collections.unmodifiableMap(eventWithoutStateClassStatsMap),
270+
Collections.unmodifiableMap(eventWithStateClassStatsMapByPlannerPhase),
271+
Collections.unmodifiableMap(plannerRuleClassStatsMap));
252272
}
253273

254274
private static class MutableStats extends Stats {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/debug/Stats.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424

2525
import javax.annotation.Nonnull;
2626
import java.util.Map;
27+
import java.util.Set;
28+
import java.util.stream.Stream;
2729

2830
public class Stats {
2931
@Nonnull
@@ -61,4 +63,14 @@ public long getOwnTimeInNs() {
6163
public Stats toImmutable() {
6264
return new Stats(ImmutableMap.copyOf(locationCountMap), totalTimeInNs, ownTimeInNs);
6365
}
66+
67+
public static Stats merge(final Stats first, final Stats second) {
68+
return new Stats(
69+
Stream.of(first.getLocationCountMap(), second.getLocationCountMap())
70+
.map(Map::entrySet)
71+
.flatMap(Set::stream)
72+
.collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue, Long::sum)),
73+
first.getTotalTimeInNs() + second.getTotalTimeInNs(),
74+
first.getOwnTimeInNs() + second.getOwnTimeInNs());
75+
}
6476
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/debug/StatsMaps.java‎

Lines changed: 67 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,29 +21,45 @@
2121
package com.apple.foundationdb.record.query.plan.cascades.debug;
2222

2323
import com.apple.foundationdb.record.query.plan.cascades.CascadesRule;
24+
import com.apple.foundationdb.record.query.plan.cascades.PlannerPhase;
2425
import com.google.common.base.Suppliers;
2526
import com.google.common.collect.ImmutableMap;
2627

2728
import javax.annotation.Nonnull;
29+
import java.util.LinkedHashMap;
2830
import java.util.Map;
31+
import java.util.Optional;
2932
import java.util.function.Supplier;
3033

3134
public class StatsMaps {
3235
@Nonnull
33-
private final Map<Class<? extends Debugger.Event>, ? extends Stats> eventClassStatsMap;
36+
private final Map<Class<? extends Debugger.Event>, ? extends Stats> eventWithoutStateClassStatsMap;
37+
@Nonnull
38+
private final Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, ? extends Stats>> eventWithStateClassStatsByPlannerPhaseMap;
3439
@Nonnull
3540
private final Map<Class<? extends CascadesRule<?>>, ? extends Stats> plannerRuleClassStatsMap;
3641

3742
@Nonnull
3843
private final Supplier<Map<Class<? extends Debugger.Event>, Stats>> immutableEventClassStatsMapSupplier;
3944
@Nonnull
45+
private final Supplier<Map<Class<? extends Debugger.Event>, Stats>> immutableEventWithoutStateClassStatsMapSupplier;
46+
@Nonnull
47+
private final Supplier<Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, Stats>>> immutableEventWithStateClassStatsByPlannerPhaseMapSupplier;
48+
@Nonnull
4049
private final Supplier<Map<Class<? extends CascadesRule<?>>, Stats>> immutablePlannerRuleClassStatsMapSupplier;
4150

42-
public StatsMaps(@Nonnull final Map<Class<? extends Debugger.Event>, ? extends Stats> eventClassStatsMap,
51+
52+
public StatsMaps(@Nonnull final Map<Class<? extends Debugger.Event>, ? extends Stats> eventWithoutStateClassStatsMap,
53+
@Nonnull final Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, ? extends Stats>> eventWithStateClassStatsByPlannerPhaseMap,
4354
@Nonnull final Map<Class<? extends CascadesRule<?>>, ? extends Stats> plannerRuleClassStatsMap) {
44-
this.eventClassStatsMap = eventClassStatsMap;
55+
56+
this.eventWithoutStateClassStatsMap = eventWithoutStateClassStatsMap;
57+
this.eventWithStateClassStatsByPlannerPhaseMap = eventWithStateClassStatsByPlannerPhaseMap;
4558
this.plannerRuleClassStatsMap = plannerRuleClassStatsMap;
59+
4660
this.immutableEventClassStatsMapSupplier = Suppliers.memoize(this::computeImmutableEventClassStatsMap);
61+
this.immutableEventWithoutStateClassStatsMapSupplier = Suppliers.memoize(this::computeImmutableEventWithoutStateClassStatsMap);
62+
this.immutableEventWithStateClassStatsByPlannerPhaseMapSupplier = Suppliers.memoize(this::computeImmutableEventWithStateClassStatsByPlannerPhaseMap);
4763
this.immutablePlannerRuleClassStatsMapSupplier = Suppliers.memoize(this::computeImmutablePlannerRuleClassStatsMap);
4864
}
4965

@@ -58,13 +74,56 @@ public Map<Class<? extends CascadesRule<?>>, Stats> getPlannerRuleClassStatsMap(
5874
}
5975

6076
@Nonnull
61-
private Map<Class<? extends Debugger.Event>, Stats> computeImmutableEventClassStatsMap() {
62-
final var eventClassStatsMapBuilder =
77+
public Map<Class<? extends Debugger.Event>, Stats> getEventWithoutStateClassStatsMap() {
78+
return immutableEventClassStatsMapSupplier.get();
79+
}
80+
81+
@Nonnull
82+
public Optional<Map<Class<? extends Debugger.EventWithState>, Stats>> getEventWithStateClassStatsMapByPlannerPhase(@Nonnull PlannerPhase plannerPhase) {
83+
return Optional.ofNullable(immutableEventWithStateClassStatsByPlannerPhaseMapSupplier.get().get(plannerPhase));
84+
}
85+
86+
private Map<Class<? extends Debugger.Event>, Stats> computeImmutableEventWithoutStateClassStatsMap() {
87+
final var eventWithoutStateClassStatsMapBuilder =
6388
ImmutableMap.<Class<? extends Debugger.Event>, Stats>builder();
64-
for (final var entry : eventClassStatsMap.entrySet()) {
65-
eventClassStatsMapBuilder.put(entry.getKey(), entry.getValue().toImmutable());
89+
90+
eventWithoutStateClassStatsMap.forEach(
91+
(eventClass, stats) -> eventWithoutStateClassStatsMapBuilder.put(eventClass, stats.toImmutable())
92+
);
93+
94+
return eventWithoutStateClassStatsMapBuilder.build();
95+
}
96+
97+
@Nonnull
98+
private Map<Class<? extends Debugger.Event>, Stats> computeImmutableEventClassStatsMap() {
99+
// Add all events not tied to a specific planner phase first
100+
Map<Class<? extends Debugger.Event>, Stats> result = new LinkedHashMap<>(this.immutableEventWithoutStateClassStatsMapSupplier.get());
101+
102+
// Merge the stats all events tied to a specific planner phase with other events
103+
for (final var eventWithStateClassStats : eventWithStateClassStatsByPlannerPhaseMap.values()) {
104+
for (final var eventWithStateClassStatsEntry : eventWithStateClassStats.entrySet()) {
105+
result.merge(
106+
eventWithStateClassStatsEntry.getKey(),
107+
eventWithStateClassStatsEntry.getValue().toImmutable(),
108+
(s1, s2) -> Stats.merge(s1, s2).toImmutable());
109+
}
110+
}
111+
112+
return ImmutableMap.copyOf(result);
113+
}
114+
115+
@Nonnull
116+
private Map<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, Stats>> computeImmutableEventWithStateClassStatsByPlannerPhaseMap() {
117+
final var eventClassStatsByPlannerPhaseMapBuilder =
118+
ImmutableMap.<PlannerPhase, Map<Class<? extends Debugger.EventWithState>, Stats>>builder();
119+
for (final var eventClassStatsByPlannerPhaseEntry : eventWithStateClassStatsByPlannerPhaseMap.entrySet()) {
120+
final Map<Class<? extends Debugger.EventWithState>, Stats> eventClassImmutableStats =
121+
eventClassStatsByPlannerPhaseEntry.getValue().entrySet().stream().collect(
122+
ImmutableMap.toImmutableMap(Map.Entry::getKey, e -> e.getValue().toImmutable())
123+
);
124+
eventClassStatsByPlannerPhaseMapBuilder.put(eventClassStatsByPlannerPhaseEntry.getKey(), eventClassImmutableStats);
66125
}
67-
return eventClassStatsMapBuilder.build();
126+
return eventClassStatsByPlannerPhaseMapBuilder.build();
68127
}
69128

70129
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/debug/StatsViewer.java‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,14 @@
2020

2121
package com.apple.foundationdb.record.query.plan.cascades.debug;
2222

23+
import com.apple.foundationdb.record.query.plan.cascades.PlannerPhase;
2324
import com.apple.foundationdb.record.util.pair.Pair;
2425
import com.google.common.collect.ImmutableMap;
2526

2627
import javax.annotation.Nonnull;
2728
import java.util.Locale;
2829
import java.util.Map;
30+
import java.util.Optional;
2931
import java.util.concurrent.TimeUnit;
3032

3133
/** Only used by debuggers to display stats.*/
@@ -39,16 +41,32 @@ public static String showStats(@Nonnull final StatsDebugger statsDebugger) {
3941
}
4042

4143
StringBuilder tableBuilder = new StringBuilder();
42-
tableBuilder.append("<table class=\"table\">");
43-
tableHeader(tableBuilder, "Event");
44-
final ImmutableMap<String, Stats> eventStatsMap =
45-
statsMaps.get().getEventClassStatsMap().entrySet()
46-
.stream()
47-
.map(entry -> Pair.of(entry.getKey().getSimpleName(), entry.getValue()))
48-
.sorted(Map.Entry.comparingByKey())
49-
.collect(ImmutableMap.toImmutableMap(Pair::getKey, Pair::getValue));
50-
tableBody(tableBuilder, eventStatsMap);
51-
tableBuilder.append("</table>");
44+
45+
final var phaseNameToStatsMap = ImmutableMap.of(
46+
"Rewriting", statsMaps.get().getEventWithStateClassStatsMapByPlannerPhase(PlannerPhase.REWRITING),
47+
"Planning", statsMaps.get().getEventWithStateClassStatsMapByPlannerPhase(PlannerPhase.PLANNING),
48+
"Unspecified", Optional.of(statsMaps.get().getEventWithoutStateClassStatsMap())
49+
);
50+
51+
for (final var phaseNameToStatsMapEntry : phaseNameToStatsMap.entrySet()) {
52+
if (phaseNameToStatsMapEntry.getValue().map(Map::isEmpty).orElse(false)) {
53+
continue;
54+
}
55+
56+
tableBuilder.append("<h4>").append(phaseNameToStatsMapEntry.getKey()).append(" Phase:</h4>");
57+
tableBuilder.append("<table class=\"table\">");
58+
tableHeader(tableBuilder, "Event");
59+
60+
final ImmutableMap<String, Stats> eventStatsMap =
61+
phaseNameToStatsMapEntry.getValue().get().entrySet()
62+
.stream()
63+
.map(entry -> Pair.of(entry.getKey().getSimpleName(), entry.getValue()))
64+
.sorted(Map.Entry.comparingByKey())
65+
.collect(ImmutableMap.toImmutableMap(Pair::getKey, Pair::getValue));
66+
67+
tableBody(tableBuilder, eventStatsMap);
68+
tableBuilder.append("</table>");
69+
}
5270

5371
final String eventProfilingString = tableBuilder.toString();
5472

0 commit comments

Comments
 (0)