Fix compensation logic to not early out if child compensations are needed (#3593)

…eded

Author: normen662

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/GroupByExpression.java

@@ -652,7 +652,8 @@ public Compensation compensate(@Nonnull final PartialMatch partialMatch,
             return Compensation.impossibleCompensation();
         }
         final var compensatedResult = compensatedResultOptional.get();
-        if (!compensatedResult.getResultCompensationFunction().isNeeded()) {
+        if (!childCompensation.isNeeded() &&
+                !compensatedResult.getResultCompensationFunction().isNeeded()) {
             return Compensation.noCompensation();
         }
 

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/LogicalTypeFilterExpression.java

@@ -205,7 +205,8 @@ public Compensation compensate(@Nonnull final PartialMatch partialMatch,
             return Compensation.impossibleCompensation();
         }
         final var compensatedResult = compensatedResultOptional.get();
-        if (!compensatedResult.getResultCompensationFunction().isNeeded()) {
+        if (!childCompensation.isNeeded() &&
+                !compensatedResult.getResultCompensationFunction().isNeeded()) {
             return Compensation.noCompensation();
         }
 

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java

@@ -880,7 +880,8 @@ public Compensation compensate(@Nonnull final PartialMatch partialMatch,
         isAnyCompensationFunctionImpossible |= compensatedResult.isCompensationImpossible();
 
         final var isCompensationNeeded =
-                !unmatchedQuantifiers.isEmpty() ||
+                childCompensation.isNeeded() ||
+                        !unmatchedQuantifiers.isEmpty() ||
                         isAnyCompensationFunctionNeeded ||
                         compensatedResult.getResultCompensationFunction().isNeeded();
 

yaml-tests/src/test/resources/aggregate-index-tests.metrics.binpb

@@ -1,8 +1,7 @@
 �
 H
 agg-index-tests5EXPLAIN select col1, sum(col2) from T1 group by col1;�
-��Ŀ�
 ���
-(@0�,80@YAISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
+�����
 ӎ�(@0��080@YAISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -15,8 +14,7 @@ digraph G {
 }�
 =
 agg-index-tests*EXPLAIN select col1 from T1 group by col1;�
-����
-h ��(60�� 8%@ZISCAN(VI1 <,>) | MAP (_ AS _0) | AGG () GROUP BY (_._0.COL1 AS _0) | MAP (_._0._0 AS COL1)�digraph G {
+��ôh ���(60��78%@ZISCAN(VI1 <,>) | MAP (_ AS _0) | AGG () GROUP BY (_._0.COL1 AS _0) | MAP (_._0._0 AS COL1)�digraph G {
   fontname=courier;
   rankdir=BT;
   splines=polyline;
@@ -32,8 +30,7 @@ digraph G {
 }�
 B
 agg-index-tests/EXPLAIN select sum(col2) from T1 group by col1;�
-�ҏ��
 ���
-(@0��-80@KAISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)�
+�����
 �ޭ(@0��680@KAISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -46,7 +43,7 @@ digraph G {
 }�
 Z
 agg-index-testsGEXPLAIN select col1, sum(col2) from T1 group by col1 order by col1 asc;�
-�����
 �ޜ(@0��+80@YAISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
+��ԛ�
 ���(@0��180@YAISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -59,7 +56,7 @@ digraph G {
 }�
 [
 agg-index-testsHEXPLAIN select col1, sum(col2) from T1 group by col1 order by col1 desc;�
-��؄�
 �Ԛ(@0Ĩ,80@aAISCAN(MV1 <,> BY_GROUP REVERSE -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�digraph G {
+�ْ��
 ���(@0��580@aAISCAN(MV1 <,> BY_GROUP REVERSE -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�digraph G {
   fontname=courier;
   rankdir=BT;
   splines=polyline;
@@ -71,8 +68,7 @@ digraph G {
 }�
 L
 agg-index-tests9EXPLAIN select col1, sum(col2) + 1 from T1 group by col1;�
-�����
 �ˌ
-(@0��,80@]AISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 + 1 AS _1)�
+��ԙ�
 ׇ�(@0��?80@]AISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 + 1 AS _1)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -85,7 +81,7 @@ digraph G {
 }�
 H
 agg-index-tests5EXPLAIN select col1, max(col2) from T1 group by col1;�
-�͕��
 ���(B0��282@WAISCAN(MV8 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)�
+�����
 ���(B0ғ?82@WAISCAN(MV8 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -98,7 +94,7 @@ digraph G {
 }�
 V
 agg-index-testsCEXPLAIN select col1, max(col2) from T1 group by col1 order by col1;�
-��ނ�
 ���(B0��082@WAISCAN(MV8 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)�
+�����
 ��(B0הL82@WAISCAN(MV8 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -111,7 +107,7 @@ digraph G {
 }�
 [
 agg-index-testsHEXPLAIN select col1, max(col2) from T1 group by col1 order by col1 desc;�
-��ҁ�
 ���(B0��082@_AISCAN(MV8 <,> BY_GROUP REVERSE -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)�digraph G {
+�����
 ���(B0��I82@_AISCAN(MV8 <,> BY_GROUP REVERSE -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)�digraph G {
   fontname=courier;
   rankdir=BT;
   splines=polyline;
@@ -123,7 +119,7 @@ digraph G {
 }�
 X
 agg-index-testsEEXPLAIN select col1, max(col2) from T1 where col1 = 10 group by col1;�
-�����
 磌(J0��:8F@rAISCAN(MV8 [EQUALS promote(@c13 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)�
+�����
 ���(J0��P8F@rAISCAN(MV8 [EQUALS promote(@c13 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -151,10 +147,10 @@ D
   5 -> 4 [ label=<&nbsp;q104> label="q104" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
   6 -> 5 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
   2 -> 1 [ label=<&nbsp;q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
-}�
+}�
 ?
-agg-index-tests,EXPLAIN select col2 from T1 where col1 = 10;�
-�����
 ��(E0��O8D@nCOVERING(MV8 [EQUALS promote(@c8 AS LONG)] -> [COL1: KEY[0], COL2: KEY[1], ID: KEY[3]]) | MAP (_.COL2 AS COL2)�
+agg-index-tests,EXPLAIN select col2 from T1 where col1 = 10;�
+�鑈�
 ���(E0���
8D@nCOVERING(MV8 [EQUALS promote(@c8 AS LONG)] -> [COL1: KEY[0], COL2: KEY[1], ID: KEY[3]]) | MAP (_.COL2 AS COL2)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -167,8 +163,7 @@ digraph G {
 }�
 R
 agg-index-tests?EXPLAIN select col2 from T1 where col1 = 10 order by col2 desc;�
-����
-X ܛ�(00��8@vCOVERING(MV8 [EQUALS promote(@c8 AS LONG)] REVERSE -> [COL1: KEY[0], COL2: KEY[1], ID: KEY[3]]) | MAP (_.COL2 AS COL2)�
+����.X ��&(00��08@vCOVERING(MV8 [EQUALS promote(@c8 AS LONG)] REVERSE -> [COL1: KEY[0], COL2: KEY[1], ID: KEY[3]]) | MAP (_.COL2 AS COL2)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -216,7 +211,7 @@ H
 }�
 X
 agg-index-testsEEXPLAIN select col1, sum(col2) from T1 USE INDEX (vi1) group by col1;�
-����B ���
(0��8@ISCAN(VI1 <,>) | MAP (_ AS _0) | AGG (sum_l(_._0.COL2) AS _0) GROUP BY (_._0.COL1 AS _0) | MAP (_._0._0 AS COL1, _._1._0 AS _1)�digraph G {
+��B ���
(0��8@ISCAN(VI1 <,>) | MAP (_ AS _0) | AGG (sum_l(_._0.COL2) AS _0) GROUP BY (_._0.COL1 AS _0) | MAP (_._0._0 AS COL1, _._1._0 AS _1)�digraph G {
   fontname=courier;
   rankdir=BT;
   splines=polyline;
@@ -417,10 +412,10 @@ H
   4 -> 3 [ label=<&nbsp;q4> label="q4" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
   5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
   2 -> 1 [ label=<&nbsp;q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
-}�
+}�
 M
-agg-index-tests:EXPLAIN select col1, max_ever(col2) from T1 group by col1;�
-��ڈ�
 ���(H0��<85@YAISCAN(MV6 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
+agg-index-tests:EXPLAIN select col1, max_ever(col2) from T1 group by col1;�
+����$�
 ���(H0���
85@YAISCAN(MV6 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -433,8 +428,8 @@ digraph G {
 }�
 M
 agg-index-tests:EXPLAIN select col1, min_ever(col2) from T1 group by col1;�
-�����
 �͟
-(>0��)8*@ZAISCAN(MV12 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
+�����
 ���
+(>0��98*@ZAISCAN(MV12 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -473,7 +468,7 @@ digraph G {
 }�
 X
 agg-index-testsEEXPLAIN select col1, sum(col2) from T1 where col1 > 15 group by col1;�
-�����
 ���(H0��98D@|AISCAN(MV1 [[GREATER_THAN promote(@c13 AS LONG)]] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
+�Ǿ��
 ���(H0��v8D@|AISCAN(MV1 [[GREATER_THAN promote(@c13 AS LONG)]] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -486,7 +481,7 @@ digraph G {
 }�
 R
 agg-index-tests?EXPLAIN select sum(col2) from T1 where col1  = 10 group by col1�
-�����
 ���(H0��:8D@fAISCAN(MV1 [EQUALS promote(@c11 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)�
+�����
 촊(H0��@8D@fAISCAN(MV1 [EQUALS promote(@c11 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)�
 digraph G {
   fontname=courier;
   rankdir=BT;
@@ -496,10 +491,24 @@ digraph G {
   3 [ label=<<table border="0" cellborder="1" cellspacing="0" cellpadding="8"><tr><td align="left">Index</td></tr><tr><td align="left">MV1</td></tr></table>> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, LONG AS COL1, LONG AS COL2)" ];
   3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
   2 -> 1 [ label=<&nbsp;q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
-}�
+}�
+x
+agg-index-testseEXPLAIN select col1 as g, sum(col2) as s from T1 use index (mv1) where col1 in (10, 20) group by col1�
+�����
 �Ʀ( 0�S8+@
�
AISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | FILTER _._0 IN promote(@c22 AS ARRAY(LONG)) | MAP (_._0 AS G, _._1 AS S)�digraph G {
+  fontname=courier;
+  rankdir=BT;
+  splines=polyline;
+  1 [ label=<<table border="0" cellborder="1" cellspacing="0" cellpadding="8"><tr><td align="left">Value Computation</td></tr><tr><td align="left">MAP (q6._0 AS G, q6._1 AS S)</td></tr></table>> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS G, LONG AS S)" ];
+  2 [ label=<<table border="0" cellborder="1" cellspacing="0" cellpadding="8"><tr><td align="left">Predicate Filter</td></tr><tr><td align="left">WHERE q2._0 IN promote(@c22 AS ARRAY(LONG))</td></tr></table>> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, LONG AS _1)" ];
+  3 [ label=<<table border="0" cellborder="1" cellspacing="0" cellpadding="8"><tr><td align="left">Index Scan</td></tr><tr><td align="left">scan type: BY_GROUP</td></tr><tr><td align="left">range: &lt;-∞, ∞&gt;</td></tr></table>> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, LONG AS _1)" ];
+  4 [ label=<<table border="0" cellborder="1" cellspacing="0" cellpadding="8"><tr><td align="left">Index</td></tr><tr><td align="left">MV1</td></tr></table>> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, LONG AS COL1, LONG AS COL2)" ];
+  3 -> 2 [ label=<&nbsp;q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
+  4 -> 3 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
+  2 -> 1 [ label=<&nbsp;q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ];
+}�
 c
-agg-index-testsPEXPLAIN select sum(col2) from T1 where col1 <= 10 group by col1 having col1 > 0;�
-��Ŝ�
 �ǥ	(H0��38E@�
AISCAN(MV1 [[GREATER_THAN promote(@c19 AS LONG) && LESS_THAN_OR_EQUALS promote(@c12 AS LONG)]] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)�digraph G {
+agg-index-testsPEXPLAIN select sum(col2) from T1 where col1 <= 10 group by col1 having col1 > 0;�
+����G�
 ���6(H0��
8E@�
AISCAN(MV1 [[GREATER_THAN promote(@c19 AS LONG) && LESS_THAN_OR_EQUALS promote(@c12 AS LONG)]] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)�digraph G {
   fontname=courier;
   rankdir=BT;
   splines=polyline;
@@ -899,10 +908,10 @@ u
     rankDir=LR;
     4 -> 3 [ color="red" style="invis" ];
   }
-}�
+}�
 �

-agg-index-tests�
EXPLAIN select b, c, d, e, max(x) from t5 where a = 0 and c = 0 group by a, b, c, d, e having b IN ('foo', 'bar') order by max(x), d�
-���ā
� ��h(]0��W85@�[IN @c38] INUNION q0 -> { AISCAN(MV18 [EQUALS promote(@c19 AS LONG), EQUALS q0, EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.D, _.B, _.C, _.E)�digraph G {
+agg-index-tests�
EXPLAIN select b, c, d, e, max(x) from t5 where a = 0 and c = 0 group by a, b, c, d, e having b IN ('foo', 'bar') order by max(x), d�
+����e� �әJ(^0��48>@�[IN @c38] INUNION q0 -> { AISCAN(MV18 [EQUALS promote(@c19 AS LONG), EQUALS q0, EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.D, _.B, _.C, _.E)�digraph G {
   fontname=courier;
   rankdir=BT;
   splines=polyline;
@@ -920,10 +929,10 @@ u
     rankDir=LR;
     5 -> 2 [ color="red" style="invis" ];
   }
-}�
+}�
 �

-agg-index-tests�
EXPLAIN select b, c, d, e, min(x) from t5 where a = 0 and c = 0 group by a, b, c, d, e having b IN ('foo', 'bar') order by min(x), d�
-�����
� ���r(]0��>85@�[IN @c38] INUNION q0 -> { AISCAN(MV19 [EQUALS promote(@c19 AS LONG), EQUALS q0, EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.D, _.B, _.C, _.E)�digraph G {
+agg-index-tests�
EXPLAIN select b, c, d, e, min(x) from t5 where a = 0 and c = 0 group by a, b, c, d, e having b IN ('foo', 'bar') order by min(x), d�
+����y� ��Y(^0��I8>@�[IN @c38] INUNION q0 -> { AISCAN(MV19 [EQUALS promote(@c19 AS LONG), EQUALS q0, EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.D, _.B, _.C, _.E)�digraph G {
   fontname=courier;
   rankdir=BT;
   splines=polyline;