Support SQL array subscript operator (#3586)

This introduces support to SQL array subscription operator, with the
following caveats:
- It works with one-dimensional arrays only.
- Similar to other vendors, such as Postgres and Oracle, it returns
`null` when the index is out-of-bound instead of raising an error.
- Uses 1-based index, as defined in SQL standard.

Example:

```sql
create table A(pk integer, x integer array, primary key(pk))
insert into A values (1, [1, 2, 3]), (2, [2, 3, 4]), (3, [3, 4, 5])
select X[2] from A where pk = 1;
-- returns 2
select X[1+1] from A where pk = 1;
-- returns 2
select X[-1] from A where pk = 1;
-- returns null
select X[1000] from A where pk = 1;
-- returns null
```

This also restructures the parsing rules for expressions such that it
prohibits more invalid combinations of binary expressions, and aligns
them to work well with the subscript operator, and adds support to `not
in` which was previously throwing a syntax error.

This fixes #3584, #3570, and partially #3588 with few limitations.

Author: hatyo

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/SubscriptValue.java

@@ -0,0 +1,202 @@
+/*
+ * SubscriptValue.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.record.query.plan.cascades.values;
+
+import com.apple.foundationdb.record.EvaluationContext;
+import com.apple.foundationdb.record.ObjectPlanHash;
+import com.apple.foundationdb.record.PlanDeserializer;
+import com.apple.foundationdb.record.PlanHashable;
+import com.apple.foundationdb.record.PlanSerializationContext;
+import com.apple.foundationdb.record.planprotos.PSubscriptValue;
+import com.apple.foundationdb.record.planprotos.PValue;
+import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase;
+import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction;
+import com.apple.foundationdb.record.query.plan.cascades.SemanticException;
+import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
+import com.apple.foundationdb.record.query.plan.cascades.typing.Typed;
+import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence;
+import com.google.auto.service.AutoService;
+import com.google.common.base.Verify;
+import com.google.common.collect.Iterables;
+import com.google.protobuf.Message;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class SubscriptValue extends AbstractValue {
+    private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Subscript-Value");
+
+    @Nonnull
+    private final Value indexValue;
+
+    @Nonnull
+    private final Value sourceValue;
+
+    @Nonnull
+    private final Type type;
+
+    public SubscriptValue(@Nonnull final Value indexValue,
+                          @Nonnull final Value sourceValue) {
+        this.indexValue = indexValue;
+        this.sourceValue = sourceValue;
+        this.type = Verify.verifyNotNull(((Type.Array)sourceValue.getResultType()).getElementType()).nullable();
+    }
+
+    @Nonnull
+    @Override
+    protected Iterable<? extends Value> computeChildren() {
+        return List.of(indexValue, sourceValue);
+    }
+
+    @Nonnull
+    @Override
+    public ExplainTokensWithPrecedence explain(@Nonnull final Iterable<Supplier<ExplainTokensWithPrecedence>> explainSuppliers) {
+        final var indexValueExplain = Iterables.get(explainSuppliers, 0).get();
+        final var sourceValueExplain = Iterables.get(explainSuppliers, 1).get();
+        return ExplainTokensWithPrecedence.of(sourceValueExplain.getExplainTokens()
+                .addOpeningSquareBracket().addOptionalWhitespace()
+                .addNested(indexValueExplain.getExplainTokens())
+                .addClosingAngledBracket());
+    }
+
+    @Nullable
+    @Override
+    public <M extends Message> Object eval(@Nullable final FDBRecordStoreBase<M> store, @Nonnull final EvaluationContext context) {
+        final var index = indexValue.eval(store, context);
+        if (index == null) {
+            return null;
+        }
+        final var source = sourceValue.eval(store, context);
+        if (source == null) {
+            return null;
+        }
+        // index is 1-based as defined in SQL standard (Foundation, Section: 4.10.2):
+        // > An array is a collection A in which each element is associated with exactly one ordinal position in A.
+        //   If n is the cardinality of A, then the ordinal position p of an element is an integer in the range 1 (one)
+        //   ≤ p ≤ n.
+        final var sourceAsList = (List<?>)source;
+        final var adjustedIndex = (int)index - 1;
+        if (adjustedIndex < 0 || adjustedIndex >= sourceAsList.size()) {
+            // this does not raise out-of-bound error.
+            return null;
+        }
+        return sourceAsList.get(adjustedIndex);
+    }
+
+    @Nonnull
+    @Override
+    public Type getResultType() {
+        return type;
+    }
+
+    @Override
+    public int hashCodeWithoutChildren() {
+        return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH);
+    }
+
+    @Nonnull
+    @Override
+    public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) {
+        return PValue.newBuilder()
+                .setSubscriptValue(toProto(serializationContext))
+                .build();
+    }
+
+    @Override
+    public int planHash(@Nonnull final PlanHashMode hashMode) {
+        return PlanHashable.objectsPlanHash(hashMode, BASE_HASH);
+    }
+
+    @Nonnull
+    @Override
+    public PSubscriptValue toProto(@Nonnull final PlanSerializationContext serializationContext) {
+        return PSubscriptValue.newBuilder()
+                .setIndex(indexValue.toValueProto(serializationContext))
+                .setSource(sourceValue.toValueProto(serializationContext))
+                .build();
+    }
+
+    @Nonnull
+    public static SubscriptValue fromProto(@Nonnull final PlanSerializationContext serializationContext,
+                                      @Nonnull final PSubscriptValue subscriptValueProto) {
+        return new SubscriptValue(Value.fromValueProto(serializationContext, subscriptValueProto.getIndex()),
+                Value.fromValueProto(serializationContext, subscriptValueProto.getSource()));
+    }
+
+    @Nonnull
+    @Override
+    @SuppressWarnings("PMD.CompareObjectsWithEquals")
+    public Value withChildren(final Iterable<? extends Value> newChildren) {
+        Verify.verify(Iterables.size(newChildren) == 2);
+        final var newIndexValue = Iterables.get(newChildren, 0);
+        final var newSourceValue = Iterables.get(newChildren, 1);
+        if (indexValue == newIndexValue && newSourceValue == sourceValue) {
+            return this;
+        }
+        return new SubscriptValue(newIndexValue, newSourceValue);
+    }
+
+    /**
+     * Deserializer.
+     */
+    @AutoService(PlanDeserializer.class)
+    public static class Deserializer implements PlanDeserializer<PSubscriptValue, SubscriptValue> {
+        @Nonnull
+        @Override
+        public Class<PSubscriptValue> getProtoMessageClass() {
+            return PSubscriptValue.class;
+        }
+
+        @Nonnull
+        @Override
+        public SubscriptValue fromProto(@Nonnull final PlanSerializationContext serializationContext,
+                                        @Nonnull final PSubscriptValue subscriptValueProto) {
+            return SubscriptValue.fromProto(serializationContext, subscriptValueProto);
+        }
+    }
+
+    /**
+     * The {@code subscript} function.
+     */
+    @AutoService(BuiltInFunction.class)
+    public static class SubscriptValueFn extends BuiltInFunction<Value> {
+        public SubscriptValueFn() {
+            super("subscript", List.of(Type.primitiveType(Type.TypeCode.INT), Type.any()), Type.any(), SubscriptValue.SubscriptValueFn::encapsulate);
+        }
+
+        @SuppressWarnings({"PMD.UnusedFormalParameter", "PMD.UnusedPrivateMethod"}) // false positive, method is used
+        private static Value encapsulate(@Nonnull BuiltInFunction<Value> ignored,
+                                         @Nonnull final List<? extends Typed> arguments) {
+            Verify.verify(arguments.size() == 2);
+            var indexValue = (Value)arguments.get(0);
+            final var indexMaxType = Type.maximumType(indexValue.getResultType(), Type.primitiveType(Type.TypeCode.INT));
+            SemanticException.check(indexMaxType != null, SemanticException.ErrorCode.INCOMPATIBLE_TYPE);
+            indexValue = PromoteValue.inject(indexValue, indexMaxType);
+
+            var sourceValue = (Value)arguments.get(1);
+            Verify.verify(sourceValue.getResultType().isArray());
+
+            return new SubscriptValue(indexValue, sourceValue);
+        }
+    }
+}

fdb-record-layer-core/src/main/proto/record_query_plan.proto

@@ -255,6 +255,7 @@ message PValue {
     PFirstOrDefaultStreamingValue first_or_default_streaming_value = 49;
     PEvaluatesToValue evaluates_to_value = 50;
     PArrayDistinctValue array_distinct_value = 51;
+    PSubscriptValue subscript_value = 52;
   }
 }
 
@@ -457,6 +458,11 @@ message PEvaluatesToValue {
   optional PEvaluation evaluation = 2;
 }
 
+message PSubscriptValue {
+  optional PValue index = 1;
+  optional PValue source = 2;
+}
+
 message PFieldValue {
   optional PValue child_value = 1;
   optional PFieldPath field_path = 2;

fdb-relational-core/src/main/antlr/RelationalParser.g4

@@ -880,7 +880,7 @@ ofTypeClause
     ;
 
 arrayConstructor
-    : LEFT_SQUARE_BRACKET expression (',' expression)* RIGHT_SQUARE_BRACKET
+    : LEFT_SQUARE_BRACKET expressions? RIGHT_SQUARE_BRACKET
     ;
 
 userVariables
@@ -1129,43 +1129,38 @@ namedFunctionArg
 
 //    Expressions, predicates
 
-// Simplified approach for expression
 expression
-    : notOperator=(NOT | '!') expression                                                #notExpression     // done
-    | expression logicalOperator expression                                             #logicalExpression // done
-    | predicate IS NOT? testValue=(TRUE | FALSE | NULL_LITERAL)                         #isExpression      // done
-    | predicate NOT? LIKE pattern=STRING_LITERAL (ESCAPE escape=STRING_LITERAL)?        #likePredicate // done
-    | predicate                                                                         #predicateExpression // done
+    : notOperator=(NOT | '!') expression                                                 #notExpression     // done
+    | EXISTS '(' query ')'                                                               #existsExpressionAtom // done
+    | expressionAtom predicate?                                                          #predicatedExpression
+    | expression logicalOperator expression                                              #logicalExpression // done
     ;
 
 predicate
-    : expressionAtom IN inList                                          #inPredicate // done
-    | left=predicate comparisonOperator right=predicate                 #binaryComparisonPredicate // done
-    | operand=predicate NOT? BETWEEN left=predicate AND right=predicate #betweenComparisonPredicate // done
-    | expressionAtom                                                    #expressionAtomPredicate // done
+    : NOT? BETWEEN left=expressionAtom AND right=expressionAtom           #betweenComparisonPredicate // done
+    | NOT? IN inList                                                      #inPredicate // done
+    | NOT? LIKE pattern=STRING_LITERAL (ESCAPE escape=STRING_LITERAL)?    #likePredicate // done
+    | IS NOT? testValue=(TRUE | FALSE | NULL_LITERAL)                     #isExpression      // done
+    ;
+
+expressionAtom
+    : constant                                                                            #constantExpressionAtom // done
+    | fullColumnName                                                                      #fullColumnNameExpressionAtom // done
+    | functionCall                                                                        #functionCallExpressionAtom // done
+    | preparedStatementParameter                                                          #preparedStatementParameterAtom // done
+    | recordConstructor                                                                   #recordConstructorExpressionAtom // done
+    | arrayConstructor                                                                    #arrayConstructorExpressionAtom // done
+    | base=expressionAtom LEFT_SQUARE_BRACKET index=expressionAtom RIGHT_SQUARE_BRACKET   #subscriptExpression // done
+    | left=expressionAtom bitOperator right=expressionAtom                                #bitExpressionAtom // done
+    | left=expressionAtom mathOperator right=expressionAtom                               #mathExpressionAtom // done
+    | left=expressionAtom comparisonOperator right=expressionAtom                         #binaryComparisonPredicate // done
     ;
 
 inList
     : '(' (queryExpressionBody | expressions) ')'
     | preparedStatementParameter
     ;
 
-// Add in ASTVisitor nullNotnull in constant
-expressionAtom
-    : constant                                                      #constantExpressionAtom // done
-    | fullColumnName                                                #fullColumnNameExpressionAtom // done
-    | functionCall                                                  #functionCallExpressionAtom // done
-    | preparedStatementParameter                                    #preparedStatementParameterAtom // done
-    | recordConstructor                                             #recordConstructorExpressionAtom // done
-    | arrayConstructor                                              #arrayConstructorExpressionAtom // done
-    | EXISTS '(' query ')'                                          #existsExpressionAtom // done
-    | '(' queryExpressionBody ')'                                   #subqueryExpressionAtom // done (unsupported)
-    | INTERVAL expression intervalType                              #intervalExpressionAtom // done (unsupported)
-    | left=expressionAtom bitOperator right=expressionAtom          #bitExpressionAtom // done
-    | left=expressionAtom mathOperator right=expressionAtom         #mathExpressionAtom // done
-    | left=expressionAtom jsonOperator right=expressionAtom         #jsonExpressionAtom // done (unsupported)
-    ;
-
 preparedStatementParameter
     : QUESTION
     | NAMED_PARAMETER

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java

@@ -446,7 +446,6 @@ public Object visitPreparedStatementParameter(@Nonnull RelationalParser.Prepared
 
     @Override
     public Object visitInPredicate(@Nonnull RelationalParser.InPredicateContext ctx) {
-        ctx.expressionAtom().accept(this);
         ctx.IN().accept(this);
 
         if (ctx.inList().preparedStatementParameter() != null) {

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ParseHelpers.java

@@ -123,14 +123,14 @@ public static String underlineParsingError(@Nonnull Recognizer<?, ?> recognizer,
 
     public static boolean isConstant(@Nonnull final RelationalParser.ExpressionsContext expressionsContext) {
         for (final var exp : expressionsContext.expression()) {
-            if (!(exp instanceof RelationalParser.PredicateExpressionContext)) {
+            if (!(exp instanceof RelationalParser.PredicatedExpressionContext)) {
                 return false;
             }
-            final var predicate = (RelationalParser.PredicateExpressionContext) exp;
-            if (!(predicate.predicate() instanceof RelationalParser.ExpressionAtomPredicateContext)) {
+            final var predicate = (RelationalParser.PredicatedExpressionContext) exp;
+            if (predicate.predicate() != null) {
                 return false;
             }
-            final var expressionAtom = ((RelationalParser.ExpressionAtomPredicateContext) predicate.predicate()).expressionAtom();
+            final var expressionAtom = predicate.expressionAtom();
             if (!(expressionAtom instanceof RelationalParser.ConstantExpressionAtomContext)) {
                 return false;
             }

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java

@@ -122,6 +122,7 @@ private static ImmutableMap<String, Function<Integer, Optional<BuiltInFunction<?
                 .put("&", argumentsCount -> BuiltInFunctionCatalog.resolve("bitand", argumentsCount))
                 .put("|", argumentsCount -> BuiltInFunctionCatalog.resolve("bitor", argumentsCount))
                 .put("^", argumentsCount -> BuiltInFunctionCatalog.resolve("bitxor", argumentsCount))
+                .put("[]", argumentsCount -> BuiltInFunctionCatalog.resolve("subscript", argumentsCount))
                 .put("bitmap_bit_position", argumentsCount -> BuiltInFunctionCatalog.resolve("bitmap_bit_position", 1 + argumentsCount))
                 .put("bitmap_bucket_offset", argumentsCount -> BuiltInFunctionCatalog.resolve("bitmap_bucket_offset", 1 + argumentsCount))
                 .put("bitmap_construct_agg", argumentsCount -> BuiltInFunctionCatalog.resolve("BITMAP_CONSTRUCT_AGG", argumentsCount))