mixed-order matching and planning (#2858)

* added new Values for each of the directional conversion functions * save point * KeyExpressionWithValue refactored * field copiers driven by QueryPredicates and Values * predicate multi maps in match infos * predicate matching works somewhat; covering index optimization coded up but unested * save point * refactored SortOrder again * fixed test cases for ordering value simplification * save point * rough code complete * 66 failures left * less than 10 failures * getting the code-checkers to work; adding m3 maps * m3 maps work * fix bug related to non-simplified fixed bindings * nullable store to some evals * addressed review comments * addressing more comments * Update fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java Co-authored-by: Youssef Hatem <[email protected]> --------- Co-authored-by: Youssef Hatem <[email protected]>
FoundationDB · Sep 5, 2024 · 09ab9fd · 09ab9fd
1 parent c08ce62
commit 09ab9fd
Show file tree

Hide file tree

Showing 189 changed files with 5,915 additions and 1,692 deletions.
diff --git a/fdb-extensions/src/main/java/com/apple/foundationdb/tuple/TupleOrdering.java b/fdb-extensions/src/main/java/com/apple/foundationdb/tuple/TupleOrdering.java
@@ -44,17 +44,19 @@ public class TupleOrdering {
      * {@link #ASC_NULLS_FIRST} corresponds to the default {@code Tuple} packing and is included for completeness.
      */
     public enum Direction {
-        ASC_NULLS_FIRST(false, false),
-        ASC_NULLS_LAST(false, true),
-        DESC_NULLS_FIRST(true, true),
-        DESC_NULLS_LAST(true, false);
+        ASC_NULLS_FIRST(false, false, "↑"),
+        ASC_NULLS_LAST(false, true, "↗"),
+        DESC_NULLS_FIRST(true, true, "↙"),
+        DESC_NULLS_LAST(true, false, "↓");
 
         private final boolean inverted;
         private final boolean counterflowNulls;
+        private final String arrowIndicator;
 
-        Direction(boolean inverted, boolean counterflowNulls) {
+        Direction(boolean inverted, boolean counterflowNulls, String arrowIndicator) {
             this.inverted = inverted;
             this.counterflowNulls = counterflowNulls;
+            this.arrowIndicator = arrowIndicator;
         }
 
         /**
@@ -73,6 +75,14 @@ public boolean isCounterflowNulls() {
             return counterflowNulls;
         }
 
+        /**
+         * Returns a string that symbolizes the meaning of this enum constant.
+         * @return the arrow indicator
+         */
+        public String getArrowIndicator() {
+            return arrowIndicator;
+        }
+
         /**
          * Get whether values are ordered ascending.
          * This is the opposite of {@link #isInverted()}, viewed from the name of the enum, as opposed to what encoding
@@ -110,6 +120,22 @@ public boolean isNullsFirst() {
         public boolean isNullsLast() {
             return inverted != counterflowNulls;
         }
+
+        @Nonnull
+        public Direction reverseDirection() {
+            switch (this) {
+                case ASC_NULLS_FIRST:
+                    return DESC_NULLS_LAST;
+                case ASC_NULLS_LAST:
+                    return DESC_NULLS_FIRST;
+                case DESC_NULLS_FIRST:
+                    return ASC_NULLS_LAST;
+                case DESC_NULLS_LAST:
+                    return ASC_NULLS_FIRST;
+                default:
+                    throw new IllegalArgumentException("cannot reverse this direction");
+            }
+        }
     }
 
     private TupleOrdering() {

diff --git a/...java/com/apple/foundationdb/record/metadata/expressions/CollateFunctionKeyExpression.java b/...java/com/apple/foundationdb/record/metadata/expressions/CollateFunctionKeyExpression.java
@@ -28,8 +28,7 @@
 import com.apple.foundationdb.record.provider.common.text.TextCollator;
 import com.apple.foundationdb.record.provider.common.text.TextCollatorRegistry;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
-import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
-import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
+import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.google.protobuf.Message;
 
@@ -77,7 +76,7 @@ public class CollateFunctionKeyExpression extends FunctionKeyExpression implemen
     @Nonnull
     private final TextCollatorRegistry collatorRegistry;
     @Nullable
-    private TextCollator invariableCollator;
+    private final TextCollator invariableCollator;
 
     protected CollateFunctionKeyExpression(@Nonnull TextCollatorRegistry collatorRegistry,
                                            @Nonnull String name, @Nonnull KeyExpression arguments) {
@@ -180,9 +179,13 @@ public int getColumnSize() {
 
     @Nonnull
     @Override
-    public Value toValue(@Nonnull final CorrelationIdentifier baseAlias,
-                         @Nonnull final Type baseType,
-                         @Nonnull final List<String> fieldNamePrefix) {
+    public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpressionVisitor<S, R> visitor) {
+        return visitor.visitExpression(this);
+    }
+
+    @Nonnull
+    @Override
+    public Value toValue(@Nonnull final List<? extends Value> argumentValues) {
         // TODO support this
         throw new UnsupportedOperationException();
     }

diff --git a/...c/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java b/...c/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java
@@ -23,12 +23,16 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
+import com.apple.foundationdb.record.RecordCoreArgumentException;
 import com.apple.foundationdb.record.RecordCoreException;
 import com.apple.foundationdb.record.RecordMetaDataProto;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
+import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
+import com.apple.foundationdb.record.query.plan.cascades.values.FunctionCatalog;
+import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.record.util.HashUtils;
 import com.apple.foundationdb.record.util.ServiceLoaderProvider;
 import com.google.protobuf.Descriptors;
@@ -257,6 +261,27 @@ public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpr
         return visitor.visitExpression(this);
     }
 
+    /**
+     * This method creates a {@link Value} based on this key expression. The caller provides the {@link Value}s
+     * that serve as the arguments to the function. In reality this method is exclusively called from the
+     * expansion visitors, e.g. {@link com.apple.foundationdb.record.query.plan.cascades.KeyExpressionExpansionVisitor},
+     * thus forming the bridge between key expressions and Cascades values.
+     * @param argumentValues the argument values
+     * @return a new {@link Value}
+     */
+    @Nonnull
+    public abstract Value toValue(@Nonnull List<? extends Value> argumentValues);
+
+    @Nonnull
+    protected Value resolveAndEncapsulateFunction(@Nonnull final String functionName,
+                                                  @Nonnull final List<? extends Value> argumentValues) {
+        final BuiltInFunction<?> builtInFunction =
+                FunctionCatalog.resolve(functionName, argumentValues.size())
+                        .orElseThrow(() -> new RecordCoreArgumentException("unknown function",
+                                LogMessageKeys.FUNCTION, getName()));
+        return (Value)builtInFunction.encapsulate(argumentValues);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {

diff --git a/.../main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpressionWithValue.java b/.../main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpressionWithValue.java
@@ -28,7 +28,6 @@
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 
 import javax.annotation.Nonnull;
-import java.util.List;
 
 /**
  * A key expression that can be represented as a single {@link Value} because it meets both of the following criteria:
@@ -41,12 +40,12 @@
 @API(API.Status.EXPERIMENTAL)
 public interface KeyExpressionWithValue extends KeyExpression {
     @Nonnull
-    default Value toValue(@Nonnull Quantifier baseQuantifier, @Nonnull List<String> fieldNamePrefix) {
-        return toValue(baseQuantifier.getAlias(), baseQuantifier.getFlowedObjectType(), fieldNamePrefix);
+    default Value toValue(@Nonnull final Quantifier baseQuantifier) {
+        return toValue(baseQuantifier.getAlias(), baseQuantifier.getFlowedObjectType());
     }
 
     @Nonnull
-    Value toValue(@Nonnull CorrelationIdentifier baseAlias, @Nonnull Type baseType, @Nonnull List<String> fieldNamePrefix);
+    Value toValue(@Nonnull CorrelationIdentifier baseAlias, @Nonnull Type baseType);
 
     @Nonnull
     @Override

diff --git a/...rc/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java b/...rc/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java
@@ -28,6 +28,7 @@
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
+import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
@@ -101,11 +102,16 @@ public RecordMetaDataProto.Value toProto() throws SerializationException {
         return proto;
     }
 
+    @Nonnull
+    @Override
+    public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpressionVisitor<S, R> visitor) {
+        return visitor.visitExpression(this);
+    }
+
     @Nonnull
     @Override
     public Value toValue(@Nonnull final CorrelationIdentifier baseAlias,
-                         @Nonnull final Type baseType,
-                         @Nonnull final List<String> fieldNamePrefix) {
+                         @Nonnull final Type baseType) {
         return LiteralValue.ofScalar(value);
     }
 

diff --git a/.../apple/foundationdb/record/metadata/expressions/LongArithmethicFunctionKeyExpression.java b/.../apple/foundationdb/record/metadata/expressions/LongArithmethicFunctionKeyExpression.java
@@ -22,24 +22,18 @@
 
 import com.apple.foundationdb.record.FunctionNames;
 import com.apple.foundationdb.record.ObjectPlanHash;
-import com.apple.foundationdb.record.RecordCoreArgumentException;
-import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction;
-import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
-import com.apple.foundationdb.record.query.plan.cascades.ScalarTranslationVisitor;
-import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
-import com.apple.foundationdb.record.query.plan.cascades.values.FunctionCatalog;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.google.auto.service.AutoService;
+import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.Message;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.function.BinaryOperator;
@@ -129,15 +123,9 @@ public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpr
 
     @Nonnull
     @Override
-    public Value toValue(@Nonnull final CorrelationIdentifier baseAlias, @Nonnull final Type baseType, @Nonnull final List<String> fieldNamePrefix) {
-        ScalarTranslationVisitor scalarTranslationVisitor = new ScalarTranslationVisitor(arguments);
-        scalarTranslationVisitor.push(ScalarTranslationVisitor.ScalarVisitorState.of(baseAlias, baseType, fieldNamePrefix));
-        List<Value> argumentValues = new ArrayList<>(arguments.getColumnSize());
-        for (KeyExpression expression : arguments.normalizeKeyForPositions()) {
-            argumentValues.add(expression.expand(scalarTranslationVisitor));
-        }
-        BuiltInFunction<?> builtInFunction = FunctionCatalog.resolve(valueFunctionName, arguments.getColumnSize()).orElseThrow(() -> new RecordCoreArgumentException("unknown function", LogMessageKeys.FUNCTION, getName()));
-        return (Value) builtInFunction.encapsulate(argumentValues);
+    public Value toValue(@Nonnull final List<? extends Value> argumentValues) {
+        Verify.verify(argumentValues.size() == arguments.getColumnSize());
+        return resolveAndEncapsulateFunction(valueFunctionName, argumentValues);
     }
 
     /**

diff --git a/...n/java/com/apple/foundationdb/record/metadata/expressions/OrderFunctionKeyExpression.java b/...n/java/com/apple/foundationdb/record/metadata/expressions/OrderFunctionKeyExpression.java
@@ -25,8 +25,8 @@
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
-import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
-import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
+import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
+import com.apple.foundationdb.record.query.plan.cascades.values.ToOrderedBytesValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.foundationdb.tuple.TupleOrdering;
@@ -91,11 +91,14 @@ public int getColumnSize() {
 
     @Nonnull
     @Override
-    public Value toValue(@Nonnull final CorrelationIdentifier baseAlias,
-                         @Nonnull final Type baseType,
-                         @Nonnull final List<String> fieldNamePrefix) {
-        // TODO need inner Value for match and Ordering info from direction.
-        throw new UnsupportedOperationException();
+    public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpressionVisitor<S, R> visitor) {
+        return visitor.visitExpression(this);
+    }
+
+    @Nonnull
+    @Override
+    public Value toValue(@Nonnull final List<? extends Value> argumentValues) {
+        return new ToOrderedBytesValue(argumentValues.get(0), direction);
     }
 
     @Nullable

diff --git a/.../main/java/com/apple/foundationdb/record/metadata/expressions/QueryableKeyExpression.java b/.../main/java/com/apple/foundationdb/record/metadata/expressions/QueryableKeyExpression.java
@@ -41,7 +41,7 @@
  * with an optional conversion of any comparison operand.
  */
 @API(API.Status.EXPERIMENTAL)
-public interface QueryableKeyExpression extends KeyExpression, KeyExpressionWithValue {
+public interface QueryableKeyExpression extends KeyExpression {
     @Nonnull
     String getName();
 

diff --git a/...main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java b/...main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java
@@ -27,6 +27,7 @@
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
+import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.RecordTypeValue;
@@ -118,7 +119,13 @@ public RecordMetaDataProto.KeyExpression toKeyExpression() {
 
     @Nonnull
     @Override
-    public Value toValue(@Nonnull final CorrelationIdentifier baseAlias, @Nonnull final Type baseType, @Nonnull final List<String> fieldNamePrefix) {
+    public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpressionVisitor<S, R> visitor) {
+        return visitor.visitExpression(this);
+    }
+
+    @Nonnull
+    @Override
+    public Value toValue(@Nonnull final CorrelationIdentifier baseAlias, @Nonnull final Type baseType) {
         return new RecordTypeValue(QuantifiedObjectValue.of(baseAlias, new Type.AnyRecord(true)));
     }
 

diff --git a/...rc/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java b/...rc/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java
@@ -28,6 +28,7 @@
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion;
 import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
+import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.record.query.plan.cascades.values.VersionValue;
@@ -108,7 +109,13 @@ public RecordMetaDataProto.KeyExpression toKeyExpression() {
 
     @Nonnull
     @Override
-    public Value toValue(@Nonnull final CorrelationIdentifier baseAlias, @Nonnull final Type baseType, @Nonnull final List<String> fieldNamePrefix) {
+    public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpressionVisitor<S, R> visitor) {
+        return visitor.visitExpression(this);
+    }
+
+    @Nonnull
+    @Override
+    public Value toValue(@Nonnull final CorrelationIdentifier baseAlias, @Nonnull final Type baseType) {
         return new VersionValue(baseAlias);
     }
 

diff --git a/...er-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/CrossProduct.java b/...er-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/CrossProduct.java
@@ -31,6 +31,7 @@
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 
@@ -174,7 +175,7 @@ public void skip(final int level) {
             }
         }
 
-        private ComplexIterable(@Nonnull final List<? extends Iterable<T>> sources) {
+        private ComplexIterable(@Nonnull final Collection<? extends Iterable<T>> sources) {
             Verify.verify(sources.size() > 1);
             this.sources = ImmutableList.copyOf(sources);
         }
@@ -243,7 +244,7 @@ public EnumeratingIterator<T> iterator() {
      *         {@code dependsOnFn} in a sense that the iterators created by this iterator will not return
      *         orderings that violate the given depends-on constraints
      */
-    public static <T> EnumeratingIterable<T> crossProduct(@Nonnull final List<? extends Iterable<T>> sources) {
+    public static <T> EnumeratingIterable<T> crossProduct(@Nonnull final Collection<? extends Iterable<T>> sources) {
         // try simple
         @Nullable
         final EnumeratingIterable<T> maybeSimpleIterable = trySimpleIterable(sources);
@@ -255,7 +256,7 @@ public static <T> EnumeratingIterable<T> crossProduct(@Nonnull final List<? exte
     }
 
     @Nullable
-    private static <T> EnumeratingIterable<T> trySimpleIterable(@Nonnull final List<? extends Iterable<T>> sources) {
+    private static <T> EnumeratingIterable<T> trySimpleIterable(@Nonnull final Collection<? extends Iterable<T>> sources) {
         if (sources.isEmpty()) {
             return EnumeratingIterable.emptyIterable();
         } else if (sources.size() == 1) {