diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index e354a71696..b8a73f14ec 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -35,7 +35,7 @@ with the new code. * **Performance** Improvement 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Performance** Improvement 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Feature** Feature 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) -* **Feature** Feature 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) +* **Feature** Support temporary table planning and execution [(Issue #2962)](https://github.com/FoundationDB/fdb-record-layer/pull/2962) * **Feature** Feature 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Feature** Feature 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Feature** Feature 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContext.java index 15a4c240f3..045b9ca1a1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContext.java @@ -127,11 +127,13 @@ public Object getBinding(@Nonnull String name) { /** * Get the value bound to a single parameter. * + * @param type the type of the parameter * @param alias the correlation identifier + * * @return the value bound to the given parameter */ - public Object getBinding(@Nonnull CorrelationIdentifier alias) { - return bindings.get(Bindings.Internal.CORRELATION.bindingName(alias.getId())); + public Object getBinding(@Nonnull final Bindings.Internal type, @Nonnull final CorrelationIdentifier alias) { + return bindings.get(type.bindingName(alias.getId())); } /** @@ -201,11 +203,13 @@ public EvaluationContext withBinding(@Nonnull String bindingName, @Nullable Obje * context included all bindings except that it will bind an additional * parameter to an additional value. * + * @param type the type of the binding. * @param alias the alias determining the binding name to add * @param value the value to bind the name to + * * @return a new EvaluationContext with the new binding */ - public EvaluationContext withBinding(@Nonnull CorrelationIdentifier alias, @Nullable Object value) { - return childBuilder().setBinding(Bindings.Internal.CORRELATION.bindingName(alias.getId()), value).build(typeRepository); + public EvaluationContext withBinding(final Bindings.Internal type, @Nonnull CorrelationIdentifier alias, @Nullable Object value) { + return childBuilder().setBinding(type.bindingName(alias.getId()), value).build(typeRepository); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/StreamGrouping.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/StreamGrouping.java index 4fddbd0148..721a920657 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/StreamGrouping.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/StreamGrouping.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.record.cursors.aggregate; +import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; @@ -183,13 +184,13 @@ private void finalizeGroup(Object nextGroup) { } private void accumulate(@Nullable Object currentObject) { - EvaluationContext nestedContext = context.withBinding(alias, currentObject); + EvaluationContext nestedContext = context.withBinding(Bindings.Internal.CORRELATION, alias, currentObject); final Object partial = aggregateValue.evalToPartial(store, nestedContext); accumulator.accumulate(partial); } private Object evalGroupingKey(@Nullable final Object currentObject) { - final EvaluationContext nestedContext = context.withBinding(alias, currentObject); + final EvaluationContext nestedContext = context.withBinding(Bindings.Internal.CORRELATION, alias, currentObject); return Objects.requireNonNull(groupingKeyValue).eval(store, nestedContext); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/PlanStringRepresentation.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/PlanStringRepresentation.java index 050692fb4d..2f783841ce 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/PlanStringRepresentation.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/PlanStringRepresentation.java @@ -45,6 +45,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryInValuesJoinPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionPlan; @@ -58,6 +59,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionOnKeyExpressionPlan; @@ -403,6 +405,14 @@ public PlanStringRepresentation visitInsertPlan(@Nonnull RecordQueryInsertPlan e .append(element.getTargetRecordType()); } + @Nonnull + @Override + public PlanStringRepresentation visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan element) { + return visit(element.getChild()) + .append(" | TEMP TABLE INSERT INTO ") + .append(element.getTargetRecordType()); + } + @Nonnull private PlanStringRepresentation visitIntersectionPlan(@Nonnull RecordQueryIntersectionPlan element) { return appendItems(element.getChildren(), " ∩ "); @@ -577,6 +587,14 @@ public PlanStringRepresentation visitSortPlan(@Nonnull RecordQuerySortPlan eleme .append(element.getKey()); } + @Nonnull + @Override + public PlanStringRepresentation visitTempTableScanPlan(@Nonnull final TempTableScanPlan element) { + return append("TEMP TABLE SCAN ([") + .append(element.getResultValue()) + .append("])"); + } + @Nonnull @Override public PlanStringRepresentation visitDefault(@Nonnull RecordQueryPlan element) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java index 27e32bdc66..4b0bd72964 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java @@ -34,11 +34,13 @@ import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementInJoinRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementInUnionRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementInsertRule; +import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTempTableInsertRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementIntersectionRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementNestedLoopJoinRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementPhysicalScanRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementSimpleSelectRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementStreamingAggregationRule; +import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTempTableScanRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTypeFilterRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementUniqueRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementUnorderedUnionRule; @@ -136,6 +138,7 @@ public class PlannerRuleSet { ); private static final List> IMPLEMENTATION_RULES = ImmutableList.of( + new ImplementTempTableScanRule(), new ImplementTypeFilterRule(), new ImplementFilterRule(), new PushTypeFilterBelowFilterRule(), @@ -169,6 +172,7 @@ public class PlannerRuleSet { new ImplementStreamingAggregationRule(), new ImplementDeleteRule(), new ImplementInsertRule(), + new ImplementTempTableInsertRule(), new ImplementUpdateRule() ); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/TempTable.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/TempTable.java new file mode 100644 index 0000000000..e021864dbf --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/TempTable.java @@ -0,0 +1,143 @@ +/* + * TempTable.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.cursors.ListCursor; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.apple.foundationdb.tuple.ByteArrayUtil2; +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.ZeroCopyByteString; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +/** + * A mutable, temporary, serializable, and in-memory buffer of {@link QueryResult}s. It is aimed to be used as a temporary + * placeholder for computation results produced by some physical operator, but can be leveraged to represent, for example, + * a SQL temporary tables as well. + */ +public class TempTable { + + @Nonnull + private final Queue underlyingBuffer; + + private TempTable() { + this(new LinkedList<>()); + } + + private TempTable(@Nonnull Queue buffer) { + this.underlyingBuffer = buffer; + } + + /** + * Add a new {@link QueryResult} element to the queue. + * @param element the new element to be added. + */ + public void add(@Nonnull QueryResult element) { + underlyingBuffer.add(element); + } + + /** + * Add a new {@link QueryResult} elements to the queue. + * @param elements the new elements to be added. + */ + public void add(@Nonnull QueryResult... elements) { + Arrays.stream(elements).forEach(this::add); + } + + + @Nonnull + public Queue getReadBuffer() { + return underlyingBuffer; + } + + @SuppressWarnings("unchecked") + @Nonnull + public RecordCursor getReadCursor(@Nullable byte[] continuation) { + return new ListCursor<>((List)getReadBuffer(), continuation); + } + + private void serializeBuffer(@Nonnull RecordMetaDataProto.PTempTable.Builder protoMessageBuilder) { + for (final var element : underlyingBuffer) { + final var elementByteString = element.toByteString(); + protoMessageBuilder.addBufferItems(elementByteString); + } + } + + @Nonnull + public RecordMetaDataProto.PTempTable toProto() { + final var builder = RecordMetaDataProto.PTempTable.newBuilder(); + serializeBuffer(builder); + return builder.build(); + } + + @Nonnull + public ByteString toByteString() { + return toProto().toByteString(); + } + + @Nonnull + public byte[] toBytes() { + return toByteString().toByteArray(); + } + + @Nonnull + public static TempTable deserialize(@Nullable Descriptors.Descriptor descriptor, @Nonnull byte[]bytes) { + return deserialize(descriptor, ZeroCopyByteString.wrap(bytes)); + } + + @Nonnull + public static TempTable deserialize(@Nullable Descriptors.Descriptor descriptor, @Nonnull ByteString byteString) { + final RecordMetaDataProto.PTempTable tempTableProto; + try { + tempTableProto = RecordMetaDataProto.PTempTable.parseFrom(byteString); + } catch (InvalidProtocolBufferException ex) { + throw new RecordCoreException("invalid bytes", ex) + .addLogInfo(LogMessageKeys.RAW_BYTES, ByteArrayUtil2.loggable(byteString.toByteArray())); + } + return fromProto(tempTableProto, descriptor); + } + + @Nonnull + public static TempTable fromProto(@Nonnull final RecordMetaDataProto.PTempTable tempTableProto, + @Nullable Descriptors.Descriptor descriptor) { + final var underlyingBuffer = new LinkedList(); + for (final var element : tempTableProto.getBufferItemsList()) { + underlyingBuffer.add(QueryResult.deserialize(descriptor, element)); + } + return new TempTable(underlyingBuffer); + } + + @Nonnull + public static TempTable newInstance() { + return new TempTable(); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/NodeInfo.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/NodeInfo.java index 51340e3deb..f5eef959e6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/NodeInfo.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/NodeInfo.java @@ -114,6 +114,11 @@ public class NodeInfo { NodeIcon.DATA_ACCESS_OPERATOR, "Scan", "A scan operator loads a set of records from the database that are within the given range of primary keys."); + public static final NodeInfo TEMP_TABLE_SCAN_OPERATOR = new NodeInfo( + "TempTableScanOperator", + NodeIcon.DATA_ACCESS_OPERATOR, + "Temp Table Scan", + "A scan operator loads a set of records from an in-memory temporary buffer."); public static final NodeInfo SCORE_FOR_RANK_OPERATOR = new NodeInfo( "ScoreForRankOperator", NodeIcon.DATA_ACCESS_OPERATOR, diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/PlannerGraph.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/PlannerGraph.java index 9350eb7d1e..20de2f4a1f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/PlannerGraph.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/PlannerGraph.java @@ -387,6 +387,26 @@ String getToolTip() { } } + /** + * Node class for temporary data objects. + */ + public static class TemporaryDataNodeWithInfo extends DataNodeWithInfo { + + public TemporaryDataNodeWithInfo(@Nonnull final Type type, @Nullable final List sources) { + super(NodeInfo.TEMPORARY_BUFFER_DATA, type, sources); + } + + public TemporaryDataNodeWithInfo(@Nonnull Type type, @Nullable final List sources, @Nonnull final Map additionalAttributes) { + super(NodeInfo.TEMPORARY_BUFFER_DATA, type, sources, additionalAttributes); + } + + @Nonnull + @Override + public String getFillColor() { + return "goldenrod2"; + } + } + /** * Node class for actual plan operators. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/DeleteExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/DeleteExpression.java index 00f19e50e9..f7d7b1ca57 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/DeleteExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/DeleteExpression.java @@ -83,7 +83,7 @@ public List getQuantifiers() { @Nonnull @Override public DeleteExpression translateCorrelations(@Nonnull final TranslationMap translationMap, @Nonnull final List translatedQuantifiers) { - return new DeleteExpression(inner, targetRecordType); + return new DeleteExpression(Iterables.getOnlyElement(translatedQuantifiers).narrow(Quantifier.ForEach.class), targetRecordType); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/InsertExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/InsertExpression.java index 001f33acce..c01e3028e0 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/InsertExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/InsertExpression.java @@ -92,7 +92,7 @@ public List getQuantifiers() { @Nonnull @Override public InsertExpression translateCorrelations(@Nonnull final TranslationMap translationMap, @Nonnull final List translatedQuantifiers) { - return new InsertExpression(inner, targetRecordType, targetType); + return new InsertExpression(Iterables.getOnlyElement(translatedQuantifiers).narrow(Quantifier.ForEach.class), targetRecordType, targetType); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TempTableInsertExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TempTableInsertExpression.java new file mode 100644 index 0000000000..ba25235758 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TempTableInsertExpression.java @@ -0,0 +1,210 @@ +/* + * TempTableInsertExpression.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.expressions; + +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.TempTable; +import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTempTableInsertRule; +import com.apple.foundationdb.record.query.plan.cascades.values.ConstantObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.apple.foundationdb.record.query.plan.cascades.explain.NodeInfo; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraphRewritable; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.ObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryAbstractDataModificationPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; + +import javax.annotation.Nonnull; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * A logical expression for inserting into a temporary memory buffer {@link TempTable}. + * This expression is used to implement a corresponding {@link TempTableInsertPlan} operator that + * does exactly that. + * + * @see ImplementTempTableInsertRule for more information. + */ +public class TempTableInsertExpression implements RelationalExpressionWithChildren, PlannerGraphRewritable { + + @Nonnull + private final Quantifier.ForEach inner; + @Nonnull + private final Value resultValue; + @Nonnull + private final Value tempTableReferenceValue; + + private TempTableInsertExpression(@Nonnull final Quantifier.ForEach inner, + @Nonnull final Value tempTableReferenceValue) { + this.inner = inner; + this.tempTableReferenceValue = tempTableReferenceValue; + Verify.verify(tempTableReferenceValue.getResultType().isRelation()); + final var innerType = ((Type.Relation)tempTableReferenceValue.getResultType()).getInnerType(); + this.resultValue = new QueriedValue(Objects.requireNonNull(innerType)); + } + + @Override + public int getRelationalChildCount() { + return 1; + } + + @Nonnull + @Override + public Set getCorrelatedToWithoutChildren() { + return tempTableReferenceValue.getCorrelatedToWithoutChildren(); + } + + @Nonnull + @Override + public List getQuantifiers() { + return ImmutableList.of(inner); + } + + @Nonnull + @Override + public TempTableInsertExpression translateCorrelations(@Nonnull final TranslationMap translationMap, + @Nonnull final List translatedQuantifiers) { + return new TempTableInsertExpression(Iterables.getOnlyElement(translatedQuantifiers).narrow(Quantifier.ForEach.class), + tempTableReferenceValue.translateCorrelations(translationMap)); + } + + @Nonnull + @Override + public Value getResultValue() { + return resultValue; + } + + @Nonnull + public TempTableInsertPlan toPlan(@Nonnull final Quantifier.Physical physicalInner) { + Verify.verify(inner.getAlias().equals(physicalInner.getAlias())); + return TempTableInsertPlan.insertPlan(physicalInner, + makeComputationValue(tempTableReferenceValue.getResultType()), + tempTableReferenceValue); + } + + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public boolean equalsWithoutChildren(@Nonnull final RelationalExpression otherExpression, + @Nonnull final AliasMap equivalencesMap) { + if (this == otherExpression) { + return true; + } + if (getClass() != otherExpression.getClass()) { + return false; + } + final TempTableInsertExpression otherInsertExpression = (TempTableInsertExpression)otherExpression; + return tempTableReferenceValue.semanticEquals(otherInsertExpression.tempTableReferenceValue, equivalencesMap); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @Override + public boolean equals(final Object other) { + return semanticEquals(other); + } + + @Override + public int hashCode() { + return semanticHashCode(); + } + + @Override + public int hashCodeWithoutChildren() { + return Objects.hash(tempTableReferenceValue); + } + + @Override + public String toString() { + return "TempTableInsert(" + tempTableReferenceValue + ")"; + } + + /** + * Create a planner graph for better visualization. + * @return the rewritten planner graph that models the target as a separate node that is connected to the + * temporary table insert expression node. + */ + @Nonnull + @Override + public PlannerGraph rewritePlannerGraph(@Nonnull final List childGraphs) { + Verify.verify(!childGraphs.isEmpty()); + + final var graphForTarget = + PlannerGraph.fromNodeAndChildGraphs( + new PlannerGraph.TemporaryDataNodeWithInfo(getResultType(), ImmutableList.of(tempTableReferenceValue.toString())), + ImmutableList.of()); + + return PlannerGraph.fromNodeInnerAndTargetForModifications( + new PlannerGraph.ModificationLogicalOperatorNode(this, + NodeInfo.MODIFICATION_OPERATOR, + ImmutableList.of("TempTableInsert"), + ImmutableMap.of()), + Iterables.getOnlyElement(childGraphs), graphForTarget); + } + + @Nonnull + private static Value makeComputationValue(@Nonnull final Type targetType) { + return ObjectValue.of(RecordQueryAbstractDataModificationPlan.currentModifiedRecordAlias(), targetType); + } + + /** + * Creates a new instance of {@link TempTableInsertExpression} that adds records to a constant-bound {@link TempTable}, + * i.e. a temporary table that is not correlated to any other plan operator. + * + * @param inner The source of the inserted records + * @param constantAlias The alias of the constant-bound temporary table. + * @param constantId The id of the constant in the constant map. + * @param type The type of the temporary table records. + * @return A new {@link TempTableInsertExpression} that adds records to a constant-bound {@link TempTable}. + */ + @Nonnull + public static TempTableInsertExpression ofConstant(@Nonnull final Quantifier.ForEach inner, + @Nonnull final CorrelationIdentifier constantAlias, + @Nonnull final String constantId, + @Nonnull final Type type) { + return new TempTableInsertExpression(inner, ConstantObjectValue.of(constantAlias, constantId, new Type.Relation(type))); + } + + /** + * Creates a new instance of {@link TempTableInsertExpression} that adds records to a correlated {@link TempTable}, + * i.e. a temporary table that is the result of a table-valued correlation. + * + * @param inner The source of the inserted records + * @param correlation The table-valued correlation. + * @param type The type of the temporary table records. + * @return A new {@link TempTableInsertExpression} that adds records to a correlated {@link TempTable}. + */ + @Nonnull + public static TempTableInsertExpression ofCorrelated(@Nonnull final Quantifier.ForEach inner, + @Nonnull final CorrelationIdentifier correlation, + @Nonnull final Type type) { + return new TempTableInsertExpression(inner, QuantifiedObjectValue.of(correlation, new Type.Relation(type))); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TempTableScanExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TempTableScanExpression.java new file mode 100644 index 0000000000..0c376ad582 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TempTableScanExpression.java @@ -0,0 +1,187 @@ +/* + * TempTableScanExpression.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2019 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.expressions; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.Bindings; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange; +import com.apple.foundationdb.record.query.plan.cascades.Compensation; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.PartialMatch; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.TempTable; +import com.apple.foundationdb.record.query.plan.cascades.explain.NodeInfo; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraphRewritable; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.ConstantObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import javax.annotation.Nonnull; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * A logical expression for scanning from a table-valued {@link Bindings.Internal#CORRELATION} + * that can correspond to a temporary memory buffer, i.e. a {@link TempTable}. + * This expression is used to implement a corresponding {@link TempTableScanPlan} operator that + * does exactly that. + */ +@API(API.Status.EXPERIMENTAL) +public class TempTableScanExpression implements RelationalExpression, PlannerGraphRewritable { + @Nonnull + private final Value tempTableReferenceValue; + @Nonnull + private final QueriedValue resultValue; + + private TempTableScanExpression(@Nonnull final Value tempTableReferenceValue) { + this.tempTableReferenceValue = tempTableReferenceValue; + final var innerType = ((Type.Relation)tempTableReferenceValue.getResultType()).getInnerType(); + this.resultValue = new QueriedValue(Objects.requireNonNull(innerType)); + } + + @Nonnull + @Override + public Value getResultValue() { + return resultValue; + } + + @Nonnull + public Value getTempTableReferenceValue() { + return tempTableReferenceValue; + } + + @Nonnull + @Override + public List getQuantifiers() { + return ImmutableList.of(); + } + + @Nonnull + @Override + public Set getCorrelatedTo() { + return tempTableReferenceValue.getCorrelatedTo(); + } + + @Nonnull + @Override + public TempTableScanExpression translateCorrelations(@Nonnull final TranslationMap translationMap, + @Nonnull final List translatedQuantifiers) { + return new TempTableScanExpression(tempTableReferenceValue.translateCorrelations(translationMap)); + } + + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public boolean equalsWithoutChildren(@Nonnull final RelationalExpression otherExpression, @Nonnull final AliasMap equivalencesMap) { + if (this == otherExpression) { + return true; + } + if (getClass() != otherExpression.getClass()) { + return false; + } + final var otherTempTableScanExpression = (TempTableScanExpression)otherExpression; + return getTempTableReferenceValue().semanticEquals(otherTempTableScanExpression.getTempTableReferenceValue(), equivalencesMap); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @Override + public boolean equals(final Object other) { + return semanticEquals(other); + } + + @Override + public int hashCode() { + return semanticHashCode(); + } + + @Override + public int hashCodeWithoutChildren() { + return Objects.hash(tempTableReferenceValue); + } + + @Override + public String toString() { + return "TempTableScan"; + } + + @Override + public Compensation compensate(@Nonnull final PartialMatch partialMatch, @Nonnull final Map boundParameterPrefixMap) { + return Compensation.noCompensation(); + } + + @Nonnull + @Override + public PlannerGraph rewritePlannerGraph(@Nonnull final List childGraphs) { + Verify.verify(childGraphs.isEmpty()); + + final PlannerGraph.DataNodeWithInfo dataNodeWithInfo = new PlannerGraph + .TemporaryDataNodeWithInfo(getResultType(), ImmutableList.of(tempTableReferenceValue.toString())); + + return PlannerGraph.fromNodeAndChildGraphs( + new PlannerGraph.LogicalOperatorNodeWithInfo(this, + NodeInfo.TEMP_TABLE_SCAN_OPERATOR, + ImmutableList.of(), + ImmutableMap.of()), + ImmutableList.of(PlannerGraph.fromNodeAndChildGraphs( + dataNodeWithInfo, + childGraphs))); + } + + /** + * Creates a new instance of {@link TempTableScanExpression} that scans records from a constant-bound {@link TempTable}, + * i.e. a temporary table that is not correlated to any other plan operator. + * + * @param constantAlias The alias of the constant-bound temporary table. + * @param constantId The id of the constant in the constant map. + * @param type The type of the temporary table records. + * @return A new {@link TempTableScanExpression} that adds records to a constant-bound {@link TempTable}. + */ + @Nonnull + public static TempTableScanExpression ofConstant(@Nonnull final CorrelationIdentifier constantAlias, + @Nonnull final String constantId, + @Nonnull final Type type) { + return new TempTableScanExpression(ConstantObjectValue.of(constantAlias, constantId, new Type.Relation(type))); + } + + /** + * Creates a new instance of {@link TempTableScanExpression} that scans records from a correlated {@link TempTable}, + * i.e. a temporary table that is the result of a table-valued correlation. + * + * @param correlation The table-valued correlation. + * @param type The type of the temporary table records. + * @return A new {@link TempTableScanExpression} that adds records to a correlated {@link TempTable}. + */ + @Nonnull + public static TempTableScanExpression ofCorrelated(@Nonnull final CorrelationIdentifier correlation, + @Nonnull final Type type) { + return new TempTableScanExpression(QuantifiedObjectValue.of(correlation, new Type.Relation(type))); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/UpdateExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/UpdateExpression.java index d52c41e658..9def3aa630 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/UpdateExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/UpdateExpression.java @@ -132,7 +132,8 @@ public UpdateExpression translateCorrelations(@Nonnull final TranslationMap tran for (final var entry : transformMap.entrySet()) { translatedTransformMapBuilder.put(entry.getKey(), entry.getValue().translateCorrelations(translationMap)); } - return new UpdateExpression(inner, targetRecordType, targetType, translatedTransformMapBuilder.build()); + return new UpdateExpression(Iterables.getOnlyElement(translatedQuantifiers).narrow(Quantifier.ForEach.class), + targetRecordType, targetType, translatedTransformMapBuilder.build()); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RecordQueryPlanMatchers.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RecordQueryPlanMatchers.java index ac4ed9735f..c0c2a1fbe7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RecordQueryPlanMatchers.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RecordQueryPlanMatchers.java @@ -71,6 +71,8 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedPrimaryKeyDistinctPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedUnionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryUpdatePlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -389,6 +391,11 @@ public static BindingMatcher scanPlan() { return ofTypeOwning(RecordQueryScanPlan.class, CollectionMatcher.empty()); } + @Nonnull + public static BindingMatcher tempTableScanPlan() { + return ofTypeOwning(TempTableScanPlan.class, CollectionMatcher.empty()); + } + @Nonnull public static BindingMatcher typeFilter(@Nonnull final BindingMatcher downstream) { return ofTypeOwning(RecordQueryTypeFilterPlan.class, any(downstream)); @@ -778,6 +785,11 @@ public static BindingMatcher insertPlan(@Nonnull final Bi return childrenPlans(RecordQueryInsertPlan.class, exactlyPlans(downstream)); } + @Nonnull + public static BindingMatcher tempTableInsertPlan(@Nonnull final BindingMatcher downstream) { + return childrenPlans(TempTableInsertPlan.class, exactlyPlans(downstream)); + } + @Nonnull public static BindingMatcher updatePlan(@Nonnull final BindingMatcher downstream) { return childrenPlans(RecordQueryUpdatePlan.class, all(downstream)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java index 6c114b6541..83d3968a72 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java @@ -27,6 +27,7 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.GroupByExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.InsertExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalDistinctExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalFilterExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalProjectionExpression; @@ -38,6 +39,7 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpressionWithPredicates; import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableScanExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.UpdateExpression; import com.apple.foundationdb.record.query.plan.cascades.predicates.QueryPredicate; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; @@ -273,8 +275,18 @@ public static BindingMatcher insertExpression(@Nonnull final B return ofTypeOwning(InsertExpression.class, only(downstream)); } + @Nonnull + public static BindingMatcher tempTableInsertExpression(@Nonnull final BindingMatcher downstream) { + return ofTypeOwning(TempTableInsertExpression.class, only(downstream)); + } + @Nonnull public static BindingMatcher updateExpression(@Nonnull final BindingMatcher downstream) { return ofTypeOwning(UpdateExpression.class, only(downstream)); } + + @Nonnull + public static BindingMatcher tempTableScanExpression() { + return ofTypeOwning(TempTableScanExpression.class, CollectionMatcher.empty()); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java index b6e8cff16d..6cacb22f47 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.GroupByExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.InsertExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalDistinctExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalFilterExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalIntersectionExpression; @@ -47,6 +48,7 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.PrimaryScanExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableScanExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.UpdateExpression; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.plans.InComparandSource; @@ -71,6 +73,8 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryInValuesJoinPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryLoadByKeysPlan; @@ -242,6 +246,12 @@ public Cardinalities visitRecordQueryInsertPlan(@Nonnull final RecordQueryInsert return fromChild(insertPlan); } + @Nonnull + @Override + public Cardinalities visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { + return fromChild(tempTableInsertPlan); + } + @Nonnull @Override public Cardinalities visitRecordQueryIntersectionOnValuesPlan(@Nonnull final RecordQueryIntersectionOnValuesPlan intersectionOnValuesPlan) { @@ -493,6 +503,12 @@ public Cardinalities visitInsertExpression(@Nonnull final InsertExpression inser return fromChild(insertExpression); } + @Nonnull + @Override + public Cardinalities visitTempTableInsertExpression(@Nonnull final TempTableInsertExpression tempTableInsertExpression) { + return fromChild(tempTableInsertExpression); + } + @Nonnull @Override public Cardinalities visitPrimaryScanExpression(@Nonnull final PrimaryScanExpression element) { @@ -557,6 +573,12 @@ public Cardinalities visitFullUnorderedScanExpression(@Nonnull final FullUnorder return Cardinalities.unknownMaxCardinality(); } + @Nonnull + @Override + public Cardinalities visitTempTableScanExpression(@Nonnull final TempTableScanExpression element) { + return Cardinalities.unknownMaxCardinality(); + } + @Nonnull @Override public Cardinalities visitGroupByExpression(@Nonnull final GroupByExpression element) { @@ -602,6 +624,12 @@ public Cardinalities visitRecordQuerySortPlan(@Nonnull final RecordQuerySortPlan return fromChild(querySortPlan); } + @Nonnull + @Override + public Cardinalities visitTempTableScanPlan(@Nonnull final TempTableScanPlan element) { + return Cardinalities.unknownMaxCardinality(); + } + @Nonnull @Override public Cardinalities evaluateAtExpression(@Nonnull RelationalExpression expression, @Nonnull List childResults) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java index dda8ff7d8e..add23c7f0e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java @@ -59,6 +59,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryInValuesJoinPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryLoadByKeysPlan; @@ -74,6 +75,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySetPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionOnKeyExpressionPlan; @@ -295,6 +297,12 @@ public Derivations visitRangePlan(@Nonnull final RecordQueryRangePlan rangePlan) return new Derivations(values, values); } + @Nonnull + @Override + public Derivations visitTempTableScanPlan(@Nonnull final TempTableScanPlan tempTableScanPlan) { + return new Derivations(ImmutableList.of(tempTableScanPlan.getResultValue()), ImmutableList.of()); + } + @Nonnull @Override public Derivations visitExplodePlan(@Nonnull final RecordQueryExplodePlan explodePlan) { @@ -327,6 +335,27 @@ public Derivations visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPl return new Derivations(resultValuesBuilder.build(), localValuesBuilder.build()); } + @Nonnull + @Override + public Derivations visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { + final Quantifier rangesOver = Iterables.getOnlyElement(tempTableInsertPlan.getQuantifiers()); + final var childDerivations = derivationsFromQuantifier(rangesOver); + final var childResultValues = childDerivations.getResultValues(); + final var computationValue = tempTableInsertPlan.getComputationValue(); + + final var resultValuesBuilder = ImmutableList.builder(); + final var localValuesBuilder = ImmutableList.builder(); + localValuesBuilder.addAll(childDerivations.getLocalValues()); + for (final var childResultValue : childResultValues) { + final var resultsTranslationMap = TranslationMap.builder() + .when(rangesOver.getAlias()).then(((sourceAlias, leafValue) -> childResultValue)) + .when(Quantifier.current()).then((sourceAlias, leafValue) -> new QueriedValue(leafValue.getResultType(), ImmutableList.of(tempTableInsertPlan.getTargetRecordType()))) + .build(); + resultValuesBuilder.add(computationValue.translateCorrelationsAndSimplify(resultsTranslationMap)); + } + return new Derivations(resultValuesBuilder.build(), localValuesBuilder.build()); + } + @Nonnull @Override public Derivations visitIntersectionOnValuesPlan(@Nonnull final RecordQueryIntersectionOnValuesPlan intersectionOnValuePlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java index d40cb4c824..7d29117724 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java @@ -46,6 +46,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryInValuesJoinPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryLoadByKeysPlan; @@ -58,6 +59,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionOnKeyExpressionPlan; @@ -193,6 +195,12 @@ public Boolean visitRangePlan(@Nonnull final RecordQueryRangePlan element) { return true; } + @Nonnull + @Override + public Boolean visitTempTableScanPlan(@Nonnull final TempTableScanPlan element) { + return false; + } + @Nonnull @Override public Boolean visitExplodePlan(@Nonnull final RecordQueryExplodePlan element) { @@ -205,6 +213,12 @@ public Boolean visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPlan) return distinctRecordsFromSingleChild(insertPlan); } + @Nonnull + @Override + public Boolean visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { + return distinctRecordsFromSingleChild(tempTableInsertPlan); + } + @Nonnull @Override public Boolean visitIntersectionOnValuesPlan(@Nonnull final RecordQueryIntersectionOnValuesPlan element) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java index 558bbef844..bb9c082ba2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java @@ -58,6 +58,8 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryInValuesJoinPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryLoadByKeysPlan; @@ -281,6 +283,12 @@ public Ordering visitRangePlan(@Nonnull final RecordQueryRangePlan element) { ImmutableSetMultimap.of()), true); } + @Nonnull + @Override + public Ordering visitTempTableScanPlan(@Nonnull final TempTableScanPlan element) { + return Ordering.empty(); + } + @Nonnull @Override public Ordering visitExplodePlan(@Nonnull final RecordQueryExplodePlan element) { @@ -293,6 +301,12 @@ public Ordering visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPlan) return Ordering.empty(); } + @Nonnull + @Override + public Ordering visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { + return orderingFromSingleChild(tempTableInsertPlan); + } + @Nonnull @Override public Ordering visitIntersectionOnValuesPlan(@Nonnull final RecordQueryIntersectionOnValuesPlan intersectionOnValuePlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java index 9fcb19f8b7..bc21d23038 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java @@ -47,6 +47,8 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryInValuesJoinPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryLoadByKeysPlan; @@ -194,6 +196,12 @@ public Optional> visitRangePlan(@Nonnull final RecordQueryRangePlan return Optional.empty(); } + @Nonnull + @Override + public Optional> visitTempTableScanPlan(@Nonnull final TempTableScanPlan element) { + return Optional.empty(); + } + @Nonnull @Override public Optional> visitExplodePlan(@Nonnull final RecordQueryExplodePlan element) { @@ -207,6 +215,13 @@ public Optional> visitInsertPlan(@Nonnull final RecordQueryInsertPla return Optional.empty(); } + @Nonnull + @Override + public Optional> visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { + // table queues do not support primary key currently. + return Optional.empty(); + } + @Nonnull @Override public Optional> visitIntersectionOnValuesPlan(@Nonnull final RecordQueryIntersectionOnValuesPlan intersectionOnValuesPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java index 33f09af631..ab18d4d30a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java @@ -44,6 +44,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryInValuesJoinPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryLoadByKeysPlan; @@ -56,6 +57,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionOnKeyExpressionPlan; @@ -182,6 +184,12 @@ public Boolean visitRangePlan(@Nonnull final RecordQueryRangePlan element) { return false; } + @Nonnull + @Override + public Boolean visitTempTableScanPlan(@Nonnull final TempTableScanPlan element) { + return false; + } + @Nonnull @Override public Boolean visitExplodePlan(@Nonnull final RecordQueryExplodePlan element) { @@ -194,6 +202,12 @@ public Boolean visitInsertPlan(@Nonnull final RecordQueryInsertPlan element) { return true; } + @Nonnull + @Override + public Boolean visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan element) { + return false; + } + @Nonnull @Override public Boolean visitIntersectionOnValuesPlan(@Nonnull final RecordQueryIntersectionOnValuesPlan intersectionOnValuesPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTempTableInsertRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTempTableInsertRule.java new file mode 100644 index 0000000000..c50c10e076 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTempTableInsertRule.java @@ -0,0 +1,78 @@ +/* + * ImplementTempTableInsertRule.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2019 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.rules; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.query.plan.cascades.CascadesRule; +import com.apple.foundationdb.record.query.plan.cascades.CascadesRuleCall; +import com.apple.foundationdb.record.query.plan.cascades.Reference; +import com.apple.foundationdb.record.query.plan.cascades.PlanPartition; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression; +import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; +import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; + +import javax.annotation.Nonnull; + +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.AnyMatcher.any; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.QuantifierMatchers.forEachQuantifierOverRef; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.ReferenceMatchers.anyPlanPartition; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.ReferenceMatchers.planPartitions; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RelationalExpressionMatchers.tempTableInsertExpression; + +/** + * A rule that implements a {@link TempTableInsertExpression} producing a {@link TempTableInsertPlan} operator. + */ +@API(API.Status.EXPERIMENTAL) +@SuppressWarnings("PMD.TooManyStaticImports") +public class ImplementTempTableInsertRule extends CascadesRule { + @Nonnull + private static final BindingMatcher innerPlanPartitionMatcher = anyPlanPartition(); + + @Nonnull + private static final BindingMatcher innerReferenceMatcher = + planPartitions(any(innerPlanPartitionMatcher)); + + private static final BindingMatcher innerQuantifierMatcher = + forEachQuantifierOverRef(innerReferenceMatcher); + + @Nonnull + private static final BindingMatcher root = + tempTableInsertExpression(innerQuantifierMatcher); + + public ImplementTempTableInsertRule() { + super(root); + } + + @Override + public void onMatch(@Nonnull final CascadesRuleCall call) { + final var innerPlanPartition = call.get(innerPlanPartitionMatcher); + final var innerReference = call.get(innerReferenceMatcher); + final var innerQuantifier = call.get(innerQuantifierMatcher); + final var insertExpression = call.get(root); + + final var physicalQuantifier = + Quantifier.physicalBuilder() + .morphFrom(innerQuantifier) + .build(call.memoizeMemberPlans(innerReference, innerPlanPartition.getPlans())); + call.yieldExpression(insertExpression.toPlan(physicalQuantifier)); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTempTableScanRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTempTableScanRule.java new file mode 100644 index 0000000000..992e6a2b39 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTempTableScanRule.java @@ -0,0 +1,50 @@ +/* + * ImplementTempTableScanRule.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.rules; + +import com.apple.foundationdb.record.query.plan.cascades.CascadesRule; +import com.apple.foundationdb.record.query.plan.cascades.CascadesRuleCall; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableScanExpression; +import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; +import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; + +import javax.annotation.Nonnull; + +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RelationalExpressionMatchers.tempTableScanExpression; + +/** + * A rule that implements a {@link TempTableScanExpression} producing a {@link TempTableScanPlan} operator. + */ +public class ImplementTempTableScanRule extends CascadesRule { + + @Nonnull + private static final BindingMatcher root = tempTableScanExpression(); + + public ImplementTempTableScanRule() { + super(root); + } + + @Override + public void onMatch(@Nonnull final CascadesRuleCall call) { + final var tempTableScanExpression = call.get(root); + call.yieldExpression(new TempTableScanPlan(tempTableScanExpression.getTempTableReferenceValue())); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java index 3662c41939..1196ca8323 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java @@ -150,6 +150,15 @@ default boolean isRecord() { return getTypeCode().equals(TypeCode.RECORD); } + /** + * Checks whether a {@link Type} is {@link Relation}. + * + * @return true if the {@link Type} is {@link Relation}, otherwise false. + */ + default boolean isRelation() { + return getTypeCode().equals(TypeCode.RELATION); + } + /** * Checks whether a {@link Type} is {@link Enum}. * diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ConstantObjectValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ConstantObjectValue.java index d49796c583..dd6ae888fd 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ConstantObjectValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ConstantObjectValue.java @@ -136,6 +136,9 @@ public Object eval(@Nullable final FDBRecordStoreBase sto Verify.verify(getResultType().isNullable()); return null; } + if (getResultType().isRelation()) { + return obj; + } final var objType = Type.fromObject(obj); final var promotionNeeded = PromoteValue.isPromotionNeeded(objType, getResultType()); if (!promotionNeeded) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/IndexEntryObjectValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/IndexEntryObjectValue.java index dfd6c279f5..751159ed41 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/IndexEntryObjectValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/IndexEntryObjectValue.java @@ -123,7 +123,7 @@ public BooleanWithConstraint equalsWithoutChildren(@Nonnull final Value other) { @Nullable @Override public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { - final var indexEntry = Objects.requireNonNull((IndexEntry)context.getBinding(indexEntryAlias)); + final var indexEntry = Objects.requireNonNull((IndexEntry)context.getBinding(Bindings.Internal.CORRELATION, indexEntryAlias)); final var tuple = (source == TupleSource.KEY ? indexEntry.getKey() : indexEntry.getValue()); var value = getForOrdinalPath(tuple, ordinalPath); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ObjectValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ObjectValue.java index 35b5ebad65..b07801c642 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ObjectValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ObjectValue.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanDeserializer; @@ -97,7 +98,7 @@ public Value replaceReferenceWithField(@Nonnull final FieldValue fieldValue) { @Nullable @Override public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { - return context.getBinding(alias); + return context.getBinding(Bindings.Internal.CORRELATION, alias); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedObjectValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedObjectValue.java index cd13ea32c5..973d9736a3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedObjectValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedObjectValue.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanDeserializer; @@ -88,7 +89,7 @@ public Value replaceReferenceWithField(@Nonnull final FieldValue fieldValue) { @Override public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { // TODO this "if" can be encoded in encapsulation code implementing type promotion rules - final var binding = (QueryResult)context.getBinding(alias); + final var binding = (QueryResult)context.getBinding(Bindings.Internal.CORRELATION, alias); if (resultType.isRecord()) { return binding.getDatum() == null ? null : binding.getMessage(); } else { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/VersionValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/VersionValue.java index b1e20f0815..9c774c811f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/VersionValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/VersionValue.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanDeserializer; @@ -68,7 +69,7 @@ public CorrelationIdentifier getAlias() { @Nullable @Override public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { - QueryResult binding = (QueryResult) context.getBinding(baseAlias); + QueryResult binding = (QueryResult) context.getBinding(Bindings.Internal.CORRELATION, baseAlias); return binding.getQueriedRecordMaybe() .map(FDBRecord::getVersion) .orElse(null); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/QueryResult.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/QueryResult.java index 3760aee2ae..a58f2bf6f7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/QueryResult.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/QueryResult.java @@ -23,10 +23,20 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.IndexEntry; import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord; +import com.apple.foundationdb.record.query.plan.serialization.PlanSerialization; +import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.tuple.Tuple; +import com.google.common.base.Verify; +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; +import com.google.protobuf.ZeroCopyByteString; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -44,28 +54,35 @@ public class QueryResult { @Nullable private final Object datum; + // TODO: this can be removed because it is (probably) redundant. The `queriedRecord` is mostly kept here to flow + // information that already exist in the `datum` such as the primary key constituents. There are however + // some extra information that are exclusive to the `queriedRecord` such as the record type and the record + // version, and the size, however this information can be carried out through modelling the plan operators + // differently such that they include any relevant information (such as record version) in the flown result + // (i.e. the `datum`) itself. @Nullable private final FDBQueriedRecord queriedRecord; + // TODO: this can be removed because it is redundant. The primary key information is encoded + // inside the `datum` object, and can be retrieved by a `Value` that knows how to reconstruct + // the primary key from it correctly. @Nullable - private final IndexEntry indexEntry; + private final Tuple primaryKey; + // transient field that amortizes the calculation of the serialized form of this immutable object. @Nullable - private final Tuple primaryKey; + private ByteString cachedByteString; + // transient field that amortizes the calculation of the serialized form of this immutable object. @Nullable - private final RecordType recordType; + private byte[] cachedBytes; private QueryResult(@Nullable final Object datum, @Nullable final FDBQueriedRecord queriedRecord, - @Nullable final IndexEntry indexEntry, - @Nullable final Tuple primaryKey, - @Nullable final RecordType recordType) { + @Nullable final Tuple primaryKey) { this.datum = datum; this.queriedRecord = queriedRecord; - this.indexEntry = indexEntry; this.primaryKey = primaryKey; - this.recordType = recordType; } /** @@ -145,7 +162,10 @@ public Optional getMessageMaybe() { @Nullable public IndexEntry getIndexEntry() { - return indexEntry; + if (queriedRecord != null) { + return queriedRecord.getIndexEntry(); + } + return null; } @Nullable @@ -155,12 +175,62 @@ public Tuple getPrimaryKey() { @Nullable public RecordType getRecordType() { - return recordType; + if (queriedRecord != null) { + return queriedRecord.getRecordType(); + } + return null; } @Nonnull public QueryResult withComputed(@Nullable final Object computed) { - return new QueryResult(computed, queriedRecord, indexEntry, primaryKey, recordType); + return new QueryResult(computed, queriedRecord, primaryKey); + } + + @SuppressWarnings("unchecked") + @Nonnull + public ByteString toByteString() { + if (cachedByteString == null) { + final var builder = RecordMetaDataProto.PQueryResult.newBuilder(); + if (datum instanceof FDBQueriedRecord) { + builder.setComplex(((FDBQueriedRecord)datum).getRecord().toByteString()); + } else if (datum instanceof Message) { + builder.setComplex(((Message)datum).toByteString()); + } else { + builder.setPrimitive(PlanSerialization.valueObjectToProto(datum)); + } + cachedByteString = builder.build().toByteString(); + } + return cachedByteString; + } + + @Nonnull + public byte[] toBytes() { + if (cachedBytes == null) { + cachedBytes = toByteString().toByteArray(); + } + return cachedBytes; + } + + + + @Nonnull + public static QueryResult deserialize(@Nullable final Descriptors.Descriptor descriptor, @Nonnull final byte[] bytes) { + return deserialize(descriptor, ZeroCopyByteString.wrap(bytes)); + } + + @Nonnull + public static QueryResult deserialize(@Nullable final Descriptors.Descriptor descriptor, @Nonnull final ByteString byteString) { + try { + final var parsed = RecordMetaDataProto.PQueryResult.parseFrom(byteString); + if (parsed.hasPrimitive()) { + return QueryResult.ofComputed(PlanSerialization.protoToValueObject(parsed.getPrimitive())); + } else { + return QueryResult.ofComputed(DynamicMessage.parseFrom(Verify.verifyNotNull(descriptor), parsed.getComplex())); + } + } catch (InvalidProtocolBufferException ex) { + throw new RecordCoreException("invalid bytes", ex) + .addLogInfo(LogMessageKeys.RAW_BYTES, ByteArrayUtil2.loggable(byteString.toByteArray())); + } } /** @@ -169,21 +239,19 @@ public QueryResult withComputed(@Nullable final Object computed) { * @return the newly created query result */ @Nonnull - public static QueryResult ofComputed(@Nullable Object computed) { - return new QueryResult(computed, null, null, null, null); + public static QueryResult ofComputed(@Nullable final Object computed) { + return new QueryResult(computed, null, null); } /** * Create a new result with the given element while inheriting other parts from a caller-provided entities. * @param computed the given computed result - * @param indexEntry an index entry (if appropriate) or {@code null} * @param primaryKey a primary key (if available) or {@code null} - * @param recordType a record type (if available) or {@code null} * @return the newly created query result */ @Nonnull - public static QueryResult ofComputed(@Nullable Object computed, @Nullable IndexEntry indexEntry, @Nullable Tuple primaryKey, @Nullable RecordType recordType) { - return new QueryResult(computed, null, indexEntry, primaryKey, recordType); + public static QueryResult ofComputed(@Nullable final Object computed, @Nullable final Tuple primaryKey) { + return new QueryResult(computed, null, primaryKey); } /** @@ -192,14 +260,12 @@ public static QueryResult ofComputed(@Nullable Object computed, @Nullable IndexE * @return the newly created query queriedRecord */ @Nonnull - public static QueryResult fromQueriedRecord(@Nullable FDBQueriedRecord queriedRecord) { + public static QueryResult fromQueriedRecord(@Nullable final FDBQueriedRecord queriedRecord) { if (queriedRecord == null) { - return new QueryResult(null, null, null, null, null); + return new QueryResult(null, null, null); } return new QueryResult(queriedRecord.getRecord(), queriedRecord, - queriedRecord.getIndexEntry(), - queriedRecord.getPrimaryKey(), - queriedRecord.getRecordType()); + queriedRecord.getPrimaryKey()); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryAbstractDataModificationPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryAbstractDataModificationPlan.java index 111a45f0c0..149f13c37e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryAbstractDataModificationPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryAbstractDataModificationPlan.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.query.plan.plans; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.ObjectPlanHash; @@ -31,7 +32,6 @@ import com.apple.foundationdb.record.planprotos.PRecordQueryAbstractDataModificationPlan; import com.apple.foundationdb.record.provider.common.StoreTimer; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; -import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; @@ -45,6 +45,7 @@ import com.apple.foundationdb.record.util.pair.Pair; import com.google.common.base.Preconditions; import com.google.common.base.Suppliers; +import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -94,7 +95,7 @@ public abstract class RecordQueryAbstractDataModificationPlan implements RecordQ private final Quantifier.Physical inner; @Nonnull private final Type innerFlowedType; - @Nonnull + @Nullable private final String targetRecordType; @Nonnull private final Type.Record targetType; @@ -129,7 +130,9 @@ public abstract class RecordQueryAbstractDataModificationPlan implements RecordQ protected RecordQueryAbstractDataModificationPlan(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PRecordQueryAbstractDataModificationPlan recordQueryAbstractDataModificationPlanProto) { this(Quantifier.Physical.fromProto(serializationContext, Objects.requireNonNull(recordQueryAbstractDataModificationPlanProto.getInner())), - Objects.requireNonNull(recordQueryAbstractDataModificationPlanProto.getTargetRecordType()), + PlanSerialization.getFieldOrNull(recordQueryAbstractDataModificationPlanProto, + PRecordQueryAbstractDataModificationPlan::hasTargetRecordType, + m -> Objects.requireNonNull(m.getTargetRecordType())), Type.Record.fromProto(serializationContext, Objects.requireNonNull(recordQueryAbstractDataModificationPlanProto.getTargetType())), PlanSerialization.getFieldOrNull(recordQueryAbstractDataModificationPlanProto, PRecordQueryAbstractDataModificationPlan::hasTransformationsTrie, @@ -142,7 +145,7 @@ protected RecordQueryAbstractDataModificationPlan(@Nonnull final PlanSerializati } protected RecordQueryAbstractDataModificationPlan(@Nonnull final Quantifier.Physical inner, - @Nonnull final String targetRecordType, + @Nullable final String targetRecordType, @Nonnull final Type.Record targetType, @Nullable final MessageHelpers.TransformationTrieNode transformationsTrie, @Nullable final MessageHelpers.CoercionTrieNode coercionTrie, @@ -192,26 +195,37 @@ public Set getDynamicTypes() { return dynamicTypesBuilder.build(); } + @Nullable + protected Descriptors.Descriptor getTargetDescriptor(@Nonnull final FDBRecordStoreBase store) { + return store.getRecordMetaData().getRecordType(Objects.requireNonNull(targetRecordType)).getDescriptor(); + } + @Nonnull @Override - @SuppressWarnings({"PMD.CloseResource", "resource"}) + @SuppressWarnings({"PMD.CloseResource", "resource", "unchecked"}) public RecordCursor executePlan(@Nonnull final FDBRecordStoreBase store, @Nonnull final EvaluationContext context, @Nullable final byte[] continuation, @Nonnull final ExecuteProperties executeProperties) { final RecordCursor results = getInnerPlan().executePlan(store, context, continuation, executeProperties.clearSkipAndLimit()); - final var targetDescriptor = store.getRecordMetaData().getRecordType(targetRecordType).getDescriptor(); + final var targetDescriptor = getTargetDescriptor(store); return results - .map(queryResult -> Pair.of(queryResult, mutateRecord(store, context, queryResult, targetDescriptor))) - .mapPipelined(pair -> saveRecordAsync(store, pair.getRight(), executeProperties.isDryRun()) - .thenApply(storedRecord -> { + .map(queryResult -> { + if (targetDescriptor == null) { + // TODO: this needs reworking when introducing SQL temp tables. + Verify.verify(coercionTrie == null && transformationsTrie == null); + return Pair.of(queryResult, (M)Preconditions.checkNotNull(queryResult.getMessage())); + } + return Pair.of(queryResult, mutateRecord(store, context, queryResult, targetDescriptor)); + }).mapPipelined(pair -> saveRecordAsync(store, context, pair.getRight(), executeProperties.isDryRun()) + .thenApply(queryResult -> { final var nestedContext = context.childBuilder() .setBinding(inner.getAlias(), pair.getLeft()) // pre-mutation - .setBinding(currentModifiedRecordAlias, storedRecord.getRecord()) // post-mutation + .setBinding(currentModifiedRecordAlias, queryResult.getMessage()) // post-mutation .build(context.getTypeRepository()); final var result = computationValue.eval(store, nestedContext); - return QueryResult.ofComputed(result, null, storedRecord.getPrimaryKey(), null); + return QueryResult.ofComputed(result, queryResult.getPrimaryKey()); }), store.getPipelineSize(getPipelineOperation())); } @@ -224,7 +238,7 @@ public M mutateRecord(@Nonnull final FDBRecordStoreBase s @Nonnull final QueryResult queryResult, @Nonnull final Descriptors.Descriptor targetDescriptor) { final var inRecord = (M)Preconditions.checkNotNull(queryResult.getMessage()); return (M)MessageHelpers.transformMessage(store, - context.withBinding(inner.getAlias(), queryResult), + context.withBinding(Bindings.Internal.CORRELATION, inner.getAlias(), queryResult), transformationsTrie, coercionTrie, targetType, @@ -235,7 +249,9 @@ public M mutateRecord(@Nonnull final FDBRecordStoreBase s } @Nonnull - public abstract CompletableFuture> saveRecordAsync(@Nonnull FDBRecordStoreBase store, @Nonnull M message, boolean isDryRun); + public abstract CompletableFuture saveRecordAsync(@Nonnull FDBRecordStoreBase store, + @Nonnull EvaluationContext context, + @Nonnull M message, boolean isDryRun); @Override public boolean isReverse() { @@ -375,8 +391,10 @@ public int getComplexity() { public PRecordQueryAbstractDataModificationPlan toRecordQueryAbstractModificationPlanProto(@Nonnull final PlanSerializationContext serializationContext) { final PRecordQueryAbstractDataModificationPlan.Builder builder = PRecordQueryAbstractDataModificationPlan.newBuilder() .setInner(inner.toProto(serializationContext)) - .setTargetRecordType(targetRecordType) .setTargetType(targetType.toProto(serializationContext)); + if (targetRecordType != null) { + builder.setTargetRecordType(targetRecordType); + } if (transformationsTrie != null) { builder.setTransformationsTrie(transformationsTrie.toProto(serializationContext)); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryFlatMapPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryFlatMapPlan.java index 7fa49e5acc..552de9988b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryFlatMapPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryFlatMapPlan.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.query.plan.plans; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.ObjectPlanHash; @@ -119,12 +120,12 @@ public RecordCursor executePlan(@Nonnull final outerContinuation -> outerQuantifier.getRangesOverPlan().executePlan(store, context, outerContinuation, nestedExecuteProperties), (outerResult, innerContinuation) -> { - final EvaluationContext fromOuterContext = context.withBinding(outerQuantifier.getAlias(), outerResult); + final EvaluationContext fromOuterContext = context.withBinding(Bindings.Internal.CORRELATION, outerQuantifier.getAlias(), outerResult); return innerQuantifier.getRangesOverPlan().executePlan(store, fromOuterContext, innerContinuation, nestedExecuteProperties) .map(innerResult -> { final EvaluationContext nestedContext = - fromOuterContext.withBinding(innerQuantifier.getAlias(), innerResult); + fromOuterContext.withBinding(Bindings.Internal.CORRELATION, innerQuantifier.getAlias(), innerResult); final var computed = resultValue.eval(store, nestedContext); return inheritOuterRecordProperties ? outerResult.withComputed(computed) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInsertPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInsertPlan.java index e6c1de71fc..7e8ea8473e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInsertPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInsertPlan.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.query.plan.plans; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PipelineOperation; import com.apple.foundationdb.record.PlanDeserializer; @@ -28,6 +29,7 @@ import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.planprotos.PRecordQueryInsertPlan; import com.apple.foundationdb.record.planprotos.PRecordQueryPlan; +import com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.query.plan.PlanStringRepresentation; @@ -83,14 +85,15 @@ public PipelineOperation getPipelineOperation() { return PipelineOperation.INSERT; } - @Nonnull @Override - public CompletableFuture> saveRecordAsync(@Nonnull final FDBRecordStoreBase store, @Nonnull final M message, final boolean isDryRun) { + public @Nonnull CompletableFuture saveRecordAsync(@Nonnull final FDBRecordStoreBase store, final @Nonnull EvaluationContext context, @Nonnull final M message, final boolean isDryRun) { + final CompletableFuture> result; if (isDryRun) { - return store.dryRunSaveRecordAsync(message, FDBRecordStoreBase.RecordExistenceCheck.ERROR_IF_EXISTS); + result = store.dryRunSaveRecordAsync(message, FDBRecordStoreBase.RecordExistenceCheck.ERROR_IF_EXISTS); } else { - return store.saveRecordAsync(message, FDBRecordStoreBase.RecordExistenceCheck.ERROR_IF_EXISTS); + result = store.saveRecordAsync(message, FDBRecordStoreBase.RecordExistenceCheck.ERROR_IF_EXISTS); } + return result.thenApply(fdbStoredRecord -> QueryResult.fromQueriedRecord(FDBQueriedRecord.stored(fdbStoredRecord))); } @Nonnull @@ -102,7 +105,7 @@ public RecordQueryInsertPlan translateCorrelations(@Nonnull final TranslationMap getTargetRecordType(), getTargetType(), getCoercionTrie(), - getComputationValue()); + getComputationValue().translateCorrelations(translationMap)); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryMapPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryMapPlan.java index f30eaa407a..614628b877 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryMapPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryMapPlan.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.query.plan.plans; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.ObjectPlanHash; @@ -94,7 +95,7 @@ public RecordCursor executePlan(@Nonnull final @Nonnull final ExecuteProperties executeProperties) { return getChild().executePlan(store, context, continuation, executeProperties) .map(innerResult -> { - final EvaluationContext nestedContext = context.withBinding(inner.getAlias(), innerResult); + final EvaluationContext nestedContext = context.withBinding(Bindings.Internal.CORRELATION, inner.getAlias(), innerResult); // Apply (map) each value to the incoming record return innerResult.withComputed(resultValue.eval(store, nestedContext)); }); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPredicatesFilterPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPredicatesFilterPlan.java index ac6a58b173..f3ea56dcbc 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPredicatesFilterPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPredicatesFilterPlan.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.query.plan.plans; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanDeserializer; @@ -109,14 +110,14 @@ protected boolean hasAsyncFilter() { @Nullable @Override protected Boolean evalFilter(@Nonnull FDBRecordStoreBase store, @Nonnull EvaluationContext context, @Nonnull QueryResult queryResult) { - final var nestedContext = context.withBinding(getInner().getAlias(), queryResult); + final var nestedContext = context.withBinding(Bindings.Internal.CORRELATION, getInner().getAlias(), queryResult); return conjunctedPredicate.eval(store, nestedContext); } @Nullable @Override protected CompletableFuture evalFilterAsync(@Nonnull FDBRecordStoreBase store, @Nonnull EvaluationContext context, @Nonnull QueryResult queryResult) { - final var nestedContext = context.withBinding(getInner().getAlias(), queryResult); + final var nestedContext = context.withBinding(Bindings.Internal.CORRELATION, getInner().getAlias(), queryResult); return new AsyncBoolean<>(false, getPredicates(), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQuerySetPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQuerySetPlan.java index b03e7647c8..3719107194 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQuerySetPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQuerySetPlan.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.record.query.plan.plans; +import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.PlanDeserializer; import com.apple.foundationdb.record.PlanHashable; @@ -395,7 +396,7 @@ public List getComparisonKeyValues() { @Override public final Function> apply(@Nonnull final FDBRecordStoreBase store, @Nonnull final EvaluationContext evaluationContext) { return queryResult -> { - final var nestedContext = evaluationContext.withBinding(baseAlias, queryResult); + final var nestedContext = evaluationContext.withBinding(Bindings.Internal.CORRELATION, baseAlias, queryResult); final var resultList = Lists.newArrayList(); for (final Value comparisonKeyValue : comparisonKeyValues) { resultList.add(comparisonKeyValue.eval(store, nestedContext)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryUpdatePlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryUpdatePlan.java index 489aa24fbd..fb230248b9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryUpdatePlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryUpdatePlan.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.query.plan.plans; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PipelineOperation; import com.apple.foundationdb.record.PlanDeserializer; @@ -29,6 +30,7 @@ import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.planprotos.PRecordQueryPlan; import com.apple.foundationdb.record.planprotos.PRecordQueryUpdatePlan; +import com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.query.plan.PlanStringRepresentation; @@ -94,14 +96,15 @@ public PipelineOperation getPipelineOperation() { return PipelineOperation.UPDATE; } - @Nonnull @Override - public CompletableFuture> saveRecordAsync(@Nonnull final FDBRecordStoreBase store, @Nonnull final M message, final boolean isDryRun) { + public @Nonnull CompletableFuture saveRecordAsync(@Nonnull final FDBRecordStoreBase store, final @Nonnull EvaluationContext context, @Nonnull final M message, final boolean isDryRun) { + final CompletableFuture> result; if (isDryRun) { - return store.dryRunSaveRecordAsync(message, FDBRecordStoreBase.RecordExistenceCheck.ERROR_IF_NOT_EXISTS_OR_RECORD_TYPE_CHANGED); + result = store.dryRunSaveRecordAsync(message, FDBRecordStoreBase.RecordExistenceCheck.ERROR_IF_NOT_EXISTS_OR_RECORD_TYPE_CHANGED); } else { - return store.saveRecordAsync(message, FDBRecordStoreBase.RecordExistenceCheck.ERROR_IF_NOT_EXISTS_OR_RECORD_TYPE_CHANGED); + result = store.saveRecordAsync(message, FDBRecordStoreBase.RecordExistenceCheck.ERROR_IF_NOT_EXISTS_OR_RECORD_TYPE_CHANGED); } + return result.thenApply(fdbStoredRecord -> QueryResult.fromQueriedRecord(FDBQueriedRecord.stored(fdbStoredRecord))); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/TempTableInsertPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/TempTableInsertPlan.java new file mode 100644 index 0000000000..d6e8be2222 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/TempTableInsertPlan.java @@ -0,0 +1,227 @@ +/* + * TempTableInsertPlan.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.plans; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PipelineOperation; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.planprotos.PRecordQueryPlan; +import com.apple.foundationdb.record.planprotos.PTempTableInsertPlan; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.PlanStringRepresentation; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.Reference; +import com.apple.foundationdb.record.query.plan.cascades.TempTable; +import com.apple.foundationdb.record.query.plan.cascades.explain.NodeInfo; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; + +/** + * A query plan that inserts records into a temporary in-memory buffer {@link TempTable}. + */ +@API(API.Status.INTERNAL) +public class TempTableInsertPlan extends RecordQueryAbstractDataModificationPlan { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Temp-Table-Insert-Plan"); + + public static final Logger LOGGER = LoggerFactory.getLogger(TempTableInsertPlan.class); + + @Nonnull + private final Value tempTableReferenceValue; + + protected TempTableInsertPlan(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PTempTableInsertPlan tempTableInsertPlanProto) { + super(serializationContext, Objects.requireNonNull(tempTableInsertPlanProto.getSuper())); + this.tempTableReferenceValue = Value.fromValueProto(serializationContext, tempTableInsertPlanProto.getTempTableReferenceValue()); + } + + private TempTableInsertPlan(@Nonnull final Quantifier.Physical inner, + @Nonnull final Value computationValue, + @Nonnull final Value tempTableReferenceValue) { + super(inner, null, (Type.Record)((Type.Relation)tempTableReferenceValue.getResultType()).getInnerType(), null, + null, computationValue, currentModifiedRecordAlias()); + this.tempTableReferenceValue = tempTableReferenceValue; + } + + @Override + public PipelineOperation getPipelineOperation() { + return PipelineOperation.INSERT; + } + + @Nullable + @Override + protected Descriptors.Descriptor getTargetDescriptor(@Nonnull final FDBRecordStoreBase ignored) { + return null; + } + + @Override + public @Nonnull CompletableFuture saveRecordAsync(@Nonnull final FDBRecordStoreBase store, + @Nonnull final EvaluationContext context, + @Nonnull final M message, + boolean isDryRun) { + // dry run is ignored since inserting into a table queue has no storage side effects. + final var queryResult = QueryResult.ofComputed(message); + final var tempTable = (TempTable)getTempTableReferenceValue().eval(store, context); + tempTable.add(queryResult); + return CompletableFuture.completedFuture(queryResult); + } + + @Nonnull + @Override + public TempTableInsertPlan translateCorrelations(@Nonnull final TranslationMap translationMap, + @Nonnull final List translatedQuantifiers) { + return new TempTableInsertPlan( + Iterables.getOnlyElement(translatedQuantifiers).narrow(Quantifier.Physical.class), + getComputationValue().translateCorrelations(translationMap), + getTempTableReferenceValue().translateCorrelations(translationMap)); + } + + @Nonnull + @Override + public TempTableInsertPlan withChild(@Nonnull final Reference childRef) { + return new TempTableInsertPlan(Quantifier.physical(childRef), + getComputationValue(), + getTempTableReferenceValue()); + } + + @Override + public int hashCodeWithoutChildren() { + return Objects.hash(BASE_HASH.planHash(PlanHashable.CURRENT_FOR_CONTINUATION), getTempTableReferenceValue(), + super.hashCodeWithoutChildren()); + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + return PlanHashable.objectsPlanHash(mode, BASE_HASH, getTempTableReferenceValue(), super.planHash(mode)); + } + + @Nonnull + @Override + public String toString() { + return PlanStringRepresentation.toString(this); + } + + /** + * Rewrite the planner graph for better visualization. + * + * @param childGraphs planner graphs of children expression that already have been computed + * + * @return the rewritten planner graph that models the filter as a node that uses the expression attribute + * to depict the record types this operator filters. + */ + @Nonnull + @Override + public PlannerGraph rewritePlannerGraph(@Nonnull final List childGraphs) { + + final var graphForTarget = + PlannerGraph.fromNodeAndChildGraphs( + new PlannerGraph.TemporaryDataNodeWithInfo(getTargetType(), ImmutableList.of(getTempTableReferenceValue().toString())), + ImmutableList.of()); + + return PlannerGraph.fromNodeInnerAndTargetForModifications( + new PlannerGraph.ModificationOperatorNodeWithInfo(this, + NodeInfo.MODIFICATION_OPERATOR, + ImmutableList.of("TempTableInsert"), + ImmutableMap.of()), + Iterables.getOnlyElement(childGraphs), graphForTarget); + } + + @Nonnull + @Override + public PTempTableInsertPlan toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PTempTableInsertPlan.newBuilder() + .setSuper(toRecordQueryAbstractModificationPlanProto(serializationContext)) + .setTempTableReferenceValue(getTempTableReferenceValue().toValueProto(serializationContext)) + .build(); + } + + @Nonnull + @Override + public PRecordQueryPlan toRecordQueryPlanProto(@Nonnull final PlanSerializationContext serializationContext) { + return PRecordQueryPlan.newBuilder().setTempTableInsertPlan(toProto(serializationContext)).build(); + } + + @Nonnull + public static TempTableInsertPlan fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PTempTableInsertPlan tempTableInsertPlanProto) { + return new TempTableInsertPlan(serializationContext, tempTableInsertPlanProto); + } + + /** + * Factory method to create a {@link TempTableInsertPlan}. + * + * @param inner an input value to transform + * @param computationValue a value to be computed based on the {@code inner} and + * {@link RecordQueryAbstractDataModificationPlan#currentModifiedRecordAlias()} + * @param tempTableReferenceValue The table queue identifier to insert into. + * + * @return a newly created {@link TempTableInsertPlan} + */ + @Nonnull + public static TempTableInsertPlan insertPlan(@Nonnull final Quantifier.Physical inner, + @Nonnull final Value computationValue, + @Nonnull final Value tempTableReferenceValue) { + return new TempTableInsertPlan(inner, computationValue, tempTableReferenceValue); + } + + @Nonnull + public Value getTempTableReferenceValue() { + return tempTableReferenceValue; + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PTempTableInsertPlan.class; + } + + @Nonnull + @Override + public TempTableInsertPlan fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PTempTableInsertPlan tempTableInsertPlanProto) { + return TempTableInsertPlan.fromProto(serializationContext, tempTableInsertPlanProto); + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/TempTableScanPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/TempTableScanPlan.java new file mode 100644 index 0000000000..6ebb1e7c4d --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/TempTableScanPlan.java @@ -0,0 +1,257 @@ +/* + * TempTableScanPlan.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.plans; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.planprotos.PTempTableScanPlan; +import com.apple.foundationdb.record.planprotos.PRecordQueryPlan; +import com.apple.foundationdb.record.provider.common.StoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.AvailableFields; +import com.apple.foundationdb.record.query.plan.PlanStringRepresentation; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.Memoizer; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.TempTable; +import com.apple.foundationdb.record.query.plan.cascades.explain.NodeInfo; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; +import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * Scans records from a table-valued correlation, corresponding for example to a temporary in-memory buffer {@link TempTable}. + */ +@API(API.Status.INTERNAL) +public class TempTableScanPlan implements RecordQueryPlanWithNoChildren { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Temp-Table-Scan-Plan"); + + @Nonnull + private final Value tempTableReferenceValue; + + public TempTableScanPlan(@Nonnull final Value tempTableReferenceValue) { + this.tempTableReferenceValue = tempTableReferenceValue; + } + + @Nonnull + @Override + public RecordCursor executePlan(@Nonnull final FDBRecordStoreBase store, + @Nonnull final EvaluationContext context, + @Nullable final byte[] continuation, + @Nonnull final ExecuteProperties executeProperties) { + final var tempTable = (TempTable)this.tempTableReferenceValue.eval(store, context); + return tempTable.getReadCursor(continuation); + } + + @Nonnull + @Override + public Set getCorrelatedTo() { + return ImmutableSet.of(); + } + + @Nonnull + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public TempTableScanPlan translateCorrelations(@Nonnull final TranslationMap translationMap, + @Nonnull final List translatedQuantifiers) { + return new TempTableScanPlan(tempTableReferenceValue.translateCorrelations(translationMap)); + } + + @Override + public boolean isReverse() { + return false; + } + + @Override + public TempTableScanPlan strictlySorted(@Nonnull Memoizer memoizer) { + return this; + } + + @Override + public boolean hasRecordScan() { + return false; + } + + @Override + public boolean hasFullRecordScan() { + return false; + } + + @Override + public boolean hasIndexScan(@Nonnull String indexName) { + return false; + } + + @Nonnull + @Override + public Set getUsedIndexes() { + return ImmutableSet.of(); + } + + @Override + public boolean hasLoadBykeys() { + return false; + } + + @Nonnull + @Override + public AvailableFields getAvailableFields() { + return AvailableFields.NO_FIELDS; + } + + @Nonnull + @Override + public Value getResultValue() { + return tempTableReferenceValue; + } + + @Nonnull + @Override + public Set getDynamicTypes() { + return ImmutableSet.of(tempTableReferenceValue.getResultType()); + } + + + @Nonnull + @Override + public String toString() { + return PlanStringRepresentation.toString(this); + } + + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public boolean equalsWithoutChildren(@Nonnull final RelationalExpression otherExpression, + @Nonnull final AliasMap equivalencesMap) { + if (this == otherExpression) { + return true; + } + if (getClass() != otherExpression.getClass()) { + return false; + } + final var otherTempTableScan = (TempTableScanPlan)otherExpression; + return tempTableReferenceValue.semanticEquals(otherTempTableScan.tempTableReferenceValue, equivalencesMap); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @Override + public boolean equals(final Object other) { + return structuralEquals(other); + } + + @Override + public int hashCode() { + return structuralHashCode(); + } + + @Override + public int hashCodeWithoutChildren() { + return Objects.hash(getResultValue()); + } + + @Override + public void logPlanStructure(StoreTimer timer) { + // nothing to increment + } + + @Override + public int getComplexity() { + return 1; + } + + @Override + public int planHash(@Nonnull PlanHashMode mode) { + switch (mode.getKind()) { + case LEGACY: + case FOR_CONTINUATION: + return PlanHashable.objectsPlanHash(mode, BASE_HASH, getResultValue()); + default: + throw new UnsupportedOperationException("Hash kind " + mode.getKind() + " is not supported"); + } + } + + @Nonnull + @Override + public PlannerGraph rewritePlannerGraph(@Nonnull final List childGraphs) { + return PlannerGraph.fromNodeAndChildGraphs( + new PlannerGraph.OperatorNodeWithInfo(this, + NodeInfo.TEMP_TABLE_SCAN_OPERATOR, + ImmutableList.of(tempTableReferenceValue.toString())), + childGraphs); + } + + @Nonnull + @Override + public PTempTableScanPlan toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PTempTableScanPlan.newBuilder() + .setTempTableReferenceValue(tempTableReferenceValue.toValueProto(serializationContext)) + .build(); + } + + @Nonnull + @Override + public PRecordQueryPlan toRecordQueryPlanProto(@Nonnull final PlanSerializationContext serializationContext) { + return PRecordQueryPlan.newBuilder().setTempTableScanPlan(toProto(serializationContext)).build(); + } + + @Nonnull + public static TempTableScanPlan fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PTempTableScanPlan tempTableScanPlanProto) { + return new TempTableScanPlan(Value.fromValueProto(serializationContext, tempTableScanPlanProto.getTempTableReferenceValue())); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PTempTableScanPlan.class; + } + + @Nonnull + @Override + public TempTableScanPlan fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PTempTableScanPlan tempTableScanPlanProto) { + return TempTableScanPlan.fromProto(serializationContext, tempTableScanPlanProto); + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerialization.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerialization.java index 2d51afed62..3f260d912f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerialization.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerialization.java @@ -24,11 +24,8 @@ import com.apple.foundationdb.record.PlanDeserializer; import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression; -import com.apple.foundationdb.record.planprotos.PComparableObject; -import com.apple.foundationdb.record.planprotos.PEnumLightValue; -import com.apple.foundationdb.record.planprotos.PFDBRecordVersion; -import com.apple.foundationdb.record.planprotos.PUUID; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion; import com.apple.foundationdb.record.util.ProtoUtils; import com.google.common.base.Verify; @@ -65,22 +62,22 @@ public class PlanSerialization { * round-trip as the instance of the actual enum is not deserializable as such. Instead, we resort to deserializing * an enum as a {@link com.apple.foundationdb.record.util.ProtoUtils.DynamicEnum}. * @param object object that also happens to be a plan fragment - * @return a {@link PComparableObject} that can be serialized. + * @return a {@link RecordMetaDataProto.PComparableObject} that can be serialized. */ @Nonnull - public static PComparableObject valueObjectToProto(@Nullable final Object object) { - final PComparableObject.Builder builder = PComparableObject.newBuilder(); + public static RecordMetaDataProto.PComparableObject valueObjectToProto(@Nullable final Object object) { + final RecordMetaDataProto.PComparableObject.Builder builder = RecordMetaDataProto.PComparableObject.newBuilder(); if (object instanceof Internal.EnumLite) { - builder.setEnumObject(PEnumLightValue.newBuilder() + builder.setEnumObject(RecordMetaDataProto.PEnumLightValue.newBuilder() .setName(object.toString()).setNumber(((Internal.EnumLite)object).getNumber())); } else if (object instanceof UUID) { final UUID uuid = (UUID)object; - builder.setUuid(PUUID.newBuilder() + builder.setUuid(RecordMetaDataProto.PUUID.newBuilder() .setMostSigBits(uuid.getMostSignificantBits()) .setLeastSigBits(uuid.getLeastSignificantBits())) .build(); } else if (object instanceof FDBRecordVersion) { - builder.setFdbRecordVersion(PFDBRecordVersion.newBuilder() + builder.setFdbRecordVersion(RecordMetaDataProto.PFDBRecordVersion.newBuilder() .setRawBytes(ByteString.copyFrom(((FDBRecordVersion)object).toBytes(false))).build()); } else if (object instanceof ByteString) { builder.setBytesAsByteString((ByteString)object); @@ -97,20 +94,20 @@ public static PComparableObject valueObjectToProto(@Nullable final Object object * should be able to deal with all kinds of objects that can appear in these cases. Note that enums do not * round-trip as the instance of the actual enum is not deserializable as such. Instead, we resort to deserializing * an enum as a {@link com.apple.foundationdb.record.util.ProtoUtils.DynamicEnum}. - * @param proto a {@link PComparableObject} that can be deserialized + * @param proto a {@link RecordMetaDataProto.PComparableObject} that can be deserialized * @return a value object */ @Nullable - public static Object protoToValueObject(@Nonnull final PComparableObject proto) { + public static Object protoToValueObject(@Nonnull final RecordMetaDataProto.PComparableObject proto) { if (proto.hasEnumObject()) { - final PEnumLightValue enumProto = Objects.requireNonNull(proto.getEnumObject()); + final RecordMetaDataProto.PEnumLightValue enumProto = Objects.requireNonNull(proto.getEnumObject()); Verify.verify(enumProto.hasNumber()); return new ProtoUtils.DynamicEnum(enumProto.getNumber(), Objects.requireNonNull(enumProto.getName())); } else if (proto.hasUuid()) { - final PUUID uuidProto = Objects.requireNonNull(proto.getUuid()); + final RecordMetaDataProto.PUUID uuidProto = Objects.requireNonNull(proto.getUuid()); return new UUID(uuidProto.getMostSigBits(), uuidProto.getLeastSigBits()); } else if (proto.hasFdbRecordVersion()) { - final PFDBRecordVersion fdbRecordVersion = Objects.requireNonNull(proto.getFdbRecordVersion()); + final RecordMetaDataProto.PFDBRecordVersion fdbRecordVersion = Objects.requireNonNull(proto.getFdbRecordVersion()); return FDBRecordVersion.fromBytes(fdbRecordVersion .getRawBytes().toByteArray(), false); } else if (proto.hasBytesAsByteString()) { diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index 0a82f40d10..4eb3b71d57 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -291,4 +291,39 @@ message Predicate { optional ConstantPredicate constant_predicate = 3; optional NotPredicate not_predicate = 4; optional ValuePredicate value_predicate = 5; +} + +message PEnumLightValue { + optional string name = 1; + optional int32 number = 2; +} + +message PUUID { + optional uint64 mostSigBits = 1; + optional uint64 leastSigBits = 2; +} + +message PFDBRecordVersion { + optional bytes raw_bytes = 1; +} + +message PComparableObject { + oneof specific_object { + Value primitive_object = 1; + PEnumLightValue enum_object = 2; + PUUID uuid = 3; + PFDBRecordVersion fdb_record_version = 4; + bytes bytes_as_byte_string = 5; + } +} + +message PQueryResult { + oneof datum { + PComparableObject primitive = 1; + bytes complex = 2; + } +} + +message PTempTable { + repeated bytes bufferItems = 2; } \ No newline at end of file diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index e4e5d8c724..8f87259809 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -971,30 +971,6 @@ message PConversionSimpleComparison { optional KeyExpression conversion = 3; } -message PComparableObject { - oneof specific_object { - Value primitive_object = 1; - PEnumLightValue enum_object = 2; - PUUID uuid = 3; - PFDBRecordVersion fdb_record_version = 4; - bytes bytes_as_byte_string = 5; - } -} - -message PEnumLightValue { - optional string name = 1; - optional int32 number = 2; -} - -message PUUID { - optional uint64 mostSigBits = 1; - optional uint64 leastSigBits = 2; -} - -message PFDBRecordVersion { - optional bytes raw_bytes = 1; -} - message PParameterComparison { enum PBindingKind { IN = 1; @@ -1193,6 +1169,8 @@ message PRecordQueryPlan { PRecordQueryUnorderedUnionPlan unordered_union_plan = 31; PRecordQueryUpdatePlan update_plan = 32; PRecordQueryDefaultOnEmptyPlan default_on_empty_plan = 33; + PTempTableScanPlan temp_table_scan_plan = 34; + PTempTableInsertPlan temp_table_insert_plan = 35; } } @@ -1370,6 +1348,13 @@ message PRecordQueryIndexPlan { optional PQueryPlanConstraint constraint = 9; } +// +// PTempTableScanPlan +// +message PTempTableScanPlan { + optional PValue temp_table_reference_value = 1; +} + message PIndexScanParameters { extensions 5000 to max; oneof specific_index_scan_parameters { @@ -1502,6 +1487,14 @@ message PRecordQueryInsertPlan { optional PRecordQueryAbstractDataModificationPlan super = 1; } +// +// PTempTableInsertPlan +// +message PTempTableInsertPlan { + optional PRecordQueryAbstractDataModificationPlan super = 1; + optional PValue temp_table_reference_value = 2; +} + // // PRecordQueryIntersectionPlan // diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBRecordStoreQueryTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBRecordStoreQueryTestBase.java index a9c7c04df9..ea63a6cd91 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBRecordStoreQueryTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBRecordStoreQueryTestBase.java @@ -418,9 +418,18 @@ protected List fetchResultValues(FDBRecordContext context, RecordQueryPla protected List fetchResultValues(FDBRecordContext context, RecordQueryPlan plan, Function rowHandler, TestHelpers.DangerousConsumer checkDiscarded, ExecuteProperties executeProperties) throws Exception { + return fetchResultValues(context, plan, rowHandler, EvaluationContext.empty(), checkDiscarded, executeProperties); + } + + protected List fetchResultValues(FDBRecordContext context, RecordQueryPlan plan, Function rowHandler, + EvaluationContext extraEvaluationContext, + TestHelpers.DangerousConsumer checkDiscarded, ExecuteProperties executeProperties) throws Exception { final var usedTypes = UsedTypesProperty.evaluate(plan); List result = new ArrayList<>(); - final var evaluationContext = EvaluationContext.forTypeRepository(TypeRepository.newBuilder().addAllTypes(usedTypes).build()); + var evaluationContext = EvaluationContext.forTypeRepository(TypeRepository.newBuilder().addAllTypes(usedTypes).build()); + for (final var binding : extraEvaluationContext.getBindings().asMappingList()) { + evaluationContext = evaluationContext.withBinding(binding.getKey(), binding.getValue()); + } try (RecordCursorIterator cursor = plan.executePlan(recordStore, evaluationContext, null, executeProperties).asIterator()) { while (cursor.hasNext()) { Message message = Verify.verifyNotNull(cursor.next()).getMessage(); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTest.java new file mode 100644 index 0000000000..82ea6bb3ab --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTest.java @@ -0,0 +1,214 @@ +/* + * TempTableTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.query; + +import com.apple.foundationdb.record.Bindings; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.TestRecords1Proto; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.query.IndexQueryabilityFilter; +import com.apple.foundationdb.record.query.expressions.Comparisons; +import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; +import com.apple.foundationdb.record.query.plan.cascades.Column; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.GraphExpansion; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.Reference; +import com.apple.foundationdb.record.query.plan.cascades.TempTable; +import com.apple.foundationdb.record.query.plan.cascades.expressions.ExplodeExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableScanExpression; +import com.apple.foundationdb.record.query.plan.cascades.predicates.ValuePredicate; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.AbstractArrayConstructorValue; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; +import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; +import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; +import com.apple.foundationdb.record.util.pair.Pair; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Message; +import org.junit.jupiter.api.BeforeEach; + +import javax.annotation.Nonnull; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.ListMatcher.only; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.QueryPredicateMatchers.valuePredicate; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.explodePlan; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.mapPlan; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.predicates; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.predicatesFilterPlan; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.tempTableInsertPlan; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.tempTableScanPlan; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.ValueMatchers.fieldValueWithFieldNames; +import static com.apple.foundationdb.record.query.plan.cascades.values.AbstractArrayConstructorValue.LightArrayConstructorValue.emptyArray; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Test suite for {@link TempTable} planning and execution. + * Particularly, testing both {@code INSERT} into and {@code SCAN} from a {@link TempTable}. + */ +public class TempTableTest extends FDBRecordStoreQueryTestBase { + + @BeforeEach + void setupPlanner() { + try (FDBRecordContext context = openContext()) { + openNestedRecordStore(context); + } + } + + @DualPlannerTest(planner = DualPlannerTest.Planner.CASCADES) + void scanTempTableWorksCorrectly() throws Exception { + try (FDBRecordContext context = openContext()) { + // select rec_no, str_value_indexed from . + final var tempTable = TempTable.newInstance(); + final var tempTableId = CorrelationIdentifier.uniqueID(); + final var plan = getTempTableScanPlan(tempTable, tempTableId, true); + assertEquals(ImmutableSet.of(Pair.of(42L, "fortySecondValue"), + Pair.of(45L, "fortyFifthValue")), collectResults(context, plan, tempTable, tempTableId)); + } + } + + + @DualPlannerTest(planner = DualPlannerTest.Planner.CASCADES) + void scanTempTableWithPredicateWorksCorrectly() throws Exception { + // select rec_no, str_value_indexed from where rec_no < 44L. + try (FDBRecordContext context = openContext()) { + final var type = Type.Record.fromDescriptor(TestRecords1Proto.MySimpleRecord.getDescriptor()); + final var tempTable = TempTable.newInstance(); + tempTable.add(QueryResult.ofComputed(item(42L, "fortySecondValue")), + QueryResult.ofComputed(item(45L, "fortyFifthValue"))); + final var tempTableId = CorrelationIdentifier.uniqueID(); + final var tempTableScanQun = Quantifier.forEach(Reference.of(TempTableScanExpression.ofConstant(tempTableId, tempTableId.getId(), type))); + final var recNoField = FieldValue.ofFieldName(tempTableScanQun.getFlowedObjectValue(), "rec_no"); + final var recNoColumn = Column.of(Optional.of("rec_no"), FieldValue.ofFieldName(tempTableScanQun.getFlowedObjectValue(), "rec_no")); + final var strValueIndexedField = Column.of(Optional.of("str_value_indexed"), FieldValue.ofFieldName(tempTableScanQun.getFlowedObjectValue(), "str_value_indexed")); + final var selectExpressionBuilder = GraphExpansion.builder() + .addAllResultColumns(ImmutableList.of(recNoColumn, strValueIndexedField)) + .addPredicate(new ValuePredicate(recNoField, new Comparisons.SimpleComparison(Comparisons.Type.LESS_THAN, 44L))) + .addQuantifier(tempTableScanQun); + final var logicalPlan = Reference.of(LogicalSortExpression.unsorted(Quantifier.forEach(Reference.of(selectExpressionBuilder.build().buildSelect())))); + final var cascadesPlanner = (CascadesPlanner)planner; + final var plan = cascadesPlanner.planGraph(() -> logicalPlan, Optional.empty(), IndexQueryabilityFilter.TRUE, EvaluationContext.empty()).getPlan(); + assertMatchesExactly(plan, mapPlan(predicatesFilterPlan(tempTableScanPlan()).where(predicates(only(valuePredicate(fieldValueWithFieldNames("rec_no"), new Comparisons.SimpleComparison(Comparisons.Type.LESS_THAN, 44L))))))); + assertEquals(ImmutableSet.of(Pair.of(42L, "fortySecondValue")), collectResults(context, plan, tempTable, tempTableId)); + } + } + + @DualPlannerTest(planner = DualPlannerTest.Planner.CASCADES) + void insertIntoTempTableWorksCorrectly() throws Exception { + // insert into values ((1, 'first', 10, 1), (2, 'second', 11, 2)) + try (FDBRecordContext context = openContext()) { + final var tempTable = TempTable.newInstance(); + final var tempTableId = CorrelationIdentifier.uniqueID(); + final var firstRecord = RecordConstructorValue.ofUnnamed( + ImmutableList.of(LiteralValue.ofScalar(1L), + LiteralValue.ofScalar("first"), + LiteralValue.ofScalar(10), + LiteralValue.ofScalar(1), + LiteralValue.ofScalar(1), + emptyArray(Type.primitiveType(Type.TypeCode.INT)))); + final var secondArray = RecordConstructorValue.ofUnnamed( + ImmutableList.of(LiteralValue.ofScalar(2L), + LiteralValue.ofScalar("second"), + LiteralValue.ofScalar(11), + LiteralValue.ofScalar(2), + LiteralValue.ofScalar(2), + emptyArray(Type.primitiveType(Type.TypeCode.INT)))); + final var explodeExpression = new ExplodeExpression(AbstractArrayConstructorValue.LightArrayConstructorValue.of(firstRecord, secondArray)); + var qun = Quantifier.forEach(Reference.of(explodeExpression)); + + qun = Quantifier.forEach(Reference.of(TempTableInsertExpression.ofConstant(qun, + tempTableId, tempTableId.getId(), Type.Record.fromDescriptor(TestRecords1Proto.MySimpleRecord.getDescriptor())))); + final var insertPlan = Reference.of(LogicalSortExpression.unsorted(qun)); + + final var cascadesPlanner = (CascadesPlanner)planner; + var plan = cascadesPlanner.planGraph(() -> insertPlan, Optional.empty(), IndexQueryabilityFilter.TRUE, EvaluationContext.empty()).getPlan(); + assertMatchesExactly(plan, tempTableInsertPlan(explodePlan())); + final ImmutableMap.Builder constants = ImmutableMap.builder(); + constants.put(tempTableId.getId(), tempTable); + final var evaluationContext = EvaluationContext.empty().withBinding(Bindings.Internal.CONSTANT, tempTableId, constants.build()); + fetchResultValues(context, plan, Function.identity(), evaluationContext, c -> { }, ExecuteProperties.SERIAL_EXECUTE); + + // select rec_no, str_value_indexed from tq1 | tq1 is a temporary table. + plan = getTempTableScanPlan(tempTable, tempTableId, false); + assertEquals(ImmutableSet.of(Pair.of(1L, "first"), + Pair.of(2L, "second")), collectResults(context, plan, tempTable, tempTableId)); + } + } + + @Nonnull + private Set> collectResults(@Nonnull FDBRecordContext context, + @Nonnull RecordQueryPlan plan, + @Nonnull TempTable tempTable, + @Nonnull CorrelationIdentifier tempTableId) throws Exception { + ImmutableSet.Builder> resultBuilder = ImmutableSet.builder(); + final ImmutableMap.Builder constants = ImmutableMap.builder(); + constants.put(tempTableId.getId(), tempTable); + final var evaluationContext = EvaluationContext.empty().withBinding(Bindings.Internal.CONSTANT, tempTableId, constants.build()); + fetchResultValues(context, plan, record -> { + final Descriptors.Descriptor recDescriptor = record.getDescriptorForType(); + Long recNo = (long) record.getField(recDescriptor.findFieldByName("rec_no")); + String strValueIndexed = (String) record.getField(recDescriptor.findFieldByName("str_value_indexed")); + resultBuilder.add(Pair.of(recNo, strValueIndexed)); + return record; + }, evaluationContext, c -> { + }, ExecuteProperties.newBuilder().setDryRun(true).build()); + return resultBuilder.build(); + } + + @Nonnull + private static Message item(long recNo, @Nonnull String strValueIndexed) { + return TestRecords1Proto.MySimpleRecord.newBuilder() + .setRecNo(recNo) + .setStrValueIndexed(strValueIndexed) + .build(); + } + + @Nonnull + private RecordQueryPlan getTempTableScanPlan(@Nonnull TempTable tempTable, @Nonnull CorrelationIdentifier tempTableId, boolean addData) { + final var type = Type.Record.fromDescriptor(TestRecords1Proto.MySimpleRecord.getDescriptor()); + if (addData) { + tempTable.add(QueryResult.ofComputed(item(42L, "fortySecondValue")), + QueryResult.ofComputed(item(45L, "fortyFifthValue"))); + } + final var tempTableScanQun = Quantifier.forEach(Reference.of(TempTableScanExpression.ofConstant(tempTableId, tempTableId.getId(), type))); + final var recNoField = Column.of(Optional.of("rec_no"), FieldValue.ofFieldName(tempTableScanQun.getFlowedObjectValue(), "rec_no")); + final var strValueIndexedField = Column.of(Optional.of("str_value_indexed"), FieldValue.ofFieldName(tempTableScanQun.getFlowedObjectValue(), "str_value_indexed")); + final var selectExpressionBuilder = GraphExpansion.builder() + .addAllResultColumns(ImmutableList.of(recNoField, strValueIndexedField)) + .addQuantifier(tempTableScanQun); + final var logicalPlan = Reference.of(LogicalSortExpression.unsorted(Quantifier.forEach(Reference.of(selectExpressionBuilder.build().buildSelect())))); + final var cascadesPlanner = (CascadesPlanner)planner; + final var plan = cascadesPlanner.planGraph(() -> logicalPlan, Optional.empty(), IndexQueryabilityFilter.TRUE, EvaluationContext.empty()).getPlan(); + assertMatchesExactly(plan, mapPlan(tempTableScanPlan())); + return plan; + } +}