Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e36dd3e

Browse files
seutseut
committed
do not use fetch projection if not needed
1 parent 480b4c0 commit e36dd3e

5 files changed

Lines changed: 166 additions & 57 deletions

File tree

‎sql/src/main/java/io/crate/operation/projectors/FetchProjector.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ public class FetchProjector implements Projector, RowDownstreamHandle {
8080

8181
private long inputCursor = 0;
8282
private boolean consumedRows = false;
83+
private boolean needInputRow = false;
8384

8485
private static final ESLogger LOGGER = Loggers.getLogger(FetchProjector.class);
8586

@@ -130,6 +131,7 @@ public FetchProjector(TransportFetchNodeAction transportFetchNodeAction,
130131
List<Input<?>> inputs = new ArrayList<>(outputSymbols.size());
131132
for (Symbol symbol : outputSymbols) {
132133
if (inputSymbols.contains(symbol)) {
134+
needInputRow = true;
133135
inputs.add(rowInputSymbolVisitor.process(symbol, collectRowContext));
134136
} else {
135137
inputs.add(rowInputSymbolVisitor.process(symbol, fetchRowContext));
@@ -260,8 +262,10 @@ public void onResponse(NodeFetchResponse response) {
260262
int idx = 0;
261263
synchronized (rowDelegateLock) {
262264
for (Row row : response.rows()) {
263-
collectRowDelegate.delegate(nodeBucket.inputRow(idx));
264265
fetchRowDelegate.delegate(row);
266+
if (needInputRow) {
267+
collectRowDelegate.delegate(nodeBucket.inputRow(idx));
268+
}
265269
if (partitionRow != null) {
266270
partitionRowDelegate.delegate(partitionRow);
267271
}

‎sql/src/main/java/io/crate/planner/consumer/QueryThenFetchConsumer.java‎

Lines changed: 134 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323

2424
import com.google.common.base.MoreObjects;
2525
import com.google.common.collect.ImmutableList;
26-
import com.google.common.collect.Lists;
2726
import io.crate.Constants;
2827
import io.crate.analyze.OrderBy;
2928
import io.crate.analyze.QueriedTable;
@@ -48,9 +47,9 @@
4847
import io.crate.planner.projection.MergeProjection;
4948
import io.crate.planner.projection.Projection;
5049
import io.crate.planner.projection.TopNProjection;
51-
import io.crate.planner.symbol.InputColumn;
52-
import io.crate.planner.symbol.Reference;
53-
import io.crate.planner.symbol.Symbol;
50+
import io.crate.planner.projection.builder.ProjectionBuilder;
51+
import io.crate.planner.projection.builder.SplitPoints;
52+
import io.crate.planner.symbol.*;
5453
import io.crate.types.DataTypes;
5554

5655
import java.util.ArrayList;
@@ -59,10 +58,10 @@
5958
public class QueryThenFetchConsumer implements Consumer {
6059

6160
private static final Visitor VISITOR = new Visitor();
61+
private static final OutputOrderReferenceCollector OUTPUT_ORDER_REFERENCE_COLLECTOR = new OutputOrderReferenceCollector();
6262
private static final ScoreReferenceDetector SCORE_REFERENCE_DETECTOR = new ScoreReferenceDetector();
6363
private static final ColumnIdent DOC_ID_COLUMN_IDENT = new ColumnIdent(DocSysColumns.DOCID.name());
6464
private static final InputColumn DEFAULT_DOC_ID_INPUT_COLUMN = new InputColumn(0, DataTypes.STRING);
65-
private static final InputColumn DEFAULT_SCORE_INPUT_COLUMN = new InputColumn(1, DataTypes.FLOAT);
6665

6766
@Override
6867
public boolean consume(AnalyzedRelation rootRelation, ConsumerContext context) {
@@ -95,37 +94,50 @@ public PlannedAnalyzedRelation visitQueriedTable(QueriedTable table, ConsumerCon
9594
return new NoopPlannedAnalyzedRelation(table);
9695
}
9796

98-
// TODO: if _score is selected, ordering by _score using a TopNProjection is only temporarily and MUST be changed.
99-
// Proposal: only order by _score if requested by user and order it on the shard
100-
boolean scoreSelected = false;
97+
boolean outputsAreAllOrdered = false;
98+
List<Projection> collectProjections = new ArrayList<>();
99+
List<Projection> mergeProjections = new ArrayList<>();
100+
List<Symbol> collectSymbols = new ArrayList<>();
101+
List<Symbol> outputSymbols = new ArrayList<>();
101102
ReferenceInfo docIdRefInfo = tableInfo.getReferenceInfo(DOC_ID_COLUMN_IDENT);
102-
List<Symbol> collectSymbols = Lists.<Symbol>newArrayList(new Reference(docIdRefInfo));
103103

104-
List<Symbol> outputSymbols = new ArrayList<>(table.querySpec().outputs().size());
105-
for (Symbol symbol : table.querySpec().outputs()) {
106-
if (SCORE_REFERENCE_DETECTOR.detect(symbol)) {
107-
scoreSelected = true;
108-
collectSymbols.add(symbol);
109-
}
110-
outputSymbols.add(DocReferenceConverter.convertIfPossible(symbol, tableInfo));
111-
}
104+
ProjectionBuilder projectionBuilder = new ProjectionBuilder(table.querySpec());
105+
SplitPoints splitPoints = projectionBuilder.getSplitPoints();
112106

113-
List<Projection> collectProjections = new ArrayList<>();
107+
// MAP/COLLECT related
114108
OrderBy orderBy = table.querySpec().orderBy();
115109
if (orderBy != null) {
116110
table.tableRelation().validateOrderBy(orderBy);
117-
for (Symbol symbol : orderBy.orderBySymbols()) {
118-
if (!collectSymbols.contains(symbol)) {
119-
// order by symbols will be resolved on collect
111+
112+
// detect if all output columns are used in orderBy,
113+
// if so, no fetch projection is needed
114+
// TODO: if no dedicated fetchPhase is needed we should stick to QAF instead
115+
OutputOrderReferenceContext outputOrderContext =
116+
OUTPUT_ORDER_REFERENCE_COLLECTOR.collect(splitPoints.leaves());
117+
outputOrderContext.collectOrderBy = true;
118+
OUTPUT_ORDER_REFERENCE_COLLECTOR.collect(orderBy.orderBySymbols(), outputOrderContext);
119+
outputsAreAllOrdered = outputOrderContext.outputsAreAllOrdered();
120+
if (outputsAreAllOrdered) {
121+
collectSymbols = splitPoints.toCollect();
122+
} else {
123+
collectSymbols.addAll(orderBy.orderBySymbols());
124+
}
125+
126+
}
127+
if (!outputsAreAllOrdered) {
128+
collectSymbols.add(0, new Reference(docIdRefInfo));
129+
for (Symbol symbol : table.querySpec().outputs()) {
130+
// _score can only be resolved during collect
131+
if (SCORE_REFERENCE_DETECTOR.detect(symbol) && !collectSymbols.contains(symbol)) {
120132
collectSymbols.add(symbol);
121133
}
134+
outputSymbols.add(DocReferenceConverter.convertIfPossible(symbol, tableInfo));
122135
}
123-
124-
MergeProjection mergeProjection = new MergeProjection(
136+
}
137+
if (orderBy != null) {
138+
MergeProjection mergeProjection = projectionBuilder.mergeProjection(
125139
collectSymbols,
126-
orderBy.orderBySymbols(),
127-
orderBy.reverseFlags(),
128-
orderBy.nullsFirst());
140+
orderBy);
129141
collectProjections.add(mergeProjection);
130142
}
131143

@@ -138,35 +150,43 @@ public PlannedAnalyzedRelation visitQueriedTable(QueriedTable table, ConsumerCon
138150
orderBy,
139151
MoreObjects.firstNonNull(table.querySpec().limit(), Constants.DEFAULT_SELECT_LIMIT) + table.querySpec().offset()
140152
);
141-
collectNode.keepContextForFetcher(true);
153+
collectNode.keepContextForFetcher(!outputsAreAllOrdered);
142154
collectNode.projections(collectProjections);
155+
// MAP/COLLECT related END
143156

144-
List<Projection> mergeProjections = new ArrayList<>(2);
145-
TopNProjection topNProjection = new TopNProjection(
146-
MoreObjects.firstNonNull(table.querySpec().limit(), Constants.DEFAULT_SELECT_LIMIT),
147-
table.querySpec().offset()
148-
);
149-
List<Symbol> outputs = new ArrayList<>(2);
150-
outputs.add(DEFAULT_DOC_ID_INPUT_COLUMN);
151-
if (scoreSelected) {
152-
outputs.add(DEFAULT_SCORE_INPUT_COLUMN);
153-
}
154-
topNProjection.outputs(outputs);
155-
mergeProjections.add(topNProjection);
156-
157-
// by default don't split fetch requests into pages/chunks,
158-
// only if record set is higher than default limit
159-
int bulkSize = FetchProjector.NO_BULK_REQUESTS;
160-
if (topNProjection.limit() > Constants.DEFAULT_SELECT_LIMIT) {
161-
bulkSize = Constants.DEFAULT_SELECT_LIMIT;
157+
// HANDLER/MERGE/FETCH related
158+
TopNProjection topNProjection;
159+
if (!outputsAreAllOrdered) {
160+
topNProjection = projectionBuilder.topNProjection(
161+
collectSymbols,
162+
null,
163+
table.querySpec().offset(),
164+
table.querySpec().limit(),
165+
null);
166+
mergeProjections.add(topNProjection);
167+
168+
// by default don't split fetch requests into pages/chunks,
169+
// only if record set is higher than default limit
170+
int bulkSize = FetchProjector.NO_BULK_REQUESTS;
171+
if (topNProjection.limit() > Constants.DEFAULT_SELECT_LIMIT) {
172+
bulkSize = Constants.DEFAULT_SELECT_LIMIT;
173+
}
174+
FetchProjection fetchProjection = new FetchProjection(
175+
DEFAULT_DOC_ID_INPUT_COLUMN, collectSymbols, outputSymbols,
176+
tableInfo.partitionedByColumns(),
177+
collectNode.executionNodes(),
178+
bulkSize,
179+
table.querySpec().isLimited());
180+
mergeProjections.add(fetchProjection);
181+
} else {
182+
topNProjection = projectionBuilder.topNProjection(
183+
collectSymbols,
184+
null,
185+
table.querySpec().offset(),
186+
table.querySpec().limit(),
187+
table.querySpec().outputs());
188+
mergeProjections.add(topNProjection);
162189
}
163-
FetchProjection fetchProjection = new FetchProjection(
164-
DEFAULT_DOC_ID_INPUT_COLUMN, collectSymbols, outputSymbols,
165-
tableInfo.partitionedByColumns(),
166-
collectNode.executionNodes(),
167-
bulkSize,
168-
table.querySpec().isLimited());
169-
mergeProjections.add(fetchProjection);
170190

171191
MergeNode localMergeNode;
172192
if (orderBy != null) {
@@ -183,6 +203,7 @@ public PlannedAnalyzedRelation visitQueriedTable(QueriedTable table, ConsumerCon
183203
collectNode,
184204
context.plannerContext());
185205
}
206+
// HANDLER/MERGE/FETCH related END
186207

187208
return new QueryThenFetch(collectNode, localMergeNode);
188209
}
@@ -193,4 +214,65 @@ protected PlannedAnalyzedRelation visitAnalyzedRelation(AnalyzedRelation relatio
193214
}
194215
}
195216

217+
static class OutputOrderReferenceContext {
218+
219+
private List<Reference> outputReferences = new ArrayList<>();
220+
private List<Reference> orderByReferences = new ArrayList<>();
221+
public boolean collectOrderBy = false;
222+
223+
public void addReference(Reference reference) {
224+
if (collectOrderBy) {
225+
orderByReferences.add(reference);
226+
} else {
227+
outputReferences.add(reference);
228+
}
229+
}
230+
231+
public boolean outputsAreAllOrdered() {
232+
return orderByReferences.containsAll(outputReferences);
233+
}
234+
235+
}
236+
237+
static class OutputOrderReferenceCollector extends SymbolVisitor<OutputOrderReferenceContext, Void> {
238+
239+
public OutputOrderReferenceContext collect(List<Symbol> symbols) {
240+
OutputOrderReferenceContext context = new OutputOrderReferenceContext();
241+
collect(symbols, context);
242+
return context;
243+
}
244+
245+
public void collect(List<Symbol> symbols, OutputOrderReferenceContext context) {
246+
for (Symbol symbol : symbols) {
247+
process(symbol, context);
248+
}
249+
}
250+
251+
@Override
252+
public Void visitAggregation(Aggregation aggregation, OutputOrderReferenceContext context) {
253+
for (Symbol symbol : aggregation.inputs()) {
254+
process(symbol, context);
255+
}
256+
return null;
257+
}
258+
259+
@Override
260+
public Void visitReference(Reference symbol, OutputOrderReferenceContext context) {
261+
context.addReference(symbol);
262+
return null;
263+
}
264+
265+
@Override
266+
public Void visitDynamicReference(DynamicReference symbol, OutputOrderReferenceContext context) {
267+
return visitReference(symbol, context);
268+
}
269+
270+
@Override
271+
public Void visitFunction(Function function, OutputOrderReferenceContext context) {
272+
for (Symbol symbol : function.arguments()) {
273+
process(symbol, context);
274+
}
275+
return null;
276+
}
277+
}
196278
}

‎sql/src/main/java/io/crate/planner/projection/builder/ProjectionBuilder.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,12 +149,11 @@ public TopNProjection topNProjection(
149149
}
150150

151151
public MergeProjection mergeProjection(
152-
Collection<Symbol> inputs,
152+
List<Symbol> inputs,
153153
OrderBy orderBy
154154
) {
155155
InputCreatingVisitor.Context context = new InputCreatingVisitor.Context(inputs);
156-
List<Symbol> outputs = inputVisitor.process(inputs, context);
157-
return new MergeProjection(outputs,
156+
return new MergeProjection(inputs,
158157
inputVisitor.process(orderBy.orderBySymbols(), context),
159158
orderBy.reverseFlags(),
160159
orderBy.nullsFirst());

‎sql/src/main/java/io/crate/planner/projection/builder/SplitPoints.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public class SplitPoints {
4040
int estimate = querySpec.outputs().size();
4141
this.querySpec = querySpec;
4242
this.toCollect = new ArrayList<>(estimate);
43-
this.aggregates = new ArrayList<>(estimate-1);
43+
this.aggregates = new ArrayList<>(estimate);
4444
this.leaves = new ArrayList<>(estimate);
4545
}
4646

‎sql/src/test/java/io/crate/planner/PlannerTest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,30 @@ public void testQueryThenFetchPlan() throws Exception {
498498
assertThat(fetchProjection.outputs().get(0), isReference("_doc['name']"));
499499
}
500500

501+
@Test
502+
public void testQueryThenFetchPlanNoFetch() throws Exception {
503+
// testing that a fetch projection is not added if all output symbols are included
504+
// at the orderBy symbols
505+
Plan plan = plan("select name from users where name = 'x' order by name limit 10");
506+
assertThat(plan, instanceOf(QueryThenFetch.class));
507+
CollectNode collectNode = ((QueryThenFetch) plan).collectNode();
508+
assertTrue(collectNode.whereClause().hasQuery());
509+
assertFalse(collectNode.isPartitioned());
510+
511+
DQLPlanNode resultNode = ((QueryThenFetch) plan).resultNode();
512+
assertThat(resultNode.outputTypes().size(), is(1));
513+
assertEquals(DataTypes.STRING, resultNode.outputTypes().get(0));
514+
515+
assertThat(resultNode, instanceOf(MergeNode.class));
516+
MergeNode mergeNode = (MergeNode) resultNode;
517+
assertTrue(mergeNode.finalProjection().isPresent());
518+
519+
Projection lastProjection = mergeNode.finalProjection().get();
520+
assertThat(lastProjection, instanceOf(TopNProjection.class));
521+
TopNProjection topNProjection = (TopNProjection) lastProjection;
522+
assertThat(topNProjection.outputs().size(), is(1));
523+
}
524+
501525
@Test
502526
public void testQueryThenFetchPlanDefaultLimit() throws Exception {
503527
QueryThenFetch plan = (QueryThenFetch)plan("select name from users");

0 commit comments

Comments
 (0)