Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 82452c7

Browse files
924060929924060929
authored
branch-3.1: [fix](coordinator) fix cte with local shuffle throw exception #52870 (#52876)
cherry pick from #52870 the CTE use MultiCastDataSink.destinations to store the destination instead of FragmentExecParams.destinations, but the old Coordinator use FragmentExecParams.destinations for local shuffle, so throw the exception: ``` [CANCELLED]Rows mismatched! Data may be lost. [Expected enqueue rows=14, Real enqueue rows=0, Detail: Type: HASH_SHUFFLE, Local Exchange Id: -10, Shuffled Map: [0:6], ] ```
1 parent 184efae commit 82452c7

2 files changed

Lines changed: 223 additions & 2 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1578,7 +1578,7 @@ private void computeMultiCastFragmentParams() throws Exception {
15781578
}
15791579
// process bucket shuffle join on fragment without scan node
15801580
while (bucketSeq < bucketNum) {
1581-
TPlanFragmentDestination dest = setDestination(destParams, params.destinations.size(),
1581+
TPlanFragmentDestination dest = setDestination(destParams, destinations.size(),
15821582
bucketSeq);
15831583
bucketSeq++;
15841584
destinations.add(dest);
@@ -1616,7 +1616,7 @@ private void computeMultiCastFragmentParams() throws Exception {
16161616
dest.fragment_instance_id = destParams.instanceExecParams.get(j).instanceId;
16171617
dest.server = toRpcHost(destParams.instanceExecParams.get(j).host);
16181618
dest.brpc_server = toBrpcHost(destParams.instanceExecParams.get(j).host);
1619-
destParams.instanceExecParams.get(j).recvrId = params.destinations.size();
1619+
destParams.instanceExecParams.get(j).recvrId = destinations.size();
16201620
destinations.add(dest);
16211621
}
16221622
}

‎fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleTest.java‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.qe;
19+
20+
import org.apache.doris.nereids.rules.RuleType;
21+
import org.apache.doris.planner.PlanFragmentId;
22+
import org.apache.doris.qe.Coordinator.FInstanceExecParam;
23+
import org.apache.doris.qe.Coordinator.FragmentExecParams;
24+
import org.apache.doris.thrift.TNetworkAddress;
25+
import org.apache.doris.utframe.TestWithFeService;
26+
27+
import com.google.common.collect.ArrayListMultimap;
28+
import com.google.common.collect.LinkedHashMultimap;
29+
import com.google.common.collect.SetMultimap;
30+
import org.junit.jupiter.api.Assertions;
31+
import org.junit.jupiter.api.Test;
32+
33+
import java.util.Collection;
34+
import java.util.Map;
35+
import java.util.Map.Entry;
36+
37+
public class LocalShuffleTest extends TestWithFeService {
38+
@Override
39+
protected int backendNum() {
40+
return 2;
41+
}
42+
43+
@Override
44+
protected void runBeforeAll() throws Exception {
45+
createDatabase("test");
46+
useDatabase("test");
47+
createTable("CREATE TABLE `oooo_oneid` (\n"
48+
+ " `identity_code` varchar(128) NULL COMMENT \"\",\n"
49+
+ " `identity_value` varchar(128) NULL COMMENT \"\",\n"
50+
+ " `oneid` bigint NULL COMMENT \"\",\n"
51+
+ " INDEX idx_oneid (`oneid`) USING INVERTED COMMENT ''\n"
52+
+ ") ENGINE=OLAP\n"
53+
+ "UNIQUE KEY(`identity_code`, `identity_value`)\n"
54+
+ "COMMENT 'oooo'\n"
55+
+ "DISTRIBUTED BY HASH(`identity_code`, `identity_value`) BUCKETS 1\n"
56+
+ "PROPERTIES (\n"
57+
+ "\"replication_allocation\" = \"tag.location.default: 2\"\n"
58+
+ ")");
59+
60+
createTable("CREATE TABLE `aaaa_target` (\n"
61+
+ " `base_id` varchar(128) NULL COMMENT \"\",\n"
62+
+ " `hash_partition_id` bigint NOT NULL COMMENT \"\",\n"
63+
+ " `j_bbbb` text NULL COMMENT \"j_bbbb\"\n"
64+
+ ") ENGINE=OLAP\n"
65+
+ "UNIQUE KEY(`base_id`, `hash_partition_id`)\n"
66+
+ "COMMENT 'aaaa'\n"
67+
+ "PARTITION BY LIST (`hash_partition_id`)\n"
68+
+ "(PARTITION p0 VALUES IN (\"0\"),\n"
69+
+ "PARTITION p1 VALUES IN (\"1\"),\n"
70+
+ "PARTITION p2 VALUES IN (\"2\"),\n"
71+
+ "PARTITION p3 VALUES IN (\"3\"),\n"
72+
+ "PARTITION p4 VALUES IN (\"4\"),\n"
73+
+ "PARTITION p5 VALUES IN (\"5\"),\n"
74+
+ "PARTITION p6 VALUES IN (\"6\"),\n"
75+
+ "PARTITION p7 VALUES IN (\"7\"),\n"
76+
+ "PARTITION p8 VALUES IN (\"8\"),\n"
77+
+ "PARTITION p9 VALUES IN (\"9\"),\n"
78+
+ "PARTITION p10 VALUES IN (\"10\"),\n"
79+
+ "PARTITION p11 VALUES IN (\"11\"),\n"
80+
+ "PARTITION p12 VALUES IN (\"12\"),\n"
81+
+ "PARTITION p13 VALUES IN (\"13\"),\n"
82+
+ "PARTITION p14 VALUES IN (\"14\"),\n"
83+
+ "PARTITION p15 VALUES IN (\"15\"),\n"
84+
+ "PARTITION p16 VALUES IN (\"16\"),\n"
85+
+ "PARTITION p17 VALUES IN (\"17\"),\n"
86+
+ "PARTITION p18 VALUES IN (\"18\"),\n"
87+
+ "PARTITION p19 VALUES IN (\"19\"))\n"
88+
+ "DISTRIBUTED BY HASH(`base_id`, `hash_partition_id`) BUCKETS 1\n"
89+
+ "PROPERTIES (\n"
90+
+ "\"replication_allocation\" = \"tag.location.default: 2\"\n"
91+
+ ");");
92+
93+
createTable("CREATE TABLE `bbbb_target` (\n"
94+
+ " `base_id` varchar(128) NULL COMMENT \"\",\n"
95+
+ " `hash_partition_id` bigint NOT NULL COMMENT \"\"\n"
96+
+ ") ENGINE=OLAP\n"
97+
+ "UNIQUE KEY(`base_id`, `hash_partition_id`)\n"
98+
+ "COMMENT 'bbbb'\n"
99+
+ "PARTITION BY LIST (`hash_partition_id`)\n"
100+
+ "(PARTITION p0 VALUES IN (\"0\"),\n"
101+
+ "PARTITION p1 VALUES IN (\"1\"),\n"
102+
+ "PARTITION p2 VALUES IN (\"2\"),\n"
103+
+ "PARTITION p3 VALUES IN (\"3\"),\n"
104+
+ "PARTITION p4 VALUES IN (\"4\"),\n"
105+
+ "PARTITION p5 VALUES IN (\"5\"),\n"
106+
+ "PARTITION p6 VALUES IN (\"6\"),\n"
107+
+ "PARTITION p7 VALUES IN (\"7\"),\n"
108+
+ "PARTITION p8 VALUES IN (\"8\"),\n"
109+
+ "PARTITION p9 VALUES IN (\"9\"),\n"
110+
+ "PARTITION p10 VALUES IN (\"10\"),\n"
111+
+ "PARTITION p11 VALUES IN (\"11\"),\n"
112+
+ "PARTITION p12 VALUES IN (\"12\"),\n"
113+
+ "PARTITION p13 VALUES IN (\"13\"),\n"
114+
+ "PARTITION p14 VALUES IN (\"14\"),\n"
115+
+ "PARTITION p15 VALUES IN (\"15\"),\n"
116+
+ "PARTITION p16 VALUES IN (\"16\"),\n"
117+
+ "PARTITION p17 VALUES IN (\"17\"),\n"
118+
+ "PARTITION p18 VALUES IN (\"18\"),\n"
119+
+ "PARTITION p19 VALUES IN (\"19\"))\n"
120+
+ "DISTRIBUTED BY HASH(`base_id`, `hash_partition_id`) BUCKETS 1\n"
121+
+ "PROPERTIES (\n"
122+
+ "\"replication_allocation\" = \"tag.location.default: 2\"\n"
123+
+ ");");
124+
125+
createTable("CREATE TABLE `cccc_target` (\n"
126+
+ " `base_id` varchar(128) NULL COMMENT \"\",\n"
127+
+ " `hash_partition_id` bigint NOT NULL COMMENT \"\"\n"
128+
+ ") ENGINE=OLAP\n"
129+
+ "UNIQUE KEY(`base_id`, `hash_partition_id`)\n"
130+
+ "COMMENT 'bbbb'\n"
131+
+ "PARTITION BY LIST (`hash_partition_id`)\n"
132+
+ "(PARTITION p0 VALUES IN (\"0\"),\n"
133+
+ "PARTITION p1 VALUES IN (\"1\"),\n"
134+
+ "PARTITION p2 VALUES IN (\"2\"),\n"
135+
+ "PARTITION p3 VALUES IN (\"3\"),\n"
136+
+ "PARTITION p4 VALUES IN (\"4\"),\n"
137+
+ "PARTITION p5 VALUES IN (\"5\"),\n"
138+
+ "PARTITION p6 VALUES IN (\"6\"),\n"
139+
+ "PARTITION p7 VALUES IN (\"7\"),\n"
140+
+ "PARTITION p8 VALUES IN (\"8\"),\n"
141+
+ "PARTITION p9 VALUES IN (\"9\"),\n"
142+
+ "PARTITION p10 VALUES IN (\"10\"),\n"
143+
+ "PARTITION p11 VALUES IN (\"11\"),\n"
144+
+ "PARTITION p12 VALUES IN (\"12\"),\n"
145+
+ "PARTITION p13 VALUES IN (\"13\"),\n"
146+
+ "PARTITION p14 VALUES IN (\"14\"),\n"
147+
+ "PARTITION p15 VALUES IN (\"15\"),\n"
148+
+ "PARTITION p16 VALUES IN (\"16\"),\n"
149+
+ "PARTITION p17 VALUES IN (\"17\"),\n"
150+
+ "PARTITION p18 VALUES IN (\"18\"),\n"
151+
+ "PARTITION p19 VALUES IN (\"19\"))\n"
152+
+ "DISTRIBUTED BY HASH(`base_id`, `hash_partition_id`) BUCKETS 1\n"
153+
+ "PROPERTIES (\n"
154+
+ "\"replication_allocation\" = \"tag.location.default: 2\"\n"
155+
+ ");");
156+
}
157+
158+
@Test
159+
public void testCteWithLocalShuffle() throws Exception {
160+
connectContext.getState().reset();
161+
connectContext.getSessionVariable().parallelPipelineTaskNum = 2;
162+
connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name());
163+
connectContext.getSessionVariable().setQueryTimeoutS(10);
164+
connectContext.getSessionVariable().setDisableJoinReorder(true);
165+
166+
String sql = "with oooo_oneid as (select identity_value from test.oooo_oneid),\n"
167+
+ " aaaa_frm_$oooo as (select j_bbbb,base_id from test.aaaa_target),\n"
168+
+ " bbbb_frm_$aaaa_target as (select * from bbbb_target where base_id in (select 1 from aaaa_frm_$oooo)),\n"
169+
+ " bbbb_cdn_1602527468_lnkb_bbbb_target$aaaa_target as (select base_id from bbbb_frm_$aaaa_target),\n"
170+
+ " bbbb_frm_$oooo as (select *\n"
171+
+ " from bbbb_target\n"
172+
+ " where base_id in (select identity_value from oooo_oneid)\n"
173+
+ " ),\n"
174+
+ " cccc_frm_$bbbb_target as (select * from cccc_target where base_id in (select 1 from bbbb_frm_$oooo))\n"
175+
+ "select\n"
176+
+ " j_bbbb in (select base_id from bbbb_cdn_1602527468_lnkb_bbbb_target$aaaa_target),\n"
177+
+ " j_bbbb in (select base_id from bbbb_cdn_1602527468_lnkb_bbbb_target$aaaa_target)\n"
178+
+ "from (select j_bbbb\n"
179+
+ " from oooo_oneid\n"
180+
+ " left outer join aaaa_frm_$oooo on (oooo_oneid.identity_value = aaaa_frm_$oooo.base_id)\n"
181+
+ " left outer join bbbb_frm_$oooo on (oooo_oneid.identity_value = bbbb_frm_$oooo.base_id)\n"
182+
+ ")final_plain_table";
183+
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql);
184+
try {
185+
stmtExecutor.execute();
186+
} catch (Throwable t) {
187+
// ignore
188+
}
189+
Coordinator coord = stmtExecutor.getCoord();
190+
Map<PlanFragmentId, FragmentExecParams> fragmentExecParamsMap = coord.getFragmentExecParamsMap();
191+
192+
for (FragmentExecParams fragmentExecParams : fragmentExecParamsMap.values()) {
193+
// skip check root fragment
194+
if (fragmentExecParams.fragment.getChildren().isEmpty()) {
195+
continue;
196+
}
197+
SetMultimap<TNetworkAddress, Integer> receiverIds = LinkedHashMultimap.create();
198+
ArrayListMultimap<TNetworkAddress, FInstanceExecParam> hostToInstances = ArrayListMultimap.create();
199+
boolean setRecvrId = false;
200+
for (FInstanceExecParam instanceExecParam : fragmentExecParams.instanceExecParams) {
201+
if (instanceExecParam.recvrId != -1) {
202+
setRecvrId = true;
203+
}
204+
receiverIds.put(instanceExecParam.host, instanceExecParam.recvrId);
205+
hostToInstances.put(instanceExecParam.host, instanceExecParam);
206+
}
207+
if (!setRecvrId) {
208+
// skip check when share broadcast hash table
209+
continue;
210+
}
211+
for (Entry<TNetworkAddress, Collection<Integer>> hostToReceiverIds : receiverIds.asMap()
212+
.entrySet()) {
213+
// if this host has 2 instances, it should contain 2 receiverId
214+
Assertions.assertEquals(
215+
hostToInstances.get(hostToReceiverIds.getKey()).size(),
216+
hostToReceiverIds.getValue().size()
217+
);
218+
}
219+
}
220+
}
221+
}

0 commit comments

Comments
 (0)