Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 2d6257b

Browse files
dangleptrdangleptr
authored
Enhance balancer (vesoft-inc#1378)
1 parent 5d10861 commit 2d6257b

25 files changed

+316
-79
lines changed

‎src/common/base/Status.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ class Status final {
113113
STATUS_GENERATOR(UserNotFound);
114114
STATUS_GENERATOR(LeaderChanged);
115115
STATUS_GENERATOR(Balanced);
116+
STATUS_GENERATOR(PartNotFound);
116117

117118
#undef STATUS_GENERATOR
118119

@@ -144,6 +145,7 @@ class Status final {
144145
kUserNotFound = 408,
145146
kLeaderChanged = 409,
146147
kBalanced = 410,
148+
kPartNotFound = 411,
147149
};
148150

149151
Code code() const {

‎src/interface/storage.thrift‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ enum ErrorCode {
3737
E_INVALID_STORE = -33,
3838
E_INVALID_PEER = -34,
3939
E_RETRY_EXHAUSTED = -35,
40+
E_TRANSFER_LEADER_FAILED = -36,
4041

4142
// meta client failed
4243
E_LOAD_META_FAILED = -41,
@@ -260,6 +261,12 @@ struct CatchUpDataReq {
260261
3: common.HostAddr target,
261262
}
262263

264+
struct CheckPeersReq {
265+
1: common.GraphSpaceID space_id,
266+
2: common.PartitionID part_id,
267+
3: list<common.HostAddr> peers,
268+
}
269+
263270
struct GetLeaderReq {
264271
}
265272

@@ -388,13 +395,14 @@ service StorageService {
388395
AdminExecResp waitingForCatchUpData(1: CatchUpDataReq req);
389396
AdminExecResp removePart(1: RemovePartReq req);
390397
AdminExecResp memberChange(1: MemberChangeReq req);
398+
AdminExecResp checkPeers(1: CheckPeersReq req);
399+
GetLeaderResp getLeaderPart(1: GetLeaderReq req);
400+
391401
// Interfaces for nebula cluster checkpoint
392402
AdminExecResp createCheckpoint(1: CreateCPRequest req);
393403
AdminExecResp dropCheckpoint(1: DropCPRequest req);
394404
AdminExecResp blockingWrites(1: BlockingSignRequest req);
395405

396-
GetLeaderResp getLeaderPart(1: GetLeaderReq req);
397-
398406
// Interfaces for key-value storage
399407
ExecResponse put(1: PutRequest req);
400408
GeneralResponse get(1: GetRequest req);

‎src/kvstore/NebulaStore.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@ class NebulaStore : public KVStore, public Handler {
7676
return 0;
7777
}
7878

79+
HostAddr address() const {
80+
return storeSvcAddr_;
81+
}
82+
7983
std::shared_ptr<folly::IOThreadPoolExecutor> getIoPool() const {
8084
return ioPool_;
8185
}

‎src/kvstore/Part.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#define KVSTORE_PART_H_
99

1010
#include "base/Base.h"
11+
#include "base/NebulaKeyUtils.h"
1112
#include "raftex/RaftPart.h"
1213
#include "kvstore/Common.h"
1314
#include "kvstore/KVEngine.h"
@@ -76,6 +77,11 @@ class Part : public raftex::RaftPart {
7677
void reset() {
7778
LOG(INFO) << idStr_ << "Clean up all wals";
7879
wal()->reset();
80+
ResultCode res = engine_->remove(NebulaKeyUtils::systemCommitKey(partId_));
81+
if (res != ResultCode::SUCCEEDED) {
82+
LOG(WARNING) << idStr_ << "Remove the committedLogId failed, error "
83+
<< static_cast<int32_t>(res);
84+
}
7985
}
8086

8187
private:

‎src/kvstore/raftex/RaftPart.cpp‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -800,7 +800,7 @@ void RaftPart::replicateLogs(folly::EventBase* eb,
800800
VLOG(2) << self->idStr_ << "Received enough response";
801801
CHECK(!result.hasException());
802802
if (tracker.slow()) {
803-
tracker.output(self->idStr_, folly::stringPrintf("Total send wals: %ld",
803+
tracker.output(self->idStr_, folly::stringPrintf("Total send logs: %ld",
804804
lastLogId - prevLogId + 1));
805805
}
806806
self->processAppendLogResponses(*result,
@@ -1089,7 +1089,8 @@ bool RaftPart::leaderElection() {
10891089
uint64_t curWeight = weight_.load();
10901090
weight_.store(curWeight * 2);
10911091
}
1092-
return resp.get_error_code() == cpp2::ErrorCode::SUCCEEDED;
1092+
return resp.get_error_code() == cpp2::ErrorCode::SUCCEEDED
1093+
&& !hosts[idx]->isLearner();
10931094
});
10941095

10951096
VLOG(2) << idStr_
@@ -1712,7 +1713,7 @@ void RaftPart::reset() {
17121713
}
17131714

17141715
AppendLogResult RaftPart::isCatchedUp(const HostAddr& peer) {
1715-
std::lock_guard<std::mutex> lck(logsLock_);
1716+
std::lock_guard<std::mutex> lck(raftLock_);
17161717
LOG(INFO) << idStr_ << "Check whether I catch up";
17171718
if (role_ != Role::LEADER) {
17181719
LOG(INFO) << idStr_ << "I am not the leader";
@@ -1743,6 +1744,24 @@ bool RaftPart::linkCurrentWAL(const char* newPath) {
17431744
return wal_->linkCurrentWAL(newPath);
17441745
}
17451746

1747+
void RaftPart::checkAndResetPeers(const std::vector<HostAddr>& peers) {
1748+
std::lock_guard<std::mutex> lck(raftLock_);
1749+
// To avoid the iterator invalid, we use another container for it.
1750+
decltype(hosts_) hosts = hosts_;
1751+
for (auto& h : hosts) {
1752+
LOG(INFO) << idStr_ << "Check host " << h->addr_;
1753+
auto it = std::find(peers.begin(), peers.end(), h->addr_);
1754+
if (it == peers.end()) {
1755+
LOG(INFO) << idStr_ << "The peer " << h->addr_ << " should not exist in my peers";
1756+
removePeer(h->addr_);
1757+
}
1758+
}
1759+
for (auto& p : peers) {
1760+
LOG(INFO) << idStr_ << "Add peer " << p << " if not exist!";
1761+
addPeer(p);
1762+
}
1763+
}
1764+
17461765
} // namespace raftex
17471766
} // namespace nebula
17481767

‎src/kvstore/raftex/RaftPart.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,11 @@ class RaftPart : public std::enable_shared_from_this<RaftPart> {
184184

185185
bool linkCurrentWAL(const char* newPath);
186186

187+
/**
188+
* Reset my peers if not equals the argument
189+
*/
190+
void checkAndResetPeers(const std::vector<HostAddr>& peers);
191+
187192
/*****************************************************
188193
*
189194
* Methods to process incoming raft requests

‎src/meta/processors/admin/AdminClient.cpp‎

Lines changed: 69 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,18 @@ folly::Future<Status> AdminClient::transLeader(GraphSpaceID spaceId,
2525
storage::cpp2::TransLeaderReq req;
2626
req.set_space_id(spaceId);
2727
req.set_part_id(partId);
28+
auto ret = getPeers(spaceId, partId);
29+
if (!ret.ok()) {
30+
return ret.status();
31+
}
32+
auto& peers = ret.value();
33+
auto it = std::find(peers.begin(), peers.end(), leader);
34+
if (it == peers.end()) {
35+
LOG(WARNING) << "Can't find part " << partId << " on " << leader;
36+
return Status::PartNotFound();
37+
}
2838
auto target = dst;
2939
if (dst == kRandomPeer) {
30-
auto ret = getPeers(spaceId, partId);
31-
if (!ret.ok()) {
32-
return ret.status();
33-
}
34-
auto& peers = ret.value();
3540
for (auto& p : peers) {
3641
if (p != leader && ActiveHostsMan::isLived(kv_, p)) {
3742
target = p;
@@ -48,6 +53,9 @@ folly::Future<Status> AdminClient::transLeader(GraphSpaceID spaceId,
4853
case storage::cpp2::ErrorCode::E_LEADER_CHANGED: {
4954
return Status::OK();
5055
}
56+
case storage::cpp2::ErrorCode::E_PART_NOT_FOUND: {
57+
return Status::PartNotFound();
58+
}
5159
default:
5260
return Status::Error("Unknown code %d",
5361
static_cast<int32_t>(resp.get_code()));
@@ -237,6 +245,60 @@ folly::Future<Status> AdminClient::removePart(GraphSpaceID spaceId,
237245
});
238246
}
239247

248+
folly::Future<Status> AdminClient::checkPeers(GraphSpaceID spaceId, PartitionID partId) {
249+
if (injector_) {
250+
return injector_->checkPeers();
251+
}
252+
storage::cpp2::CheckPeersReq req;
253+
req.set_space_id(spaceId);
254+
req.set_part_id(partId);
255+
auto ret = getPeers(spaceId, partId);
256+
if (!ret.ok()) {
257+
return ret.status();
258+
}
259+
auto peers = std::move(ret).value();
260+
std::vector<nebula::cpp2::HostAddr> thriftPeers;
261+
thriftPeers.resize(peers.size());
262+
std::transform(peers.begin(), peers.end(), thriftPeers.begin(), [this](const auto& h) {
263+
return toThriftHost(h);
264+
});
265+
req.set_peers(std::move(thriftPeers));
266+
folly::Promise<Status> pro;
267+
auto fut = pro.getFuture();
268+
std::vector<folly::Future<Status>> futures;
269+
for (auto& p : peers) {
270+
auto f = getResponse(p, req, [] (auto client, auto request) {
271+
return client->future_checkPeers(request);
272+
}, [] (auto&& resp) -> Status {
273+
if (resp.get_code() == storage::cpp2::ErrorCode::SUCCEEDED) {
274+
return Status::OK();
275+
} else {
276+
return Status::Error("Add part failed! code=%d",
277+
static_cast<int32_t>(resp.get_code()));
278+
}
279+
});
280+
futures.emplace_back(std::move(f));
281+
}
282+
folly::collectAll(std::move(futures)).thenTry([p = std::move(pro)] (auto&& t) mutable {
283+
if (t.hasException()) {
284+
p.setValue(Status::Error("Check failed!"));
285+
} else {
286+
auto v = std::move(t).value();
287+
for (auto& resp : v) {
288+
// The exception has been catched inside getResponse.
289+
CHECK(!resp.hasException());
290+
auto st = std::move(resp).value();
291+
if (!st.ok()) {
292+
p.setValue(st);
293+
return;
294+
}
295+
}
296+
p.setValue(Status::OK());
297+
}
298+
});
299+
return fut;
300+
}
301+
240302
template<typename Request,
241303
typename RemoteFunc,
242304
typename RespGenerator>
@@ -360,7 +422,6 @@ void AdminClient::getResponse(
360422
// this task failed forever.
361423
index = leaderIndex;
362424
hosts.emplace_back(leader);
363-
return;
364425
}
365426
LOG(INFO) << "Return leader change from " << hosts[index]
366427
<< ", new leader is " << leader
@@ -499,6 +560,8 @@ folly::Future<Status> AdminClient::getLeaderDist(HostLeaderMap* result) {
499560
}
500561

501562
p.setValue(Status::OK());
563+
}).thenError([p = std::move(promise)] (auto&& e) mutable {
564+
p.setValue(Status::Error("Get leader failed, %s", e.what().c_str()));
502565
});
503566

504567
return future;

‎src/meta/processors/admin/AdminClient.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class FaultInjector {
3131
virtual folly::Future<Status> memberChange() = 0;
3232
virtual folly::Future<Status> updateMeta() = 0;
3333
virtual folly::Future<Status> removePart() = 0;
34+
virtual folly::Future<Status> checkPeers() = 0;
3435
virtual folly::Future<Status> getLeaderDist(HostLeaderMap* hostLeaderMap) = 0;
3536
virtual folly::Future<Status> createSnapshot() = 0;
3637
virtual folly::Future<Status> dropSnapshot() = 0;
@@ -93,6 +94,9 @@ class AdminClient {
9394
PartitionID partId,
9495
const HostAddr& host);
9596

97+
folly::Future<Status> checkPeers(GraphSpaceID spaceId,
98+
PartitionID partId);
99+
96100
folly::Future<Status> getLeaderDist(HostLeaderMap* result);
97101

98102
folly::Future<Status> createSnapshot(GraphSpaceID spaceId, const std::string& name);

‎src/meta/processors/admin/BalancePlan.cpp‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ void BalancePlan::invoke() {
7373
task.invoke();
7474
}
7575
}; // onFinished
76-
tasks_[taskIndex].onError_ = [this, i, j]() {
76+
tasks_[taskIndex].onError_ = [this, i, j, taskIndex]() {
7777
bool finished = false;
7878
bool stopped = false;
7979
{
@@ -84,13 +84,19 @@ void BalancePlan::invoke() {
8484
finished = true;
8585
LOG(INFO) << "Balance " << id_ << " failed!";
8686
}
87+
stopped = stopped_;
8788
}
8889
if (finished) {
8990
CHECK_EQ(j, this->buckets_[i].size() - 1);
9091
saveInStore(true);
9192
onFinished_();
9293
} else if (j + 1 < this->buckets_[i].size()) {
9394
auto& task = this->tasks_[this->buckets_[i][j + 1]];
95+
if (tasks_[taskIndex].spaceId_ == task.spaceId_
96+
&& tasks_[taskIndex].partId_ == task.partId_) {
97+
LOG(INFO) << "Skip the task for the same partId " << task.partId_;
98+
task.ret_ = BalanceTask::Result::FAILED;
99+
}
94100
if (stopped) {
95101
task.ret_ = BalanceTask::Result::INVALID;
96102
}
@@ -166,8 +172,10 @@ bool BalancePlan::recovery(bool resume) {
166172
task.startTimeMs_ = std::get<3>(tup);
167173
task.endTimeMs_ = std::get<4>(tup);
168174
if (resume && task.ret_ != BalanceTask::Result::SUCCEEDED) {
169-
// Resume the failed task.
170-
task.ret_ = BalanceTask::Result::IN_PROGRESS;
175+
// Resume the failed task, skip the in-progress and invalid tasks
176+
if (task.ret_ == BalanceTask::Result::FAILED) {
177+
task.ret_ = BalanceTask::Result::IN_PROGRESS;
178+
}
171179
task.status_ = BalanceTask::Status::START;
172180
if (ActiveHostsMan::isLived(kv_, task.src_)) {
173181
task.srcLived_ = true;

0 commit comments

Comments
 (0)