Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit fa7862e

Browse files
panda-sheepcritical27
panda-sheep
and
critical27
authored
[raft] clean part rocksdb data when raftpart::reset (vesoft-inc#2522)
* clean rocksdb part data when raftpart::reset * use removeRange * fix clang format AllowShortBlocksOnASingleLine:never * address cangfengzhs's comment * address critical27's comment Co-authored-by: Doodle <[email protected]>
1 parent 76a97d8 commit fa7862e

File tree

6 files changed

+288
-15
lines changed

6 files changed

+288
-15
lines changed

‎src/common/utils/NebulaKeyUtils.cpp‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,32 @@ bool NebulaKeyUtils::isValidVidLen(size_t vIdLen, const VertexID& srcVId, const
1919
}
2020

2121
// static
22-
std::string NebulaKeyUtils::vertexKey(size_t vIdLen,
23-
PartitionID partId,
24-
const VertexID& vId,
25-
TagID tagId) {
22+
std::string NebulaKeyUtils::firstKey(const std::string& prefix, size_t count) {
23+
std::string key;
24+
key.reserve(prefix.size() + count);
25+
key.append(prefix).append(count, '\0');
26+
return key;
27+
}
28+
29+
// static
30+
std::string NebulaKeyUtils::lastKey(const std::string& prefix, size_t count) {
31+
std::string key;
32+
key.reserve(prefix.size() + count);
33+
key.append(prefix).append(count, '\377');
34+
return key;
35+
}
36+
37+
// static
38+
std::string NebulaKeyUtils::vertexKey(
39+
size_t vIdLen, PartitionID partId, const VertexID& vId, TagID tagId, char pad) {
2640
CHECK_GE(vIdLen, vId.size());
2741
int32_t item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kVertex);
2842

2943
std::string key;
3044
key.reserve(kVertexLen + vIdLen);
3145
key.append(reinterpret_cast<const char*>(&item), sizeof(int32_t))
3246
.append(vId.data(), vId.size())
33-
.append(vIdLen - vId.size(), '\0')
47+
.append(vIdLen - vId.size(), pad)
3448
.append(reinterpret_cast<const char*>(&tagId), sizeof(TagID));
3549
return key;
3650
}

‎src/common/utils/NebulaKeyUtils.h‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,23 @@ class NebulaKeyUtils final {
4141
*/
4242
static bool isValidVidLen(size_t vIdLen, const VertexID& srcvId, const VertexID& dstvId = "");
4343

44+
/**
45+
* Generate the first key with prefix.
46+
* count means the number of count '\0' is filled after the prefix.
47+
* */
48+
static std::string firstKey(const std::string& prefix, size_t count);
49+
50+
/**
51+
* Generate the last key with prefix.
52+
* count means the number of count '\377' is filled after the prefix.
53+
* */
54+
static std::string lastKey(const std::string& prefix, size_t count);
55+
4456
/**
4557
* Generate vertex key for kv store
4658
* */
47-
static std::string vertexKey(size_t vIdLen, PartitionID partId, const VertexID& vId, TagID tagId);
59+
static std::string vertexKey(
60+
size_t vIdLen, PartitionID partId, const VertexID& vId, TagID tagId, char pad = '\0');
4861

4962
static std::string edgeKey(size_t vIdLen,
5063
PartitionID partId,

‎src/kvstore/NebulaStore.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,8 @@ std::shared_ptr<Part> NebulaStore::newPart(GraphSpaceID spaceId,
346346
workers_,
347347
snapshot_,
348348
clientMan_,
349-
diskMan_);
349+
diskMan_,
350+
getSpaceVidLen(spaceId));
350351
std::vector<HostAddr> peers;
351352
if (defaultPeers.empty()) {
352353
// pull the information from meta

‎src/kvstore/Part.cpp‎

Lines changed: 47 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66

77
#include "kvstore/Part.h"
88

9+
#include "common/utils/IndexKeyUtils.h"
910
#include "common/utils/NebulaKeyUtils.h"
11+
#include "common/utils/OperationKeyUtils.h"
1012
#include "kvstore/LogEncoder.h"
1113
#include "kvstore/RocksEngineConfig.h"
1214

@@ -27,7 +29,8 @@ Part::Part(GraphSpaceID spaceId,
2729
std::shared_ptr<folly::Executor> handlers,
2830
std::shared_ptr<raftex::SnapshotManager> snapshotMan,
2931
std::shared_ptr<RaftClient> clientMan,
30-
std::shared_ptr<DiskManager> diskMan)
32+
std::shared_ptr<DiskManager> diskMan,
33+
int32_t vIdLen)
3134
: RaftPart(FLAGS_cluster_id,
3235
spaceId,
3336
partId,
@@ -42,7 +45,8 @@ Part::Part(GraphSpaceID spaceId,
4245
spaceId_(spaceId),
4346
partId_(partId),
4447
walPath_(walPath),
45-
engine_(engine) {}
48+
engine_(engine),
49+
vIdLen_(vIdLen) {}
4650

4751
std::pair<LogID, TermID> Part::lastCommittedLogId() {
4852
std::string val;
@@ -420,11 +424,47 @@ bool Part::preProcessLog(LogID logId, TermID termId, ClusterID clusterId, const
420424
}
421425

422426
void Part::cleanup() {
423-
LOG(INFO) << idStr_ << "Clean rocksdb commit key";
424-
auto res = engine_->remove(NebulaKeyUtils::systemCommitKey(partId_));
425-
if (res != nebula::cpp2::ErrorCode::SUCCEEDED) {
426-
LOG(WARNING) << idStr_ << "Remove the committedLogId failed, error "
427-
<< static_cast<int32_t>(res);
427+
LOG(INFO) << idStr_ << "Clean rocksdb part data";
428+
// Remove the vertex, edge, index, systemCommitKey, operation data under the part
429+
const auto& vertexPre = NebulaKeyUtils::vertexPrefix(partId_);
430+
auto ret = engine_->removeRange(NebulaKeyUtils::firstKey(vertexPre, vIdLen_),
431+
NebulaKeyUtils::lastKey(vertexPre, vIdLen_));
432+
if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
433+
LOG(ERROR) << idStr_ << "Remove the part vertex data failed, error "
434+
<< static_cast<int32_t>(ret);
435+
return;
436+
}
437+
438+
const auto& edgePre = NebulaKeyUtils::edgePrefix(partId_);
439+
ret = engine_->removeRange(NebulaKeyUtils::firstKey(edgePre, vIdLen_),
440+
NebulaKeyUtils::lastKey(edgePre, vIdLen_));
441+
if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
442+
LOG(ERROR) << idStr_ << "Remove the part edge data failed, error" << static_cast<int32_t>(ret);
443+
return;
444+
}
445+
446+
const auto& indexPre = IndexKeyUtils::indexPrefix(partId_);
447+
ret = engine_->removeRange(NebulaKeyUtils::firstKey(indexPre, sizeof(IndexID)),
448+
NebulaKeyUtils::lastKey(indexPre, sizeof(IndexID)));
449+
if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
450+
LOG(ERROR) << idStr_ << "Remove the part index data failed, error "
451+
<< static_cast<int32_t>(ret);
452+
return;
453+
}
454+
455+
const auto& operationPre = OperationKeyUtils::operationPrefix(partId_);
456+
ret = engine_->removeRange(NebulaKeyUtils::firstKey(operationPre, sizeof(int64_t)),
457+
NebulaKeyUtils::lastKey(operationPre, sizeof(int64_t)));
458+
if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
459+
LOG(ERROR) << idStr_ << "Remove the part operation data failed, error "
460+
<< static_cast<int32_t>(ret);
461+
return;
462+
}
463+
464+
ret = engine_->remove(NebulaKeyUtils::systemCommitKey(partId_));
465+
if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
466+
LOG(ERROR) << idStr_ << "Remove the part system commit data failed, error "
467+
<< static_cast<int32_t>(ret);
428468
}
429469
return;
430470
}

‎src/kvstore/Part.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ class Part : public raftex::RaftPart {
3434
std::shared_ptr<folly::Executor> handlers,
3535
std::shared_ptr<raftex::SnapshotManager> snapshotMan,
3636
std::shared_ptr<RaftClient> clientMan,
37-
std::shared_ptr<DiskManager> diskMan);
37+
std::shared_ptr<DiskManager> diskMan,
38+
int32_t vIdLen);
3839

3940
virtual ~Part() { LOG(INFO) << idStr_ << "~Part()"; }
4041

@@ -114,6 +115,7 @@ class Part : public raftex::RaftPart {
114115

115116
private:
116117
KVEngine* engine_ = nullptr;
118+
int32_t vIdLen_;
117119
};
118120

119121
} // namespace kvstore

‎src/kvstore/test/PartTest.cpp‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,62 @@
99

1010
#include "common/base/Base.h"
1111
#include "common/fs/TempDir.h"
12+
#include "common/utils/IndexKeyUtils.h"
13+
#include "common/utils/NebulaKeyUtils.h"
14+
#include "common/utils/OperationKeyUtils.h"
1215
#include "kvstore/Part.h"
16+
#include "kvstore/RocksEngine.h"
1317

1418
namespace nebula {
1519
namespace kvstore {
1620

21+
const int32_t kDefaultVIdLen = 8;
22+
23+
void checkVertexData(RocksEngine* engine,
24+
PartitionID partId,
25+
int expectNum,
26+
bool checkVal = false) {
27+
std::string vertexPrefix = NebulaKeyUtils::vertexPrefix(partId);
28+
std::unique_ptr<KVIterator> iter;
29+
auto code = engine->prefix(vertexPrefix, &iter);
30+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code);
31+
int32_t num = 0;
32+
while (iter->valid()) {
33+
num++;
34+
if (checkVal) {
35+
ASSERT_EQ(iter->val().str(), folly::stringPrintf("val%d", num));
36+
}
37+
iter->next();
38+
}
39+
ASSERT_EQ(num, expectNum);
40+
}
41+
42+
void checkEdgeData(RocksEngine* engine, PartitionID partId, int expectNum) {
43+
std::string edgePrefix = NebulaKeyUtils::edgePrefix(partId);
44+
std::unique_ptr<KVIterator> iter;
45+
auto code = engine->prefix(edgePrefix, &iter);
46+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code);
47+
int32_t num = 0;
48+
while (iter->valid()) {
49+
num++;
50+
iter->next();
51+
}
52+
ASSERT_EQ(num, expectNum);
53+
}
54+
55+
void checkIndexData(RocksEngine* engine, PartitionID partId, int expectNum) {
56+
std::string indexPrefix = IndexKeyUtils::indexPrefix(partId);
57+
std::unique_ptr<KVIterator> iter;
58+
auto code = engine->prefix(indexPrefix, &iter);
59+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code);
60+
int32_t num = 0;
61+
while (iter->valid()) {
62+
num++;
63+
iter->next();
64+
}
65+
ASSERT_EQ(num, expectNum);
66+
}
67+
1768
TEST(PartTest, RocksTest) {
1869
fs::TempDir dataPath("/tmp/rocksdb_test.XXXXXX");
1970
rocksdb::Options options;
@@ -46,6 +97,158 @@ TEST(PartTest, RocksTest) {
4697
delete db;
4798
}
4899

100+
TEST(PartTest, KeyOrderTest) {
101+
fs::TempDir dataPath("/tmp/KeyOrderTest.XXXXXX");
102+
auto engine = std::make_unique<RocksEngine>(0, kDefaultVIdLen, dataPath.path());
103+
104+
std::vector<KV> data;
105+
PartitionID partId = 1;
106+
107+
// build vertex data in part 1, 2
108+
while (partId < 3) {
109+
auto key1 = NebulaKeyUtils::vertexKey(kDefaultVIdLen, partId, "", 0);
110+
data.emplace_back(key1, folly::stringPrintf("val%d", 1));
111+
112+
auto key2 = NebulaKeyUtils::vertexKey(kDefaultVIdLen, partId, "", INT_MAX);
113+
data.emplace_back(key2, folly::stringPrintf("val%d", 2));
114+
115+
auto key3 = NebulaKeyUtils::vertexKey(kDefaultVIdLen, partId, "ffffff", INT_MAX, '\377');
116+
data.emplace_back(key3, folly::stringPrintf("val%d", 3));
117+
118+
auto key4 = NebulaKeyUtils::vertexKey(kDefaultVIdLen, partId, "", INT_MAX, '\377');
119+
data.emplace_back(key4, folly::stringPrintf("val%d", 4));
120+
121+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, engine->multiPut(data));
122+
data.clear();
123+
partId++;
124+
}
125+
{
126+
partId = 1;
127+
while (partId < 3) {
128+
checkVertexData(engine.get(), partId, 4, true);
129+
partId++;
130+
}
131+
}
132+
}
133+
134+
TEST(PartTest, PartCleanTest) {
135+
fs::TempDir dataPath("/tmp/PartCleanTest.XXXXXX");
136+
auto engine = std::make_unique<RocksEngine>(0, kDefaultVIdLen, dataPath.path());
137+
138+
std::vector<KV> data;
139+
PartitionID partId = 1;
140+
141+
// build vertex data in part 1, 2
142+
while (partId < 3) {
143+
TagID tagId = 1;
144+
for (int i = 0; i < 10; i++) {
145+
auto key = NebulaKeyUtils::vertexKey(kDefaultVIdLen, partId, std::to_string(i), tagId);
146+
data.emplace_back(key, folly::stringPrintf("val%d", i));
147+
}
148+
tagId = 2;
149+
for (int i = 0; i < 10; i++) {
150+
auto key = NebulaKeyUtils::vertexKey(kDefaultVIdLen, partId, std::to_string(i), tagId);
151+
data.emplace_back(key, folly::stringPrintf("val%d", i));
152+
}
153+
154+
EdgeType edgetype = 3;
155+
for (int i = 0; i < 10; i++) {
156+
auto key = NebulaKeyUtils::edgeKey(
157+
kDefaultVIdLen, partId, std::to_string(i), edgetype, 0, std::to_string(i));
158+
data.emplace_back(key, folly::stringPrintf("val%d", i));
159+
}
160+
IndexID indexId = 5;
161+
for (int i = 0; i < 10; i++) {
162+
auto key =
163+
IndexKeyUtils::vertexIndexKey(kDefaultVIdLen, partId, indexId, std::to_string(i), "123");
164+
data.emplace_back(key, folly::stringPrintf("val%d", i));
165+
}
166+
167+
data.emplace_back(NebulaKeyUtils::systemCommitKey(partId), "123");
168+
data.emplace_back(NebulaKeyUtils::systemPartKey(partId), "");
169+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, engine->multiPut(data));
170+
data.clear();
171+
partId++;
172+
}
173+
174+
{
175+
partId = 1;
176+
while (partId < 2) {
177+
checkVertexData(engine.get(), partId, 20);
178+
checkEdgeData(engine.get(), partId, 10);
179+
checkIndexData(engine.get(), partId, 10);
180+
{
181+
std::string val1;
182+
auto code1 = engine->get(NebulaKeyUtils::systemCommitKey(partId), &val1);
183+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code1);
184+
ASSERT_EQ("123", val1);
185+
186+
std::string val2;
187+
auto code2 = engine->get(NebulaKeyUtils::systemPartKey(partId), &val2);
188+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code2);
189+
}
190+
191+
partId++;
192+
}
193+
194+
{
195+
// remove range part::clean data
196+
partId = 1;
197+
198+
const auto& vertexPre = NebulaKeyUtils::vertexPrefix(partId);
199+
auto ret = engine->removeRange(NebulaKeyUtils::firstKey(vertexPre, kDefaultVIdLen),
200+
NebulaKeyUtils::lastKey(vertexPre, kDefaultVIdLen));
201+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret);
202+
203+
const auto& edgePre = NebulaKeyUtils::edgePrefix(partId);
204+
ret = engine->removeRange(NebulaKeyUtils::firstKey(edgePre, kDefaultVIdLen),
205+
NebulaKeyUtils::lastKey(edgePre, kDefaultVIdLen));
206+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret);
207+
208+
const auto& indexPre = IndexKeyUtils::indexPrefix(partId);
209+
ret = engine->removeRange(NebulaKeyUtils::firstKey(indexPre, sizeof(IndexID)),
210+
NebulaKeyUtils::lastKey(indexPre, sizeof(IndexID)));
211+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret);
212+
213+
const auto& operationPre = OperationKeyUtils::operationPrefix(partId);
214+
ret = engine->removeRange(NebulaKeyUtils::firstKey(operationPre, sizeof(int64_t)),
215+
NebulaKeyUtils::lastKey(operationPre, sizeof(int64_t)));
216+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret);
217+
218+
ret = engine->remove(NebulaKeyUtils::systemCommitKey(partId));
219+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, ret);
220+
}
221+
222+
{
223+
// check data again
224+
partId = 1;
225+
checkVertexData(engine.get(), partId, 0);
226+
checkEdgeData(engine.get(), partId, 0);
227+
checkIndexData(engine.get(), partId, 0);
228+
std::string val1;
229+
auto code1 = engine->get(NebulaKeyUtils::systemCommitKey(partId), &val1);
230+
ASSERT_NE(nebula::cpp2::ErrorCode::SUCCEEDED, code1);
231+
232+
std::string val2;
233+
auto code2 = engine->get(NebulaKeyUtils::systemPartKey(partId), &val2);
234+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code2);
235+
}
236+
{
237+
partId = 2;
238+
checkVertexData(engine.get(), partId, 20);
239+
checkEdgeData(engine.get(), partId, 10);
240+
checkIndexData(engine.get(), partId, 10);
241+
std::string val1;
242+
auto code1 = engine->get(NebulaKeyUtils::systemCommitKey(partId), &val1);
243+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code1);
244+
245+
std::string val2;
246+
auto code2 = engine->get(NebulaKeyUtils::systemPartKey(partId), &val2);
247+
ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, code2);
248+
}
249+
}
250+
}
251+
49252
} // namespace kvstore
50253
} // namespace nebula
51254

0 commit comments

Comments
 (0)