Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit d79ed19

Browse files
laura-dingdangleptr
laura-ding
authored and
dangleptr
committed
Add StatsManager in graphd (vesoft-inc#1301)
1 parent 0a29cd0 commit d79ed19

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+564
-287
lines changed

‎src/common/filter/test/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ set(FILTER_TEST_LIBS
22
$<TARGET_OBJECTS:filter_obj>
33
$<TARGET_OBJECTS:storage_client>
44
$<TARGET_OBJECTS:meta_client>
5+
$<TARGET_OBJECTS:stats_obj>
6+
$<TARGET_OBJECTS:time_obj>
57
$<TARGET_OBJECTS:schema_obj>
68
$<TARGET_OBJECTS:storage_thrift_obj>
79
$<TARGET_OBJECTS:meta_thrift_obj>

‎src/common/stats/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ nebula_add_library(
22
stats_obj
33
OBJECT
44
StatsManager.cpp
5+
Stats.cpp
56
)
67

78
nebula_add_subdirectory(test)

‎src/common/stats/Stats.cpp‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/* Copyright (c) 2019 vesoft inc. All rights reserved.
2+
*
3+
* This source code is licensed under Apache 2.0 License,
4+
* attached with Common Clause Condition 1.0, found in the LICENSES directory.
5+
*/
6+
7+
#include "stats/StatsManager.h"
8+
#include "stats/Stats.h"
9+
10+
DEFINE_int32(histogram_bucketSize, 1000, "The width of each bucket");
11+
DEFINE_uint32(histogram_min, 1, "The smallest value for the bucket range");
12+
DEFINE_uint32(histogram_max, 1000 * 1000, "The largest value for the bucket range");
13+
14+
namespace nebula {
15+
namespace stats {
16+
17+
Stats::Stats(const std::string& serverName, const std::string& moduleName) {
18+
qpsStatId_ = StatsManager::registerStats(serverName + "_" + moduleName + "_qps");
19+
errorQpsStatId_ = StatsManager::registerStats(serverName + "_" + moduleName + "_error_qps");
20+
latencyStatId_ = StatsManager::registerHisto(serverName + "_" + moduleName + "_latency",
21+
FLAGS_histogram_bucketSize, FLAGS_histogram_min, FLAGS_histogram_max);
22+
}
23+
24+
// static
25+
void Stats::addStatsValue(const Stats *stats, bool ok, int64_t latency, uint32_t count) {
26+
if (stats == nullptr) {
27+
return;
28+
}
29+
if (ok && stats->getQpsStatId() != 0) {
30+
StatsManager::addValue(stats->getQpsStatId(), count);
31+
}
32+
if (!ok && stats->getErrorQpsStatId() != 0) {
33+
StatsManager::addValue(stats->getErrorQpsStatId(), count);
34+
}
35+
if (stats->getLatencyStatId() != 0) {
36+
StatsManager::addValue(stats->getLatencyStatId(), latency);
37+
}
38+
}
39+
40+
int32_t Stats::getQpsStatId() const {
41+
return qpsStatId_;
42+
}
43+
44+
int32_t Stats::getErrorQpsStatId() const {
45+
return errorQpsStatId_;
46+
}
47+
48+
int32_t Stats::getLatencyStatId() const {
49+
return latencyStatId_;
50+
}
51+
52+
} // namespace stats
53+
} // namespace nebula
54+

‎src/common/stats/Stats.h‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/* Copyright (c) 2019 vesoft inc. All rights reserved.
2+
*
3+
* This source code is licensed under Apache 2.0 License,
4+
* attached with Common Clause Condition 1.0, found in the LICENSES directory.
5+
*/
6+
7+
#ifndef COMMON_STATS_STATS_H_
8+
#define COMMON_STATS_STATS_H_
9+
10+
#include "stats/StatsManager.h"
11+
12+
namespace nebula {
13+
namespace stats {
14+
15+
class Stats final {
16+
public:
17+
Stats() = default;
18+
~Stats() = default;
19+
20+
explicit Stats(const std::string& serverName, const std::string& moduleName);
21+
22+
public:
23+
static void addStatsValue(const Stats *stats, bool ok, int64_t latency = 0, uint32_t count = 1);
24+
25+
int32_t getQpsStatId() const;
26+
27+
int32_t getErrorQpsStatId() const;
28+
29+
int32_t getLatencyStatId() const;
30+
31+
private:
32+
int32_t qpsStatId_{0};
33+
int32_t errorQpsStatId_{0};
34+
int32_t latencyStatId_{0};
35+
};
36+
} // namespace stats
37+
} // namespace nebula
38+
39+
#endif // COMMON_STATS_STATS_H_

‎src/common/stats/StatsManager.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ int32_t StatsManager::registerHisto(folly::StringPiece counterName,
6666
StatsManager::VT bucketSize,
6767
StatsManager::VT min,
6868
StatsManager::VT max) {
69+
LOG(INFO) << "registerHisto, bucketSize: " << bucketSize
70+
<< ", min: " << min << ", max: " << max;
6971
using std::chrono::seconds;
7072

7173
auto& sm = get();

‎src/daemons/CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ nebula_add_executable(
1515
$<TARGET_OBJECTS:storage_client>
1616
$<TARGET_OBJECTS:storage_thrift_obj>
1717
$<TARGET_OBJECTS:meta_client>
18+
$<TARGET_OBJECTS:stats_obj>
19+
$<TARGET_OBJECTS:time_obj>
1820
$<TARGET_OBJECTS:meta_thrift_obj>
1921
$<TARGET_OBJECTS:common_thrift_obj>
2022
$<TARGET_OBJECTS:thrift_obj>
@@ -59,6 +61,8 @@ nebula_add_executable(
5961
$<TARGET_OBJECTS:http_client_obj>
6062
$<TARGET_OBJECTS:storage_client>
6163
$<TARGET_OBJECTS:meta_client>
64+
$<TARGET_OBJECTS:stats_obj>
65+
$<TARGET_OBJECTS:time_obj>
6266
$<TARGET_OBJECTS:meta_thrift_obj>
6367
$<TARGET_OBJECTS:common_thrift_obj>
6468
$<TARGET_OBJECTS:thrift_obj>
@@ -98,6 +102,8 @@ nebula_add_executable(
98102
$<TARGET_OBJECTS:storage_client>
99103
$<TARGET_OBJECTS:storage_thrift_obj>
100104
$<TARGET_OBJECTS:meta_client>
105+
$<TARGET_OBJECTS:stats_obj>
106+
$<TARGET_OBJECTS:time_obj>
101107
$<TARGET_OBJECTS:meta_thrift_obj>
102108
$<TARGET_OBJECTS:raftex_obj>
103109
$<TARGET_OBJECTS:raftex_thrift_obj>

‎src/graph/DeleteVertexExecutor.cpp‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ void DeleteVertexExecutor::execute() {
3838
auto *runner = ectx()->rctx()->runner();
3939
auto cb = [this] (auto &&resp) {
4040
if (!resp.ok()) {
41-
DCHECK(onError_);
42-
onError_(Status::Error("Internal Error"));
41+
doError(Status::Error("Internal Error"),
42+
ectx()->getGraphStats()->getDeleteVertexStats());
4343
return;
4444
}
4545
auto rpcResp = std::move(resp).value();
@@ -63,8 +63,8 @@ void DeleteVertexExecutor::execute() {
6363

6464
auto error = [this] (auto &&e) {
6565
LOG(ERROR) << "Exception caught: " << e.what();
66-
DCHECK(onError_);
67-
onError_(Status::Error("Internal error"));
66+
doError(Status::Error("Internal Error"),
67+
ectx()->getGraphStats()->getDeleteVertexStats());
6868
return;
6969
};
7070
std::move(future).via(runner).thenValue(cb).thenError(error);
@@ -76,8 +76,8 @@ void DeleteVertexExecutor::deleteEdges(std::vector<storage::cpp2::EdgeKey>* edge
7676
auto cb = [this] (auto &&resp) {
7777
auto completeness = resp.completeness();
7878
if (completeness != 100) {
79-
DCHECK(onError_);
80-
onError_(Status::Error("Internal Error"));
79+
doError(Status::Error("Internal Error"),
80+
ectx()->getGraphStats()->getDeleteVertexStats());
8181
return;
8282
}
8383
deleteVertex();
@@ -86,8 +86,8 @@ void DeleteVertexExecutor::deleteEdges(std::vector<storage::cpp2::EdgeKey>* edge
8686

8787
auto error = [this] (auto &&e) {
8888
LOG(ERROR) << "Exception caught: " << e.what();
89-
DCHECK(onError_);
90-
onError_(Status::Error("Internal error"));
89+
doError(Status::Error("Internal Error"),
90+
ectx()->getGraphStats()->getDeleteVertexStats());
9191
return;
9292
};
9393
std::move(future).via(runner).thenValue(cb).thenError(error);
@@ -98,19 +98,18 @@ void DeleteVertexExecutor::deleteVertex() {
9898
auto *runner = ectx()->rctx()->runner();
9999
auto cb = [this] (auto &&resp) {
100100
if (!resp.ok()) {
101-
DCHECK(onError_);
102-
onError_(Status::Error("Internal Error"));
101+
doError(Status::Error("Internal Error"),
102+
ectx()->getGraphStats()->getDeleteVertexStats());
103103
return;
104104
}
105-
DCHECK(onFinish_);
106-
onFinish_(Executor::ProcessControl::kNext);
105+
doFinish(Executor::ProcessControl::kNext, ectx()->getGraphStats()->getDeleteVertexStats());
107106
return;
108107
};
109108

110109
auto error = [this] (auto &&e) {
111110
LOG(ERROR) << "Exception caught: " << e.what();
112-
DCHECK(onError_);
113-
onError_(Status::Error("Internal error"));
111+
doError(Status::Error("Internal Error"),
112+
ectx()->getGraphStats()->getDeleteVertexStats());
114113
return;
115114
};
116115
std::move(future).via(runner).thenValue(cb).thenError(error);

‎src/graph/ExecutionContext.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "base/Base.h"
1111
#include "cpp/helpers.h"
1212
#include "graph/RequestContext.h"
13+
#include "graph/GraphStats.h"
1314
#include "parser/SequentialSentences.h"
1415
#include "meta/SchemaManager.h"
1516
#include "meta/ClientBasedGflagsManager.h"
@@ -33,13 +34,15 @@ class ExecutionContext final : public cpp::NonCopyable, public cpp::NonMovable {
3334
meta::SchemaManager *sm,
3435
meta::ClientBasedGflagsManager *gflagsManager,
3536
storage::StorageClient *storage,
36-
meta::MetaClient *metaClient) {
37+
meta::MetaClient *metaClient,
38+
GraphStats* stats) {
3739
rctx_ = std::move(rctx);
3840
sm_ = sm;
3941
gflagsManager_ = gflagsManager;
4042
storageClient_ = storage;
4143
metaClient_ = metaClient;
4244
variableHolder_ = std::make_unique<VariableHolder>();
45+
stats_ = stats;
4346
}
4447

4548
~ExecutionContext();
@@ -68,13 +71,18 @@ class ExecutionContext final : public cpp::NonCopyable, public cpp::NonMovable {
6871
return metaClient_;
6972
}
7073

74+
GraphStats* getGraphStats() const {
75+
return stats_;
76+
}
77+
7178
private:
7279
RequestContextPtr rctx_;
7380
meta::SchemaManager *sm_{nullptr};
7481
meta::ClientBasedGflagsManager *gflagsManager_{nullptr};
7582
storage::StorageClient *storageClient_{nullptr};
7683
meta::MetaClient *metaClient_{nullptr};
7784
std::unique_ptr<VariableHolder> variableHolder_;
85+
GraphStats *stats_{nullptr};
7886
};
7987

8088
} // namespace graph

‎src/graph/ExecutionEngine.cpp‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,15 @@ Status ExecutionEngine::init(std::shared_ptr<folly::IOThreadPoolExecutor> ioExec
2828
if (!addrs.ok()) {
2929
return addrs.status();
3030
}
31-
metaClient_ = std::make_unique<meta::MetaClient>(ioExecutor, std::move(addrs.value()));
31+
32+
stats_ = std::make_unique<GraphStats>();
33+
34+
metaClient_ = std::make_unique<meta::MetaClient>(ioExecutor,
35+
std::move(addrs.value()),
36+
HostAddr(0, 0),
37+
0,
38+
false,
39+
stats_->getMetaClientStats());
3240
// load data try 3 time
3341
bool loadDataOk = metaClient_->waitForMetadReady(3);
3442
if (!loadDataOk) {
@@ -41,7 +49,9 @@ Status ExecutionEngine::init(std::shared_ptr<folly::IOThreadPoolExecutor> ioExec
4149

4250
gflagsManager_ = std::make_unique<meta::ClientBasedGflagsManager>(metaClient_.get());
4351

44-
storage_ = std::make_unique<storage::StorageClient>(ioExecutor, metaClient_.get());
52+
storage_ = std::make_unique<storage::StorageClient>(ioExecutor,
53+
metaClient_.get(),
54+
stats_->getStorageClientStats());
4555
return Status::OK();
4656
}
4757

@@ -50,7 +60,8 @@ void ExecutionEngine::execute(RequestContextPtr rctx) {
5060
schemaManager_.get(),
5161
gflagsManager_.get(),
5262
storage_.get(),
53-
metaClient_.get());
63+
metaClient_.get(),
64+
stats_.get());
5465
// TODO(dutor) add support to plan cache
5566
auto plan = new ExecutionPlan(std::move(ectx));
5667

‎src/graph/ExecutionEngine.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "meta/ClientBasedGflagsManager.h"
1616
#include "meta/client/MetaClient.h"
1717
#include "network/NetworkUtils.h"
18+
#include "graph/GraphStats.h"
1819
#include <folly/executors/IOThreadPoolExecutor.h>
1920

2021
/**
@@ -45,6 +46,7 @@ class ExecutionEngine final : public cpp::NonCopyable, public cpp::NonMovable {
4546
std::unique_ptr<meta::ClientBasedGflagsManager> gflagsManager_;
4647
std::unique_ptr<storage::StorageClient> storage_;
4748
std::unique_ptr<meta::MetaClient> metaClient_;
49+
std::unique_ptr<GraphStats> stats_;
4850
};
4951

5052
} // namespace graph

0 commit comments

Comments
 (0)