From 7334e38ea2029f1901d040b642d82b83d6af1d1f Mon Sep 17 00:00:00 2001 From: XieHan Date: Sun, 21 Feb 2021 01:58:36 +0800 Subject: [PATCH 01/75] add CommChannel --- src/kernel/Communicator.cc | 328 +++++++++++++++++++++++++++++++++---- src/kernel/Communicator.h | 28 +++- 2 files changed, 318 insertions(+), 38 deletions(-) diff --git a/src/kernel/Communicator.cc b/src/kernel/Communicator.cc index 388909a052..934d59e699 100644 --- a/src/kernel/Communicator.cc +++ b/src/kernel/Communicator.cc @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -43,6 +42,7 @@ struct CommConnEntry CommConnection *conn; long long seq; int sockfd; + short channel_flag; #define CONN_STATE_CONNECTING 0 #define CONN_STATE_CONNECTED 1 #define CONN_STATE_RECEIVING 2 @@ -51,7 +51,7 @@ struct CommConnEntry #define CONN_STATE_KEEPALIVE 5 #define CONN_STATE_CLOSING 6 #define CONN_STATE_ERROR 7 - int state; + short state; int error; int ref; struct iovec *write_iov; @@ -119,6 +119,45 @@ static int __create_ssl(SSL_CTX *ssl_ctx, struct CommConnEntry *entry) return -1; } +#ifndef IOV_MAX +# ifdef UIO_MAXIOV +# define IOV_MAX UIO_MAXIOV +# else +# define IOV_MAX 1024 +# endif +#endif + +static int __send_vectors(struct iovec vectors[], int cnt, + struct CommConnEntry *entry) +{ + ssize_t n; + int i; + + while (cnt > 0) + { + n = writev(entry->sockfd, vectors, cnt <= IOV_MAX ? cnt : IOV_MAX); + if (n < 0) + return errno == EAGAIN ? cnt : -1; + + for (i = 0; i < cnt; i++) + { + if ((size_t)n >= vectors[i].iov_len) + n -= vectors[i].iov_len; + else + { + vectors[i].iov_base = (char *)vectors[i].iov_base + n; + vectors[i].iov_len -= n; + break; + } + } + + vectors += i; + cnt -= i; + } + + return 0; +} + int CommTarget::init(const struct sockaddr *addr, socklen_t addrlen, int connect_timeout, int response_timeout) { @@ -275,6 +314,11 @@ class CommServiceTarget : public CommTarget friend class Communicator; }; +CommConnection *CommSession::get_connection() const +{ + return this->entry ? entry->conn : NULL; +} + CommSession::~CommSession() { struct CommConnEntry *entry; @@ -304,6 +348,21 @@ CommSession::~CommSession() ((CommServiceTarget *)target)->decref(); } +class CommMessageOutEmpty : public CommMessageOut +{ +private: + virtual int encode(struct iovec *vectors, int max) + { + return 0; + } +}; + +CommMessageOut *CommChannel::message_out() +{ + static CommMessageOutEmpty empty; + return ∅ +} + inline int Communicator::first_timeout(CommSession *session) { int timeout = session->target->response_timeout; @@ -389,30 +448,10 @@ int Communicator::send_message_sync(struct iovec vectors[], int cnt, CommSession *session = entry->session; CommService *service; int timeout; - ssize_t n; - int i; - - while (cnt > 0) - { - n = writev(entry->sockfd, vectors, cnt <= IOV_MAX ? cnt : IOV_MAX); - if (n < 0) - return errno == EAGAIN ? cnt : -1; - - for (i = 0; i < cnt; i++) - { - if ((size_t)n >= vectors[i].iov_len) - n -= vectors[i].iov_len; - else - { - vectors[i].iov_base = (char *)vectors[i].iov_base + n; - vectors[i].iov_len -= n; - break; - } - } - vectors += i; - cnt -= i; - } + cnt = __send_vectors(vectors, cnt, entry); + if (cnt != 0) + return cnt; service = entry->service; if (service) @@ -529,7 +568,7 @@ int Communicator::send_message(struct CommConnEntry *entry) } end = vectors + cnt; - if (!entry->ssl) + if (!entry->ssl || cnt == 0) { cnt = this->send_message_sync(vectors, cnt, entry); if (cnt <= 0) @@ -837,12 +876,10 @@ struct CommConnEntry *Communicator::accept_conn(CommServiceTarget *target, CommService *service) { struct CommConnEntry *entry; - size_t size; if (__set_fd_nonblock(target->sockfd) >= 0) { - size = offsetof(struct CommConnEntry, mutex); - entry = (struct CommConnEntry *)malloc(size); + entry = (struct CommConnEntry *)malloc(sizeof (struct CommConnEntry)); if (entry) { entry->conn = service->new_connection(target->sockfd); @@ -855,6 +892,7 @@ struct CommConnEntry *Communicator::accept_conn(CommServiceTarget *target, entry->ssl = NULL; entry->sockfd = target->sockfd; entry->state = CONN_STATE_CONNECTED; + entry->channel_flag = 0; entry->ref = 1; return entry; } @@ -996,7 +1034,9 @@ void Communicator::handle_connect_result(struct poller_result *res) target->release(); session->handle(state, res->error); - this->release_conn(entry); + if (__sync_sub_and_fetch(&entry->ref, 1) == 0) + this->release_conn(entry); + break; } } @@ -1185,7 +1225,7 @@ int Communicator::create_service_session(struct CommConnEntry *entry) session->passive = 1; entry->session = session; session->target = target; - session->conn = entry->conn; + session->entry = entry; session->seq = entry->seq++; session->out = NULL; session->in = NULL; @@ -1257,7 +1297,117 @@ int Communicator::partial_written(size_t n, void *context) void Communicator::callback(struct poller_result *res, void *context) { Communicator *comm = (Communicator *)context; - msgqueue_put(res, comm->queue); + struct CommConnEntry *entry; + CommSession *session; + int state; + + switch (res->data.operation) + { + case PD_OP_READ: + case PD_OP_WRITE: + case PD_OP_CONNECT: + case PD_OP_SSL_CONNECT: + entry = (struct CommConnEntry *)res->data.context; + if (entry->channel_flag) + break; + + default: + msgqueue_put(res, comm->queue); + return; + } + + session = entry->session; + if (entry->state == CONN_STATE_CONNECTING) + { + comm->handle_connect_result(res); + if (entry->ref == 1) + comm->release_conn(entry); + else if (entry->state == CONN_STATE_RECEIVING) + ((CommChannel *)session)->handle_established(); + } + else if (entry->state == CONN_STATE_SUCCESS) + { + ((CommChannel *)session)->handle_in(session->in); + session->in = NULL; + if (session->timeout == 0) + entry->state = CONN_STATE_CLOSING; + else + { + session->timeout = -1; + session->begin_time.tv_nsec = -1; + entry->state = CONN_STATE_RECEIVING; + } + } + else + { + if (res->data.operation == PD_OP_WRITE) + { + close(entry->sockfd); + free(entry->write_iov); + switch (res->state) + { + case PR_ST_FINISHED: + state = CS_STATE_SUCCESS; + break; + case PR_ST_ERROR: + state = CS_STATE_ERROR; + break; + case PR_ST_DELETED: + case PR_ST_STOPPED: + state = CS_STATE_STOPPED; + break; + } + + free(entry); + entry = session->entry; + session->handle(state, res->error); + session = entry->session; + } + else + { + switch (res->state) + { + case PR_ST_FINISHED: + if (session->in) + { + res->error = ECONNRESET; + case PR_ST_ERROR: + state = CS_STATE_ERROR; + } + else + state = CS_STATE_SUCCESS; + + break; + + case PR_ST_DELETED: + if (entry->state == CONN_STATE_CLOSING) + state = CS_STATE_SUCCESS; + else + case PR_ST_STOPPED: + state = CS_STATE_STOPPED; + + break; + } + + if (entry->channel_flag == 1) + { + ((CommChannel *)session)->handle_terminated(); + entry->state = state; + entry->error = res->error; + } + else + state = CS_STATE_SHUTDOWN; + } + + if (__sync_sub_and_fetch(&entry->ref, 1) == 0) + { + entry->target->release(); + session->handle(state, res->error); + comm->release_conn(entry); + } + } + + free(res); } void *Communicator::accept(const struct sockaddr *addr, socklen_t addrlen, @@ -1341,7 +1491,7 @@ int Communicator::create_poller(size_t poller_threads) int Communicator::init(size_t poller_threads, size_t handler_threads) { - if (poller_threads == 0) + if (poller_threads == 0 || handler_threads == 0) { errno = EINVAL; return -1; @@ -1421,6 +1571,7 @@ struct CommConnEntry *Communicator::launch_conn(CommSession *session, entry->ssl = NULL; entry->sockfd = sockfd; entry->state = CONN_STATE_CONNECTING; + entry->channel_flag = 0; entry->ref = 1; return entry; } @@ -1471,7 +1622,7 @@ int Communicator::request_idle_conn(CommSession *session, CommTarget *target) if (entry) { entry->session = session; - session->conn = entry->conn; + session->entry = entry; session->seq = entry->seq++; session->out = session->message_out(); if (session->out) @@ -1514,7 +1665,7 @@ int Communicator::request(CommSession *session, CommTarget *target) entry = this->launch_conn(session, target); if (entry) { - session->conn = entry->conn; + session->entry = entry; session->seq = entry->seq++; data.operation = PD_OP_CONNECT; data.fd = entry->sockfd; @@ -1526,7 +1677,7 @@ int Communicator::request(CommSession *session, CommTarget *target) this->release_conn(entry); } - session->conn = NULL; + session->entry = NULL; session->seq = 0; return -1; } @@ -1657,6 +1808,111 @@ int Communicator::reply(CommSession *session) return 0; } +int Communicator::establish(CommChannel *channel, CommTarget *target) +{ + struct CommConnEntry *entry; + struct poller_data data; + + entry = this->launch_conn(channel, target); + if (entry) + { + entry->channel_flag = 1; + entry->ref++; + + channel->target = target; + channel->entry = entry; + channel->out = NULL; + channel->in = NULL; + channel->seq = 0; + + data.operation = PD_OP_CONNECT; + data.fd = entry->sockfd; + data.ssl = NULL; + data.context = entry; + if (mpoller_add(&data, target->connect_timeout, this->mpoller) >= 0) + return 0; + + this->release_conn(entry); + } + + return -1; +} + +void Communicator::prep_send(CommSession *session, CommChannel *channel) +{ + session->target = channel->target; + session->entry = channel->entry; + session->in = NULL; + session->seq = 0; +} + +int Communicator::send(CommMessageOut *msg, CommSession *session, + CommChannel *channel) +{ + struct CommConnEntry *entry = channel->entry; + struct iovec vectors[ENCODE_IOV_MAX]; + struct iovec *end; + int cnt; + + session->out = msg; + cnt = msg->encode(vectors, ENCODE_IOV_MAX); + if ((unsigned int)cnt > ENCODE_IOV_MAX) + { + if (cnt > ENCODE_IOV_MAX) + errno = EOVERFLOW; + return -1; + } + + end = vectors + cnt; + if (!entry->ssl || cnt == 0) + { + cnt = __send_vectors(vectors, cnt, entry); + if (cnt <= 0) + return cnt; + } + + entry = (struct CommConnEntry *)malloc(sizeof (struct CommConnEntry)); + if (entry) + { + entry->sockfd = dup(channel->entry->sockfd); + if (entry->sockfd >= 0) + { + entry->session = session; + entry->ssl = channel->entry->ssl; + entry->state = CONN_STATE_KEEPALIVE; + entry->channel_flag = 1; + if (this->send_message_async(end - cnt, cnt, entry) >= 0) + { + __sync_add_and_fetch(&channel->entry->ref, 1); + return 1; + } + + close(entry->sockfd); + } + + free(entry); + } + + return -1; +} + +void Communicator::shutdown(CommChannel *channel) +{ + struct CommConnEntry *entry = channel->entry; + int errno_bak = errno; + + entry->channel_flag = 2; + mpoller_del(entry->sockfd, this->mpoller); + if (__sync_sub_and_fetch(&entry->ref, 1) == 0) + { + entry->target->release(); + channel->handle(entry->state, entry->error); + this->release_conn(entry); + } + + errno = errno_bak; +} + int Communicator::sleep(SleepSession *session) { struct timespec value; diff --git a/src/kernel/Communicator.h b/src/kernel/Communicator.h index 32339c5b78..1254116ca6 100644 --- a/src/kernel/Communicator.h +++ b/src/kernel/Communicator.h @@ -119,6 +119,7 @@ class CommMessageIn : private poller_message_t #define CS_STATE_ERROR 1 #define CS_STATE_STOPPED 2 #define CS_STATE_TOREPLY 3 /* for service session only. */ +#define CS_STATE_SHUTDOWN 4 /* for channel only */ class CommSession { @@ -133,14 +134,14 @@ class CommSession protected: CommTarget *get_target() const { return this->target; } - CommConnection *get_connection() const { return this->conn; } CommMessageOut *get_message_out() const { return this->out; } CommMessageIn *get_message_in() const { return this->in; } long long get_seq() const { return this->seq; } + CommConnection *get_connection() const; private: CommTarget *target; - CommConnection *conn; + struct CommConnEntry *entry; CommMessageOut *out; CommMessageIn *in; long long seq; @@ -229,6 +230,24 @@ class CommService friend class Communicator; }; +class CommChannel : public CommSession +{ +private: + virtual CommMessageIn *message_in() = 0; + virtual int keep_alive_timeout() { return -1; } + virtual int first_timeout() { return -1; } + virtual void handle_established() = 0; + virtual void handle_in(CommMessageIn *msg) = 0; + virtual void handle_terminated() { } + virtual void handle(int state, int error) = 0; + +private: + virtual CommMessageOut *message_out(); /* final */ + virtual CommMessageOut *get_message_out() { return NULL; } /* deleted */ + + friend class Communicator; +}; + #define SS_STATE_COMPLETE 0 #define SS_STATE_ERROR 1 #define SS_STATE_DISRUPTED 2 @@ -262,6 +281,11 @@ class Communicator int bind(CommService *service); void unbind(CommService *service); + int establish(CommChannel *channel, CommTarget *target); + void prep_send(CommSession *session, CommChannel *channel); + int send(CommMessageOut *msg, CommSession *session, CommChannel *channel); + void shutdown(CommChannel *channel); + int sleep(SleepSession *session); int io_bind(IOService *service); From c5453f34f76cb0413ce9dac2b4bb150f514b29c6 Mon Sep 17 00:00:00 2001 From: XieHan Date: Tue, 30 Mar 2021 18:22:55 +0800 Subject: [PATCH 02/75] update code --- src/kernel/Communicator.cc | 50 +++++++++++++++++++++----------------- src/kernel/Communicator.h | 24 ++++++++++++++---- tutorial/CMakeLists.txt | 1 + 3 files changed, 48 insertions(+), 27 deletions(-) diff --git a/src/kernel/Communicator.cc b/src/kernel/Communicator.cc index 934d59e699..7e4843efe0 100644 --- a/src/kernel/Communicator.cc +++ b/src/kernel/Communicator.cc @@ -314,11 +314,6 @@ class CommServiceTarget : public CommTarget friend class Communicator; }; -CommConnection *CommSession::get_connection() const -{ - return this->entry ? entry->conn : NULL; -} - CommSession::~CommSession() { struct CommConnEntry *entry; @@ -363,6 +358,18 @@ CommMessageOut *CommChannel::message_out() return ∅ } +CommMessageOut *CommSessionOut::message_out() +{ + errno = EPERM; + return NULL; +} + +CommMessageIn *CommSessionOut::message_in() +{ + errno = EPERM; + return NULL; +} + inline int Communicator::first_timeout(CommSession *session) { int timeout = session->target->response_timeout; @@ -1225,7 +1232,7 @@ int Communicator::create_service_session(struct CommConnEntry *entry) session->passive = 1; entry->session = session; session->target = target; - session->entry = entry; + session->conn = entry->conn; session->seq = entry->seq++; session->out = NULL; session->in = NULL; @@ -1359,7 +1366,7 @@ void Communicator::callback(struct poller_result *res, void *context) } free(entry); - entry = session->entry; + entry = ((CommSessionOut *)session)->entry; session->handle(state, res->error); session = entry->session; } @@ -1622,7 +1629,7 @@ int Communicator::request_idle_conn(CommSession *session, CommTarget *target) if (entry) { entry->session = session; - session->entry = entry; + session->conn = entry->conn; session->seq = entry->seq++; session->out = session->message_out(); if (session->out) @@ -1665,7 +1672,7 @@ int Communicator::request(CommSession *session, CommTarget *target) entry = this->launch_conn(session, target); if (entry) { - session->entry = entry; + session->conn = entry->conn; session->seq = entry->seq++; data.operation = PD_OP_CONNECT; data.fd = entry->sockfd; @@ -1677,7 +1684,7 @@ int Communicator::request(CommSession *session, CommTarget *target) this->release_conn(entry); } - session->entry = NULL; + session->conn = NULL; session->seq = 0; return -1; } @@ -1838,15 +1845,7 @@ int Communicator::establish(CommChannel *channel, CommTarget *target) return -1; } -void Communicator::prep_send(CommSession *session, CommChannel *channel) -{ - session->target = channel->target; - session->entry = channel->entry; - session->in = NULL; - session->seq = 0; -} - -int Communicator::send(CommMessageOut *msg, CommSession *session, +int Communicator::send(CommMessageOut *msg, CommSessionOut *session, CommChannel *channel) { struct CommConnEntry *entry = channel->entry; @@ -1854,6 +1853,11 @@ int Communicator::send(CommMessageOut *msg, CommSession *session, struct iovec *end; int cnt; + session->target = channel->target; + session->entry = channel->entry; + session->in = NULL; + session->seq = 0; + session->out = msg; cnt = msg->encode(vectors, ENCODE_IOV_MAX); if ((unsigned int)cnt > ENCODE_IOV_MAX) @@ -1867,8 +1871,10 @@ int Communicator::send(CommMessageOut *msg, CommSession *session, if (!entry->ssl || cnt == 0) { cnt = __send_vectors(vectors, cnt, entry); - if (cnt <= 0) - return cnt; + if (cnt == 0) + return 1; + else if (cnt < 0) + return -1; } entry = (struct CommConnEntry *)malloc(sizeof (struct CommConnEntry)); @@ -1884,7 +1890,7 @@ int Communicator::send(CommMessageOut *msg, CommSession *session, if (this->send_message_async(end - cnt, cnt, entry) >= 0) { __sync_add_and_fetch(&channel->entry->ref, 1); - return 1; + return 0; } close(entry->sockfd); diff --git a/src/kernel/Communicator.h b/src/kernel/Communicator.h index 1254116ca6..a5803e1959 100644 --- a/src/kernel/Communicator.h +++ b/src/kernel/Communicator.h @@ -134,14 +134,14 @@ class CommSession protected: CommTarget *get_target() const { return this->target; } + CommConnection *get_connection() const { return this->conn; } CommMessageOut *get_message_out() const { return this->out; } CommMessageIn *get_message_in() const { return this->in; } long long get_seq() const { return this->seq; } - CommConnection *get_connection() const; private: CommTarget *target; - struct CommConnEntry *entry; + CommConnection *conn; CommMessageOut *out; CommMessageIn *in; long long seq; @@ -243,8 +243,22 @@ class CommChannel : public CommSession private: virtual CommMessageOut *message_out(); /* final */ - virtual CommMessageOut *get_message_out() { return NULL; } /* deleted */ + CommMessageOut *get_message_out() { return NULL; } /* deleted */ + +private: + struct CommConnEntry *entry; + friend class Communicator; +}; + +class CommSessionOut : public CommSession +{ +private: + virtual CommMessageOut *message_out(); /* final */ + virtual CommMessageIn *message_in(); /* final */ + CommMessageIn *get_message_in() { return NULL; } /* deleted */ +private: + struct CommConnEntry *entry; friend class Communicator; }; @@ -282,8 +296,8 @@ class Communicator void unbind(CommService *service); int establish(CommChannel *channel, CommTarget *target); - void prep_send(CommSession *session, CommChannel *channel); - int send(CommMessageOut *msg, CommSession *session, CommChannel *channel); + int send(CommMessageOut *msg, CommSessionOut *session, + CommChannel *channel); void shutdown(CommChannel *channel); int sleep(SleepSession *session); diff --git a/tutorial/CMakeLists.txt b/tutorial/CMakeLists.txt index c3c08722a9..7415daf137 100644 --- a/tutorial/CMakeLists.txt +++ b/tutorial/CMakeLists.txt @@ -23,6 +23,7 @@ else () endif () set(TUTORIAL_LIST + test_channel tutorial-01-wget tutorial-02-redis_cli tutorial-03-wget_to_redis From 76b3bffb80597e951e4467031415bb0466e6831f Mon Sep 17 00:00:00 2001 From: XieHan Date: Tue, 30 Mar 2021 18:23:28 +0800 Subject: [PATCH 03/75] update code --- tutorial/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tutorial/CMakeLists.txt b/tutorial/CMakeLists.txt index 7415daf137..c3c08722a9 100644 --- a/tutorial/CMakeLists.txt +++ b/tutorial/CMakeLists.txt @@ -23,7 +23,6 @@ else () endif () set(TUTORIAL_LIST - test_channel tutorial-01-wget tutorial-02-redis_cli tutorial-03-wget_to_redis From a44e0af4ac57e4a858ea43da87cf7c135e1b6d7f Mon Sep 17 00:00:00 2001 From: XieHan Date: Sat, 24 Apr 2021 01:32:05 +0800 Subject: [PATCH 04/75] update Channel interface --- src/kernel/Communicator.cc | 30 +++++++++--------------------- src/kernel/Communicator.h | 25 ++++++++++++++++++------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/src/kernel/Communicator.cc b/src/kernel/Communicator.cc index ed46035af4..fb602d7afb 100644 --- a/src/kernel/Communicator.cc +++ b/src/kernel/Communicator.cc @@ -358,18 +358,6 @@ CommMessageOut *CommChannel::message_out() return ∅ } -CommMessageOut *CommSessionOut::message_out() -{ - errno = EPERM; - return NULL; -} - -CommMessageIn *CommSessionOut::message_in() -{ - errno = EPERM; - return NULL; -} - inline int Communicator::first_timeout(CommSession *session) { int timeout = session->target->response_timeout; @@ -1366,7 +1354,7 @@ void Communicator::callback(struct poller_result *res, void *context) } free(entry); - entry = ((CommSessionOut *)session)->entry; + entry = ((TransSession *)session)->channel->entry; session->handle(state, res->error); session = entry->session; } @@ -1498,7 +1486,7 @@ int Communicator::create_poller(size_t poller_threads) int Communicator::init(size_t poller_threads, size_t handler_threads) { - if (poller_threads == 0 || handler_threads == 0) + if (poller_threads == 0) { errno = EINVAL; return -1; @@ -1845,21 +1833,21 @@ int Communicator::establish(CommChannel *channel, CommTarget *target) return -1; } -int Communicator::send(CommMessageOut *msg, CommSessionOut *session, - CommChannel *channel) +int Communicator::send(TransSession *session, CommChannel *channel) { struct CommConnEntry *entry = channel->entry; struct iovec vectors[ENCODE_IOV_MAX]; struct iovec *end; int cnt; - session->target = channel->target; - session->entry = channel->entry; - session->in = NULL; + session->channel = channel; session->seq = 0; + session->in = NULL; + session->out = session->message_out(); + if (!session->out) + return -1; - session->out = msg; - cnt = msg->encode(vectors, ENCODE_IOV_MAX); + cnt = session->out->encode(vectors, ENCODE_IOV_MAX); if ((unsigned int)cnt > ENCODE_IOV_MAX) { if (cnt > ENCODE_IOV_MAX) diff --git a/src/kernel/Communicator.h b/src/kernel/Communicator.h index e3c69a53e2..586ea6db9e 100644 --- a/src/kernel/Communicator.h +++ b/src/kernel/Communicator.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -258,15 +259,26 @@ class CommChannel : public CommSession friend class Communicator; }; -class CommSessionOut : public CommSession +class TransSession : public CommSession { private: - virtual CommMessageOut *message_out(); /* final */ - virtual CommMessageIn *message_in(); /* final */ - CommMessageIn *get_message_in() { return NULL; } /* deleted */ + virtual CommMessageOut *message_out() + { + errno = ENOSYS; + return NULL; + } + + virtual CommMessageIn *message_in() + { + errno = ENOSYS; + return NULL; + } + +protected: + CommChannel *get_channel() const { return this->channel; } private: - struct CommConnEntry *entry; + CommChannel *channel; friend class Communicator; }; @@ -304,8 +316,7 @@ class Communicator void unbind(CommService *service); int establish(CommChannel *channel, CommTarget *target); - int send(CommMessageOut *msg, CommSessionOut *session, - CommChannel *channel); + int send(TransSession *session, CommChannel *channel); void shutdown(CommChannel *channel); int sleep(SleepSession *session); From 0b1f90c81f982a0b01f088c965f73393809b83e2 Mon Sep 17 00:00:00 2001 From: XieHan Date: Fri, 30 Apr 2021 16:58:27 +0800 Subject: [PATCH 05/75] fix HTTP 100 continue --- src/protocol/HttpMessage.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/protocol/HttpMessage.cc b/src/protocol/HttpMessage.cc index d9b07f1e8b..6094cde0b8 100644 --- a/src/protocol/HttpMessage.cc +++ b/src/protocol/HttpMessage.cc @@ -342,11 +342,14 @@ int HttpResponse::append(const void *buf, size_t *size) { int ret = HttpMessage::append(buf, size); - if (ret > 0 && *http_parser_get_code(this->parser) == '1') + if (ret > 0) { - http_parser_deinit(this->parser); - http_parser_init(1, this->parser); - ret = 0; + if (strcmp(http_parser_get_code(this->parser), "100") == 0) + { + http_parser_deinit(this->parser); + http_parser_init(1, this->parser); + ret = 0; + } } return ret; From 194beb21994e0620014ee056ee5535c3ca55a924 Mon Sep 17 00:00:00 2001 From: XieHan Date: Fri, 7 May 2021 17:10:52 +0800 Subject: [PATCH 06/75] add CommScheduler's Channel functions --- src/kernel/CommScheduler.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/kernel/CommScheduler.h b/src/kernel/CommScheduler.h index 62a43501fe..65d003fc75 100644 --- a/src/kernel/CommScheduler.h +++ b/src/kernel/CommScheduler.h @@ -171,6 +171,33 @@ class CommScheduler this->comm.io_unbind(service); } + /* for channels. */ + int establish(CommChannel *channel, CommSchedObject *object, + int wait_timeout, CommTarget **target) + { + int ret = -1; + + *target = object->acquire(wait_timeout); + if (*target) + { + ret = this->comm.establish(channel, *target); + if (ret < 0) + (*target)->release(); + } + + return ret; + } + + int send(TransSession *session, CommChannel *channel) + { + return this->comm.send(session, channel); + } + + void shutdown(CommChannel *channel) + { + this->comm.shutdown(channel); + } + public: int is_handler_thread() const { From 23bc4f3ee3c13555cba413c63a08cfaaa438fb55 Mon Sep 17 00:00:00 2001 From: XieHan Date: Sun, 9 May 2021 02:00:15 +0800 Subject: [PATCH 07/75] add TransRequest module --- CMakeLists_Headers.txt | 1 + src/kernel/CMakeLists.txt | 1 + src/kernel/TransRequest.cc | 59 ++++++++++++++++++++ src/kernel/TransRequest.h | 109 +++++++++++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+) create mode 100644 src/kernel/TransRequest.cc create mode 100644 src/kernel/TransRequest.h diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index eb3ffce76c..e764067b32 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -7,6 +7,7 @@ set(COMMON_KERNEL_HEADERS src/kernel/SleepRequest.h src/kernel/ExecRequest.h src/kernel/IORequest.h + src/kernel/TransRequest.h src/kernel/Executor.h src/kernel/list.h src/kernel/mpoller.h diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index 209f0ec07c..683c78295e 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -17,6 +17,7 @@ set(SRC msgqueue.c thrdpool.c CommRequest.cc + TransRequest.cc CommScheduler.cc Communicator.cc Executor.cc diff --git a/src/kernel/TransRequest.cc b/src/kernel/TransRequest.cc new file mode 100644 index 0000000000..950366b0cb --- /dev/null +++ b/src/kernel/TransRequest.cc @@ -0,0 +1,59 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Xie Han (xiehan@sogou-inc.com) +*/ + +#include +#include "CommScheduler.h" +#include "TransRequest.h" + +void ChanRequest::dispatch() +{ + if (!this->established) + { + if (this->scheduler->establish(this, this->object, this->wait_timeout, + &this->target) < 0) + { + this->state = CS_STATE_ERROR; + this->error = errno; + this->subtask_done(); + } + } + else + this->scheduler->shutdown(this); +} + +void TransRequest::dispatch() +{ + int ret = this->scheduler->send(this, this->channel); + + if (ret == 0) + return; + + if (ret > 0) + { + this->state = CS_STATE_SUCCESS; + this->error = 0; + } + else + { + this->state = CS_STATE_ERROR; + this->error = errno; + } + + this->subtask_done(); +} + diff --git a/src/kernel/TransRequest.h b/src/kernel/TransRequest.h new file mode 100644 index 0000000000..b83549c265 --- /dev/null +++ b/src/kernel/TransRequest.h @@ -0,0 +1,109 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Xie Han (xiehan@sogou-inc.com) +*/ + +#ifndef _TRANSREQUEST_H_ +#define _TRANSREQUEST_H_ + +#include "SubTask.h" +#include "Communicator.h" +#include "CommScheduler.h" + +class ChanRequest : public SubTask, public CommChannel +{ +public: + ChanRequest(CommSchedObject *object, CommScheduler *scheduler) + { + this->scheduler = scheduler; + this->object = object; + this->wait_timeout = 0; + this->established = 0; + } + + CommSchedObject *get_request_object() const { return this->object; } + void set_request_object(CommSchedObject *object) { this->object = object; } + int get_wait_timeout() const { return this->wait_timeout; } + void set_wait_timeout(int timeout) { this->wait_timeout = timeout; } + +public: + virtual void dispatch(); + +protected: + int state; + int error; + +protected: + CommTarget *target; + +protected: + int established; + int wait_timeout; + CommSchedObject *object; + CommScheduler *scheduler; + +protected: + virtual void handle_established() + { + this->state = CS_STATE_SUCCESS; + this->error = 0; + this->established = 1; + this->subtask_done(); + } + + virtual void handle(int state, int error) + { + this->state = state; + this->error = error; + this->established = 0; + this->subtask_done(); + } +}; + +class TransRequest : public SubTask, public TransSession +{ +public: + TransRequest(CommChannel *channel, CommScheduler *scheduler) + { + this->scheduler = scheduler; + this->channel = channel; + } + + CommChannel *get_request_channel() const { return this->channel; } + void set_request_channel(CommChannel *channel) { this->channel = channel; } + +public: + virtual void dispatch(); + +protected: + int state; + int error; + +protected: + CommChannel *channel; + CommScheduler *scheduler; + +protected: + virtual void handle(int state, int error) + { + this->state = state; + this->error = error; + this->subtask_done(); + } +}; + +#endif + From 92787e6061f2554368efbaccced02e73477f87eb Mon Sep 17 00:00:00 2001 From: XieHan Date: Wed, 26 May 2021 16:17:11 +0800 Subject: [PATCH 08/75] fix channel send --- src/kernel/Communicator.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/kernel/Communicator.cc b/src/kernel/Communicator.cc index fb602d7afb..a2b8fce38b 100644 --- a/src/kernel/Communicator.cc +++ b/src/kernel/Communicator.cc @@ -1841,6 +1841,7 @@ int Communicator::send(TransSession *session, CommChannel *channel) int cnt; session->channel = channel; + session->target = channel->target; session->seq = 0; session->in = NULL; session->out = session->message_out(); From 14b0e65eef2fe388b680ea57927fcc9d63370d08 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Wed, 9 Jun 2021 20:53:49 +0800 Subject: [PATCH 09/75] add WebSocket --- CMakeLists_Headers.txt | 6 + docs/tutorial-14-websocket_cli.md | 220 ++++++++++++ src/client/WFWebSocketClient.h | 114 ++++++ src/factory/CMakeLists.txt | 2 +- src/factory/WFChannel.h | 97 ++++++ src/factory/WFChannel.inl | 485 ++++++++++++++++++++++++++ src/factory/WebSocketTaskImpl.cc | 225 ++++++++++++ src/manager/CMakeLists.txt | 1 + src/manager/WFCondition.cc | 203 +++++++++++ src/manager/WFCondition.h | 49 +++ src/manager/WFGlobal.cc | 18 +- src/protocol/CMakeLists.txt | 5 +- src/protocol/WebSocketMessage.cc | 299 ++++++++++++++++ src/protocol/WebSocketMessage.h | 84 +++++ src/protocol/websocket_parser.c | 239 +++++++++++++ src/protocol/websocket_parser.h | 123 +++++++ tutorial/CMakeLists.txt | 1 + tutorial/tutorial-14-websocket_cli.cc | 84 +++++ 18 files changed, 2241 insertions(+), 14 deletions(-) create mode 100644 docs/tutorial-14-websocket_cli.md create mode 100644 src/client/WFWebSocketClient.h create mode 100644 src/factory/WFChannel.h create mode 100644 src/factory/WFChannel.inl create mode 100644 src/factory/WebSocketTaskImpl.cc create mode 100644 src/manager/WFCondition.cc create mode 100644 src/manager/WFCondition.h create mode 100644 src/protocol/WebSocketMessage.cc create mode 100644 src/protocol/WebSocketMessage.h create mode 100644 src/protocol/websocket_parser.c create mode 100644 src/protocol/websocket_parser.h create mode 100644 tutorial/tutorial-14-websocket_cli.cc diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index 25cf9e891a..8008699f71 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -55,11 +55,14 @@ set(INCLUDE_HEADERS src/protocol/mysql_types.h src/protocol/mysql_byteorder.h src/protocol/SSLWrapper.h + src/protocol/WebSocketMessage.h + src/protocol/websocket_parser.h src/server/WFServer.h src/server/WFHttpServer.h src/server/WFRedisServer.h src/server/WFMySQLServer.h src/client/WFMySQLConnection.h + src/client/WFWebSocketClient.h src/manager/DNSCache.h src/manager/WFGlobal.h src/manager/UpstreamManager.h @@ -68,6 +71,7 @@ set(INCLUDE_HEADERS src/manager/WFFuture.h src/manager/WFFacilities.h src/manager/WFFacilities.inl + src/manager/WFCondition.h src/util/EncodeStream.h src/util/LRUCache.h src/util/StringUtil.h @@ -84,6 +88,8 @@ set(INCLUDE_HEADERS src/factory/WFAlgoTaskFactory.inl src/factory/Workflow.h src/factory/WFOperator.h + src/factory/WFChannel.h + src/factory/WFChannel.inl src/nameservice/WFNameService.h src/nameservice/WFDNSResolver.h src/nameservice/WFServiceGovernance.h diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md new file mode 100644 index 0000000000..4936d33968 --- /dev/null +++ b/docs/tutorial-14-websocket_cli.md @@ -0,0 +1,220 @@ +# Workflow第一个双工通信客户端：websocket_cli + +# 示例代码 + +[tutorial-14-websocket_cli.cc](/tutorial/tutorial-14-websocket_cli.cc) + +# 关于websocket_cli + +这是一个可以收发**WebSocket**协议的client，也是第一个在**Workflow**上实现的双工通信协议。 + +运行方式：**./websocket_cli \** + +URL格式：**ws://host:port** + +- port缺省值为80； +- 如果是ssl，URL格式为：wss://host:port +- ssl的port缺省值为443； + +# 创建并启动WebSocket任务 + +首先需要创建一个``WebSocketClient``对象，并且通过``init()``函数进行初始化。具体接口参考：[WFWebSocketClient.h](/src/client/WFWebSocketClient.h) + +```cpp +WebSocketClient client(process); +client.init(URL); +... +``` + +构造client时需要传入``process()``函数，这是我们收消息的处理函数，类型为：``std::function``，与**Workflow**其他``process()``的语义类似。 + +``init()``中需要传入**URL**，如果**UR**L非法，则``init()``会返回-1表示**URL**解析失败。 + +然后就可以通过client创建任务了，用法与**Workflow**其他协议类似： + +```cpp +WFWebSocketTask *task = client.create_websocket_task(callback); +WebSocketFrame *msg = text_task->get_msg(); +msg->set_text_data("This is Workflow websocket client."); +task->start(); +``` + +我们通过``create_websocket_task()``的接口创建一个往外发数据的任务，并传入回调函数**callback**。task可以通过``get_msg()``接口拿到要发送的消息体，我们往消息体里填入要发送的数据。因为**WebSocket**协议支持**文本**和**二进制**的数据，所以我们使用``set_text_data()``接口，表示传入的是文本消息。最后把任务**start**起来。task相关的具体接口可以查看：[WFChannel.h](/src/factory/WFChannel.h) + +```cpp +using WFWebSocketTask = WFChannelTask; +using websocket_callback_t = std::function; +using websocket_process_t = std::function; +``` + +# 发消息 + +**WebSocket**的消息体为``WebSocketFrame``，可以在[WebSocketMessage.h](/src/protocol/WebSocketMessage.h)查看到具体接口： + +```cpp +class WebSocketFrame : public ProtocolMessage +{ +public: + bool set_opcode(int opcode); + int get_opcode() const; + + void set_masking_key(uint32_t masking_key); + + bool set_text_data(const char *data); + bool set_text_data(const char *data, size_t size, bool fin); + + bool set_binary_data(const char *data, size_t size); + bool set_binary_data(const char *data, size_t size, bool fin); + + bool get_data(const char **data, size_t *size) const; + + bool finished() const; + + ... +}; +``` +#### 1. opcode + +协议中几种我们会接触到的数据包类型： +- WebSocketFrameText +- WebSocketFrameBinary +- WebSocketFramePing +- WebSocketFramePong + +一般我们发送数据的时候用的是前两种（**文本**和**二进制**），无需手动指定**opcode**，而如果需要手动发送**PING**包时则需要通过``bool set_opcode(int opcode)``指定为**PING**包；发送完PING包之后根据**WebSocket**协议，对方会给我们回一个**PONG**包，我们会在``process()``里拿到。 + +#### 2. masking_key + +**WebSocket**协议的文本和二进制数据都需要经过一个掩码加密，用户可以不填，也可以通过``void set_masking_key(uint32_t masking_key)``手动指定； + +#### 3. data + +数据包括两种： +- **文本**，通过``set_text_data()``这类接口设置； +- **二进制**，通过``set_binary_data()``这类接口设置； + +注意这些均为**非拷贝接口**，消息在发出之前需要用户来保证data在内存的生命周期； + +这两类接口都有一个带``bool fin``参数的接口，表示本消息是否finish。因为**WebSocket**协议的数据包允许分段传输，如果你要发送一个完整的消息想分多次发送，则可以使用带``bool fin``的接口，并且把``fin``值设置为``false``。 + +#### 4. callback + +只要消息发送完毕，就会回到我们创建task时传入的回调函数，此时我们得知消息是否发送成功。 + +一个简单的例子： + +```cpp +void send_callback(WFWebSocketTask *task) +{ + if (task->get_state() != WFT_STATE_SUCCESS) + fprintf(stderr, "Task send error: %d\n", task->get_error()); +} +``` + +# 收消息 + +每次收到server发来的消息，``process()``都会被调起。如何在收到文本数据之后把数据打印出来？ + +一个简单的例子： + +```cpp +void process(WFWebSocketTask *task) +{ + const char *data; + size_t size; + + if (task->get_msg()->get_opcode() == WebSocketFrameText) + { + task->get_msg()->get_data(&data, &size); + ... + } +} +``` + +#### 1. 参数 + +``process()``函数里拿到的参数``WFWebSocketTask *task``，与callback回调函数里拿到的类型是一样的，因此用法也非常类似： + +- 可以通过``get_msg()``拿到对应的数据，也就是上述的``WebSocketFrame``； +- 可以通过msg上的接口``get_opcode()``判断是什么类型的数据包，``process()``可能收到的数据包类型包括：**WebSocketFrameText**、**WebSocketFrameBinary**、**WebSocketFramePong**； + +#### 2. data + +无论是**文本**还是**二进制**，都由``bool get_data(const char **data, size_t *size) const``拿收到的数据。 + +#### 3. fin + +由于数据可以分段发送，因此我们可以通过``bool finished() const``判断该完整的消息是否结束。如果没有结束，则用户需要自行把data里的数据拷走，等消息结束之后进行完整消息的处理。 + +更多接口细节可以查看[websocket_parser.h ](/src/protocol/websocket_parser.h) + +# 关闭client + +根据**WebSocket**协议，用户需要发起一个close包已告诉对方以示断开连接。 + +一个简单的例子： + +```cpp +WFFacilities::WaitGroup wg(1); + +WFWebSocketTask *task = client.create_close_task([&wg](WFWebSocketTask *task) { + wg.done(); +}); + +task->start(); +wait_group.wait(); +``` + +这里发起了一个close任务，由于close是异步的，因此在``task->start()``之后当前线程会退出，我们在当前线程结合一个了``wait_group``进行不占线程的阻塞，并在close任务的回调函数里唤醒，然后当前线程就可以安全删除client实例和退出了。 + +需要注意的是，如果不主动发起close任务，直接删除client实例，那么底层使用的那个网络连接还会存在，直到超时或其他原因断开。 + +# websocket_cli的参数 + +``WebSocketClient``的构造函数有两个，除了刚才介绍的传入``process()``函数的接口以外，还可以传入client的参数： + +```cpp +class WebSocketClient +{ +public: + WebSocketClient(const struct WFWebSocketParams *params, + websocket_process_t process); + WebSocketClient(websocket_process_t process); + ... +``` + +其中，参数的定义如下： + +```cpp +struct WFWebSocketParams +{ + int idle_timeout; // client保持长连接的空闲时间，超过idle_timeout没有数据过来会自动断开。默认：10s + int ping_interval; // client自动发ping的时间间隔，用于做心跳，保持与远端的连接。默认：-1，不自动发ping(功能开发中) + size_t size_limit; // 每个数据包的大小限制，超过的话会拿到错误码1009(WSStatusCodeTooLarge)。默认：不限制 + bool random_masking_key; // WebSocket协议中数据包的掩码，框架帮每次自动随机生成一个。默认：不自动生成(功能开发中) +}; +``` + +如果不传入参数，会使用默认参数来构造client。 + +# 进阶版：注意事项！ + +#### 1. 与Workflow原有用法的差异 + +由于**WebSocket**协议是**Workflow**中首个实现的双工通信协议，因此有些差异是必须强调的： + +1. **WebSocket**协议的收发都是使用**poller**线程，因此websocket_cli用户需要把**poller**线程数改大点，参考：[about-config.md](/docs/about-config.md) +2. **process**函数中所有的消息都是由同一个线程串行执行的； +3. 回调函数**callback**的执行不一定在同一个线程； + +#### 2. 时序性保证 + +[**发消息**] + +消息发送顺序取决于发送任务调起的时机是否顺序，因此用户可以把要发送的任务串到一个series里做串行的保证，也可以在上一个任务的callback里发起一下一个任务。 + +但如果没有顺序发送的保证，那么往外发的``WFWebSocketTask``也可以被放到任何一个任务流图里，随具体业务逻辑顺序调起。 + +[**收消息**] + +用于收消息的``process()``函数是保证被按收包顺序调起的，且保证前一个消息的process()执行完毕，下一个process才会调起。 diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h new file mode 100644 index 0000000000..89df518d06 --- /dev/null +++ b/src/client/WFWebSocketClient.h @@ -0,0 +1,114 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WFWEBSOCKETCLIENT_H_ +#define _WFWEBSOCKETCLIENT_H_ + +#include +#include +#include "WFGlobal.h" +#include "HttpUtil.h" +#include "HttpMessage.h" +#include "WFChannel.h" +#include "WebSocketMessage.h" + +struct WFWebSocketParams +{ + int idle_timeout; + int ping_interval; + size_t size_limit; + bool random_masking_key; +}; + +static constexpr struct WFWebSocketParams WEBSOCKET_PARAMS_DEFAULT = +{ + .idle_timeout = WS_HANDSHAKE_TIMEOUT, + .ping_interval = -1, + .size_limit = (size_t)-1, + .random_masking_key = false, +}; + +class WebSocketClient +{ +public: + int init(const std::string& url); + WFWebSocketTask *create_websocket_task(websocket_callback_t cb); + WFWebSocketTask *create_close_task(websocket_callback_t cb); + +private: + ComplexWebSocketChannel *channel; + struct WFWebSocketParams params; + +public: + WebSocketClient(const struct WFWebSocketParams *params, + websocket_process_t process); + WebSocketClient(websocket_process_t process) : + WebSocketClient(&WEBSOCKET_PARAMS_DEFAULT, std::move(process)) + { } +}; + +inline WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) +{ + return new ComplexWebSocketOutTask(this->channel, WFGlobal::get_scheduler(), + std::move(cb)); +} + +inline int WebSocketClient::init(const std::string& url) +{ + ParsedURI uri; + if (URIParser::parse(url, uri) != 0) + return -1; + + this->channel->set_uri(uri); + return 0; +} + +inline WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) +{ + ComplexWebSocketOutTask *close_task; + close_task = new ComplexWebSocketOutTask(this->channel, + WFGlobal::get_scheduler(), + std::move(cb)); + protocol::WebSocketFrame *msg = close_task->get_msg(); + msg->set_opcode(WebSocketFrameConnectionClose); + return close_task; +} + +WebSocketClient::WebSocketClient(const struct WFWebSocketParams *params, + websocket_process_t process) +{ + this->params = *params; + this->channel = new ComplexWebSocketChannel(NULL, WFGlobal::get_scheduler(), + std::move(process)); + this->channel->set_idle_timeout(this->params.idle_timeout); + this->channel->set_size_limit(this->params.size_limit); + + this->channel->set_callback([this](WFChannel *channel) + { + pthread_mutex_lock(&this->channel->mutex); + if (this->channel->is_established() == 0) + { + this->channel->set_state(WFT_STATE_SYS_ERROR); + this->channel->set_sending(false); + } + pthread_mutex_unlock(&this->channel->mutex); + }); +} + +#endif + diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index ba89c630da..5b24712792 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -6,6 +6,7 @@ set(SRC HttpTaskImpl.cc RedisTaskImpl.cc MySQLTaskImpl.cc + WebSocketTaskImpl.cc WFTaskFactory.cc Workflow.cc ) @@ -17,6 +18,5 @@ if (KAFKA STREQUAL "y") KafkaTaskImpl.cc ) add_library("factory_kafka" OBJECT ${SRC}) - set_property(SOURCE KafkaTaskImpl.cc APPEND PROPERTY COMPILE_OPTIONS "-fno-rtti") endif () diff --git a/src/factory/WFChannel.h b/src/factory/WFChannel.h new file mode 100644 index 0000000000..c54e834e07 --- /dev/null +++ b/src/factory/WFChannel.h @@ -0,0 +1,97 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) +*/ + +#ifndef _WFCHANNEL_H_ +#define _WFCHANNEL_H_ + +#include "CommScheduler.h" +#include "TransRequest.h" +#include "Workflow.h" +#include "WebSocketMessage.h" + +template +class WFChannelTask : public TransRequest +{ +public: + void start() + { + assert(!series_of(this)); + Workflow::start_series_work(this, nullptr); + } + + void dismiss() + { + assert(!series_of(this)); + delete this; + } + +public: + MSG *get_msg() + { + return &this->msg; + } + +public: + void *user_data; + +public: + int get_state() const { return this->state; } + int get_error() const { return this->error; } + + void set_callback(std::function *)> cb) + { + this->callback = std::move(cb); + } + +protected: + virtual SubTask *done() + { + SeriesWork *series = series_of(this); + + if (this->callback) + this->callback(this); + + delete this; + return series->pop(); + } + +protected: + MSG msg; + std::function *)> callback; + +public: + WFChannelTask(CommChannel *channel, CommScheduler *scheduler, + std::function *)>&& cb) : + TransRequest(channel, scheduler), + callback(std::move(cb)) + { + } + +protected: + virtual ~WFChannelTask() { } +}; + +using WFWebSocketTask = WFChannelTask; +using websocket_callback_t = std::function; +using websocket_process_t = std::function; + +#include "WFChannel.inl" + +#endif + diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl new file mode 100644 index 0000000000..fc4e2aa268 --- /dev/null +++ b/src/factory/WFChannel.inl @@ -0,0 +1,485 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) +*/ + +#include +#include "TransRequest.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFCondition.h" +#include "WFNameService.h" +#include "RouteManager.h" +#include "WFGlobal.h" +#include "EndpointParams.h" +#include "HttpUtil.h" +#include "HttpMessage.h" +#include "WFChannel.h" + +template +class WFChannelOutTask : public WFChannelTask +{ +protected: + virtual MSG *message_out() { return &this->msg; } + +public: + WFChannelOutTask(CommChannel *channel, CommScheduler *scheduler, + std::function *)>&& cb) : + WFChannelTask(channel, scheduler, std::move(cb)) + { + } + +protected: + virtual ~WFChannelOutTask() { } +}; + +template +class WFChannelInTask : public WFChannelTask +{ +protected: + virtual void dispatch() + { + this->state = WFT_STATE_SUCCESS; + this->error = 0; + this->process(this); + this->subtask_done(); + } + +public: + WFChannelInTask(CommChannel *channel, CommScheduler *scheduler, + std::function *)>& proc) : + WFChannelTask(channel, scheduler, nullptr), + process(proc) + { + } + +protected: + std::function *)>& process; + +protected: + virtual ~WFChannelInTask() { } +}; + +template +class WFChannel : public ChanRequest +{ +public: + int get_state() const { return this->state; } + int get_error() const { return this->error; } + bool is_established() const { return this->established; } + + void set_callback(std::function *)>&& cb) + { + this->callback = std::move(cb); + } + +protected: + virtual CommMessageIn *message_in() + { + this->session = this->new_session(); + return this->session->get_msg(); + } + + virtual WFChannelTask *new_session() + { + auto *task = new WFChannelInTask(this, this->scheduler, + this->process); + Workflow::create_series_work(task, nullptr); + return task; + } + + virtual void handle_in(CommMessageIn *in) + { + if (this->session) + this->session->dispatch(); + this->session = NULL; + } + + virtual SubTask *done() + { + if (this->callback) + this->callback(this); + + return series_of(this)->pop(); + } + +protected: + std::function *)> process; + std::function *)> callback; + +private: + WFChannelTask *session; + +public: + WFChannel(CommSchedObject *object, CommScheduler *scheduler, + std::function *)>&& process) : + ChanRequest(object, scheduler), + process(std::move(process)) + { + this->session = NULL; + } + +protected: + virtual ~WFChannel() { } +}; + +/**********WFComplexChannel for sequentially establish and send**********/ + +template +class WFComplexChannel : public WFChannel +{ +public: + void set_uri(const ParsedURI& uri) { this->uri = uri; } + const ParsedURI *get_uri() const { return &this->uri; } + + int get_error() const { return this->error; } + + void set_state(int state) { this->state = state; } + int get_state() const { return this->state; } + + void set_sending(bool sending) { this->sending = sending; } + bool get_sending() const { return this->sending; } + +protected: + virtual void dispatch(); + virtual SubTask *done(); + virtual void handle_terminated(); + virtual WFRouterTask *route(); + void router_callback(WFRouterTask *task); + +public: + pthread_mutex_t mutex; + WFCondition condition; + +protected: + bool sending; + WFRouterTask *router_task; + ParsedURI uri; + WFNSPolicy *ns_policy; + RouteManager::RouteResult route_result; + +public: + WFComplexChannel(CommSchedObject *object, CommScheduler *scheduler, + std::function *)>&& process) : + WFChannel(object, scheduler, std::move(process)), + mutex(PTHREAD_MUTEX_INITIALIZER) + { + this->state = WFT_STATE_UNDEFINED; + this->error = 0; + this->sending = false; + } + +protected: + virtual ~WFComplexChannel() { } +}; + +template +void WFComplexChannel::dispatch() +{ + if (this->object) + return this->WFChannel::dispatch(); + + if (this->state == WFT_STATE_UNDEFINED) + { + this->router_task = this->route(); + series_of(this)->push_front(this); + series_of(this)->push_front(this->router_task); + } + + this->subtask_done(); +} + +template +SubTask *WFComplexChannel::done() +{ + SeriesWork *series = series_of(this); + + if (this->established == 1) + { + if (this->state == WFT_STATE_SYS_ERROR) + this->ns_policy->failed(&this->route_result, NULL, this->target); + else + this->ns_policy->success(&this->route_result, NULL, this->target); + } + + if (this->router_task) + { + this->router_task = NULL; + return series->pop(); + } + + if (this->callback) + this->callback(this); + + if (this->state == WFT_STATE_SUCCESS) + this->state = WFT_STATE_UNDEFINED; + + if (this->established == 0 && this->state == WFT_STATE_SYS_ERROR) // sending == false + delete this; + + return series->pop(); +} + +template +void WFComplexChannel::handle_terminated() +{ + bool shutdown = false; + + pthread_mutex_lock(&this->mutex); + Workflow::create_series_work(this, nullptr); + + if (this->sending == false) + { + this->sending = true; + shutdown = true; + } else { + WFCounterTask *counter = this->condition.create_wait_task(nullptr); + series_of(this)->push_front(this); + series_of(this)->push_front(counter); + } + pthread_mutex_unlock(&this->mutex); + + if (shutdown == true) + this->dispatch(); +} + +template +WFRouterTask *WFComplexChannel::route() +{ + auto&& cb = std::bind(&WFComplexChannel::router_callback, + this, std::placeholders::_1); + struct WFNSParams params = { + .type = TT_TCP, + .uri = this->uri, + .info = "", + .fixed_addr = true, + .retry_times = 0, + .tracing = NULL, + }; + + WFNameService *ns = WFGlobal::get_name_service(); + this->ns_policy = ns->get_policy(this->uri.host ? this->uri.host : ""); + return this->ns_policy->create_router_task(¶ms, cb); +} + +template +void WFComplexChannel::router_callback(WFRouterTask *task) +{ + if (task->get_state() == WFT_STATE_SUCCESS) + { + this->route_result = std::move(*task->get_result()); + this->set_request_object(this->route_result.request_object); + } + else + { + this->state = task->get_state(); + this->error = task->get_error(); + } +} + +/**********ComplexChannelOutTask for complex channel and upgrade()**********/ + +template +class ComplexChannelOutTask : public WFChannelOutTask +{ +protected: + virtual void dispatch(); + virtual SubTask *done(); + virtual SubTask *upgrade(); + void upgrade_callback(WFCounterTask *task); + +protected: + bool ready; + +public: + ComplexChannelOutTask(WFComplexChannel *channel, CommScheduler *scheduler, + std::function *)>&& cb) : + WFChannelOutTask(channel, scheduler, std::move(cb)) + { + this->ready = true; + } + +protected: + virtual ~ComplexChannelOutTask() { } +}; + +template +void ComplexChannelOutTask::dispatch() +{ + bool should_send = false; + auto *channel = (WFComplexChannel *)this->get_request_channel(); + + if (this->state == WFT_STATE_SYS_ERROR || + channel->get_state() == WFT_STATE_SYS_ERROR) + { + return this->subtask_done(); + } + + pthread_mutex_lock(&channel->mutex); + + switch (channel->get_state()) + { + case WFT_STATE_UNDEFINED: + if (channel->get_sending() == false) + { + series_of(this)->push_front(this); + series_of(this)->push_front(channel); + channel->set_sending(true); + this->ready = false; + } + else if (this->ready == false) + { + SubTask *upgrade_task = this->upgrade(); + series_of(this)->push_front(this); + series_of(this)->push_front(upgrade_task); + } + else + { + WFCounterTask *counter = channel->condition.create_wait_task( + [this](WFCounterTask *task) + { + auto *channel = (WFComplexChannel *)this->get_request_channel(); + channel->set_state(WFT_STATE_SUCCESS); + this->ready = true; + }); + series_of(this)->push_front(this); + series_of(this)->push_front(counter); + this->ready = false; + } + break; + + case WFT_STATE_SUCCESS: + if (channel->get_sending() == false) + { + channel->set_sending(true); + should_send = true; + } + else + { + WFCounterTask *counter = channel->condition.create_wait_task( + [this](WFCounterTask *task) + { + auto *channel = (WFComplexChannel *)this->get_request_channel(); + channel->set_state(WFT_STATE_SUCCESS); + this->ready = true; + }); + series_of(this)->push_front(this); + series_of(this)->push_front(counter); + this->ready = false; + } + break; + + default: + this->state = channel->get_state(); + this->error = channel->get_error(); + break; + } + + pthread_mutex_unlock(&channel->mutex); + + if (should_send == true) + return this->WFChannelOutTask::dispatch(); + + return this->subtask_done(); +} + +template +SubTask *ComplexChannelOutTask::done() +{ + auto *channel = (WFComplexChannel *)this->get_request_channel(); + + if (channel->get_state() == WFT_STATE_UNDEFINED || + channel->get_state() == WFT_STATE_SUCCESS) + { + if (this->ready != true) + return series_of(this)->pop(); + } + else + { + this->state = channel->get_state(); + this->error = channel->get_error(); + } + + pthread_mutex_lock(&channel->mutex); + channel->set_sending(false); + channel->condition.signal(); + pthread_mutex_unlock(&channel->mutex); + + return WFChannelOutTask::done(); +} + +template +SubTask *ComplexChannelOutTask::upgrade() +{ + WFCounterTask *counter = new WFCounterTask(0, [this](WFCounterTask *task) + { + auto *channel = (WFComplexChannel *)this->get_request_channel(); + + pthread_mutex_lock(&channel->mutex); + channel->set_state(WFT_STATE_SUCCESS); + this->ready = true; + channel->set_sending(false); + channel->condition.signal(); + pthread_mutex_unlock(&channel->mutex); + }); + + return counter; +} + +/**********WebSocket task impl**********/ + +class ComplexWebSocketChannel : public WFComplexChannel +{ +public: + void set_idle_timeout(int timeout) { this->idle_timeout = timeout; } + void set_size_limit(size_t size_limit) { this->size_limit = size_limit; } + +protected: + CommMessageIn *message_in(); + void handle_in(CommMessageIn *in); + virtual int first_timeout(); + virtual WFWebSocketTask *new_session(); + +private: + int idle_timeout; + size_t size_limit; + +public: + ComplexWebSocketChannel(CommSchedObject *object, CommScheduler *scheduler, + websocket_process_t&& process) : + WFComplexChannel(object, scheduler, + std::move(process)) + { + } +}; + +class ComplexWebSocketOutTask : public ComplexChannelOutTask +{ +protected: + virtual SubTask *upgrade(); + virtual SubTask *done(); + +public: + ComplexWebSocketOutTask(ComplexWebSocketChannel *channel, CommScheduler *scheduler, + websocket_callback_t&& cb) : + ComplexChannelOutTask(channel, + scheduler, + std::move(cb)) + { + } +}; + diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc new file mode 100644 index 0000000000..8b28859546 --- /dev/null +++ b/src/factory/WebSocketTaskImpl.cc @@ -0,0 +1,225 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include "WFTask.h" +#include "WFGlobal.h" +#include "WFChannel.h" + +#define WS_HTTP_SEC_KEY_K "Sec-WebSocket-Key" +#define WS_HTTP_SEC_KEY_V "dGhlIHNhbXBsZSBub25jZQ==" +#define WS_HTTP_SEC_PROTOCOL_K "Sec-WebSocket-Protocol" +#define WS_HTTP_SEC_PROTOCOL_V "chat" +#define WS_HTTP_SEC_VERSION_K "Sec-WebSocket-Version" +#define WS_HTTP_SEC_VERSION_V "13" + +using namespace protocol; + +class ComplexWebSocketInTask : public WFChannelInTask +{ +protected: + virtual void dispatch(); + virtual SubTask *done(); + +public: + ComplexWebSocketInTask(ComplexWebSocketChannel *channel, + CommScheduler *scheduler, + websocket_process_t& proc) : + WFChannelInTask(channel, scheduler, proc) + { + } +}; + +void ComplexWebSocketInTask::dispatch() +{ + const websocket_parser_t *parser = this->get_msg()->get_parser(); + + if (parser->opcode != WebSocketFrameConnectionClose && + parser->status_code != WSStatusCodeUndefined) + { + this->state = WFT_STATE_SYS_ERROR; + this->error = parser->status_code; + } + else + { + this->state = WFT_STATE_SUCCESS; + this->error = 0; + this->process(this); + } + + this->subtask_done(); +} + +SubTask *ComplexWebSocketInTask::done() +{ + SeriesWork *series = series_of(this); + const websocket_parser_t *parser = this->get_msg()->get_parser(); + auto *channel = (ComplexWebSocketChannel *)this->get_request_channel(); + + if ((parser->opcode == WebSocketFrameConnectionClose && + !channel->is_established()) || + parser->status_code != WSStatusCodeUndefined) + { + auto *close_task = new ComplexWebSocketOutTask(channel, + WFGlobal::get_scheduler(), + nullptr); + WebSocketFrame *msg = close_task->get_msg(); + msg->set_opcode(WebSocketFrameConnectionClose); + msg->set_data(parser); + series->push_front(close_task); + } + else if (parser->opcode == WebSocketFramePing) + { + auto *pong_task = new ComplexWebSocketOutTask(channel, + WFGlobal::get_scheduler(), + nullptr); + WebSocketFrame *msg = pong_task->get_msg(); + msg->set_opcode(WebSocketFramePong); + msg->set_data(parser); + series->push_front(pong_task); + } + + if (this->callback) + this->callback(this); + + delete this; + return series->pop(); +} + +SubTask *ComplexWebSocketOutTask::done() +{ + SeriesWork *series = series_of(this); + auto *channel = (ComplexWebSocketChannel *)this->get_request_channel(); + + if (channel->get_state() == WFT_STATE_UNDEFINED || + channel->get_state() == WFT_STATE_SUCCESS) + { + if (this->ready != true) + return series_of(this)->pop(); + } + else + { + this->state = channel->get_state(); + this->error = channel->get_error(); + } + + const websocket_parser_t *parser = this->get_msg()->get_parser(); + + if (parser->opcode == WebSocketFrameConnectionClose && + this->get_state() == WFT_STATE_SUCCESS && + channel->is_established()) + { + series->push_front(this); + series->push_front(channel); + return series->pop(); + } + + pthread_mutex_lock(&channel->mutex); + channel->set_sending(false); + channel->condition.signal(); + pthread_mutex_unlock(&channel->mutex); + + return WFChannelOutTask::done(); +} + +SubTask *ComplexWebSocketOutTask::upgrade() +{ + auto *channel = (ComplexWebSocketChannel *)this->get_request_channel(); + + auto *http_task = new WFChannelOutTask(this->channel, + WFGlobal::get_scheduler(), + [this](WFChannelTask *task) + { + if (task->get_state() == WFT_STATE_SYS_ERROR) + { + this->state = WFT_STATE_SYS_ERROR; + this->error = task->get_error(); + } + + this->ready = true; + }); + HttpRequest *req = http_task->get_msg(); + req->set_method(HttpMethodGet); + req->set_http_version("HTTP/1.1"); + req->set_request_uri("/"); + req->add_header_pair("Host", channel->get_uri()->host); + req->add_header_pair("Upgrade", "websocket"); + req->add_header_pair("Connection", "Upgrade"); + req->add_header_pair(WS_HTTP_SEC_KEY_K, WS_HTTP_SEC_KEY_V); + req->add_header_pair(WS_HTTP_SEC_PROTOCOL_K, WS_HTTP_SEC_PROTOCOL_V); + req->add_header_pair(WS_HTTP_SEC_VERSION_K, WS_HTTP_SEC_VERSION_V); + + return http_task; +} + +CommMessageIn *ComplexWebSocketChannel::message_in() +{ + if (this->state == WFT_STATE_UNDEFINED) + return new HttpResponse; + + return WFComplexChannel::message_in(); +} + +void ComplexWebSocketChannel::handle_in(CommMessageIn *in) +{ + bool parse_websocket = false; + + pthread_mutex_lock(&this->mutex); + + if (this->state == WFT_STATE_UNDEFINED) + { + HttpResponse *resp = static_cast(in); + + if (strcmp(resp->get_status_code(), "101") == 0) + this->state = WFT_STATE_SUCCESS; + else + this->state = WFT_STATE_TASK_ERROR; + + delete resp; + + this->sending = false; + } + else if (this->state == WFT_STATE_SUCCESS) + parse_websocket = true; + + pthread_mutex_unlock(&this->mutex); + + if (!parse_websocket) // so this is equal to should_count + { + pthread_mutex_lock(&this->mutex); + this->condition.signal(); + pthread_mutex_unlock(&this->mutex); + return; + } + + WFComplexChannel::handle_in(in); +} + +int ComplexWebSocketChannel::first_timeout() +{ + return this->idle_timeout; +} + +WFWebSocketTask *ComplexWebSocketChannel::new_session() +{ + auto *task = new ComplexWebSocketInTask(this, this->scheduler, + this->process); + Workflow::create_series_work(task, nullptr); + task->get_msg()->set_size_limit(this->size_limit); + return task; +} + diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index 6a282556a2..f6bd6f73f2 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -5,6 +5,7 @@ set(SRC DNSCache.cc UpstreamManager.cc RouteManager.cc + WFCondition.cc WFGlobal.cc ) diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc new file mode 100644 index 0000000000..60eb2bf3b6 --- /dev/null +++ b/src/manager/WFCondition.cc @@ -0,0 +1,203 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" +#include "WFCondition.h" + +class WFTimedWaitTask; + +class WFWaitTask : public WFCounterTask +{ +public: + void set_timer(WFTimedWaitTask *timer) { this->timer = timer; } + void clear_timer_waiter(); + + struct task_entry + { + struct list_head list; + WFWaitTask *ptr; + } entry; + +protected: + virtual void dispatch(); + virtual SubTask *done(); + +private: + WFTimedWaitTask *timer; + +public: + WFWaitTask(std::function&& cb) : + WFCounterTask(1, std::move(cb)) + { + this->timer = NULL; + this->entry.list.next = NULL; + this->entry.ptr = this; + } +}; + +class WFTimedWaitTask : public __WFTimerTask +{ +public: + WFTimedWaitTask(WFWaitTask *wait_task, std::mutex *mutex, + const struct timespec *value, + CommScheduler *scheduler, + std::function cb) : + __WFTimerTask(value, scheduler, std::move(cb)) + { + this->mutex = mutex; + this->wait_task = wait_task; + } + + void clear_wait_task(); + +protected: + virtual SubTask *done(); + +private: + std::mutex *mutex; + WFWaitTask *wait_task; +}; + +void WFWaitTask::dispatch() +{ + if (this->timer) + timer->dispatch(); + + this->WFCounterTask::count(); +} + +SubTask *WFWaitTask::done() +{ + SeriesWork *series = series_of(this); + + // TODO: data move + + WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, + [this](WFTimerTask *task) { + if (this->callback) + this->callback(this); + delete this; + }); + series->push_front(switch_task); + + return series->pop(); +} + +void WFWaitTask::clear_timer_waiter() +{ + if (this->timer) + timer->clear_wait_task(); +} + +SubTask *WFTimedWaitTask::done() +{ + this->mutex->lock(); + if (this->wait_task && this->wait_task->entry.list.next) + { + list_del(&this->wait_task->entry.list); + this->wait_task->count(); + } + this->mutex->unlock(); + + SeriesWork *series = series_of(this); + + if (this->callback) + this->callback(this); + + delete this; + return series->pop(); +} + +void WFTimedWaitTask::clear_wait_task() +{ + this->mutex->lock(); + this->wait_task = NULL; + this->mutex->unlock(); +} + +WFCounterTask *WFCondition::create_wait_task(counter_callback_t cb) +{ + WFWaitTask *task = new WFWaitTask(std::move(cb)); + + this->mutex.lock(); + list_add_tail(&task->entry.list, &this->waiter_list); + this->mutex.unlock(); + + return task; +} + +WFCounterTask *WFCondition::create_timedwait_task(const struct timespec *abstime, + counter_callback_t cb) +{ + WFWaitTask *waiter = new WFWaitTask(std::move(cb)); + WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &this->mutex, abstime, + WFGlobal::get_scheduler(), + nullptr); + waiter->set_timer(task); + + this->mutex.lock(); + list_add_tail(&waiter->entry.list, &this->waiter_list); + this->mutex.unlock(); + + return waiter; +} + +void WFCondition::signal() +{ + WFWaitTask *task; + struct list_head *pos; + struct WFWaitTask::task_entry *entry; + + this->mutex.lock(); + + if (!list_empty(&this->waiter_list)) + { + pos = this->waiter_list.next; + entry = list_entry(pos, struct WFWaitTask::task_entry, list); + task = entry->ptr; + list_del(pos); + task->clear_timer_waiter(); + task->count(); + } + + this->mutex.unlock(); +} + +void WFCondition::broadcast() +{ + WFWaitTask *task; + struct list_head *pos, *tmp; + struct WFWaitTask::task_entry *entry; + + this->mutex.lock(); + if (!list_empty(&this->waiter_list)) + { + list_for_each_safe(pos, tmp, &this->waiter_list) + { + entry = list_entry(pos, struct WFWaitTask::task_entry, list); + task = entry->ptr; + list_del(pos); + task->count(); + } + } + this->mutex.unlock(); +} + diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h new file mode 100644 index 0000000000..2e2e1f3c5e --- /dev/null +++ b/src/manager/WFCondition.h @@ -0,0 +1,49 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WFCONDITION_H_ +#define _WFCONDITION_H_ + +#include +#include +#include +#include "list.h" +#include "WFTask.h" + +class WFCondition +{ +public: + WFCondition() + { + INIT_LIST_HEAD(&this->waiter_list); + } + + WFCounterTask *create_wait_task(std::function cb); + WFCounterTask *create_timedwait_task(const struct timespec *abstime, + std::function cb); + void signal(); + void broadcast(); + +public: + std::mutex mutex; +private: + struct list_head waiter_list; +}; + +#endif + diff --git a/src/manager/WFGlobal.cc b/src/manager/WFGlobal.cc index f16a8f055f..f1580aabbe 100644 --- a/src/manager/WFGlobal.cc +++ b/src/manager/WFGlobal.cc @@ -146,17 +146,18 @@ __WFGlobal::__WFGlobal() : settings_(GLOBAL_SETTINGS_DEFAULT) static_scheme_port_["MySQL"] = "3306"; static_scheme_port_["MYSQL"] = "3306"; - static_scheme_port_["mysqls"] = "3306"; - static_scheme_port_["Mysqls"] = "3306"; - static_scheme_port_["MySqls"] = "3306"; - static_scheme_port_["MySQLs"] = "3306"; - static_scheme_port_["MYSQLs"] = "3306"; - static_scheme_port_["MYSQLS"] = "3306"; - static_scheme_port_["kafka"] = "9092"; static_scheme_port_["Kafka"] = "9092"; static_scheme_port_["KAFKA"] = "9092"; + static_scheme_port_["ws"] = "80"; + static_scheme_port_["Ws"] = "80"; + static_scheme_port_["WS"] = "80"; + + static_scheme_port_["wss"] = "443"; + static_scheme_port_["Wss"] = "443"; + static_scheme_port_["WSs"] = "443"; + sync_count_ = 0; sync_max_ = 0; } @@ -727,9 +728,6 @@ static inline const char *__get_task_error_string(int error) case WFT_ERR_MYSQL_QUERY_NOT_SET: return "MySQL Query Not Set"; - case WFT_ERR_MYSQL_SSL_NOT_SUPPORTED: - return "MySQL SSL Not Supported"; - case WFT_ERR_KAFKA_PARSE_RESPONSE_FAILED: return "Kafka parse response failed"; diff --git a/src/protocol/CMakeLists.txt b/src/protocol/CMakeLists.txt index 969e380dbd..a92effc1b9 100644 --- a/src/protocol/CMakeLists.txt +++ b/src/protocol/CMakeLists.txt @@ -7,10 +7,12 @@ set(SRC mysql_stream.c mysql_parser.c mysql_byteorder.c + websocket_parser.c MySQLMessage.cc MySQLResult.cc HttpMessage.cc RedisMessage.cc + WebSocketMessage.cc HttpUtil.cc SSLWrapper.cc ) @@ -25,8 +27,5 @@ if (KAFKA STREQUAL "y") KafkaResult.cc ) add_library("protocol_kafka" OBJECT ${SRC}) - set_property(SOURCE KafkaMessage.cc APPEND PROPERTY COMPILE_OPTIONS "-fno-rtti") - set_property(SOURCE KafkaDataTypes.cc APPEND PROPERTY COMPILE_OPTIONS "-fno-rtti") - set_property(SOURCE KafkaResult.cc APPEND PROPERTY COMPILE_OPTIONS "-fno-rtti") endif () diff --git a/src/protocol/WebSocketMessage.cc b/src/protocol/WebSocketMessage.cc new file mode 100644 index 0000000000..fe11f10cd2 --- /dev/null +++ b/src/protocol/WebSocketMessage.cc @@ -0,0 +1,299 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include +#include "WebSocketMessage.h" + +namespace protocol +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + +static inline void int2store(unsigned char *T, uint16_t A) +{ + memcpy(T, &A, sizeof(A)); +} + +static inline void int8store(unsigned char *T, uint64_t A) +{ + memcpy(T, &A, sizeof(A)); +} + +#elif __BYTE_ORDER == __BIG_ENDIAN + +static inline void int2store(unsigned char *T, uint16_t A) +{ + uint def_temp = A; + *(T) = (unsigned char)(def_temp); + *(T + 1) = (unsigned char)(def_temp >> 8); +} + +static inline void int4store(unsigned char *T, uint32_t A) +{ + *(T) = (unsigned char)(A); + *(T + 1) = (unsigned char)(A >> 8); + *(T + 2) = (unsigned char)(A >> 16); + *(T + 3) = (unsigned char)(A >> 24); +} + +static inline void int8store(unsigned char *T, uint64_t A) +{ + uint def_temp = (uint)A, def_temp2 = (uint)(A >> 32); + int4store(T, def_temp); + int4store(T + 4, def_temp2); +} + +#else +# error "unknown byte order" +#endif + +WebSocketFrame::WebSocketFrame(WebSocketFrame&& msg) : + ProtocolMessage(std::move(msg)) +{ + this->parser = msg.parser; + msg.parser = NULL; +} + +WebSocketFrame& WebSocketFrame::operator = (WebSocketFrame&& msg) +{ + if (&msg != this) + { + *(ProtocolMessage *)this = std::move(msg); + if (this->parser) + { + websocket_parser_deinit(this->parser); + delete this->parser; + } + + this->parser = msg.parser; + msg.parser = NULL; + } + + return *this; +} + +int WebSocketFrame::append(const void *buf, size_t *size) +{ + int ret = websocket_parser_append_message(buf, size, this->parser); + + if (this->parser->payload_length > this->size_limit) + { + this->parser->status_code = WSStatusCodeTooLarge; + return 1; // don`t need websocket_parser_parse() + } + + if (ret == 1) + websocket_parser_parse(this->parser); + + return ret; +} + +int WebSocketFrame::encode(struct iovec vectors[], int max) +{ + unsigned char *p = this->parser->header_buf; + int cnt = 0; + + if (this->parser->opcode == WebSocketFramePing || + this->parser->opcode == WebSocketFramePong || + this->parser->opcode == WebSocketFrameConnectionClose) + { + this->parser->fin = 1; + } + else if (!this->parser->fin) + { + this->parser->opcode = WebSocketFrameContinuation; + } + + *p = (this->parser->fin << 7) | this->parser->opcode; + p++; + + if (this->parser->payload_length < 126) + { + *p = (unsigned char)this->parser->payload_length; + p++; + } + else if (this->parser->payload_length < 65536) + { + *p = 126; + p++; + int2store(p, this->parser->payload_length); + p += 2; + } + else + { + *p = 127; + p++; + int8store(p, this->parser->payload_length); + p += 8; + } + + vectors[cnt].iov_base = this->parser->header_buf; + vectors[cnt].iov_len = p - this->parser->header_buf; + cnt++; + + p = this->parser->header_buf + 1; + *p = *p | (this->parser->mask << 7); + + if (!this->parser->is_server) + { + vectors[cnt].iov_base = this->parser->masking_key; + vectors[cnt].iov_len = WS_MASKING_KEY_LENGTH; + cnt++; + } + + if (this->parser->payload_length) + { + websocket_parser_mask_data(this->parser); + vectors[cnt].iov_base = this->parser->payload_data; + vectors[cnt].iov_len = this->parser->payload_length; + cnt++; + } + + return cnt; +} + +bool WebSocketFrame::set_opcode(int opcode) +{ + if (opcode < WebSocketFrameContinuation || opcode > WebSocketFramePong) + return false; + + this->parser->opcode = opcode; + return true; +} + +int WebSocketFrame::get_opcode() const +{ + return this->parser->opcode; +} + +void WebSocketFrame::set_masking_key(uint32_t masking_key) +{ + this->parser->mask = 1; + memcpy(this->parser->masking_key, &masking_key, WS_MASKING_KEY_LENGTH); +} + +uint32_t WebSocketFrame::get_masking_key() const +{ + if (!this->parser->mask) + return atoi((char *)this->parser->masking_key); + + return 0; +} + +bool WebSocketFrame::set_binary_data(const char *data, size_t size) +{ + return this->set_binary_data(data, size, true); +} + +bool WebSocketFrame::set_binary_data(const char *data, size_t size, bool fin) +{ + bool ret = true; + + this->parser->opcode = WebSocketFrameBinary; + this->parser->fin = fin; + + if (this->parser->payload_length && this->parser->payload_data) + { + ret = false; + free(this->parser->payload_data); + } + + this->parser->payload_data = (char *)malloc(size); + memcpy(this->parser->payload_data, data, size); + this->parser->payload_length = size; + + return ret; +} + +bool WebSocketFrame::set_text_data(const char *data) +{ + return set_text_data(data, strlen(data), true); +} + +bool WebSocketFrame::set_text_data(const char *data, size_t size, bool fin) +{ + bool ret = true; + + this->parser->opcode = WebSocketFrameText; + this->parser->fin = fin; + + if (this->parser->payload_length && this->parser->payload_data) + { + ret = false; + free(this->parser->payload_data); + } + + this->parser->payload_data = (char *)malloc(size); + memcpy(this->parser->payload_data, data, size); + this->parser->payload_length = size; + + return ret; +} + +bool WebSocketFrame::set_data(const websocket_parser_t *parser) +{ + bool ret = true; + unsigned char *p; + + if (this->parser->payload_length && this->parser->payload_data) + { + ret = false; + free(this->parser->payload_data); + } + +// this->parser->status_code = parser->status_code; + this->parser->payload_length = parser->payload_length; + + if (this->parser->opcode == WebSocketFrameConnectionClose && + parser->status_code != WSStatusCodeUndefined) + { + this->parser->payload_length += 2; + } + + this->parser->payload_data = malloc(this->parser->payload_length); + p = (unsigned char *)this->parser->payload_data; + + if (this->parser->opcode == WebSocketFrameConnectionClose && + parser->status_code != WSStatusCodeUndefined) + { + int2store(p, parser->status_code); + p += 2; + } + + memcpy(p, parser->payload_data, parser->payload_length); + + return ret; +} + +bool WebSocketFrame::get_data(const char **data, size_t *size) const +{ + if (!this->parser->payload_length || !this->parser->payload_data) + return false; + + *data = (char *)this->parser->payload_data; + *size = this->parser->payload_length; + return true; +} + +bool WebSocketFrame::finished() const +{ + return this->parser->fin; +} + +} // end namespace protocol + diff --git a/src/protocol/WebSocketMessage.h b/src/protocol/WebSocketMessage.h new file mode 100644 index 0000000000..e834be2a48 --- /dev/null +++ b/src/protocol/WebSocketMessage.h @@ -0,0 +1,84 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WEBSOCKETMESSAGE_H_ +#define _WEBSOCKETMESSAGE_H_ + +#include +#include +#include +#include "ProtocolMessage.h" +#include "websocket_parser.h" + +namespace protocol +{ + +#define WS_HANDSHAKE_TIMEOUT 10 * 1000 + +class WebSocketFrame : public ProtocolMessage +{ +public: + bool set_opcode(int opcode); + int get_opcode() const; + + void set_masking_key(uint32_t masking_key); + + bool set_text_data(const char *data); + bool set_text_data(const char *data, size_t size, bool fin); + + bool set_binary_data(const char *data, size_t size); + bool set_binary_data(const char *data, size_t size, bool fin); + + bool get_data(const char **data, size_t *size) const; + + bool finished() const; + +public: + void set_client() { this->parser->is_server = 0; } + void set_server() { this->parser->is_server = 1; } + const websocket_parser_t *get_parser() { return this->parser; } + bool set_data(const websocket_parser_t *parser); + uint32_t get_masking_key() const; + +private: + websocket_parser_t *parser; + +protected: + virtual int encode(struct iovec vectors[], int max); + virtual int append(const void *buf, size_t *size); + +public: + WebSocketFrame() : parser(new websocket_parser_t) + { + websocket_parser_init(this->parser); + } + + virtual ~WebSocketFrame() + { + websocket_parser_deinit(this->parser); + delete this->parser; + } + + WebSocketFrame(WebSocketFrame&& msg); + WebSocketFrame& operator = (WebSocketFrame&& msg); +}; + +} // end namespace protocol + +#endif + diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c new file mode 100644 index 0000000000..25e1db6ad5 --- /dev/null +++ b/src/protocol/websocket_parser.c @@ -0,0 +1,239 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include +#include +#include +#include "websocket_parser.h" + +void websocket_parser_init(websocket_parser_t *parser) +{ + parser->fin = 0; + parser->mask = 0; + parser->opcode = -1; + parser->nleft = WS_MASKING_KEY_LENGTH; + parser->payload_length = 0; + parser->payload_data = NULL; + parser->nreceived = 0; + parser->is_server = 0; + parser->status_code = WSStatusCodeUndefined; + memset(parser->masking_key, 0, WS_MASKING_KEY_LENGTH); + memset(parser->header_buf, 0, WS_HEADER_LENGTH_MAX); +} + +void websocket_parser_deinit(websocket_parser_t *parser) +{ + if (parser->payload_length != 0) + free(parser->payload_data); +} + +// 4: FIN 0 0 0 4:opcode | 1: MASK 7:PAYLOAD_LENGTH | +// 0or16or64 : extend PAYLOAD_LENGTH | 0or32 : MASK_KEY | +// PAYLOAD_LENGTH: PAYLOAD_DATA +int websocket_parser_append_message(const void *buf, size_t *n, + websocket_parser_t *parser) +{ + const unsigned char *p = (const unsigned char *)buf; + const unsigned char *buf_end = (const unsigned char *)buf + *n; + + int header_length_min = parser->is_server ? WS_CLIENT_LENGTH_MIN : + WS_SERVER_LENGTH_MIN; + +// if (parser->payload_length == 0) // receiving header + if (parser->nreceived < header_length_min) + { + memcpy(parser->header_buf + parser->nreceived, p, + WS_HEADER_LENGTH_MAX - parser->nreceived); + + if (parser->nreceived + *n < header_length_min) + { + parser->nreceived += *n; + return 0; + } + + parser->fin = *p >> 7; + parser->opcode = *p & 0xF; + p++; + parser->mask = *p >> 7; + parser->payload_length = *p & 0x7F; + p++; + } + + if (parser->masking_key_offset == 0) // receive payload_length + { + if (parser->payload_length == 126 && + parser->nreceived + *n >= header_length_min + 2) + { + uint16_t *len_ptr = (uint16_t *)p; + parser->payload_length = ntohs(*len_ptr); + p += 2; + parser->masking_key_offset = 4; + } + else if (parser->payload_length == 127 && + parser->nreceived + *n >= header_length_min + 8) + { + uint64_t *len_ptr = (uint64_t *)p; + parser->payload_length = (((uint64_t) ntohl(*len_ptr)) << 32) + + ntohl(*len_ptr >> 32); + p += 8; + parser->masking_key_offset = 10; + } + else + { + parser->nreceived += *n; + parser->masking_key_offset = 2; + } + + if (parser->masking_key_offset == 0) + return 0; // continue for length + } + + if (!parser->payload_data) // receive masking_key if needed + { + if (parser->mask && parser->nleft) + { +// if (parser->masking_key_offset + 4 < buf_end) + if (buf_end - p < parser->nleft) + { + parser->nleft -= buf_end - p; + parser->nreceived += *n; + return 0; + } + + memcpy(parser->masking_key, + parser->header_buf + parser->masking_key_offset, + WS_MASKING_KEY_LENGTH); + p += parser->nleft; + parser->nreceived += parser->nleft; + } + + parser->nleft = parser->payload_length; + } + + parser->payload_data = malloc(parser->payload_length); + + if (buf_end - p < parser->nleft) + { + memcpy(parser->payload_data + parser->payload_length - parser->nleft, + p, buf_end - p); + parser->nleft -= buf_end - p; + return 0; + } + else + { + memcpy(parser->payload_data + parser->payload_length - parser->nleft, + p, parser->nleft); + p += parser->nleft; + *n -= buf_end - p; + return 1; + } +} + +int websocket_parser_parse(websocket_parser_t *parser) +{ + if (parser->opcode < WebSocketFrameContinuation || + (parser->opcode < WebSocketFrameConnectionClose && + parser->opcode > WebSocketFrameBinary) || + parser->opcode > WebSocketFramePong) + { + parser->status_code = WSStatusCodeProtocolError; + return -1; + } + + unsigned char *p = (unsigned char *)parser->payload_data; + + if (parser->opcode == WebSocketFrameConnectionClose) + { + uint16_t *ptr = (uint16_t *)p; + parser->status_code = ntohs(*ptr); + parser->payload_data = p + 2; + } + + websocket_parser_mask_data(parser); + + if (parser->opcode == WebSocketFrameText && + !utf8_check(p, parser->payload_length)) + { + parser->status_code = WSStatusCodeUnsupportedData; + return -1; + } + + return 0; +} + +void websocket_parser_mask_data(websocket_parser_t *parser) +{ + if (!parser->mask) + return; + + unsigned long long i; + unsigned char *p = (unsigned char *)parser->payload_data; + + for (i = 0; i < parser->payload_length; i++) + *p++ ^= parser->masking_key[i % 4]; + + return; +} + +//https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c +unsigned char *utf8_check(unsigned char *s, size_t len) +{ + unsigned char *end = s + len; + while (*s && s != end) { + if (*s < 0x80) + /* 0xxxxxxx */ + s++; + else if ((s[0] & 0xe0) == 0xc0) { + /* 110XXXXx 10xxxxxx */ + if ((s[1] & 0xc0) != 0x80 || + (s[0] & 0xfe) == 0xc0) /* overlong? */ + return s; + else + s += 2; + } else if ((s[0] & 0xf0) == 0xe0) { + /* 1110XXXX 10Xxxxxx 10xxxxxx */ + if ((s[1] & 0xc0) != 0x80 || + (s[2] & 0xc0) != 0x80 || + (s[0] == 0xe0 && (s[1] & 0xe0) == 0x80) || /* overlong? */ + (s[0] == 0xed && (s[1] & 0xe0) == 0xa0) || /* surrogate? */ + (s[0] == 0xef && s[1] == 0xbf && + (s[2] & 0xfe) == 0xbe)) /* U+FFFE or U+FFFF? */ + return s; + else + s += 3; + } else if ((s[0] & 0xf8) == 0xf0) { + /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */ + if ((s[1] & 0xc0) != 0x80 || + (s[2] & 0xc0) != 0x80 || + (s[3] & 0xc0) != 0x80 || + (s[0] == 0xf0 && (s[1] & 0xf0) == 0x80) || /* overlong? */ + (s[0] == 0xf4 && s[1] > 0x8f) || s[0] > 0xf4) /* > U+10FFFF? */ + return s; + else + s += 4; + } else + return s; + } + + if (s == end) + return s; + + return NULL; +} + diff --git a/src/protocol/websocket_parser.h b/src/protocol/websocket_parser.h new file mode 100644 index 0000000000..7f9d9c17c9 --- /dev/null +++ b/src/protocol/websocket_parser.h @@ -0,0 +1,123 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WEBSOCKET_PARSER_H_ +#define _WEBSOCKET_PARSER_H_ + +#define WS_HEADER_LENGTH_MAX 14 +#define WS_SERVER_LENGTH_MIN 2 +#define WS_CLIENT_LENGTH_MIN 6 +#define WS_MASKING_KEY_LENGTH 4 + +#include +#include + +enum +{ + WebSocketFrameContinuation = 0, + WebSocketFrameText = 1, + WebSocketFrameBinary = 2, + WebSocketFrameConnectionClose = 8, + WebSocketFramePing = 9, + WebSocketFramePong = 10, +}; + +enum +{ + WSStatusCodeUndefined = 0, + + WSStatusCodeNormal = 1000, + WSStatusCodeGoingAway = 1001, + WSStatusCodeProtocolError = 1002, + WSStatusCodeUnsupported = 1003, + WSStatusCodeReserved = 1004, // reserved + WSStatusCodeNoStatus = 1005, // reserved + WSStatusCodeAbnomal = 1006, // reserved + WSStatusCodeUnsupportedData = 1007, + WSStatusCodePolicyViolation = 1008, + WSStatusCodeTooLarge = 1009, + WSStatusCodeMissExtention = 1010, + WSStatusCodeInternalError = 1011, +// WSStatusCodeServiceRestart = 1012, +// WSStatusCodeTryAgainLater = 1013, + WSStatusCodeTLSHandshake = 1015, // reserved + + WSStatusCodeProtocolMax = 2999, + + WSStatusCodeIANAMin = 3000, + WSStatusCodeIANAMax = 3999, + + WSStatusCodeUserMin = 4000, + WSStatusCodeUserMax = 4999, +}; + +typedef struct __websocket_parser +{ + char fin; + char mask; + int opcode; + unsigned char masking_key[WS_MASKING_KEY_LENGTH]; + unsigned long long payload_length; + unsigned char header_buf[WS_HEADER_LENGTH_MAX]; + void *payload_data; + unsigned long long nreceived; + int masking_key_offset; + int nleft; + int is_server; + int status_code; +} websocket_parser_t; + +#ifdef __cplusplus +extern "C" +{ +#endif + +void websocket_parser_init(websocket_parser_t *parser); +void websocket_parser_deinit(websocket_parser_t *parser); +int websocket_parser_append_message(const void *buf, size_t *n, + websocket_parser_t *parser); + +int websocket_parser_decode_payload_length(websocket_parser_t *parser); + +void websocket_parser_encode_payload_length(websocket_parser_t *parser); + +int websocket_parser_parse(websocket_parser_t *parser); + +void websocket_parser_mask_data(websocket_parser_t *parser); + +unsigned char *utf8_check(unsigned char *s, size_t len); + +#ifdef __cplusplus +} +#endif + +static inline void websocket_set_mask_key(unsigned int masking_key, + websocket_parser_t *parser) +{ + uint32_t *key = (uint32_t *)parser->masking_key; + *key = masking_key; + parser->mask = 1; +} + +static inline void websocket_set_opcode(int opcode, websocket_parser_t *parser) +{ + parser->opcode = opcode; +} + +#endif + diff --git a/tutorial/CMakeLists.txt b/tutorial/CMakeLists.txt index 56593d01b8..186e11708f 100644 --- a/tutorial/CMakeLists.txt +++ b/tutorial/CMakeLists.txt @@ -35,6 +35,7 @@ set(TUTORIAL_LIST tutorial-09-http_file_server tutorial-11-graph_task tutorial-12-mysql_cli + tutorial-14-websocket_cli ) if (APPLE) diff --git a/tutorial/tutorial-14-websocket_cli.cc b/tutorial/tutorial-14-websocket_cli.cc new file mode 100644 index 0000000000..480dcfad84 --- /dev/null +++ b/tutorial/tutorial-14-websocket_cli.cc @@ -0,0 +1,84 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include "workflow/WFFacilities.h" +#include "workflow/WFWebSocketClient.h" +#include "workflow/WebSocketMessage.h" + +#include +#include +#include +#include +#include "unistd.h" + +using namespace protocol; + +void process(WFWebSocketTask *task) +{ + const char *data; + size_t size; + + if (task->get_msg()->get_opcode() == WebSocketFrameText) + { + task->get_msg()->get_data(&data, &size); + fprintf(stderr, "get text message: [%.*s]\n", (int)size, data); + } + else + { + fprintf(stderr, "opcode=%d\n", task->get_msg()->get_opcode()); + } +} + +int main(int argc, char *argv[]) +{ + if (argc != 2) + { + fprintf(stderr, "USAGE: %s \n url format: ws://host:ip\n", argv[0]); + return 0; + } + + WebSocketClient client(process); + client.init(argv[1]); + + WFFacilities::WaitGroup wg(1); + auto *task = client.create_websocket_task([&wg, &client] (WFWebSocketTask *task) + { + fprintf(stderr, "send callback() state=%d error=%d\n", + task->get_state(), task->get_error()); + + if (task->get_state() != WFT_STATE_SUCCESS) + { + wg.done(); + return; + } + + auto *close_task = client.create_close_task([&wg] (WFWebSocketTask *task) { + wg.done(); + }); + + series_of(task)->push_back(close_task); + }); + + WebSocketFrame *msg = task->get_msg(); + msg->set_text_data("This is Workflow websocket client."); + task->start(); + + wg.wait(); + + return 0; +} From b3ad666b9618ac165aec4499f9a14a379f9e143c Mon Sep 17 00:00:00 2001 From: XieHan Date: Fri, 11 Jun 2021 01:24:33 +0800 Subject: [PATCH 10/75] fix conflict --- src/protocol/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/protocol/CMakeLists.txt b/src/protocol/CMakeLists.txt index 96d220fcf4..833d9c0ebe 100644 --- a/src/protocol/CMakeLists.txt +++ b/src/protocol/CMakeLists.txt @@ -8,12 +8,9 @@ set(SRC mysql_stream.c mysql_parser.c mysql_byteorder.c -<<<<<<< HEAD websocket_parser.c -======= DnsMessage.cc DnsUtil.cc ->>>>>>> 0465a939fa66b870e2fdddba73510f5d298385b4 MySQLMessage.cc MySQLResult.cc HttpMessage.cc From 8bfabd786579372bed0ec6367396af1e2f1ae593 Mon Sep 17 00:00:00 2001 From: XieHan Date: Fri, 11 Jun 2021 01:26:09 +0800 Subject: [PATCH 11/75] fix CMake --- CMakeLists_Headers.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index ad5749d725..7a1587384c 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -67,7 +67,6 @@ set(INCLUDE_HEADERS src/server/WFMySQLServer.h src/client/WFMySQLConnection.h src/client/WFWebSocketClient.h - src/manager/DNSCache.h src/client/WFDnsClient.h src/manager/DnsCache.h src/manager/WFGlobal.h From 3b558b7572d6ec44586f897de100f313cf3831e3 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Sat, 12 Jun 2021 10:48:38 +0800 Subject: [PATCH 12/75] add create_ping_task --- src/client/WFWebSocketClient.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h index 89df518d06..0cc778d543 100644 --- a/src/client/WFWebSocketClient.h +++ b/src/client/WFWebSocketClient.h @@ -48,6 +48,7 @@ class WebSocketClient public: int init(const std::string& url); WFWebSocketTask *create_websocket_task(websocket_callback_t cb); + WFWebSocketTask *create_ping_task(websocket_callback_t cb); WFWebSocketTask *create_close_task(websocket_callback_t cb); private: @@ -78,14 +79,29 @@ inline int WebSocketClient::init(const std::string& url) return 0; } +inline WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t cb) +{ + ComplexWebSocketOutTask *ping_task; + ping_task = new ComplexWebSocketOutTask(this->channel, + WFGlobal::get_scheduler(), + std::move(cb)); + + protocol::WebSocketFrame *msg = ping_task->get_msg(); + msg->set_opcode(WebSocketFramePing); + + return ping_task; +} + inline WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) { ComplexWebSocketOutTask *close_task; close_task = new ComplexWebSocketOutTask(this->channel, WFGlobal::get_scheduler(), std::move(cb)); + protocol::WebSocketFrame *msg = close_task->get_msg(); msg->set_opcode(WebSocketFrameConnectionClose); + return close_task; } From 28444b2498f86775c431c371837c4bf0f8ed825e Mon Sep 17 00:00:00 2001 From: liyingxin Date: Sat, 12 Jun 2021 11:22:53 +0800 Subject: [PATCH 13/75] Update tutorial-14-websocket_cli.md --- docs/tutorial-14-websocket_cli.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md index 4936d33968..fc4040dca0 100644 --- a/docs/tutorial-14-websocket_cli.md +++ b/docs/tutorial-14-websocket_cli.md @@ -150,7 +150,7 @@ void process(WFWebSocketTask *task) # 关闭client -根据**WebSocket**协议，用户需要发起一个close包已告诉对方以示断开连接。 +根据**WebSocket**协议，用户需要发起一个**close**包已告诉对方以示断开连接。 一个简单的例子： @@ -199,21 +199,31 @@ struct WFWebSocketParams # 进阶版：注意事项！ +websocket_connect_close websocket_read_write + #### 1. 与Workflow原有用法的差异 由于**WebSocket**协议是**Workflow**中首个实现的双工通信协议，因此有些差异是必须强调的： -1. **WebSocket**协议的收发都是使用**poller**线程，因此websocket_cli用户需要把**poller**线程数改大点，参考：[about-config.md](/docs/about-config.md) +1. **WebSocket**协议的收发都是使用**poller**线程，因此websocket_cli用户需要把**poller**线程数改大点，同理可以把handler线程数改小。参考：[about-config.md](/docs/about-config.md) 2. **process**函数中所有的消息都是由同一个线程串行执行的； 3. 回调函数**callback**的执行不一定在同一个线程； -#### 2. 时序性保证 +#### 2. 连接的生命周期 + +只有在第一个任务发出的时候，连接才会真正被建立。因此如果只希望监听server而没有写消息需求的用户依然需要手动发一个**PING**，让内部建立连接。可以通过client的``create_ping_task()``接口创建一个**PING** task，该回调函数里可以通过**state**判断连接是否可用，如果等于**WF_STATE_SUCCESS**,则表示``process()``里已经随时可以接收server来的消息了。 + +前面提到，需要发起**CLOSE** task关闭连接，回到该回调函数时则表示连接已关闭。 + +#### 3. 时序性保证 [**发消息**] -消息发送顺序取决于发送任务调起的时机是否顺序，因此用户可以把要发送的任务串到一个series里做串行的保证，也可以在上一个任务的callback里发起一下一个任务。 +消息发送顺序取决于发送任务调起的顺序。因此发消息保序有两种方式： +- 可以把要发送的任务串到一个series里做串行的保证 +- 也可以在上一个任务的callback里发起下一个任务 -但如果没有顺序发送的保证，那么往外发的``WFWebSocketTask``也可以被放到任何一个任务流图里，随具体业务逻辑顺序调起。 +但如果没有保证顺序发送的需求，那么往外发的``WFWebSocketTask``也可以被放到任何一个任务流图里，随具体业务逻辑顺序调起。 [**收消息**] From 730d86a6539ef6100e30e8685a048d6446175db7 Mon Sep 17 00:00:00 2001 From: liyingxin Date: Sat, 12 Jun 2021 11:39:22 +0800 Subject: [PATCH 14/75] Update tutorial-14-websocket_cli.md --- docs/tutorial-14-websocket_cli.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md index fc4040dca0..080ba20c97 100644 --- a/docs/tutorial-14-websocket_cli.md +++ b/docs/tutorial-14-websocket_cli.md @@ -199,19 +199,19 @@ struct WFWebSocketParams # 进阶版：注意事项！ -websocket_connect_close websocket_read_write +websocket_connect_close websocket_read_write #### 1. 与Workflow原有用法的差异 由于**WebSocket**协议是**Workflow**中首个实现的双工通信协议，因此有些差异是必须强调的： -1. **WebSocket**协议的收发都是使用**poller**线程，因此websocket_cli用户需要把**poller**线程数改大点，同理可以把handler线程数改小。参考：[about-config.md](/docs/about-config.md) +1. **WebSocket**协议的收发都是使用**poller**线程，因此websocket_cli用户需要把**poller**线程数改大点，同理可以把**handler**线程数改小。参考：[about-config.md](/docs/about-config.md) 2. **process**函数中所有的消息都是由同一个线程串行执行的； 3. 回调函数**callback**的执行不一定在同一个线程； #### 2. 连接的生命周期 -只有在第一个任务发出的时候，连接才会真正被建立。因此如果只希望监听server而没有写消息需求的用户依然需要手动发一个**PING**，让内部建立连接。可以通过client的``create_ping_task()``接口创建一个**PING** task，该回调函数里可以通过**state**判断连接是否可用，如果等于**WF_STATE_SUCCESS**,则表示``process()``里已经随时可以接收server来的消息了。 +只有在第一个任务发出的时候，连接才会真正被建立。因此如果只希望监听server而没有写消息需求的用户依然需要手动发一个**PING**，让内部建立连接。可以通过client的``create_ping_task()``接口创建一个**PING** task，该回调函数里可以通过**state**判断连接是否可用，如果等于**WFT_STATE_SUCCESS**,则表示``process()``里已经随时可以接收server来的消息了。 前面提到，需要发起**CLOSE** task关闭连接，回到该回调函数时则表示连接已关闭。 @@ -227,4 +227,4 @@ struct WFWebSocketParams [**收消息**] -用于收消息的``process()``函数是保证被按收包顺序调起的，且保证前一个消息的process()执行完毕，下一个process才会调起。 +用于收消息的``process()``函数是保证被按收包顺序调起的，且保证前一个消息的process执行完毕，下一个process才会调起，因此用户无需担心收消息的顺序问题。 From cb3bf78ebaea74a9eac006bb282dcf19f17c9736 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Wed, 16 Jun 2021 20:27:49 +0800 Subject: [PATCH 15/75] update codes and count() --- src/factory/WFChannel.inl | 4 ++-- src/manager/WFCondition.cc | 27 +++++++++++++++++++++------ src/manager/WFCondition.h | 9 ++++----- tutorial/tutorial-14-websocket_cli.cc | 2 ++ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index fc4e2aa268..54da3960ad 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -449,8 +449,8 @@ public: void set_size_limit(size_t size_limit) { this->size_limit = size_limit; } protected: - CommMessageIn *message_in(); - void handle_in(CommMessageIn *in); + virtual CommMessageIn *message_in(); + virtual void handle_in(CommMessageIn *in); virtual int first_timeout(); virtual WFWebSocketTask *new_session(); diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index 60eb2bf3b6..52ea7efaf8 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -109,14 +109,20 @@ void WFWaitTask::clear_timer_waiter() SubTask *WFTimedWaitTask::done() { + WFWaitTask *tmp = NULL; + this->mutex->lock(); if (this->wait_task && this->wait_task->entry.list.next) { list_del(&this->wait_task->entry.list); - this->wait_task->count(); + tmp = this->wait_task; + this->wait_task = NULL; } this->mutex->unlock(); + if (tmp) + tmp->count(); + SeriesWork *series = series_of(this); if (this->callback) @@ -162,7 +168,7 @@ WFCounterTask *WFCondition::create_timedwait_task(const struct timespec *abstime void WFCondition::signal() { - WFWaitTask *task; + WFWaitTask *task = NULL; struct list_head *pos; struct WFWaitTask::task_entry *entry; @@ -175,10 +181,12 @@ void WFCondition::signal() task = entry->ptr; list_del(pos); task->clear_timer_waiter(); - task->count(); } this->mutex.unlock(); + + if (task) + task->count(); } void WFCondition::broadcast() @@ -186,6 +194,7 @@ void WFCondition::broadcast() WFWaitTask *task; struct list_head *pos, *tmp; struct WFWaitTask::task_entry *entry; + LIST_HEAD(tmp_list); this->mutex.lock(); if (!list_empty(&this->waiter_list)) @@ -193,11 +202,17 @@ void WFCondition::broadcast() list_for_each_safe(pos, tmp, &this->waiter_list) { entry = list_entry(pos, struct WFWaitTask::task_entry, list); - task = entry->ptr; - list_del(pos); - task->count(); + list_move_tail(pos, &tmp_list); } } this->mutex.unlock(); + + while (!list_empty(&tmp_list)) + { + entry = list_entry(tmp_list.next, struct WFWaitTask::task_entry, list); + task = entry->ptr; + list_del(&entry->list); + task->count(); + } } diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h index 2e2e1f3c5e..9f996dc0a6 100644 --- a/src/manager/WFCondition.h +++ b/src/manager/WFCondition.h @@ -28,19 +28,18 @@ class WFCondition { public: - WFCondition() - { - INIT_LIST_HEAD(&this->waiter_list); - } - WFCounterTask *create_wait_task(std::function cb); WFCounterTask *create_timedwait_task(const struct timespec *abstime, std::function cb); void signal(); void broadcast(); +public: + WFCondition() { INIT_LIST_HEAD(&this->waiter_list); } + public: std::mutex mutex; + private: struct list_head waiter_list; }; diff --git a/tutorial/tutorial-14-websocket_cli.cc b/tutorial/tutorial-14-websocket_cli.cc index 480dcfad84..ba3bee7204 100644 --- a/tutorial/tutorial-14-websocket_cli.cc +++ b/tutorial/tutorial-14-websocket_cli.cc @@ -67,10 +67,12 @@ int main(int argc, char *argv[]) return; } + auto *timer_task = WFTaskFactory::create_timer_task(3000000 /* 3s */, nullptr); auto *close_task = client.create_close_task([&wg] (WFWebSocketTask *task) { wg.done(); }); + series_of(task)->push_back(timer_task); series_of(task)->push_back(close_task); }); From dc7db6fc4bfddae4c2e35e661d57397fe3343d36 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Mon, 21 Jun 2021 14:00:38 +0800 Subject: [PATCH 16/75] fix conflict with master --- CMakeLists_Headers.txt | 18 ++++++++---------- src/protocol/CMakeLists.txt | 8 ++++++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index 8008699f71..a7680d56d4 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -7,7 +7,6 @@ set(COMMON_KERNEL_HEADERS src/kernel/SleepRequest.h src/kernel/ExecRequest.h src/kernel/IORequest.h - src/kernel/TransRequest.h src/kernel/Executor.h src/kernel/list.h src/kernel/mpoller.h @@ -37,7 +36,7 @@ else () endif () set(INCLUDE_HEADERS - src/algorithm/DNSRoutine.h + src/algorithm/DnsRoutine.h src/algorithm/MapReduce.h src/algorithm/MapReduce.inl src/protocol/ProtocolMessage.h @@ -55,15 +54,17 @@ set(INCLUDE_HEADERS src/protocol/mysql_types.h src/protocol/mysql_byteorder.h src/protocol/SSLWrapper.h - src/protocol/WebSocketMessage.h - src/protocol/websocket_parser.h + src/protocol/dns_parser.h + src/protocol/DnsMessage.h + src/protocol/DnsUtil.h src/server/WFServer.h + src/server/WFDnsServer.h src/server/WFHttpServer.h src/server/WFRedisServer.h src/server/WFMySQLServer.h src/client/WFMySQLConnection.h - src/client/WFWebSocketClient.h - src/manager/DNSCache.h + src/client/WFDnsClient.h + src/manager/DnsCache.h src/manager/WFGlobal.h src/manager/UpstreamManager.h src/manager/RouteManager.h @@ -71,7 +72,6 @@ set(INCLUDE_HEADERS src/manager/WFFuture.h src/manager/WFFacilities.h src/manager/WFFacilities.inl - src/manager/WFCondition.h src/util/EncodeStream.h src/util/LRUCache.h src/util/StringUtil.h @@ -88,10 +88,8 @@ set(INCLUDE_HEADERS src/factory/WFAlgoTaskFactory.inl src/factory/Workflow.h src/factory/WFOperator.h - src/factory/WFChannel.h - src/factory/WFChannel.inl src/nameservice/WFNameService.h - src/nameservice/WFDNSResolver.h + src/nameservice/WFDnsResolver.h src/nameservice/WFServiceGovernance.h src/nameservice/UpstreamPolicies.h ) diff --git a/src/protocol/CMakeLists.txt b/src/protocol/CMakeLists.txt index a92effc1b9..8830cc4a50 100644 --- a/src/protocol/CMakeLists.txt +++ b/src/protocol/CMakeLists.txt @@ -2,17 +2,18 @@ cmake_minimum_required(VERSION 3.6) project(protocol) set(SRC + dns_parser.c http_parser.c redis_parser.c mysql_stream.c mysql_parser.c mysql_byteorder.c - websocket_parser.c + DnsMessage.cc + DnsUtil.cc MySQLMessage.cc MySQLResult.cc HttpMessage.cc RedisMessage.cc - WebSocketMessage.cc HttpUtil.cc SSLWrapper.cc ) @@ -27,5 +28,8 @@ if (KAFKA STREQUAL "y") KafkaResult.cc ) add_library("protocol_kafka" OBJECT ${SRC}) + set_property(SOURCE KafkaMessage.cc APPEND PROPERTY COMPILE_OPTIONS "-fno-rtti") + set_property(SOURCE KafkaDataTypes.cc APPEND PROPERTY COMPILE_OPTIONS "-fno-rtti") + set_property(SOURCE KafkaResult.cc APPEND PROPERTY COMPILE_OPTIONS "-fno-rtti") endif () From 1da36dc98c3177535adee00815a514bdd7a7ccd3 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Mon, 21 Jun 2021 14:06:28 +0800 Subject: [PATCH 17/75] fix conflict with master --- CMakeLists_Headers.txt | 8 ++++++++ src/protocol/CMakeLists.txt | 2 ++ 2 files changed, 10 insertions(+) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index a7680d56d4..ab029bb643 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -7,6 +7,7 @@ set(COMMON_KERNEL_HEADERS src/kernel/SleepRequest.h src/kernel/ExecRequest.h src/kernel/IORequest.h + src/kernel/TransRequest.h src/kernel/Executor.h src/kernel/list.h src/kernel/mpoller.h @@ -57,12 +58,15 @@ set(INCLUDE_HEADERS src/protocol/dns_parser.h src/protocol/DnsMessage.h src/protocol/DnsUtil.h + src/protocol/WebSocketMessage.h + src/protocol/websocket_parser.h src/server/WFServer.h src/server/WFDnsServer.h src/server/WFHttpServer.h src/server/WFRedisServer.h src/server/WFMySQLServer.h src/client/WFMySQLConnection.h + src/client/WFWebSocketClient.h src/client/WFDnsClient.h src/manager/DnsCache.h src/manager/WFGlobal.h @@ -72,6 +76,7 @@ set(INCLUDE_HEADERS src/manager/WFFuture.h src/manager/WFFacilities.h src/manager/WFFacilities.inl + src/manager/WFCondition.h src/util/EncodeStream.h src/util/LRUCache.h src/util/StringUtil.h @@ -88,6 +93,8 @@ set(INCLUDE_HEADERS src/factory/WFAlgoTaskFactory.inl src/factory/Workflow.h src/factory/WFOperator.h + src/factory/WFChannel.h + src/factory/WFChannel.inl src/nameservice/WFNameService.h src/nameservice/WFDnsResolver.h src/nameservice/WFServiceGovernance.h @@ -106,3 +113,4 @@ if(KAFKA STREQUAL "y") src/factory/KafkaTaskImpl.inl ) endif() + diff --git a/src/protocol/CMakeLists.txt b/src/protocol/CMakeLists.txt index 8830cc4a50..b5a6c8fc26 100644 --- a/src/protocol/CMakeLists.txt +++ b/src/protocol/CMakeLists.txt @@ -8,12 +8,14 @@ set(SRC mysql_stream.c mysql_parser.c mysql_byteorder.c + websocket_parser.c DnsMessage.cc DnsUtil.cc MySQLMessage.cc MySQLResult.cc HttpMessage.cc RedisMessage.cc + WebSocketMessage.cc HttpUtil.cc SSLWrapper.cc ) From c8ac577db3c256b939ede0928d8a43dba1bcb279 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Wed, 23 Jun 2021 02:55:15 +0800 Subject: [PATCH 18/75] add Semaphore and SemTaskFactory --- CMakeLists_Headers.txt | 3 +- src/factory/CMakeLists.txt | 2 + src/factory/WFChannel.inl | 18 +- src/factory/WFSemTaskFactory.cc | 287 +++++++++++++++++++++++++++++++ src/factory/WFSemTaskFactory.h | 76 ++++++++ src/factory/WFSemaphore.cc | 136 +++++++++++++++ src/factory/WFSemaphore.h | 164 ++++++++++++++++++ src/factory/WebSocketTaskImpl.cc | 4 +- src/manager/CMakeLists.txt | 1 - src/manager/WFCondition.cc | 14 +- src/manager/WFCondition.h | 6 +- test/CMakeLists.txt | 1 + test/condition_unittest.cc | 108 ++++++++++++ 13 files changed, 798 insertions(+), 22 deletions(-) create mode 100644 src/factory/WFSemTaskFactory.cc create mode 100644 src/factory/WFSemTaskFactory.h create mode 100644 src/factory/WFSemaphore.cc create mode 100644 src/factory/WFSemaphore.h create mode 100644 test/condition_unittest.cc diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index ab029bb643..8360f87838 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -76,7 +76,6 @@ set(INCLUDE_HEADERS src/manager/WFFuture.h src/manager/WFFacilities.h src/manager/WFFacilities.inl - src/manager/WFCondition.h src/util/EncodeStream.h src/util/LRUCache.h src/util/StringUtil.h @@ -91,8 +90,10 @@ set(INCLUDE_HEADERS src/factory/WFTaskFactory.inl src/factory/WFAlgoTaskFactory.h src/factory/WFAlgoTaskFactory.inl + src/factory/WFSemTaskFactory.h src/factory/Workflow.h src/factory/WFOperator.h + src/factory/WFSemaphore.h src/factory/WFChannel.h src/factory/WFChannel.inl src/nameservice/WFNameService.h diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index ec43f9a7dd..07213e34ab 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -9,6 +9,8 @@ set(SRC MySQLTaskImpl.cc WebSocketTaskImpl.cc WFTaskFactory.cc + WFSemaphore.cc + WFSemTaskFactory.cc Workflow.cc ) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 54da3960ad..7664a4cb4c 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -21,7 +21,9 @@ #include "TransRequest.h" #include "WFTask.h" #include "WFTaskFactory.h" -#include "WFCondition.h" +//#include "WFCondition.h" +#include "WFSemaphore.h" +#include "WFSemTaskFactory.h" #include "WFNameService.h" #include "RouteManager.h" #include "WFGlobal.h" @@ -247,7 +249,7 @@ void WFComplexChannel::handle_terminated() this->sending = true; shutdown = true; } else { - WFCounterTask *counter = this->condition.create_wait_task(nullptr); + SubTask *counter = WFSemTaskFactory::create_wait_task(&this->condition, nullptr); series_of(this)->push_front(this); series_of(this)->push_front(counter); } @@ -349,8 +351,8 @@ void ComplexChannelOutTask::dispatch() } else { - WFCounterTask *counter = channel->condition.create_wait_task( - [this](WFCounterTask *task) + SubTask *counter = WFSemTaskFactory::create_wait_task(&channel->condition, + [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); channel->set_state(WFT_STATE_SUCCESS); @@ -370,8 +372,8 @@ void ComplexChannelOutTask::dispatch() } else { - WFCounterTask *counter = channel->condition.create_wait_task( - [this](WFCounterTask *task) + SubTask *counter = WFSemTaskFactory::create_wait_task(&channel->condition, + [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); channel->set_state(WFT_STATE_SUCCESS); @@ -416,7 +418,7 @@ SubTask *ComplexChannelOutTask::done() pthread_mutex_lock(&channel->mutex); channel->set_sending(false); - channel->condition.signal(); + channel->condition.signal(NULL); pthread_mutex_unlock(&channel->mutex); return WFChannelOutTask::done(); @@ -433,7 +435,7 @@ SubTask *ComplexChannelOutTask::upgrade() channel->set_state(WFT_STATE_SUCCESS); this->ready = true; channel->set_sending(false); - channel->condition.signal(); + channel->condition.signal(NULL); pthread_mutex_unlock(&channel->mutex); }); diff --git a/src/factory/WFSemTaskFactory.cc b/src/factory/WFSemTaskFactory.cc new file mode 100644 index 0000000000..d054ea5c8a --- /dev/null +++ b/src/factory/WFSemTaskFactory.cc @@ -0,0 +1,287 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" +#include "WFSemTaskFactory.h" + +class __WFCondition : public WFCondition +{ +public: + __WFCondition(const std::string& str) : + name(str) + { } + + struct rb_node rb; + std::string name; +}; + +class __ConditionMap +{ +public: + void signal(const std::string& name, void *msg); + void broadcast(const std::string& name, void *msg); + + WFMailboxTask *create(const std::string& name, + mailbox_callback_t&& cb); + WFMailboxTask *create(const std::string& name, + const struct timespec *abstime, + mailbox_callback_t&& cb); + + WFMailboxTask *create_switch(const std::string& name, + mailbox_callback_t&& cb); + WFMailboxTask *create_switch(const std::string& name, + const struct timespec *abstime, + mailbox_callback_t&& cb); + +public: + static __ConditionMap *get_instance() + { + static __ConditionMap kInstance; + return &kInstance; + } + + virtual ~__ConditionMap(); + +private: + __ConditionMap() + { + this->condition_map.rb_node = NULL; + } + + __WFCondition *find_condition(const std::string& name); + struct rb_root condition_map; + std::mutex mutex; +}; + +void __ConditionMap::signal(const std::string& name, void *msg) +{ + __WFCondition *cond = this->find_condition(name); + + cond->signal(msg); +} + +void __ConditionMap::broadcast(const std::string& name, void *msg) +{ + __WFCondition *cond = this->find_condition(name); + + cond->broadcast(msg); +} + +WFMailboxTask *__ConditionMap::create(const std::string& name, + mailbox_callback_t&& cb) +{ + __WFCondition *cond = this->find_condition(name); + + return WFSemTaskFactory::create_wait_task(cond, std::move(cb)); +} + +WFMailboxTask *__ConditionMap::create(const std::string& name, + const struct timespec *abstime, + mailbox_callback_t&& cb) +{ + __WFCondition *cond = this->find_condition(name); + + return WFSemTaskFactory::create_timedwait_task(cond, abstime, + std::move(cb)); +} + +WFMailboxTask *__ConditionMap::create_switch(const std::string& name, + mailbox_callback_t&& cb) +{ + __WFCondition *cond = this->find_condition(name); + + return WFSemTaskFactory::create_switch_wait_task(cond, std::move(cb)); +} + +WFMailboxTask *__ConditionMap::create_switch(const std::string& name, + const struct timespec *abstime, + mailbox_callback_t&& cb) +{ + __WFCondition *cond = this->find_condition(name); + + return WFSemTaskFactory::create_switch_timedwait_task(cond, abstime, + std::move(cb)); +} + +__ConditionMap::~__ConditionMap() +{ + __WFCondition *cond; + WFWaitTask *task; + struct list_head *pos; + struct list_head *tmp; + + while (this->condition_map.rb_node) + { + cond = rb_entry(this->condition_map.rb_node, __WFCondition, rb); + list_for_each_safe(pos, tmp, &cond->waiter_list) + { + task = list_entry(pos, WFWaitTask, list); + list_del(pos); + delete task; + } + + rb_erase(this->condition_map.rb_node, &this->condition_map); + delete cond; + } +} + +__WFCondition *__ConditionMap::find_condition(const std::string& name) +{ + struct rb_node **p = &this->condition_map.rb_node; + struct rb_node *parent = NULL; + __WFCondition *cond; + + this->mutex.lock(); + while (*p) + { + parent = *p; + cond = rb_entry(*p, __WFCondition, rb); + + if (name < cond->name) + p = &(*p)->rb_left; + else if (name > cond->name) + p = &(*p)->rb_right; + else + break; + } + + if (*p == NULL) + { + cond = new __WFCondition(name); + rb_link_node(&cond->rb, parent, p); + rb_insert_color(&cond->rb, &this->condition_map); + } + + this->mutex.unlock(); + + return cond; +} + +/////////////// factory api /////////////// +/* +SubTask *acquire_by_name(const std::string& name, mailbox_callback_t callback) +{ + return __SemaphoreMap::get_instance()->acquire(name, std::move(callback)); +} + +void release_by_name(const std::string& name, void *msg) +{ + return __SemaphoreMap::get_instance()->release(name, msg); +} +*/ +void WFSemTaskFactory::signal_by_name(const std::string& name, void *msg) +{ + return __ConditionMap::get_instance()->signal(name, msg); +} + +void WFSemTaskFactory::broadcast_by_name(const std::string& name, void *msg) +{ + return __ConditionMap::get_instance()->broadcast(name, msg); +} + +WFMailboxTask *WFSemTaskFactory::create_wait_task(const std::string& name, + mailbox_callback_t callback) +{ + return __ConditionMap::get_instance()->create(name, std::move(callback)); +} + +WFMailboxTask *WFSemTaskFactory::create_timedwait_task(const std::string& name, + const struct timespec *abstime, + mailbox_callback_t callback) +{ + return __ConditionMap::get_instance()->create(name, abstime, + std::move(callback)); +} + +WFMailboxTask *WFSemTaskFactory::create_switch_wait_task(const std::string& name, + mailbox_callback_t callback) +{ + return __ConditionMap::get_instance()->create_switch(name, std::move(callback)); +} + +WFMailboxTask *WFSemTaskFactory::create_switch_timedwait_task(const std::string& name, + const struct timespec *abstime, + mailbox_callback_t callback) +{ + return __ConditionMap::get_instance()->create_switch(name, abstime, + std::move(callback)); +} + +WFMailboxTask *WFSemTaskFactory::create_wait_task(WFCondition *cond, + mailbox_callback_t callback) +{ + WFWaitTask *task = new WFWaitTask(std::move(callback)); + + cond->mutex.lock(); + list_add_tail(&task->list, &cond->waiter_list); + cond->mutex.unlock(); + + return task; +} + +WFMailboxTask *WFSemTaskFactory::create_timedwait_task(WFCondition *cond, + const struct timespec *abstime, + mailbox_callback_t callback) +{ + WFWaitTask *waiter = new WFWaitTask(std::move(callback)); + WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &cond->mutex, abstime, + WFGlobal::get_scheduler(), + nullptr); + waiter->set_timer(task); + + cond->mutex.lock(); + list_add_tail(&waiter->list, &cond->waiter_list); + cond->mutex.unlock(); + + return waiter; +} + +WFMailboxTask *WFSemTaskFactory::create_switch_wait_task(WFCondition *cond, + mailbox_callback_t callback) +{ + WFWaitTask *task = new WFSwitchWaitTask(std::move(callback)); + + cond->mutex.lock(); + list_add_tail(&task->list, &cond->waiter_list); + cond->mutex.unlock(); + + return task; +} + +WFMailboxTask *WFSemTaskFactory::create_switch_timedwait_task(WFCondition *cond, + const struct timespec *abstime, + mailbox_callback_t callback) +{ + WFSwitchWaitTask *waiter = new WFSwitchWaitTask(std::move(callback)); + WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &cond->mutex, abstime, + WFGlobal::get_scheduler(), + nullptr); + waiter->set_timer(task); + + cond->mutex.lock(); + list_add_tail(&waiter->list, &cond->waiter_list); + cond->mutex.unlock(); + + return waiter; +} diff --git a/src/factory/WFSemTaskFactory.h b/src/factory/WFSemTaskFactory.h new file mode 100644 index 0000000000..3a91f78772 --- /dev/null +++ b/src/factory/WFSemTaskFactory.h @@ -0,0 +1,76 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WFSEMTASKFACTORY_H_ +#define _WFSEMTASKFACTORY_H_ + +#include +#include +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" +#include "WFSemaphore.h" + +class WFSemTaskFactory +{ +public: + // use semaphore by name + static SubTask *acquire_by_name(const std::string& name, mailbox_callback_t cb); + + static void release_by_name(const std::string& name, void *msg); + + // use condition by name + static void signal_by_name(const std::string& name, void *msg); + + static void broadcast_by_name(const std::string& name, void *msg); + + static WFMailboxTask *create_wait_task(const std::string& name, + mailbox_callback_t callback); + + static WFMailboxTask *create_timedwait_task(const std::string& name, + const struct timespec *abstime, + mailbox_callback_t callback); + + static WFMailboxTask *create_switch_wait_task(const std::string& name, + mailbox_callback_t callback); + + static WFMailboxTask *create_switch_timedwait_task(const std::string& name, + const struct timespec *abstime, + mailbox_callback_t callback); + + // use condition by ptr + static WFMailboxTask *create_wait_task(WFCondition *cond, + mailbox_callback_t callback); + + static WFMailboxTask *create_timedwait_task(WFCondition *cond, + const struct timespec *abstime, + mailbox_callback_t callback); + + static WFMailboxTask *create_switch_wait_task(WFCondition *cond, + mailbox_callback_t callback); + + static WFMailboxTask *create_switch_timedwait_task(WFCondition *cond, + const struct timespec *abstime, + mailbox_callback_t callback); +}; + +#endif + diff --git a/src/factory/WFSemaphore.cc b/src/factory/WFSemaphore.cc new file mode 100644 index 0000000000..e944af90a7 --- /dev/null +++ b/src/factory/WFSemaphore.cc @@ -0,0 +1,136 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Liu Kai (liukaidx@sogou-inc.com) +*/ + +#include +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFSemaphore.h" + +/////////////// Semaphore Impl /////////////// + +WFMailboxTask *WFSemaphore::acquire(std::function cb) +{ + WFSemaphoreTask *task = new WFSemaphoreTask(std::move(cb)); + + if (--this->concurrency >= 0) + { + task->count(); + } + else + { + this->mutex.lock(); + list_add_tail(&task->list, &this->waiter_list); + this->mutex.unlock(); + } + + return task; +} + +void WFSemaphore::release(void *msg) +{ + WFSemaphoreTask *task; + struct list_head *pos; + + this->mutex.lock(); + + if (++this->concurrency <= 0)// && !list_empty(&this->waiter_list)) + { + pos = this->waiter_list.next; + task = list_entry(pos, WFSemaphoreTask, list); + list_del(pos); + task->send(msg); + } + + this->mutex.unlock(); +} + +/////////////// Wait tasks Impl /////////////// + +void WFWaitTask::dispatch() +{ + if (this->timer) + timer->dispatch(); + + this->WFMailboxTask::count(); +} + +void WFWaitTask::clear_timer_waiter() +{ + if (this->timer) + timer->clear_wait_task(); +} + +SubTask *WFTimedWaitTask::done() +{ + this->mutex->lock(); + if (this->wait_task && this->wait_task->list.next) + { + list_del(&this->wait_task->list); + this->wait_task->set_error(ETIMEDOUT); + this->wait_task->count(); + this->wait_task = NULL; + } + this->mutex->unlock(); + + delete this; + return NULL; +} + +/////////////// Condition Impl /////////////// + +void WFCondition::signal(void *msg) +{ + WFWaitTask *task; + struct list_head *pos; + + this->mutex.lock(); + + if (!list_empty(&this->waiter_list)) + { + pos = this->waiter_list.next; + task = list_entry(pos, WFWaitTask, list); + list_del(pos); + task->clear_timer_waiter(); + task->send(msg); + } + + this->mutex.unlock(); +} + +void WFCondition::broadcast(void *msg) +{ + WFWaitTask *task; + struct list_head *pos, *tmp; + + this->mutex.lock(); + if (!list_empty(&this->waiter_list)) + { + list_for_each_safe(pos, tmp, &this->waiter_list) + { + task = list_entry(pos, WFWaitTask, list); + list_del(pos); + task->send(msg); + } + } + this->mutex.unlock(); +} + diff --git a/src/factory/WFSemaphore.h b/src/factory/WFSemaphore.h new file mode 100644 index 0000000000..eabd4888c2 --- /dev/null +++ b/src/factory/WFSemaphore.h @@ -0,0 +1,164 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Liu Kai (liukaidx@sogou-inc.com) +*/ + +#ifndef _WFSEMAPHORE_H_ +#define _WFSEMAPHORE_H_ + +#include +#include +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" + +class WFSemaphore +{ +public: + WFMailboxTask *acquire(std::function cb); + void release(void *msg); + +public: + std::mutex mutex; + struct list_head waiter_list; + +public: + WFSemaphore(int value) + { + INIT_LIST_HEAD(&this->waiter_list); + this->concurrency = value; +// this->size = value; + } + +private: + std::atomic concurrency; +// int size; +}; + +class WFCondition : public WFSemaphore +{ +public: + void signal(void *msg); + void broadcast(void *msg); + +public: + WFCondition() : WFSemaphore(1) { } + WFCondition(int value) : WFSemaphore(value) { } + ~WFCondition() { } +}; + +class WFSemaphoreTask : public WFMailboxTask +{ +public: + WFSemaphoreTask(std::function&& cb) : + WFMailboxTask(std::move(cb)) +// WFMailboxTask(&this->msg, 1, std::move(cb)) + { + this->list.next = NULL; + } + + virtual ~WFSemaphoreTask() { } + +public: + struct list_head list; + +private: + void *msg; // TODO: user_data instead +}; + +class WFTimedWaitTask; + +class WFWaitTask : public WFSemaphoreTask +{ +public: + void set_timer(WFTimedWaitTask *timer) { this->timer = timer; } + void clear_timer_waiter(); + void set_error(int error) { this->error = error; } + +protected: + void dispatch(); + +private: + WFTimedWaitTask *timer; + +public: + WFWaitTask(std::function&& cb) : + WFSemaphoreTask(std::move(cb)) + { + this->timer = NULL; + } + + virtual ~WFWaitTask() { } +}; + +class WFTimedWaitTask : public __WFTimerTask +{ +public: + WFTimedWaitTask(WFWaitTask *wait_task, std::mutex *mutex, + const struct timespec *value, + CommScheduler *scheduler, + std::function cb) : + __WFTimerTask(value, scheduler, std::move(cb)) + { + this->mutex = mutex; + this->wait_task = wait_task; + } + + void clear_wait_task() + { + this->mutex->lock(); + this->wait_task = NULL; + this->mutex->unlock(); + } + +protected: + virtual SubTask *done(); + +private: + std::mutex *mutex; + WFWaitTask *wait_task; +}; + +class WFSwitchWaitTask : public WFWaitTask +{ +public: + WFSwitchWaitTask(std::function&& cb) : + WFWaitTask(std::move(cb)) + { } + +protected: + SubTask *done() + { + SeriesWork *series = series_of(this); + + WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, + [this](WFTimerTask *task) { + if (this->callback) + this->callback(this); + delete this; + }); + series->push_front(switch_task); + + return series->pop(); + } +}; + +#endif + diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc index 8b28859546..4b278ddb23 100644 --- a/src/factory/WebSocketTaskImpl.cc +++ b/src/factory/WebSocketTaskImpl.cc @@ -130,7 +130,7 @@ SubTask *ComplexWebSocketOutTask::done() pthread_mutex_lock(&channel->mutex); channel->set_sending(false); - channel->condition.signal(); + channel->condition.signal(NULL); pthread_mutex_unlock(&channel->mutex); return WFChannelOutTask::done(); @@ -201,7 +201,7 @@ void ComplexWebSocketChannel::handle_in(CommMessageIn *in) if (!parse_websocket) // so this is equal to should_count { pthread_mutex_lock(&this->mutex); - this->condition.signal(); + this->condition.signal(NULL); pthread_mutex_unlock(&this->mutex); return; } diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index 5571ade8e8..91af981ca5 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -5,7 +5,6 @@ set(SRC DnsCache.cc UpstreamManager.cc RouteManager.cc - WFCondition.cc WFGlobal.cc ) diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index 52ea7efaf8..12c5d62190 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -24,7 +24,7 @@ class WFTimedWaitTask; -class WFWaitTask : public WFCounterTask +class WFWaitTask : public WFMailboxTask { public: void set_timer(WFTimedWaitTask *timer) { this->timer = timer; } @@ -44,8 +44,8 @@ class WFWaitTask : public WFCounterTask WFTimedWaitTask *timer; public: - WFWaitTask(std::function&& cb) : - WFCounterTask(1, std::move(cb)) + WFWaitTask(std::function&& cb) : + WFMailboxTask(&this->user_data, 1, std::move(cb)) { this->timer = NULL; this->entry.list.next = NULL; @@ -81,7 +81,7 @@ void WFWaitTask::dispatch() if (this->timer) timer->dispatch(); - this->WFCounterTask::count(); + this->WFMailboxTask::count(); } SubTask *WFWaitTask::done() @@ -139,7 +139,7 @@ void WFTimedWaitTask::clear_wait_task() this->mutex->unlock(); } -WFCounterTask *WFCondition::create_wait_task(counter_callback_t cb) +WFMailboxTask *WFCondition::create_wait_task(mailbox_callback_t cb) { WFWaitTask *task = new WFWaitTask(std::move(cb)); @@ -150,8 +150,8 @@ WFCounterTask *WFCondition::create_wait_task(counter_callback_t cb) return task; } -WFCounterTask *WFCondition::create_timedwait_task(const struct timespec *abstime, - counter_callback_t cb) +WFMailboxTask *WFCondition::create_timedwait_task(const struct timespec *abstime, + mailbox_callback_t cb) { WFWaitTask *waiter = new WFWaitTask(std::move(cb)); WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &this->mutex, abstime, diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h index 9f996dc0a6..9c343f3158 100644 --- a/src/manager/WFCondition.h +++ b/src/manager/WFCondition.h @@ -28,9 +28,9 @@ class WFCondition { public: - WFCounterTask *create_wait_task(std::function cb); - WFCounterTask *create_timedwait_task(const struct timespec *abstime, - std::function cb); + WFMailboxTask *create_wait_task(std::function cb); + WFMailboxTask *create_timedwait_task(const struct timespec *abstime, + std::function cb); void signal(); void broadcast(); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c673b168fb..1eb5fa8bab 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,6 +39,7 @@ set(TEST_LIST memory_unittest upstream_unittest dns_unittest + condition_unittest ) if (APPLE) diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc new file mode 100644 index 0000000000..16a96c827c --- /dev/null +++ b/test/condition_unittest.cc @@ -0,0 +1,108 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include +#include +#include +#include +#include +#include "workflow/WFTask.h" +#include "workflow/WFTaskFactory.h" +//#include "workflow/WFCondition.h" +#include "workflow/WFSemaphore.h" +#include "workflow/WFSemTaskFactory.h" +#include "workflow/WFFacilities.h" + +TEST(condition_unittest, signal) +{ + WFCondition cond; + std::mutex mutex; + int ret = 1979; + int *ptr = &ret; + + auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + *ptr = 1412; + fprintf(stderr, "task1 callback\n"); + }); + SeriesWork *series1 = Workflow::create_series_work(task1, nullptr); + + auto *task2 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + *ptr = 2; + fprintf(stderr, "task2 callback\n"); + }); + SeriesWork *series2 = Workflow::create_series_work(task2, nullptr); + + series1->start(); + series2->start(); + + mutex.lock(); + cond.signal(NULL); + mutex.unlock(); + usleep(10000); + EXPECT_EQ(ret, 1412); + cond.signal(NULL); + usleep(10000); + EXPECT_EQ(ret, 2); +} + +TEST(condition_unittest, broadcast) +{ + WFCondition cond; + std::mutex mutex; + int ret = 0; + int *ptr = &ret; + + auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + (*ptr)++; + fprintf(stderr, "task1 callback\n"); + }); + SeriesWork *series1 = Workflow::create_series_work(task1, nullptr); + + auto *task2 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + (*ptr)++; + fprintf(stderr, "task2 callback\n"); + }); + SeriesWork *series2 = Workflow::create_series_work(task2, nullptr); + + series1->start(); + series2->start(); + + cond.broadcast(NULL); + usleep(10000); + EXPECT_EQ(ret, 2); +} + +TEST(condition_unittest, timedwait) +{ + WFFacilities::WaitGroup wait_group(1); + struct timespec ts; + ts.tv_sec = 2; + ts.tv_nsec = 0; + + auto *task1 = WFSemTaskFactory::create_timedwait_task("timedwait", &ts, + [&wait_group](WFMailboxTask *task) { + EXPECT_EQ(task->get_error(), ETIMEDOUT); + wait_group.done(); + }); + + Workflow::start_series_work(task1, nullptr); + wait_group.wait(); + usleep(10000); +} + From 3b6edf1c4a1880500d2e8bec8a1b67d7d9777e10 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Wed, 23 Jun 2021 16:37:06 +0800 Subject: [PATCH 19/75] update timedwait --- src/factory/WFChannel.inl | 15 +++++++------ src/factory/WFSemTaskFactory.cc | 2 ++ src/factory/WFSemaphore.h | 4 +--- test/condition_unittest.cc | 38 ++++++++++++++++++++------------- 4 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 7664a4cb4c..d4d66f5f51 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -239,6 +239,7 @@ SubTask *WFComplexChannel::done() template void WFComplexChannel::handle_terminated() { + WFMailboxTask *waiter; bool shutdown = false; pthread_mutex_lock(&this->mutex); @@ -249,9 +250,10 @@ void WFComplexChannel::handle_terminated() this->sending = true; shutdown = true; } else { - SubTask *counter = WFSemTaskFactory::create_wait_task(&this->condition, nullptr); + waiter = WFSemTaskFactory::create_switch_wait_task(&this->condition, + nullptr); series_of(this)->push_front(this); - series_of(this)->push_front(counter); + series_of(this)->push_front(waiter); } pthread_mutex_unlock(&this->mutex); @@ -322,6 +324,7 @@ protected: template void ComplexChannelOutTask::dispatch() { + WFMailboxTask *waiter; bool should_send = false; auto *channel = (WFComplexChannel *)this->get_request_channel(); @@ -351,7 +354,7 @@ void ComplexChannelOutTask::dispatch() } else { - SubTask *counter = WFSemTaskFactory::create_wait_task(&channel->condition, + waiter = WFSemTaskFactory::create_switch_wait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); @@ -359,7 +362,7 @@ void ComplexChannelOutTask::dispatch() this->ready = true; }); series_of(this)->push_front(this); - series_of(this)->push_front(counter); + series_of(this)->push_front(waiter); this->ready = false; } break; @@ -372,7 +375,7 @@ void ComplexChannelOutTask::dispatch() } else { - SubTask *counter = WFSemTaskFactory::create_wait_task(&channel->condition, + waiter = WFSemTaskFactory::create_switch_wait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); @@ -380,7 +383,7 @@ void ComplexChannelOutTask::dispatch() this->ready = true; }); series_of(this)->push_front(this); - series_of(this)->push_front(counter); + series_of(this)->push_front(waiter); this->ready = false; } break; diff --git a/src/factory/WFSemTaskFactory.cc b/src/factory/WFSemTaskFactory.cc index d054ea5c8a..8ac8b0763a 100644 --- a/src/factory/WFSemTaskFactory.cc +++ b/src/factory/WFSemTaskFactory.cc @@ -18,8 +18,10 @@ #include #include +#include #include #include "list.h" +#include "rbtree.h" #include "WFTask.h" #include "WFTaskFactory.h" #include "WFGlobal.h" diff --git a/src/factory/WFSemaphore.h b/src/factory/WFSemaphore.h index eabd4888c2..ea60a20122 100644 --- a/src/factory/WFSemaphore.h +++ b/src/factory/WFSemaphore.h @@ -121,11 +121,9 @@ class WFTimedWaitTask : public __WFTimerTask this->wait_task = wait_task; } - void clear_wait_task() + void clear_wait_task() // must called within this mutex { - this->mutex->lock(); this->wait_task = NULL; - this->mutex->unlock(); } protected: diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index 16a96c827c..10abc9e7d2 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -33,19 +33,18 @@ TEST(condition_unittest, signal) { WFCondition cond; std::mutex mutex; - int ret = 1979; + int ret = 3; int *ptr = &ret; auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { - *ptr = 1412; - fprintf(stderr, "task1 callback\n"); + *ptr = 1; }); - SeriesWork *series1 = Workflow::create_series_work(task1, nullptr); auto *task2 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { *ptr = 2; - fprintf(stderr, "task2 callback\n"); }); + + SeriesWork *series1 = Workflow::create_series_work(task1, nullptr); SeriesWork *series2 = Workflow::create_series_work(task2, nullptr); series1->start(); @@ -54,10 +53,10 @@ TEST(condition_unittest, signal) mutex.lock(); cond.signal(NULL); mutex.unlock(); - usleep(10000); - EXPECT_EQ(ret, 1412); + usleep(1000); + EXPECT_EQ(ret, 1); cond.signal(NULL); - usleep(10000); + usleep(1000); EXPECT_EQ(ret, 2); } @@ -70,13 +69,11 @@ TEST(condition_unittest, broadcast) auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { (*ptr)++; - fprintf(stderr, "task1 callback\n"); }); SeriesWork *series1 = Workflow::create_series_work(task1, nullptr); auto *task2 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { (*ptr)++; - fprintf(stderr, "task2 callback\n"); }); SeriesWork *series2 = Workflow::create_series_work(task2, nullptr); @@ -84,25 +81,36 @@ TEST(condition_unittest, broadcast) series2->start(); cond.broadcast(NULL); - usleep(10000); + usleep(1000); EXPECT_EQ(ret, 2); } TEST(condition_unittest, timedwait) { - WFFacilities::WaitGroup wait_group(1); + WFFacilities::WaitGroup wait_group(2); struct timespec ts; - ts.tv_sec = 2; + ts.tv_sec = 1; ts.tv_nsec = 0; - auto *task1 = WFSemTaskFactory::create_timedwait_task("timedwait", &ts, + auto *task1 = WFSemTaskFactory::create_timedwait_task("timedwait1", &ts, [&wait_group](WFMailboxTask *task) { EXPECT_EQ(task->get_error(), ETIMEDOUT); wait_group.done(); }); + auto *task2 = WFSemTaskFactory::create_timedwait_task("timedwait2", &ts, + [&wait_group](WFMailboxTask *task) { + EXPECT_EQ(task->get_error(), 0); + EXPECT_TRUE(strcmp((char *)task->user_data, "wake up!!") == 0); + wait_group.done(); + }); + Workflow::start_series_work(task1, nullptr); + Workflow::start_series_work(task2, nullptr); + + usleep(1000); + char msg[10] = "wake up!!"; + WFSemTaskFactory::signal_by_name("timedwait2", msg); wait_group.wait(); - usleep(10000); } From 604eb723828cdace683781e35eb2ac853883ddef Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 24 Jun 2021 00:33:27 +0800 Subject: [PATCH 20/75] update msg --- src/factory/WFSemTaskFactory.cc | 40 +++++++++++++++++++++++---------- src/factory/WFSemaphore.cc | 18 ++++++++++----- src/factory/WFSemaphore.h | 14 +++++++----- test/condition_unittest.cc | 6 ++++- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/src/factory/WFSemTaskFactory.cc b/src/factory/WFSemTaskFactory.cc index 8ac8b0763a..ed76ed5729 100644 --- a/src/factory/WFSemTaskFactory.cc +++ b/src/factory/WFSemTaskFactory.cc @@ -32,9 +32,17 @@ class __WFCondition : public WFCondition public: __WFCondition(const std::string& str) : name(str) - { } + { + this->node.ptr = this; + } + +public: + struct entry + { + struct rb_node rb; + __WFCondition *ptr; + } node; - struct rb_node rb; std::string name; }; @@ -132,13 +140,19 @@ __ConditionMap::~__ConditionMap() WFWaitTask *task; struct list_head *pos; struct list_head *tmp; + struct WFSemaphoreTask::entry *node; + struct __WFCondition::entry *cond_node; while (this->condition_map.rb_node) { - cond = rb_entry(this->condition_map.rb_node, __WFCondition, rb); + cond_node = rb_entry(this->condition_map.rb_node, + struct __WFCondition::entry, rb); + cond = cond_node->ptr; + list_for_each_safe(pos, tmp, &cond->waiter_list) { - task = list_entry(pos, WFWaitTask, list); + node = list_entry(pos, struct WFSemaphoreTask::entry, list); + task = (WFWaitTask *)node->ptr; list_del(pos); delete task; } @@ -150,15 +164,17 @@ __ConditionMap::~__ConditionMap() __WFCondition *__ConditionMap::find_condition(const std::string& name) { + __WFCondition *cond; + struct __WFCondition::entry *cond_node; struct rb_node **p = &this->condition_map.rb_node; struct rb_node *parent = NULL; - __WFCondition *cond; this->mutex.lock(); while (*p) { parent = *p; - cond = rb_entry(*p, __WFCondition, rb); + cond_node = rb_entry(*p, struct __WFCondition::entry, rb); + cond = cond_node->ptr; if (name < cond->name) p = &(*p)->rb_left; @@ -171,8 +187,8 @@ __WFCondition *__ConditionMap::find_condition(const std::string& name) if (*p == NULL) { cond = new __WFCondition(name); - rb_link_node(&cond->rb, parent, p); - rb_insert_color(&cond->rb, &this->condition_map); + rb_link_node(&cond->node.rb, parent, p); + rb_insert_color(&cond->node.rb, &this->condition_map); } this->mutex.unlock(); @@ -236,7 +252,7 @@ WFMailboxTask *WFSemTaskFactory::create_wait_task(WFCondition *cond, WFWaitTask *task = new WFWaitTask(std::move(callback)); cond->mutex.lock(); - list_add_tail(&task->list, &cond->waiter_list); + list_add_tail(&task->node.list, &cond->waiter_list); cond->mutex.unlock(); return task; @@ -253,7 +269,7 @@ WFMailboxTask *WFSemTaskFactory::create_timedwait_task(WFCondition *cond, waiter->set_timer(task); cond->mutex.lock(); - list_add_tail(&waiter->list, &cond->waiter_list); + list_add_tail(&waiter->node.list, &cond->waiter_list); cond->mutex.unlock(); return waiter; @@ -265,7 +281,7 @@ WFMailboxTask *WFSemTaskFactory::create_switch_wait_task(WFCondition *cond, WFWaitTask *task = new WFSwitchWaitTask(std::move(callback)); cond->mutex.lock(); - list_add_tail(&task->list, &cond->waiter_list); + list_add_tail(&task->node.list, &cond->waiter_list); cond->mutex.unlock(); return task; @@ -282,7 +298,7 @@ WFMailboxTask *WFSemTaskFactory::create_switch_timedwait_task(WFCondition *cond, waiter->set_timer(task); cond->mutex.lock(); - list_add_tail(&waiter->list, &cond->waiter_list); + list_add_tail(&waiter->node.list, &cond->waiter_list); cond->mutex.unlock(); return waiter; diff --git a/src/factory/WFSemaphore.cc b/src/factory/WFSemaphore.cc index e944af90a7..3d5c564549 100644 --- a/src/factory/WFSemaphore.cc +++ b/src/factory/WFSemaphore.cc @@ -38,7 +38,7 @@ WFMailboxTask *WFSemaphore::acquire(std::function cb) else { this->mutex.lock(); - list_add_tail(&task->list, &this->waiter_list); + list_add_tail(&task->node.list, &this->waiter_list); this->mutex.unlock(); } @@ -49,13 +49,15 @@ void WFSemaphore::release(void *msg) { WFSemaphoreTask *task; struct list_head *pos; + struct WFSemaphoreTask::entry *node; this->mutex.lock(); if (++this->concurrency <= 0)// && !list_empty(&this->waiter_list)) { pos = this->waiter_list.next; - task = list_entry(pos, WFSemaphoreTask, list); + node = list_entry(pos, struct WFSemaphoreTask::entry, list); + task = node->ptr; list_del(pos); task->send(msg); } @@ -82,9 +84,9 @@ void WFWaitTask::clear_timer_waiter() SubTask *WFTimedWaitTask::done() { this->mutex->lock(); - if (this->wait_task && this->wait_task->list.next) + if (this->wait_task && this->wait_task->node.list.next) { - list_del(&this->wait_task->list); + list_del(&this->wait_task->node.list); this->wait_task->set_error(ETIMEDOUT); this->wait_task->count(); this->wait_task = NULL; @@ -101,13 +103,15 @@ void WFCondition::signal(void *msg) { WFWaitTask *task; struct list_head *pos; + struct WFSemaphoreTask::entry *node; this->mutex.lock(); if (!list_empty(&this->waiter_list)) { pos = this->waiter_list.next; - task = list_entry(pos, WFWaitTask, list); + node = list_entry(pos, struct WFSemaphoreTask::entry, list); + task = (WFWaitTask *)node->ptr; list_del(pos); task->clear_timer_waiter(); task->send(msg); @@ -120,13 +124,15 @@ void WFCondition::broadcast(void *msg) { WFWaitTask *task; struct list_head *pos, *tmp; + struct WFSemaphoreTask::entry *node; this->mutex.lock(); if (!list_empty(&this->waiter_list)) { list_for_each_safe(pos, tmp, &this->waiter_list) { - task = list_entry(pos, WFWaitTask, list); + node = list_entry(pos, struct WFSemaphoreTask::entry, list); + task = (WFWaitTask *)node->ptr; list_del(pos); task->send(msg); } diff --git a/src/factory/WFSemaphore.h b/src/factory/WFSemaphore.h index ea60a20122..236a787ade 100644 --- a/src/factory/WFSemaphore.h +++ b/src/factory/WFSemaphore.h @@ -68,19 +68,23 @@ class WFSemaphoreTask : public WFMailboxTask { public: WFSemaphoreTask(std::function&& cb) : - WFMailboxTask(std::move(cb)) -// WFMailboxTask(&this->msg, 1, std::move(cb)) + WFMailboxTask(&this->msg, 1, std::move(cb)) { - this->list.next = NULL; + this->node.list.next = NULL; + this->node.ptr = this; } virtual ~WFSemaphoreTask() { } public: - struct list_head list; + struct entry + { + struct list_head list; + WFSemaphoreTask *ptr; + } node; private: - void *msg; // TODO: user_data instead + void *msg; }; class WFTimedWaitTask; diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index 10abc9e7d2..9b6003b172 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -101,7 +101,11 @@ TEST(condition_unittest, timedwait) auto *task2 = WFSemTaskFactory::create_timedwait_task("timedwait2", &ts, [&wait_group](WFMailboxTask *task) { EXPECT_EQ(task->get_error(), 0); - EXPECT_TRUE(strcmp((char *)task->user_data, "wake up!!") == 0); + void **msg; + size_t n; + msg = task->get_mailbox(&n); + EXPECT_EQ(n, 1); + EXPECT_TRUE(strcmp((char *)*msg, "wake up!!") == 0); wait_group.done(); }); From 2bbfceb825bce2afe99f7a5df3ff5c9558741edb Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 24 Jun 2021 00:50:48 +0800 Subject: [PATCH 21/75] remove useless api --- src/factory/WFSemTaskFactory.cc | 11 +---------- src/factory/WFSemTaskFactory.h | 5 ----- src/factory/WFSemaphore.h | 2 -- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/src/factory/WFSemTaskFactory.cc b/src/factory/WFSemTaskFactory.cc index ed76ed5729..52dc098dd0 100644 --- a/src/factory/WFSemTaskFactory.cc +++ b/src/factory/WFSemTaskFactory.cc @@ -197,17 +197,7 @@ __WFCondition *__ConditionMap::find_condition(const std::string& name) } /////////////// factory api /////////////// -/* -SubTask *acquire_by_name(const std::string& name, mailbox_callback_t callback) -{ - return __SemaphoreMap::get_instance()->acquire(name, std::move(callback)); -} -void release_by_name(const std::string& name, void *msg) -{ - return __SemaphoreMap::get_instance()->release(name, msg); -} -*/ void WFSemTaskFactory::signal_by_name(const std::string& name, void *msg) { return __ConditionMap::get_instance()->signal(name, msg); @@ -303,3 +293,4 @@ WFMailboxTask *WFSemTaskFactory::create_switch_timedwait_task(WFCondition *cond, return waiter; } + diff --git a/src/factory/WFSemTaskFactory.h b/src/factory/WFSemTaskFactory.h index 3a91f78772..f5cab04581 100644 --- a/src/factory/WFSemTaskFactory.h +++ b/src/factory/WFSemTaskFactory.h @@ -32,11 +32,6 @@ class WFSemTaskFactory { public: - // use semaphore by name - static SubTask *acquire_by_name(const std::string& name, mailbox_callback_t cb); - - static void release_by_name(const std::string& name, void *msg); - // use condition by name static void signal_by_name(const std::string& name, void *msg); diff --git a/src/factory/WFSemaphore.h b/src/factory/WFSemaphore.h index 236a787ade..a61801b7a2 100644 --- a/src/factory/WFSemaphore.h +++ b/src/factory/WFSemaphore.h @@ -44,12 +44,10 @@ class WFSemaphore { INIT_LIST_HEAD(&this->waiter_list); this->concurrency = value; -// this->size = value; } private: std::atomic concurrency; -// int size; }; class WFCondition : public WFSemaphore From ba25878bf37e1b3ca3433186b75be1b86428e072 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 25 Jun 2021 16:59:07 +0800 Subject: [PATCH 22/75] update WFSemaphore: get/create_wait_task/post --- src/factory/WFChannel.inl | 8 +-- src/factory/WFSemTaskFactory.cc | 107 ++++++-------------------------- src/factory/WFSemTaskFactory.h | 33 +++------- src/factory/WFSemaphore.cc | 69 ++++++++++++++------ src/factory/WFSemaphore.h | 58 +++++++---------- 5 files changed, 107 insertions(+), 168 deletions(-) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index d4d66f5f51..d8a2c7de49 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -250,8 +250,8 @@ void WFComplexChannel::handle_terminated() this->sending = true; shutdown = true; } else { - waiter = WFSemTaskFactory::create_switch_wait_task(&this->condition, - nullptr); + waiter = WFSemTaskFactory::create_wait_task(&this->condition, + nullptr); series_of(this)->push_front(this); series_of(this)->push_front(waiter); } @@ -354,7 +354,7 @@ void ComplexChannelOutTask::dispatch() } else { - waiter = WFSemTaskFactory::create_switch_wait_task(&channel->condition, + waiter = WFSemTaskFactory::create_wait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); @@ -375,7 +375,7 @@ void ComplexChannelOutTask::dispatch() } else { - waiter = WFSemTaskFactory::create_switch_wait_task(&channel->condition, + waiter = WFSemTaskFactory::create_wait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); diff --git a/src/factory/WFSemTaskFactory.cc b/src/factory/WFSemTaskFactory.cc index 52dc098dd0..c39c98384c 100644 --- a/src/factory/WFSemTaskFactory.cc +++ b/src/factory/WFSemTaskFactory.cc @@ -52,17 +52,11 @@ class __ConditionMap void signal(const std::string& name, void *msg); void broadcast(const std::string& name, void *msg); - WFMailboxTask *create(const std::string& name, - mailbox_callback_t&& cb); - WFMailboxTask *create(const std::string& name, + WFWaitTask *create(const std::string& name, + wait_callback_t&& cb); + WFWaitTask *create(const std::string& name, const struct timespec *abstime, - mailbox_callback_t&& cb); - - WFMailboxTask *create_switch(const std::string& name, - mailbox_callback_t&& cb); - WFMailboxTask *create_switch(const std::string& name, - const struct timespec *abstime, - mailbox_callback_t&& cb); + wait_callback_t&& cb); public: static __ConditionMap *get_instance() @@ -98,17 +92,17 @@ void __ConditionMap::broadcast(const std::string& name, void *msg) cond->broadcast(msg); } -WFMailboxTask *__ConditionMap::create(const std::string& name, - mailbox_callback_t&& cb) +WFWaitTask *__ConditionMap::create(const std::string& name, + wait_callback_t&& cb) { __WFCondition *cond = this->find_condition(name); return WFSemTaskFactory::create_wait_task(cond, std::move(cb)); } -WFMailboxTask *__ConditionMap::create(const std::string& name, +WFWaitTask *__ConditionMap::create(const std::string& name, const struct timespec *abstime, - mailbox_callback_t&& cb) + wait_callback_t&& cb) { __WFCondition *cond = this->find_condition(name); @@ -116,28 +110,10 @@ WFMailboxTask *__ConditionMap::create(const std::string& name, std::move(cb)); } -WFMailboxTask *__ConditionMap::create_switch(const std::string& name, - mailbox_callback_t&& cb) -{ - __WFCondition *cond = this->find_condition(name); - - return WFSemTaskFactory::create_switch_wait_task(cond, std::move(cb)); -} - -WFMailboxTask *__ConditionMap::create_switch(const std::string& name, - const struct timespec *abstime, - mailbox_callback_t&& cb) -{ - __WFCondition *cond = this->find_condition(name); - - return WFSemTaskFactory::create_switch_timedwait_task(cond, abstime, - std::move(cb)); -} - __ConditionMap::~__ConditionMap() { __WFCondition *cond; - WFWaitTask *task; + WFCondWaitTask *task; struct list_head *pos; struct list_head *tmp; struct WFSemaphoreTask::entry *node; @@ -152,7 +128,7 @@ __ConditionMap::~__ConditionMap() list_for_each_safe(pos, tmp, &cond->waiter_list) { node = list_entry(pos, struct WFSemaphoreTask::entry, list); - task = (WFWaitTask *)node->ptr; + task = (WFCondWaitTask *)node->ptr; list_del(pos); delete task; } @@ -208,38 +184,24 @@ void WFSemTaskFactory::broadcast_by_name(const std::string& name, void *msg) return __ConditionMap::get_instance()->broadcast(name, msg); } -WFMailboxTask *WFSemTaskFactory::create_wait_task(const std::string& name, - mailbox_callback_t callback) +WFWaitTask *WFSemTaskFactory::create_wait_task(const std::string& name, + wait_callback_t callback) { return __ConditionMap::get_instance()->create(name, std::move(callback)); } -WFMailboxTask *WFSemTaskFactory::create_timedwait_task(const std::string& name, +WFWaitTask *WFSemTaskFactory::create_timedwait_task(const std::string& name, const struct timespec *abstime, - mailbox_callback_t callback) + wait_callback_t callback) { return __ConditionMap::get_instance()->create(name, abstime, std::move(callback)); } -WFMailboxTask *WFSemTaskFactory::create_switch_wait_task(const std::string& name, - mailbox_callback_t callback) -{ - return __ConditionMap::get_instance()->create_switch(name, std::move(callback)); -} - -WFMailboxTask *WFSemTaskFactory::create_switch_timedwait_task(const std::string& name, - const struct timespec *abstime, - mailbox_callback_t callback) -{ - return __ConditionMap::get_instance()->create_switch(name, abstime, - std::move(callback)); -} - -WFMailboxTask *WFSemTaskFactory::create_wait_task(WFCondition *cond, - mailbox_callback_t callback) +WFWaitTask *WFSemTaskFactory::create_wait_task(WFCondition *cond, + wait_callback_t callback) { - WFWaitTask *task = new WFWaitTask(std::move(callback)); + WFCondWaitTask *task = new WFCondWaitTask(std::move(callback)); cond->mutex.lock(); list_add_tail(&task->node.list, &cond->waiter_list); @@ -248,40 +210,11 @@ WFMailboxTask *WFSemTaskFactory::create_wait_task(WFCondition *cond, return task; } -WFMailboxTask *WFSemTaskFactory::create_timedwait_task(WFCondition *cond, +WFWaitTask *WFSemTaskFactory::create_timedwait_task(WFCondition *cond, const struct timespec *abstime, - mailbox_callback_t callback) -{ - WFWaitTask *waiter = new WFWaitTask(std::move(callback)); - WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &cond->mutex, abstime, - WFGlobal::get_scheduler(), - nullptr); - waiter->set_timer(task); - - cond->mutex.lock(); - list_add_tail(&waiter->node.list, &cond->waiter_list); - cond->mutex.unlock(); - - return waiter; -} - -WFMailboxTask *WFSemTaskFactory::create_switch_wait_task(WFCondition *cond, - mailbox_callback_t callback) -{ - WFWaitTask *task = new WFSwitchWaitTask(std::move(callback)); - - cond->mutex.lock(); - list_add_tail(&task->node.list, &cond->waiter_list); - cond->mutex.unlock(); - - return task; -} - -WFMailboxTask *WFSemTaskFactory::create_switch_timedwait_task(WFCondition *cond, - const struct timespec *abstime, - mailbox_callback_t callback) + wait_callback_t callback) { - WFSwitchWaitTask *waiter = new WFSwitchWaitTask(std::move(callback)); + WFCondWaitTask *waiter = new WFCondWaitTask(std::move(callback)); WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &cond->mutex, abstime, WFGlobal::get_scheduler(), nullptr); diff --git a/src/factory/WFSemTaskFactory.h b/src/factory/WFSemTaskFactory.h index f5cab04581..403e0810b4 100644 --- a/src/factory/WFSemTaskFactory.h +++ b/src/factory/WFSemTaskFactory.h @@ -37,34 +37,21 @@ class WFSemTaskFactory static void broadcast_by_name(const std::string& name, void *msg); - static WFMailboxTask *create_wait_task(const std::string& name, - mailbox_callback_t callback); + static WFWaitTask *create_wait_task(const std::string& name, + wait_callback_t callback); - static WFMailboxTask *create_timedwait_task(const std::string& name, - const struct timespec *abstime, - mailbox_callback_t callback); - - static WFMailboxTask *create_switch_wait_task(const std::string& name, - mailbox_callback_t callback); - - static WFMailboxTask *create_switch_timedwait_task(const std::string& name, - const struct timespec *abstime, - mailbox_callback_t callback); + static WFWaitTask *create_timedwait_task(const std::string& name, + const struct timespec *abstime, + wait_callback_t callback); // use condition by ptr - static WFMailboxTask *create_wait_task(WFCondition *cond, - mailbox_callback_t callback); - - static WFMailboxTask *create_timedwait_task(WFCondition *cond, - const struct timespec *abstime, - mailbox_callback_t callback); + static WFWaitTask *create_wait_task(WFCondition *cond, + wait_callback_t callback); - static WFMailboxTask *create_switch_wait_task(WFCondition *cond, - mailbox_callback_t callback); + static WFWaitTask *create_timedwait_task(WFCondition *cond, + const struct timespec *abstime, + wait_callback_t callback); - static WFMailboxTask *create_switch_timedwait_task(WFCondition *cond, - const struct timespec *abstime, - mailbox_callback_t callback); }; #endif diff --git a/src/factory/WFSemaphore.cc b/src/factory/WFSemaphore.cc index 3d5c564549..e31175e590 100644 --- a/src/factory/WFSemaphore.cc +++ b/src/factory/WFSemaphore.cc @@ -27,25 +27,40 @@ /////////////// Semaphore Impl /////////////// -WFMailboxTask *WFSemaphore::acquire(std::function cb) +bool WFSemaphore::get() { - WFSemaphoreTask *task = new WFSemaphoreTask(std::move(cb)); - if (--this->concurrency >= 0) + return true; + + WFSemaphoreTask *task = new WFSemaphoreTask(nullptr); + + this->mutex.lock(); + list_add_tail(&task->node.list, &this->waiter_list); + this->mutex.unlock(); + + return true; +} + +WFWaitTask *WFSemaphore::create_wait_task(std::function cb) +{ + WFSemaphoreTask *task = NULL; + struct list_head *pos; + struct WFSemaphoreTask::entry *node; + + this->mutex.lock(); + if (!list_empty(&this->waiter_list)) { - task->count(); - } - else - { - this->mutex.lock(); - list_add_tail(&task->node.list, &this->waiter_list); - this->mutex.unlock(); + pos = this->waiter_list.next; + node = list_entry(pos, struct WFSemaphoreTask::entry, list); + task = node->ptr; + task->set_callback(std::move(cb)); } + this->mutex.unlock(); return task; } -void WFSemaphore::release(void *msg) +void WFSemaphore::post(void *msg) { WFSemaphoreTask *task; struct list_head *pos; @@ -62,20 +77,38 @@ void WFSemaphore::release(void *msg) task->send(msg); } + if (this->concurrency > this->total) + this->concurrency = this->total; + this->mutex.unlock(); } /////////////// Wait tasks Impl /////////////// -void WFWaitTask::dispatch() +void WFCondWaitTask::dispatch() { if (this->timer) timer->dispatch(); - this->WFMailboxTask::count(); + this->WFWaitTask::count(); +} + +SubTask *WFCondWaitTask::done() +{ + SeriesWork *series = series_of(this); + + WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, + [this](WFTimerTask *task) { + if (this->callback) + this->callback(this); + delete this; + }); + series->push_front(switch_task); + + return series->pop(); } -void WFWaitTask::clear_timer_waiter() +void WFCondWaitTask::clear_timer_waiter() { if (this->timer) timer->clear_wait_task(); @@ -101,7 +134,7 @@ SubTask *WFTimedWaitTask::done() void WFCondition::signal(void *msg) { - WFWaitTask *task; + WFCondWaitTask *task; struct list_head *pos; struct WFSemaphoreTask::entry *node; @@ -111,7 +144,7 @@ void WFCondition::signal(void *msg) { pos = this->waiter_list.next; node = list_entry(pos, struct WFSemaphoreTask::entry, list); - task = (WFWaitTask *)node->ptr; + task = (WFCondWaitTask *)node->ptr; list_del(pos); task->clear_timer_waiter(); task->send(msg); @@ -122,7 +155,7 @@ void WFCondition::signal(void *msg) void WFCondition::broadcast(void *msg) { - WFWaitTask *task; + WFCondWaitTask *task; struct list_head *pos, *tmp; struct WFSemaphoreTask::entry *node; @@ -132,7 +165,7 @@ void WFCondition::broadcast(void *msg) list_for_each_safe(pos, tmp, &this->waiter_list) { node = list_entry(pos, struct WFSemaphoreTask::entry, list); - task = (WFWaitTask *)node->ptr; + task = (WFCondWaitTask *)node->ptr; list_del(pos); task->send(msg); } diff --git a/src/factory/WFSemaphore.h b/src/factory/WFSemaphore.h index a61801b7a2..84f79da2e6 100644 --- a/src/factory/WFSemaphore.h +++ b/src/factory/WFSemaphore.h @@ -29,25 +29,34 @@ #include "WFTaskFactory.h" #include "WFGlobal.h" +using WFWaitTask = WFMailboxTask; +using wait_callback_t = mailbox_callback_t; + class WFSemaphore { public: - WFMailboxTask *acquire(std::function cb); - void release(void *msg); + bool get(); + WFWaitTask *create_wait_task(std::function cb); + void post(void *msg); public: std::mutex mutex; struct list_head waiter_list; - + public: WFSemaphore(int value) { + if (value <= 0) + value = 1; + INIT_LIST_HEAD(&this->waiter_list); this->concurrency = value; + this->total = value; } private: std::atomic concurrency; + int total; }; class WFCondition : public WFSemaphore @@ -59,14 +68,14 @@ class WFCondition : public WFSemaphore public: WFCondition() : WFSemaphore(1) { } WFCondition(int value) : WFSemaphore(value) { } - ~WFCondition() { } + virtual ~WFCondition() { } }; -class WFSemaphoreTask : public WFMailboxTask +class WFSemaphoreTask : public WFWaitTask { public: - WFSemaphoreTask(std::function&& cb) : - WFMailboxTask(&this->msg, 1, std::move(cb)) + WFSemaphoreTask(std::function&& cb) : + WFWaitTask(&this->msg, 1, std::move(cb)) { this->node.list.next = NULL; this->node.ptr = this; @@ -87,7 +96,7 @@ class WFSemaphoreTask : public WFMailboxTask class WFTimedWaitTask; -class WFWaitTask : public WFSemaphoreTask +class WFCondWaitTask : public WFSemaphoreTask { public: void set_timer(WFTimedWaitTask *timer) { this->timer = timer; } @@ -96,24 +105,25 @@ class WFWaitTask : public WFSemaphoreTask protected: void dispatch(); + SubTask *done(); private: WFTimedWaitTask *timer; public: - WFWaitTask(std::function&& cb) : + WFCondWaitTask(std::function&& cb) : WFSemaphoreTask(std::move(cb)) { this->timer = NULL; } - virtual ~WFWaitTask() { } + virtual ~WFCondWaitTask() { } }; class WFTimedWaitTask : public __WFTimerTask { public: - WFTimedWaitTask(WFWaitTask *wait_task, std::mutex *mutex, + WFTimedWaitTask(WFCondWaitTask *wait_task, std::mutex *mutex, const struct timespec *value, CommScheduler *scheduler, std::function cb) : @@ -133,31 +143,7 @@ class WFTimedWaitTask : public __WFTimerTask private: std::mutex *mutex; - WFWaitTask *wait_task; -}; - -class WFSwitchWaitTask : public WFWaitTask -{ -public: - WFSwitchWaitTask(std::function&& cb) : - WFWaitTask(std::move(cb)) - { } - -protected: - SubTask *done() - { - SeriesWork *series = series_of(this); - - WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, - [this](WFTimerTask *task) { - if (this->callback) - this->callback(this); - delete this; - }); - series->push_front(switch_task); - - return series->pop(); - } + WFCondWaitTask *wait_task; }; #endif From 2a3b6029e72a799710caf5284da86cc9e65c66f1 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Mon, 28 Jun 2021 20:31:13 +0800 Subject: [PATCH 23/75] update Semaphore and add unittest --- CMakeLists_Headers.txt | 3 +- src/factory/CMakeLists.txt | 1 - src/factory/WFSemTaskFactory.cc | 6 +- src/manager/CMakeLists.txt | 1 + src/{factory => manager}/WFSemaphore.cc | 55 +++++++------ src/manager/WFSemaphore.h | 80 +++++++++++++++++++ .../WFSemaphore.h => manager/WFSemaphore.inl} | 53 +----------- test/condition_unittest.cc | 46 ++++++++++- 8 files changed, 162 insertions(+), 83 deletions(-) rename src/{factory => manager}/WFSemaphore.cc (78%) create mode 100644 src/manager/WFSemaphore.h rename src/{factory/WFSemaphore.h => manager/WFSemaphore.inl} (70%) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index eef2247435..3aabf79b4f 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -79,6 +79,8 @@ set(INCLUDE_HEADERS src/manager/WFFuture.h src/manager/WFFacilities.h src/manager/WFFacilities.inl + src/manager/WFSemaphore.h + src/manager/WFSemaphore.inl src/util/EncodeStream.h src/util/LRUCache.h src/util/StringUtil.h @@ -96,7 +98,6 @@ set(INCLUDE_HEADERS src/factory/WFSemTaskFactory.h src/factory/Workflow.h src/factory/WFOperator.h - src/factory/WFSemaphore.h src/factory/WFChannel.h src/factory/WFChannel.inl src/nameservice/WFNameService.h diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index 07213e34ab..06a5d38b11 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -9,7 +9,6 @@ set(SRC MySQLTaskImpl.cc WebSocketTaskImpl.cc WFTaskFactory.cc - WFSemaphore.cc WFSemTaskFactory.cc Workflow.cc ) diff --git a/src/factory/WFSemTaskFactory.cc b/src/factory/WFSemTaskFactory.cc index c39c98384c..bf8c5cd4a0 100644 --- a/src/factory/WFSemTaskFactory.cc +++ b/src/factory/WFSemTaskFactory.cc @@ -125,7 +125,7 @@ __ConditionMap::~__ConditionMap() struct __WFCondition::entry, rb); cond = cond_node->ptr; - list_for_each_safe(pos, tmp, &cond->waiter_list) + list_for_each_safe(pos, tmp, &cond->wait_list) { node = list_entry(pos, struct WFSemaphoreTask::entry, list); task = (WFCondWaitTask *)node->ptr; @@ -204,7 +204,7 @@ WFWaitTask *WFSemTaskFactory::create_wait_task(WFCondition *cond, WFCondWaitTask *task = new WFCondWaitTask(std::move(callback)); cond->mutex.lock(); - list_add_tail(&task->node.list, &cond->waiter_list); + list_add_tail(&task->node.list, &cond->wait_list); cond->mutex.unlock(); return task; @@ -221,7 +221,7 @@ WFWaitTask *WFSemTaskFactory::create_timedwait_task(WFCondition *cond, waiter->set_timer(task); cond->mutex.lock(); - list_add_tail(&waiter->node.list, &cond->waiter_list); + list_add_tail(&waiter->node.list, &cond->wait_list); cond->mutex.unlock(); return waiter; diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index 91af981ca5..d53c827f52 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -5,6 +5,7 @@ set(SRC DnsCache.cc UpstreamManager.cc RouteManager.cc + WFSemaphore.cc WFGlobal.cc ) diff --git a/src/factory/WFSemaphore.cc b/src/manager/WFSemaphore.cc similarity index 78% rename from src/factory/WFSemaphore.cc rename to src/manager/WFSemaphore.cc index e31175e590..3788f35388 100644 --- a/src/factory/WFSemaphore.cc +++ b/src/manager/WFSemaphore.cc @@ -14,6 +14,7 @@ limitations under the License. Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) Liu Kai (liukaidx@sogou-inc.com) */ @@ -35,10 +36,10 @@ bool WFSemaphore::get() WFSemaphoreTask *task = new WFSemaphoreTask(nullptr); this->mutex.lock(); - list_add_tail(&task->node.list, &this->waiter_list); + list_add_tail(&task->node.list, &this->get_list); this->mutex.unlock(); - return true; + return false; } WFWaitTask *WFSemaphore::create_wait_task(std::function cb) @@ -48,39 +49,41 @@ WFWaitTask *WFSemaphore::create_wait_task(std::function cb) struct WFSemaphoreTask::entry *node; this->mutex.lock(); - if (!list_empty(&this->waiter_list)) + if (!list_empty(&this->get_list)) { - pos = this->waiter_list.next; + pos = this->get_list.next; + list_move_tail(pos, &this->wait_list); node = list_entry(pos, struct WFSemaphoreTask::entry, list); task = node->ptr; task->set_callback(std::move(cb)); } - this->mutex.unlock(); + this->mutex.unlock(); return task; } void WFSemaphore::post(void *msg) { - WFSemaphoreTask *task; + WFSemaphoreTask *task = NULL; struct list_head *pos; struct WFSemaphoreTask::entry *node; this->mutex.lock(); - if (++this->concurrency <= 0)// && !list_empty(&this->waiter_list)) + if (++this->concurrency <= 0) { - pos = this->waiter_list.next; + if (!list_empty(&this->wait_list)) + pos = this->wait_list.next; + else + pos = this->get_list.next; node = list_entry(pos, struct WFSemaphoreTask::entry, list); task = node->ptr; list_del(pos); - task->send(msg); } - if (this->concurrency > this->total) - this->concurrency = this->total; - this->mutex.unlock(); + if (task) + task->send(msg); } /////////////// Wait tasks Impl /////////////// @@ -134,23 +137,23 @@ SubTask *WFTimedWaitTask::done() void WFCondition::signal(void *msg) { - WFCondWaitTask *task; + WFCondWaitTask *task = NULL; struct list_head *pos; struct WFSemaphoreTask::entry *node; this->mutex.lock(); - - if (!list_empty(&this->waiter_list)) + if (!list_empty(&this->wait_list)) { - pos = this->waiter_list.next; + pos = this->wait_list.next; node = list_entry(pos, struct WFSemaphoreTask::entry, list); task = (WFCondWaitTask *)node->ptr; list_del(pos); task->clear_timer_waiter(); - task->send(msg); } this->mutex.unlock(); + if (task) + task->send(msg); } void WFCondition::broadcast(void *msg) @@ -158,18 +161,24 @@ void WFCondition::broadcast(void *msg) WFCondWaitTask *task; struct list_head *pos, *tmp; struct WFSemaphoreTask::entry *node; + LIST_HEAD(tmp_list); this->mutex.lock(); - if (!list_empty(&this->waiter_list)) + if (!list_empty(&this->wait_list)) { - list_for_each_safe(pos, tmp, &this->waiter_list) + list_for_each_safe(pos, tmp, &this->wait_list) { - node = list_entry(pos, struct WFSemaphoreTask::entry, list); - task = (WFCondWaitTask *)node->ptr; - list_del(pos); - task->send(msg); + list_move_tail(pos, &tmp_list); } } + this->mutex.unlock(); + while (!list_empty(&tmp_list)) + { + node = list_entry(tmp_list.next, struct WFSemaphoreTask::entry, list); + task = (WFCondWaitTask *)node->ptr; + list_del(&node->list); + task->send(msg); + } } diff --git a/src/manager/WFSemaphore.h b/src/manager/WFSemaphore.h new file mode 100644 index 0000000000..927b037f8f --- /dev/null +++ b/src/manager/WFSemaphore.h @@ -0,0 +1,80 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) + Liu Kai (liukaidx@sogou-inc.com) +*/ + +#ifndef _WFSEMAPHORE_H_ +#define _WFSEMAPHORE_H_ + +#include +#include +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" + +using WFWaitTask = WFMailboxTask; +using wait_callback_t = mailbox_callback_t; + +class WFSemaphore +{ +public: + bool get(); + WFWaitTask *create_wait_task(std::function cb); + void post(void *msg); + +public: + std::mutex mutex; + struct list_head get_list; + struct list_head wait_list; + +public: + WFSemaphore(int value) + { + if (value <= 0) + value = 1; + + INIT_LIST_HEAD(&this->get_list); + INIT_LIST_HEAD(&this->wait_list); + this->concurrency = value; + this->total = value; + } + +private: + std::atomic concurrency; + int total; +}; + +class WFCondition : public WFSemaphore +{ +public: + void signal(void *msg); + void broadcast(void *msg); + +public: + WFCondition() : WFSemaphore(1) { } + WFCondition(int value) : WFSemaphore(value) { } + virtual ~WFCondition() { } +}; + +#include "WFSemaphore.inl" + +#endif + diff --git a/src/factory/WFSemaphore.h b/src/manager/WFSemaphore.inl similarity index 70% rename from src/factory/WFSemaphore.h rename to src/manager/WFSemaphore.inl index 84f79da2e6..8c79b848c5 100644 --- a/src/factory/WFSemaphore.h +++ b/src/manager/WFSemaphore.inl @@ -14,62 +14,13 @@ limitations under the License. Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) Liu Kai (liukaidx@sogou-inc.com) */ -#ifndef _WFSEMAPHORE_H_ -#define _WFSEMAPHORE_H_ - -#include -#include -#include -#include #include "list.h" #include "WFTask.h" #include "WFTaskFactory.h" -#include "WFGlobal.h" - -using WFWaitTask = WFMailboxTask; -using wait_callback_t = mailbox_callback_t; - -class WFSemaphore -{ -public: - bool get(); - WFWaitTask *create_wait_task(std::function cb); - void post(void *msg); - -public: - std::mutex mutex; - struct list_head waiter_list; - -public: - WFSemaphore(int value) - { - if (value <= 0) - value = 1; - - INIT_LIST_HEAD(&this->waiter_list); - this->concurrency = value; - this->total = value; - } - -private: - std::atomic concurrency; - int total; -}; - -class WFCondition : public WFSemaphore -{ -public: - void signal(void *msg); - void broadcast(void *msg); - -public: - WFCondition() : WFSemaphore(1) { } - WFCondition(int value) : WFSemaphore(value) { } - virtual ~WFCondition() { } -}; class WFSemaphoreTask : public WFWaitTask { @@ -146,5 +97,3 @@ class WFTimedWaitTask : public __WFTimerTask WFCondWaitTask *wait_task; }; -#endif - diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index 9b6003b172..e920db6e55 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -24,7 +24,6 @@ #include #include "workflow/WFTask.h" #include "workflow/WFTaskFactory.h" -//#include "workflow/WFCondition.h" #include "workflow/WFSemaphore.h" #include "workflow/WFSemTaskFactory.h" #include "workflow/WFFacilities.h" @@ -33,11 +32,13 @@ TEST(condition_unittest, signal) { WFCondition cond; std::mutex mutex; + WFFacilities::WaitGroup wg(1); int ret = 3; int *ptr = &ret; - auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&wg, &ptr](WFMailboxTask *) { *ptr = 1; + wg.done(); }); auto *task2 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { @@ -53,7 +54,7 @@ TEST(condition_unittest, signal) mutex.lock(); cond.signal(NULL); mutex.unlock(); - usleep(1000); + wg.wait(); EXPECT_EQ(ret, 1); cond.signal(NULL); usleep(1000); @@ -118,3 +119,42 @@ TEST(condition_unittest, timedwait) wait_group.wait(); } +#define work usleep + +TEST(condition_unittest, semaphore) +{ + int sem_concurrency = 1; + int task_concurrency = 3; + WFSemaphore sem(sem_concurrency); + WFFacilities::WaitGroup wg(task_concurrency); + + for (int i = 0; i < task_concurrency; i ++) + { + auto *t = WFTaskFactory::create_timer_task(i * 1, [&sem, &wg](WFTimerTask *task) { + uint64_t id = (uint64_t)series_of(task)->get_context(); + if (!sem.get()) + { + auto *waiter = sem.create_wait_task([&sem, &wg](WFWaitTask *task) { + uint64_t id = (uint64_t)series_of(task)->get_context(); + work(3); + wg.done(); + sem.post(reinterpret_cast(id)); + }); + series_of(task)->push_back(waiter); + } + else + { + work(3); + wg.done(); + sem.post(reinterpret_cast(id)); + } + }); + + SeriesWork *series = Workflow::create_series_work(t, nullptr); + series->set_context(reinterpret_cast(i)); + series->start(); + } + + wg.wait(); +} + From c07410a4255ba957175cb4d79a625bfa071b4000 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Wed, 30 Jun 2021 03:17:29 +0800 Subject: [PATCH 24/75] support resources for Semaphore --- src/manager/WFSemaphore.cc | 16 +++++++++++----- src/manager/WFSemaphore.h | 27 ++++++++++++++++++++------- src/manager/WFSemaphore.inl | 13 +++++++++++++ test/condition_unittest.cc | 27 +++++++++++++++++---------- 4 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/manager/WFSemaphore.cc b/src/manager/WFSemaphore.cc index 3788f35388..9013dfec13 100644 --- a/src/manager/WFSemaphore.cc +++ b/src/manager/WFSemaphore.cc @@ -28,14 +28,18 @@ /////////////// Semaphore Impl /////////////// -bool WFSemaphore::get() +bool WFSemaphore::get(void **dest) { + this->mutex.lock(); if (--this->concurrency >= 0) + { + *dest = this->resources[--this->index]; + this->mutex.unlock(); return true; + } WFSemaphoreTask *task = new WFSemaphoreTask(nullptr); - this->mutex.lock(); list_add_tail(&task->node.list, &this->get_list); this->mutex.unlock(); @@ -62,7 +66,7 @@ WFWaitTask *WFSemaphore::create_wait_task(std::function cb) return task; } -void WFSemaphore::post(void *msg) +void WFSemaphore::post(void *src) { WFSemaphoreTask *task = NULL; struct list_head *pos; @@ -81,9 +85,11 @@ void WFSemaphore::post(void *msg) list_del(pos); } + this->resources[this->index++] = src; + this->mutex.unlock(); if (task) - task->send(msg); + task->send(src); } /////////////// Wait tasks Impl /////////////// @@ -92,7 +98,7 @@ void WFCondWaitTask::dispatch() { if (this->timer) timer->dispatch(); - + this->WFWaitTask::count(); } diff --git a/src/manager/WFSemaphore.h b/src/manager/WFSemaphore.h index 927b037f8f..e5e8b9ffa7 100644 --- a/src/manager/WFSemaphore.h +++ b/src/manager/WFSemaphore.h @@ -36,9 +36,9 @@ using wait_callback_t = mailbox_callback_t; class WFSemaphore { public: - bool get(); + bool get(void **dest); WFWaitTask *create_wait_task(std::function cb); - void post(void *msg); + void post(void *src); public: std::mutex mutex; @@ -46,7 +46,7 @@ class WFSemaphore struct list_head wait_list; public: - WFSemaphore(int value) + WFSemaphore(int value, void **resources) { if (value <= 0) value = 1; @@ -55,23 +55,36 @@ class WFSemaphore INIT_LIST_HEAD(&this->wait_list); this->concurrency = value; this->total = value; + this->index = value; + this->resources = resources; +// this->resources = new void *[value]; } + virtual ~WFSemaphore() { /* delete []this->resources; */ } + private: std::atomic concurrency; int total; + int index; + +protected: + void **resources; }; class WFCondition : public WFSemaphore -{ +{ public: void signal(void *msg); void broadcast(void *msg); public: - WFCondition() : WFSemaphore(1) { } - WFCondition(int value) : WFSemaphore(value) { } - virtual ~WFCondition() { } + WFCondition() : WFSemaphore(1, new void *[1]) { } + WFCondition(int value) : WFSemaphore(value, new void *[value]) { } + virtual ~WFCondition() { delete []this->resources;} + + bool get(void **dest) = delete; + WFWaitTask *create_wait_task(wait_callback_t cb) = delete; + void post(void *src) = delete; }; #include "WFSemaphore.inl" diff --git a/src/manager/WFSemaphore.inl b/src/manager/WFSemaphore.inl index 8c79b848c5..f6c4f89a53 100644 --- a/src/manager/WFSemaphore.inl +++ b/src/manager/WFSemaphore.inl @@ -24,6 +24,19 @@ class WFSemaphoreTask : public WFWaitTask { +public: + void start() + { + assert(!series_of(this)); + Workflow::start_series_work(this, nullptr); + } + + void dismiss() + { + assert(!series_of(this)); + delete this; + } + public: WFSemaphoreTask(std::function&& cb) : WFWaitTask(&this->msg, 1, std::move(cb)) diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index e920db6e55..5e32b986c0 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -119,34 +119,42 @@ TEST(condition_unittest, timedwait) wait_group.wait(); } -#define work usleep - TEST(condition_unittest, semaphore) { int sem_concurrency = 1; int task_concurrency = 3; - WFSemaphore sem(sem_concurrency); + const char *words[3] = {"workflow", "srpc", "pyworkflow"}; + WFSemaphore sem(sem_concurrency, (void **)words); WFFacilities::WaitGroup wg(task_concurrency); - for (int i = 0; i < task_concurrency; i ++) + for (int i = 0; i < task_concurrency; i++) { auto *t = WFTaskFactory::create_timer_task(i * 1, [&sem, &wg](WFTimerTask *task) { uint64_t id = (uint64_t)series_of(task)->get_context(); - if (!sem.get()) + void *dest_res; + + if (!sem.get(&dest_res)) { auto *waiter = sem.create_wait_task([&sem, &wg](WFWaitTask *task) { uint64_t id = (uint64_t)series_of(task)->get_context(); - work(3); + void **msg; + size_t n; + msg = task->get_mailbox(&n); + fprintf(stderr, "%lu after wait. get resource:%s\n", id, (char *)*msg); + usleep(10); wg.done(); - sem.post(reinterpret_cast(id)); + sem.post(*msg); + //sem.post(reinterpret_cast(id)); }); series_of(task)->push_back(waiter); } else { - work(3); + fprintf(stderr, "%lu no wait. get resource: %s\n", id, (char *)dest_res); + usleep(10); wg.done(); - sem.post(reinterpret_cast(id)); + //sem.post(reinterpret_cast(id)); + sem.post(dest_res); } }); @@ -154,7 +162,6 @@ TEST(condition_unittest, semaphore) series->set_context(reinterpret_cast(i)); series->start(); } - wg.wait(); } From b747038f709cdb31fcb791d367bea880d58d2ab3 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 2 Jul 2021 20:24:57 +0800 Subject: [PATCH 25/75] merge conflicts --- CMakeLists_Headers.txt | 13 + docs/tutorial-14-websocket_cli.md | 230 ++++++++++++ src/client/WFWebSocketClient.h | 130 +++++++ src/factory/CMakeLists.txt | 3 +- src/factory/WFChannel.h | 97 +++++ src/factory/WFChannel.inl | 490 ++++++++++++++++++++++++++ src/factory/WFSemTaskFactory.cc | 229 ++++++++++++ src/factory/WFSemTaskFactory.h | 58 +++ src/factory/WebSocketTaskImpl.cc | 225 ++++++++++++ src/kernel/CMakeLists.txt | 1 + src/kernel/CommScheduler.h | 27 ++ src/kernel/Communicator.cc | 313 ++++++++++++++-- src/kernel/Communicator.h | 49 +++ src/kernel/TransRequest.cc | 59 ++++ src/kernel/TransRequest.h | 109 ++++++ src/manager/CMakeLists.txt | 1 + src/manager/WFCondition.cc | 218 ++++++++++++ src/manager/WFCondition.h | 48 +++ src/manager/WFGlobal.cc | 18 +- src/manager/WFSemaphore.cc | 190 ++++++++++ src/manager/WFSemaphore.h | 93 +++++ src/manager/WFSemaphore.inl | 112 ++++++ src/protocol/CMakeLists.txt | 2 + src/protocol/WebSocketMessage.cc | 299 ++++++++++++++++ src/protocol/WebSocketMessage.h | 84 +++++ src/protocol/websocket_parser.c | 239 +++++++++++++ src/protocol/websocket_parser.h | 123 +++++++ test/CMakeLists.txt | 1 + test/condition_unittest.cc | 167 +++++++++ tutorial/CMakeLists.txt | 2 + tutorial/tutorial-14-websocket_cli.cc | 86 +++++ 31 files changed, 3673 insertions(+), 43 deletions(-) create mode 100644 docs/tutorial-14-websocket_cli.md create mode 100644 src/client/WFWebSocketClient.h create mode 100644 src/factory/WFChannel.h create mode 100644 src/factory/WFChannel.inl create mode 100644 src/factory/WFSemTaskFactory.cc create mode 100644 src/factory/WFSemTaskFactory.h create mode 100644 src/factory/WebSocketTaskImpl.cc create mode 100644 src/kernel/TransRequest.cc create mode 100644 src/kernel/TransRequest.h create mode 100644 src/manager/WFCondition.cc create mode 100644 src/manager/WFCondition.h create mode 100644 src/manager/WFSemaphore.cc create mode 100644 src/manager/WFSemaphore.h create mode 100644 src/manager/WFSemaphore.inl create mode 100644 src/protocol/WebSocketMessage.cc create mode 100644 src/protocol/WebSocketMessage.h create mode 100644 src/protocol/websocket_parser.c create mode 100644 src/protocol/websocket_parser.h create mode 100644 test/condition_unittest.cc create mode 100644 tutorial/tutorial-14-websocket_cli.cc diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index a7680d56d4..3aabf79b4f 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -7,6 +7,7 @@ set(COMMON_KERNEL_HEADERS src/kernel/SleepRequest.h src/kernel/ExecRequest.h src/kernel/IORequest.h + src/kernel/TransRequest.h src/kernel/Executor.h src/kernel/list.h src/kernel/mpoller.h @@ -57,12 +58,18 @@ set(INCLUDE_HEADERS src/protocol/dns_parser.h src/protocol/DnsMessage.h src/protocol/DnsUtil.h + src/protocol/WebSocketMessage.h + src/protocol/websocket_parser.h + src/protocol/dns_parser.h + src/protocol/DnsMessage.h + src/protocol/DnsUtil.h src/server/WFServer.h src/server/WFDnsServer.h src/server/WFHttpServer.h src/server/WFRedisServer.h src/server/WFMySQLServer.h src/client/WFMySQLConnection.h + src/client/WFWebSocketClient.h src/client/WFDnsClient.h src/manager/DnsCache.h src/manager/WFGlobal.h @@ -72,6 +79,8 @@ set(INCLUDE_HEADERS src/manager/WFFuture.h src/manager/WFFacilities.h src/manager/WFFacilities.inl + src/manager/WFSemaphore.h + src/manager/WFSemaphore.inl src/util/EncodeStream.h src/util/LRUCache.h src/util/StringUtil.h @@ -86,8 +95,11 @@ set(INCLUDE_HEADERS src/factory/WFTaskFactory.inl src/factory/WFAlgoTaskFactory.h src/factory/WFAlgoTaskFactory.inl + src/factory/WFSemTaskFactory.h src/factory/Workflow.h src/factory/WFOperator.h + src/factory/WFChannel.h + src/factory/WFChannel.inl src/nameservice/WFNameService.h src/nameservice/WFDnsResolver.h src/nameservice/WFServiceGovernance.h @@ -106,3 +118,4 @@ if(KAFKA STREQUAL "y") src/factory/KafkaTaskImpl.inl ) endif() + diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md new file mode 100644 index 0000000000..080ba20c97 --- /dev/null +++ b/docs/tutorial-14-websocket_cli.md @@ -0,0 +1,230 @@ +# Workflow第一个双工通信客户端：websocket_cli + +# 示例代码 + +[tutorial-14-websocket_cli.cc](/tutorial/tutorial-14-websocket_cli.cc) + +# 关于websocket_cli + +这是一个可以收发**WebSocket**协议的client，也是第一个在**Workflow**上实现的双工通信协议。 + +运行方式：**./websocket_cli \** + +URL格式：**ws://host:port** + +- port缺省值为80； +- 如果是ssl，URL格式为：wss://host:port +- ssl的port缺省值为443； + +# 创建并启动WebSocket任务 + +首先需要创建一个``WebSocketClient``对象，并且通过``init()``函数进行初始化。具体接口参考：[WFWebSocketClient.h](/src/client/WFWebSocketClient.h) + +```cpp +WebSocketClient client(process); +client.init(URL); +... +``` + +构造client时需要传入``process()``函数，这是我们收消息的处理函数，类型为：``std::function``，与**Workflow**其他``process()``的语义类似。 + +``init()``中需要传入**URL**，如果**UR**L非法，则``init()``会返回-1表示**URL**解析失败。 + +然后就可以通过client创建任务了，用法与**Workflow**其他协议类似： + +```cpp +WFWebSocketTask *task = client.create_websocket_task(callback); +WebSocketFrame *msg = text_task->get_msg(); +msg->set_text_data("This is Workflow websocket client."); +task->start(); +``` + +我们通过``create_websocket_task()``的接口创建一个往外发数据的任务，并传入回调函数**callback**。task可以通过``get_msg()``接口拿到要发送的消息体，我们往消息体里填入要发送的数据。因为**WebSocket**协议支持**文本**和**二进制**的数据，所以我们使用``set_text_data()``接口，表示传入的是文本消息。最后把任务**start**起来。task相关的具体接口可以查看：[WFChannel.h](/src/factory/WFChannel.h) + +```cpp +using WFWebSocketTask = WFChannelTask; +using websocket_callback_t = std::function; +using websocket_process_t = std::function; +``` + +# 发消息 + +**WebSocket**的消息体为``WebSocketFrame``，可以在[WebSocketMessage.h](/src/protocol/WebSocketMessage.h)查看到具体接口： + +```cpp +class WebSocketFrame : public ProtocolMessage +{ +public: + bool set_opcode(int opcode); + int get_opcode() const; + + void set_masking_key(uint32_t masking_key); + + bool set_text_data(const char *data); + bool set_text_data(const char *data, size_t size, bool fin); + + bool set_binary_data(const char *data, size_t size); + bool set_binary_data(const char *data, size_t size, bool fin); + + bool get_data(const char **data, size_t *size) const; + + bool finished() const; + + ... +}; +``` +#### 1. opcode + +协议中几种我们会接触到的数据包类型： +- WebSocketFrameText +- WebSocketFrameBinary +- WebSocketFramePing +- WebSocketFramePong + +一般我们发送数据的时候用的是前两种（**文本**和**二进制**），无需手动指定**opcode**，而如果需要手动发送**PING**包时则需要通过``bool set_opcode(int opcode)``指定为**PING**包；发送完PING包之后根据**WebSocket**协议，对方会给我们回一个**PONG**包，我们会在``process()``里拿到。 + +#### 2. masking_key + +**WebSocket**协议的文本和二进制数据都需要经过一个掩码加密，用户可以不填，也可以通过``void set_masking_key(uint32_t masking_key)``手动指定； + +#### 3. data + +数据包括两种： +- **文本**，通过``set_text_data()``这类接口设置； +- **二进制**，通过``set_binary_data()``这类接口设置； + +注意这些均为**非拷贝接口**，消息在发出之前需要用户来保证data在内存的生命周期； + +这两类接口都有一个带``bool fin``参数的接口，表示本消息是否finish。因为**WebSocket**协议的数据包允许分段传输，如果你要发送一个完整的消息想分多次发送，则可以使用带``bool fin``的接口，并且把``fin``值设置为``false``。 + +#### 4. callback + +只要消息发送完毕，就会回到我们创建task时传入的回调函数，此时我们得知消息是否发送成功。 + +一个简单的例子： + +```cpp +void send_callback(WFWebSocketTask *task) +{ + if (task->get_state() != WFT_STATE_SUCCESS) + fprintf(stderr, "Task send error: %d\n", task->get_error()); +} +``` + +# 收消息 + +每次收到server发来的消息，``process()``都会被调起。如何在收到文本数据之后把数据打印出来？ + +一个简单的例子： + +```cpp +void process(WFWebSocketTask *task) +{ + const char *data; + size_t size; + + if (task->get_msg()->get_opcode() == WebSocketFrameText) + { + task->get_msg()->get_data(&data, &size); + ... + } +} +``` + +#### 1. 参数 + +``process()``函数里拿到的参数``WFWebSocketTask *task``，与callback回调函数里拿到的类型是一样的，因此用法也非常类似： + +- 可以通过``get_msg()``拿到对应的数据，也就是上述的``WebSocketFrame``； +- 可以通过msg上的接口``get_opcode()``判断是什么类型的数据包，``process()``可能收到的数据包类型包括：**WebSocketFrameText**、**WebSocketFrameBinary**、**WebSocketFramePong**； + +#### 2. data + +无论是**文本**还是**二进制**，都由``bool get_data(const char **data, size_t *size) const``拿收到的数据。 + +#### 3. fin + +由于数据可以分段发送，因此我们可以通过``bool finished() const``判断该完整的消息是否结束。如果没有结束，则用户需要自行把data里的数据拷走，等消息结束之后进行完整消息的处理。 + +更多接口细节可以查看[websocket_parser.h ](/src/protocol/websocket_parser.h) + +# 关闭client + +根据**WebSocket**协议，用户需要发起一个**close**包已告诉对方以示断开连接。 + +一个简单的例子： + +```cpp +WFFacilities::WaitGroup wg(1); + +WFWebSocketTask *task = client.create_close_task([&wg](WFWebSocketTask *task) { + wg.done(); +}); + +task->start(); +wait_group.wait(); +``` + +这里发起了一个close任务，由于close是异步的，因此在``task->start()``之后当前线程会退出，我们在当前线程结合一个了``wait_group``进行不占线程的阻塞，并在close任务的回调函数里唤醒，然后当前线程就可以安全删除client实例和退出了。 + +需要注意的是，如果不主动发起close任务，直接删除client实例，那么底层使用的那个网络连接还会存在，直到超时或其他原因断开。 + +# websocket_cli的参数 + +``WebSocketClient``的构造函数有两个，除了刚才介绍的传入``process()``函数的接口以外，还可以传入client的参数： + +```cpp +class WebSocketClient +{ +public: + WebSocketClient(const struct WFWebSocketParams *params, + websocket_process_t process); + WebSocketClient(websocket_process_t process); + ... +``` + +其中，参数的定义如下： + +```cpp +struct WFWebSocketParams +{ + int idle_timeout; // client保持长连接的空闲时间，超过idle_timeout没有数据过来会自动断开。默认：10s + int ping_interval; // client自动发ping的时间间隔，用于做心跳，保持与远端的连接。默认：-1，不自动发ping(功能开发中) + size_t size_limit; // 每个数据包的大小限制，超过的话会拿到错误码1009(WSStatusCodeTooLarge)。默认：不限制 + bool random_masking_key; // WebSocket协议中数据包的掩码，框架帮每次自动随机生成一个。默认：不自动生成(功能开发中) +}; +``` + +如果不传入参数，会使用默认参数来构造client。 + +# 进阶版：注意事项！ + +websocket_connect_close websocket_read_write + +#### 1. 与Workflow原有用法的差异 + +由于**WebSocket**协议是**Workflow**中首个实现的双工通信协议，因此有些差异是必须强调的： + +1. **WebSocket**协议的收发都是使用**poller**线程，因此websocket_cli用户需要把**poller**线程数改大点，同理可以把**handler**线程数改小。参考：[about-config.md](/docs/about-config.md) +2. **process**函数中所有的消息都是由同一个线程串行执行的； +3. 回调函数**callback**的执行不一定在同一个线程； + +#### 2. 连接的生命周期 + +只有在第一个任务发出的时候，连接才会真正被建立。因此如果只希望监听server而没有写消息需求的用户依然需要手动发一个**PING**，让内部建立连接。可以通过client的``create_ping_task()``接口创建一个**PING** task，该回调函数里可以通过**state**判断连接是否可用，如果等于**WFT_STATE_SUCCESS**,则表示``process()``里已经随时可以接收server来的消息了。 + +前面提到，需要发起**CLOSE** task关闭连接，回到该回调函数时则表示连接已关闭。 + +#### 3. 时序性保证 + +[**发消息**] + +消息发送顺序取决于发送任务调起的顺序。因此发消息保序有两种方式： +- 可以把要发送的任务串到一个series里做串行的保证 +- 也可以在上一个任务的callback里发起下一个任务 + +但如果没有保证顺序发送的需求，那么往外发的``WFWebSocketTask``也可以被放到任何一个任务流图里，随具体业务逻辑顺序调起。 + +[**收消息**] + +用于收消息的``process()``函数是保证被按收包顺序调起的，且保证前一个消息的process执行完毕，下一个process才会调起，因此用户无需担心收消息的顺序问题。 diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h new file mode 100644 index 0000000000..0cc778d543 --- /dev/null +++ b/src/client/WFWebSocketClient.h @@ -0,0 +1,130 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WFWEBSOCKETCLIENT_H_ +#define _WFWEBSOCKETCLIENT_H_ + +#include +#include +#include "WFGlobal.h" +#include "HttpUtil.h" +#include "HttpMessage.h" +#include "WFChannel.h" +#include "WebSocketMessage.h" + +struct WFWebSocketParams +{ + int idle_timeout; + int ping_interval; + size_t size_limit; + bool random_masking_key; +}; + +static constexpr struct WFWebSocketParams WEBSOCKET_PARAMS_DEFAULT = +{ + .idle_timeout = WS_HANDSHAKE_TIMEOUT, + .ping_interval = -1, + .size_limit = (size_t)-1, + .random_masking_key = false, +}; + +class WebSocketClient +{ +public: + int init(const std::string& url); + WFWebSocketTask *create_websocket_task(websocket_callback_t cb); + WFWebSocketTask *create_ping_task(websocket_callback_t cb); + WFWebSocketTask *create_close_task(websocket_callback_t cb); + +private: + ComplexWebSocketChannel *channel; + struct WFWebSocketParams params; + +public: + WebSocketClient(const struct WFWebSocketParams *params, + websocket_process_t process); + WebSocketClient(websocket_process_t process) : + WebSocketClient(&WEBSOCKET_PARAMS_DEFAULT, std::move(process)) + { } +}; + +inline WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) +{ + return new ComplexWebSocketOutTask(this->channel, WFGlobal::get_scheduler(), + std::move(cb)); +} + +inline int WebSocketClient::init(const std::string& url) +{ + ParsedURI uri; + if (URIParser::parse(url, uri) != 0) + return -1; + + this->channel->set_uri(uri); + return 0; +} + +inline WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t cb) +{ + ComplexWebSocketOutTask *ping_task; + ping_task = new ComplexWebSocketOutTask(this->channel, + WFGlobal::get_scheduler(), + std::move(cb)); + + protocol::WebSocketFrame *msg = ping_task->get_msg(); + msg->set_opcode(WebSocketFramePing); + + return ping_task; +} + +inline WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) +{ + ComplexWebSocketOutTask *close_task; + close_task = new ComplexWebSocketOutTask(this->channel, + WFGlobal::get_scheduler(), + std::move(cb)); + + protocol::WebSocketFrame *msg = close_task->get_msg(); + msg->set_opcode(WebSocketFrameConnectionClose); + + return close_task; +} + +WebSocketClient::WebSocketClient(const struct WFWebSocketParams *params, + websocket_process_t process) +{ + this->params = *params; + this->channel = new ComplexWebSocketChannel(NULL, WFGlobal::get_scheduler(), + std::move(process)); + this->channel->set_idle_timeout(this->params.idle_timeout); + this->channel->set_size_limit(this->params.size_limit); + + this->channel->set_callback([this](WFChannel *channel) + { + pthread_mutex_lock(&this->channel->mutex); + if (this->channel->is_established() == 0) + { + this->channel->set_state(WFT_STATE_SYS_ERROR); + this->channel->set_sending(false); + } + pthread_mutex_unlock(&this->channel->mutex); + }); +} + +#endif + diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index ebe58fa6f1..5410551da0 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -4,7 +4,7 @@ project(factory) set(SRC WFGraphTask.cc DnsTaskImpl.cc - WFTaskFactory.cc + WFSemTaskFactory.cc Workflow.cc HttpTaskImpl.cc ) @@ -30,5 +30,4 @@ if (KAFKA STREQUAL "y") KafkaTaskImpl.cc ) add_library("factory_kafka" OBJECT ${SRC}) - set_property(SOURCE KafkaTaskImpl.cc APPEND PROPERTY COMPILE_OPTIONS "-fno-rtti") endif () diff --git a/src/factory/WFChannel.h b/src/factory/WFChannel.h new file mode 100644 index 0000000000..c54e834e07 --- /dev/null +++ b/src/factory/WFChannel.h @@ -0,0 +1,97 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) +*/ + +#ifndef _WFCHANNEL_H_ +#define _WFCHANNEL_H_ + +#include "CommScheduler.h" +#include "TransRequest.h" +#include "Workflow.h" +#include "WebSocketMessage.h" + +template +class WFChannelTask : public TransRequest +{ +public: + void start() + { + assert(!series_of(this)); + Workflow::start_series_work(this, nullptr); + } + + void dismiss() + { + assert(!series_of(this)); + delete this; + } + +public: + MSG *get_msg() + { + return &this->msg; + } + +public: + void *user_data; + +public: + int get_state() const { return this->state; } + int get_error() const { return this->error; } + + void set_callback(std::function *)> cb) + { + this->callback = std::move(cb); + } + +protected: + virtual SubTask *done() + { + SeriesWork *series = series_of(this); + + if (this->callback) + this->callback(this); + + delete this; + return series->pop(); + } + +protected: + MSG msg; + std::function *)> callback; + +public: + WFChannelTask(CommChannel *channel, CommScheduler *scheduler, + std::function *)>&& cb) : + TransRequest(channel, scheduler), + callback(std::move(cb)) + { + } + +protected: + virtual ~WFChannelTask() { } +}; + +using WFWebSocketTask = WFChannelTask; +using websocket_callback_t = std::function; +using websocket_process_t = std::function; + +#include "WFChannel.inl" + +#endif + diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl new file mode 100644 index 0000000000..d8a2c7de49 --- /dev/null +++ b/src/factory/WFChannel.inl @@ -0,0 +1,490 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) +*/ + +#include +#include "TransRequest.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +//#include "WFCondition.h" +#include "WFSemaphore.h" +#include "WFSemTaskFactory.h" +#include "WFNameService.h" +#include "RouteManager.h" +#include "WFGlobal.h" +#include "EndpointParams.h" +#include "HttpUtil.h" +#include "HttpMessage.h" +#include "WFChannel.h" + +template +class WFChannelOutTask : public WFChannelTask +{ +protected: + virtual MSG *message_out() { return &this->msg; } + +public: + WFChannelOutTask(CommChannel *channel, CommScheduler *scheduler, + std::function *)>&& cb) : + WFChannelTask(channel, scheduler, std::move(cb)) + { + } + +protected: + virtual ~WFChannelOutTask() { } +}; + +template +class WFChannelInTask : public WFChannelTask +{ +protected: + virtual void dispatch() + { + this->state = WFT_STATE_SUCCESS; + this->error = 0; + this->process(this); + this->subtask_done(); + } + +public: + WFChannelInTask(CommChannel *channel, CommScheduler *scheduler, + std::function *)>& proc) : + WFChannelTask(channel, scheduler, nullptr), + process(proc) + { + } + +protected: + std::function *)>& process; + +protected: + virtual ~WFChannelInTask() { } +}; + +template +class WFChannel : public ChanRequest +{ +public: + int get_state() const { return this->state; } + int get_error() const { return this->error; } + bool is_established() const { return this->established; } + + void set_callback(std::function *)>&& cb) + { + this->callback = std::move(cb); + } + +protected: + virtual CommMessageIn *message_in() + { + this->session = this->new_session(); + return this->session->get_msg(); + } + + virtual WFChannelTask *new_session() + { + auto *task = new WFChannelInTask(this, this->scheduler, + this->process); + Workflow::create_series_work(task, nullptr); + return task; + } + + virtual void handle_in(CommMessageIn *in) + { + if (this->session) + this->session->dispatch(); + this->session = NULL; + } + + virtual SubTask *done() + { + if (this->callback) + this->callback(this); + + return series_of(this)->pop(); + } + +protected: + std::function *)> process; + std::function *)> callback; + +private: + WFChannelTask *session; + +public: + WFChannel(CommSchedObject *object, CommScheduler *scheduler, + std::function *)>&& process) : + ChanRequest(object, scheduler), + process(std::move(process)) + { + this->session = NULL; + } + +protected: + virtual ~WFChannel() { } +}; + +/**********WFComplexChannel for sequentially establish and send**********/ + +template +class WFComplexChannel : public WFChannel +{ +public: + void set_uri(const ParsedURI& uri) { this->uri = uri; } + const ParsedURI *get_uri() const { return &this->uri; } + + int get_error() const { return this->error; } + + void set_state(int state) { this->state = state; } + int get_state() const { return this->state; } + + void set_sending(bool sending) { this->sending = sending; } + bool get_sending() const { return this->sending; } + +protected: + virtual void dispatch(); + virtual SubTask *done(); + virtual void handle_terminated(); + virtual WFRouterTask *route(); + void router_callback(WFRouterTask *task); + +public: + pthread_mutex_t mutex; + WFCondition condition; + +protected: + bool sending; + WFRouterTask *router_task; + ParsedURI uri; + WFNSPolicy *ns_policy; + RouteManager::RouteResult route_result; + +public: + WFComplexChannel(CommSchedObject *object, CommScheduler *scheduler, + std::function *)>&& process) : + WFChannel(object, scheduler, std::move(process)), + mutex(PTHREAD_MUTEX_INITIALIZER) + { + this->state = WFT_STATE_UNDEFINED; + this->error = 0; + this->sending = false; + } + +protected: + virtual ~WFComplexChannel() { } +}; + +template +void WFComplexChannel::dispatch() +{ + if (this->object) + return this->WFChannel::dispatch(); + + if (this->state == WFT_STATE_UNDEFINED) + { + this->router_task = this->route(); + series_of(this)->push_front(this); + series_of(this)->push_front(this->router_task); + } + + this->subtask_done(); +} + +template +SubTask *WFComplexChannel::done() +{ + SeriesWork *series = series_of(this); + + if (this->established == 1) + { + if (this->state == WFT_STATE_SYS_ERROR) + this->ns_policy->failed(&this->route_result, NULL, this->target); + else + this->ns_policy->success(&this->route_result, NULL, this->target); + } + + if (this->router_task) + { + this->router_task = NULL; + return series->pop(); + } + + if (this->callback) + this->callback(this); + + if (this->state == WFT_STATE_SUCCESS) + this->state = WFT_STATE_UNDEFINED; + + if (this->established == 0 && this->state == WFT_STATE_SYS_ERROR) // sending == false + delete this; + + return series->pop(); +} + +template +void WFComplexChannel::handle_terminated() +{ + WFMailboxTask *waiter; + bool shutdown = false; + + pthread_mutex_lock(&this->mutex); + Workflow::create_series_work(this, nullptr); + + if (this->sending == false) + { + this->sending = true; + shutdown = true; + } else { + waiter = WFSemTaskFactory::create_wait_task(&this->condition, + nullptr); + series_of(this)->push_front(this); + series_of(this)->push_front(waiter); + } + pthread_mutex_unlock(&this->mutex); + + if (shutdown == true) + this->dispatch(); +} + +template +WFRouterTask *WFComplexChannel::route() +{ + auto&& cb = std::bind(&WFComplexChannel::router_callback, + this, std::placeholders::_1); + struct WFNSParams params = { + .type = TT_TCP, + .uri = this->uri, + .info = "", + .fixed_addr = true, + .retry_times = 0, + .tracing = NULL, + }; + + WFNameService *ns = WFGlobal::get_name_service(); + this->ns_policy = ns->get_policy(this->uri.host ? this->uri.host : ""); + return this->ns_policy->create_router_task(¶ms, cb); +} + +template +void WFComplexChannel::router_callback(WFRouterTask *task) +{ + if (task->get_state() == WFT_STATE_SUCCESS) + { + this->route_result = std::move(*task->get_result()); + this->set_request_object(this->route_result.request_object); + } + else + { + this->state = task->get_state(); + this->error = task->get_error(); + } +} + +/**********ComplexChannelOutTask for complex channel and upgrade()**********/ + +template +class ComplexChannelOutTask : public WFChannelOutTask +{ +protected: + virtual void dispatch(); + virtual SubTask *done(); + virtual SubTask *upgrade(); + void upgrade_callback(WFCounterTask *task); + +protected: + bool ready; + +public: + ComplexChannelOutTask(WFComplexChannel *channel, CommScheduler *scheduler, + std::function *)>&& cb) : + WFChannelOutTask(channel, scheduler, std::move(cb)) + { + this->ready = true; + } + +protected: + virtual ~ComplexChannelOutTask() { } +}; + +template +void ComplexChannelOutTask::dispatch() +{ + WFMailboxTask *waiter; + bool should_send = false; + auto *channel = (WFComplexChannel *)this->get_request_channel(); + + if (this->state == WFT_STATE_SYS_ERROR || + channel->get_state() == WFT_STATE_SYS_ERROR) + { + return this->subtask_done(); + } + + pthread_mutex_lock(&channel->mutex); + + switch (channel->get_state()) + { + case WFT_STATE_UNDEFINED: + if (channel->get_sending() == false) + { + series_of(this)->push_front(this); + series_of(this)->push_front(channel); + channel->set_sending(true); + this->ready = false; + } + else if (this->ready == false) + { + SubTask *upgrade_task = this->upgrade(); + series_of(this)->push_front(this); + series_of(this)->push_front(upgrade_task); + } + else + { + waiter = WFSemTaskFactory::create_wait_task(&channel->condition, + [this](WFMailboxTask *task) + { + auto *channel = (WFComplexChannel *)this->get_request_channel(); + channel->set_state(WFT_STATE_SUCCESS); + this->ready = true; + }); + series_of(this)->push_front(this); + series_of(this)->push_front(waiter); + this->ready = false; + } + break; + + case WFT_STATE_SUCCESS: + if (channel->get_sending() == false) + { + channel->set_sending(true); + should_send = true; + } + else + { + waiter = WFSemTaskFactory::create_wait_task(&channel->condition, + [this](WFMailboxTask *task) + { + auto *channel = (WFComplexChannel *)this->get_request_channel(); + channel->set_state(WFT_STATE_SUCCESS); + this->ready = true; + }); + series_of(this)->push_front(this); + series_of(this)->push_front(waiter); + this->ready = false; + } + break; + + default: + this->state = channel->get_state(); + this->error = channel->get_error(); + break; + } + + pthread_mutex_unlock(&channel->mutex); + + if (should_send == true) + return this->WFChannelOutTask::dispatch(); + + return this->subtask_done(); +} + +template +SubTask *ComplexChannelOutTask::done() +{ + auto *channel = (WFComplexChannel *)this->get_request_channel(); + + if (channel->get_state() == WFT_STATE_UNDEFINED || + channel->get_state() == WFT_STATE_SUCCESS) + { + if (this->ready != true) + return series_of(this)->pop(); + } + else + { + this->state = channel->get_state(); + this->error = channel->get_error(); + } + + pthread_mutex_lock(&channel->mutex); + channel->set_sending(false); + channel->condition.signal(NULL); + pthread_mutex_unlock(&channel->mutex); + + return WFChannelOutTask::done(); +} + +template +SubTask *ComplexChannelOutTask::upgrade() +{ + WFCounterTask *counter = new WFCounterTask(0, [this](WFCounterTask *task) + { + auto *channel = (WFComplexChannel *)this->get_request_channel(); + + pthread_mutex_lock(&channel->mutex); + channel->set_state(WFT_STATE_SUCCESS); + this->ready = true; + channel->set_sending(false); + channel->condition.signal(NULL); + pthread_mutex_unlock(&channel->mutex); + }); + + return counter; +} + +/**********WebSocket task impl**********/ + +class ComplexWebSocketChannel : public WFComplexChannel +{ +public: + void set_idle_timeout(int timeout) { this->idle_timeout = timeout; } + void set_size_limit(size_t size_limit) { this->size_limit = size_limit; } + +protected: + virtual CommMessageIn *message_in(); + virtual void handle_in(CommMessageIn *in); + virtual int first_timeout(); + virtual WFWebSocketTask *new_session(); + +private: + int idle_timeout; + size_t size_limit; + +public: + ComplexWebSocketChannel(CommSchedObject *object, CommScheduler *scheduler, + websocket_process_t&& process) : + WFComplexChannel(object, scheduler, + std::move(process)) + { + } +}; + +class ComplexWebSocketOutTask : public ComplexChannelOutTask +{ +protected: + virtual SubTask *upgrade(); + virtual SubTask *done(); + +public: + ComplexWebSocketOutTask(ComplexWebSocketChannel *channel, CommScheduler *scheduler, + websocket_callback_t&& cb) : + ComplexChannelOutTask(channel, + scheduler, + std::move(cb)) + { + } +}; + diff --git a/src/factory/WFSemTaskFactory.cc b/src/factory/WFSemTaskFactory.cc new file mode 100644 index 0000000000..bf8c5cd4a0 --- /dev/null +++ b/src/factory/WFSemTaskFactory.cc @@ -0,0 +1,229 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include +#include +#include +#include "list.h" +#include "rbtree.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" +#include "WFSemTaskFactory.h" + +class __WFCondition : public WFCondition +{ +public: + __WFCondition(const std::string& str) : + name(str) + { + this->node.ptr = this; + } + +public: + struct entry + { + struct rb_node rb; + __WFCondition *ptr; + } node; + + std::string name; +}; + +class __ConditionMap +{ +public: + void signal(const std::string& name, void *msg); + void broadcast(const std::string& name, void *msg); + + WFWaitTask *create(const std::string& name, + wait_callback_t&& cb); + WFWaitTask *create(const std::string& name, + const struct timespec *abstime, + wait_callback_t&& cb); + +public: + static __ConditionMap *get_instance() + { + static __ConditionMap kInstance; + return &kInstance; + } + + virtual ~__ConditionMap(); + +private: + __ConditionMap() + { + this->condition_map.rb_node = NULL; + } + + __WFCondition *find_condition(const std::string& name); + struct rb_root condition_map; + std::mutex mutex; +}; + +void __ConditionMap::signal(const std::string& name, void *msg) +{ + __WFCondition *cond = this->find_condition(name); + + cond->signal(msg); +} + +void __ConditionMap::broadcast(const std::string& name, void *msg) +{ + __WFCondition *cond = this->find_condition(name); + + cond->broadcast(msg); +} + +WFWaitTask *__ConditionMap::create(const std::string& name, + wait_callback_t&& cb) +{ + __WFCondition *cond = this->find_condition(name); + + return WFSemTaskFactory::create_wait_task(cond, std::move(cb)); +} + +WFWaitTask *__ConditionMap::create(const std::string& name, + const struct timespec *abstime, + wait_callback_t&& cb) +{ + __WFCondition *cond = this->find_condition(name); + + return WFSemTaskFactory::create_timedwait_task(cond, abstime, + std::move(cb)); +} + +__ConditionMap::~__ConditionMap() +{ + __WFCondition *cond; + WFCondWaitTask *task; + struct list_head *pos; + struct list_head *tmp; + struct WFSemaphoreTask::entry *node; + struct __WFCondition::entry *cond_node; + + while (this->condition_map.rb_node) + { + cond_node = rb_entry(this->condition_map.rb_node, + struct __WFCondition::entry, rb); + cond = cond_node->ptr; + + list_for_each_safe(pos, tmp, &cond->wait_list) + { + node = list_entry(pos, struct WFSemaphoreTask::entry, list); + task = (WFCondWaitTask *)node->ptr; + list_del(pos); + delete task; + } + + rb_erase(this->condition_map.rb_node, &this->condition_map); + delete cond; + } +} + +__WFCondition *__ConditionMap::find_condition(const std::string& name) +{ + __WFCondition *cond; + struct __WFCondition::entry *cond_node; + struct rb_node **p = &this->condition_map.rb_node; + struct rb_node *parent = NULL; + + this->mutex.lock(); + while (*p) + { + parent = *p; + cond_node = rb_entry(*p, struct __WFCondition::entry, rb); + cond = cond_node->ptr; + + if (name < cond->name) + p = &(*p)->rb_left; + else if (name > cond->name) + p = &(*p)->rb_right; + else + break; + } + + if (*p == NULL) + { + cond = new __WFCondition(name); + rb_link_node(&cond->node.rb, parent, p); + rb_insert_color(&cond->node.rb, &this->condition_map); + } + + this->mutex.unlock(); + + return cond; +} + +/////////////// factory api /////////////// + +void WFSemTaskFactory::signal_by_name(const std::string& name, void *msg) +{ + return __ConditionMap::get_instance()->signal(name, msg); +} + +void WFSemTaskFactory::broadcast_by_name(const std::string& name, void *msg) +{ + return __ConditionMap::get_instance()->broadcast(name, msg); +} + +WFWaitTask *WFSemTaskFactory::create_wait_task(const std::string& name, + wait_callback_t callback) +{ + return __ConditionMap::get_instance()->create(name, std::move(callback)); +} + +WFWaitTask *WFSemTaskFactory::create_timedwait_task(const std::string& name, + const struct timespec *abstime, + wait_callback_t callback) +{ + return __ConditionMap::get_instance()->create(name, abstime, + std::move(callback)); +} + +WFWaitTask *WFSemTaskFactory::create_wait_task(WFCondition *cond, + wait_callback_t callback) +{ + WFCondWaitTask *task = new WFCondWaitTask(std::move(callback)); + + cond->mutex.lock(); + list_add_tail(&task->node.list, &cond->wait_list); + cond->mutex.unlock(); + + return task; +} + +WFWaitTask *WFSemTaskFactory::create_timedwait_task(WFCondition *cond, + const struct timespec *abstime, + wait_callback_t callback) +{ + WFCondWaitTask *waiter = new WFCondWaitTask(std::move(callback)); + WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &cond->mutex, abstime, + WFGlobal::get_scheduler(), + nullptr); + waiter->set_timer(task); + + cond->mutex.lock(); + list_add_tail(&waiter->node.list, &cond->wait_list); + cond->mutex.unlock(); + + return waiter; +} + diff --git a/src/factory/WFSemTaskFactory.h b/src/factory/WFSemTaskFactory.h new file mode 100644 index 0000000000..403e0810b4 --- /dev/null +++ b/src/factory/WFSemTaskFactory.h @@ -0,0 +1,58 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WFSEMTASKFACTORY_H_ +#define _WFSEMTASKFACTORY_H_ + +#include +#include +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" +#include "WFSemaphore.h" + +class WFSemTaskFactory +{ +public: + // use condition by name + static void signal_by_name(const std::string& name, void *msg); + + static void broadcast_by_name(const std::string& name, void *msg); + + static WFWaitTask *create_wait_task(const std::string& name, + wait_callback_t callback); + + static WFWaitTask *create_timedwait_task(const std::string& name, + const struct timespec *abstime, + wait_callback_t callback); + + // use condition by ptr + static WFWaitTask *create_wait_task(WFCondition *cond, + wait_callback_t callback); + + static WFWaitTask *create_timedwait_task(WFCondition *cond, + const struct timespec *abstime, + wait_callback_t callback); + +}; + +#endif + diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc new file mode 100644 index 0000000000..4b278ddb23 --- /dev/null +++ b/src/factory/WebSocketTaskImpl.cc @@ -0,0 +1,225 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include "WFTask.h" +#include "WFGlobal.h" +#include "WFChannel.h" + +#define WS_HTTP_SEC_KEY_K "Sec-WebSocket-Key" +#define WS_HTTP_SEC_KEY_V "dGhlIHNhbXBsZSBub25jZQ==" +#define WS_HTTP_SEC_PROTOCOL_K "Sec-WebSocket-Protocol" +#define WS_HTTP_SEC_PROTOCOL_V "chat" +#define WS_HTTP_SEC_VERSION_K "Sec-WebSocket-Version" +#define WS_HTTP_SEC_VERSION_V "13" + +using namespace protocol; + +class ComplexWebSocketInTask : public WFChannelInTask +{ +protected: + virtual void dispatch(); + virtual SubTask *done(); + +public: + ComplexWebSocketInTask(ComplexWebSocketChannel *channel, + CommScheduler *scheduler, + websocket_process_t& proc) : + WFChannelInTask(channel, scheduler, proc) + { + } +}; + +void ComplexWebSocketInTask::dispatch() +{ + const websocket_parser_t *parser = this->get_msg()->get_parser(); + + if (parser->opcode != WebSocketFrameConnectionClose && + parser->status_code != WSStatusCodeUndefined) + { + this->state = WFT_STATE_SYS_ERROR; + this->error = parser->status_code; + } + else + { + this->state = WFT_STATE_SUCCESS; + this->error = 0; + this->process(this); + } + + this->subtask_done(); +} + +SubTask *ComplexWebSocketInTask::done() +{ + SeriesWork *series = series_of(this); + const websocket_parser_t *parser = this->get_msg()->get_parser(); + auto *channel = (ComplexWebSocketChannel *)this->get_request_channel(); + + if ((parser->opcode == WebSocketFrameConnectionClose && + !channel->is_established()) || + parser->status_code != WSStatusCodeUndefined) + { + auto *close_task = new ComplexWebSocketOutTask(channel, + WFGlobal::get_scheduler(), + nullptr); + WebSocketFrame *msg = close_task->get_msg(); + msg->set_opcode(WebSocketFrameConnectionClose); + msg->set_data(parser); + series->push_front(close_task); + } + else if (parser->opcode == WebSocketFramePing) + { + auto *pong_task = new ComplexWebSocketOutTask(channel, + WFGlobal::get_scheduler(), + nullptr); + WebSocketFrame *msg = pong_task->get_msg(); + msg->set_opcode(WebSocketFramePong); + msg->set_data(parser); + series->push_front(pong_task); + } + + if (this->callback) + this->callback(this); + + delete this; + return series->pop(); +} + +SubTask *ComplexWebSocketOutTask::done() +{ + SeriesWork *series = series_of(this); + auto *channel = (ComplexWebSocketChannel *)this->get_request_channel(); + + if (channel->get_state() == WFT_STATE_UNDEFINED || + channel->get_state() == WFT_STATE_SUCCESS) + { + if (this->ready != true) + return series_of(this)->pop(); + } + else + { + this->state = channel->get_state(); + this->error = channel->get_error(); + } + + const websocket_parser_t *parser = this->get_msg()->get_parser(); + + if (parser->opcode == WebSocketFrameConnectionClose && + this->get_state() == WFT_STATE_SUCCESS && + channel->is_established()) + { + series->push_front(this); + series->push_front(channel); + return series->pop(); + } + + pthread_mutex_lock(&channel->mutex); + channel->set_sending(false); + channel->condition.signal(NULL); + pthread_mutex_unlock(&channel->mutex); + + return WFChannelOutTask::done(); +} + +SubTask *ComplexWebSocketOutTask::upgrade() +{ + auto *channel = (ComplexWebSocketChannel *)this->get_request_channel(); + + auto *http_task = new WFChannelOutTask(this->channel, + WFGlobal::get_scheduler(), + [this](WFChannelTask *task) + { + if (task->get_state() == WFT_STATE_SYS_ERROR) + { + this->state = WFT_STATE_SYS_ERROR; + this->error = task->get_error(); + } + + this->ready = true; + }); + HttpRequest *req = http_task->get_msg(); + req->set_method(HttpMethodGet); + req->set_http_version("HTTP/1.1"); + req->set_request_uri("/"); + req->add_header_pair("Host", channel->get_uri()->host); + req->add_header_pair("Upgrade", "websocket"); + req->add_header_pair("Connection", "Upgrade"); + req->add_header_pair(WS_HTTP_SEC_KEY_K, WS_HTTP_SEC_KEY_V); + req->add_header_pair(WS_HTTP_SEC_PROTOCOL_K, WS_HTTP_SEC_PROTOCOL_V); + req->add_header_pair(WS_HTTP_SEC_VERSION_K, WS_HTTP_SEC_VERSION_V); + + return http_task; +} + +CommMessageIn *ComplexWebSocketChannel::message_in() +{ + if (this->state == WFT_STATE_UNDEFINED) + return new HttpResponse; + + return WFComplexChannel::message_in(); +} + +void ComplexWebSocketChannel::handle_in(CommMessageIn *in) +{ + bool parse_websocket = false; + + pthread_mutex_lock(&this->mutex); + + if (this->state == WFT_STATE_UNDEFINED) + { + HttpResponse *resp = static_cast(in); + + if (strcmp(resp->get_status_code(), "101") == 0) + this->state = WFT_STATE_SUCCESS; + else + this->state = WFT_STATE_TASK_ERROR; + + delete resp; + + this->sending = false; + } + else if (this->state == WFT_STATE_SUCCESS) + parse_websocket = true; + + pthread_mutex_unlock(&this->mutex); + + if (!parse_websocket) // so this is equal to should_count + { + pthread_mutex_lock(&this->mutex); + this->condition.signal(NULL); + pthread_mutex_unlock(&this->mutex); + return; + } + + WFComplexChannel::handle_in(in); +} + +int ComplexWebSocketChannel::first_timeout() +{ + return this->idle_timeout; +} + +WFWebSocketTask *ComplexWebSocketChannel::new_session() +{ + auto *task = new ComplexWebSocketInTask(this, this->scheduler, + this->process); + Workflow::create_series_work(task, nullptr); + task->get_msg()->set_size_limit(this->size_limit); + return task; +} + diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index 209f0ec07c..683c78295e 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -17,6 +17,7 @@ set(SRC msgqueue.c thrdpool.c CommRequest.cc + TransRequest.cc CommScheduler.cc Communicator.cc Executor.cc diff --git a/src/kernel/CommScheduler.h b/src/kernel/CommScheduler.h index 62a43501fe..65d003fc75 100644 --- a/src/kernel/CommScheduler.h +++ b/src/kernel/CommScheduler.h @@ -171,6 +171,33 @@ class CommScheduler this->comm.io_unbind(service); } + /* for channels. */ + int establish(CommChannel *channel, CommSchedObject *object, + int wait_timeout, CommTarget **target) + { + int ret = -1; + + *target = object->acquire(wait_timeout); + if (*target) + { + ret = this->comm.establish(channel, *target); + if (ret < 0) + (*target)->release(); + } + + return ret; + } + + int send(TransSession *session, CommChannel *channel) + { + return this->comm.send(session, channel); + } + + void shutdown(CommChannel *channel) + { + this->comm.shutdown(channel); + } + public: int is_handler_thread() const { diff --git a/src/kernel/Communicator.cc b/src/kernel/Communicator.cc index 8770bd7473..9ffaade5cd 100644 --- a/src/kernel/Communicator.cc +++ b/src/kernel/Communicator.cc @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -43,6 +42,7 @@ struct CommConnEntry CommConnection *conn; long long seq; int sockfd; + short channel_flag; #define CONN_STATE_CONNECTING 0 #define CONN_STATE_CONNECTED 1 #define CONN_STATE_RECEIVING 2 @@ -51,7 +51,7 @@ struct CommConnEntry #define CONN_STATE_KEEPALIVE 5 #define CONN_STATE_CLOSING 6 #define CONN_STATE_ERROR 7 - int state; + short state; int error; int ref; struct iovec *write_iov; @@ -119,6 +119,45 @@ static int __create_ssl(SSL_CTX *ssl_ctx, struct CommConnEntry *entry) return -1; } +#ifndef IOV_MAX +# ifdef UIO_MAXIOV +# define IOV_MAX UIO_MAXIOV +# else +# define IOV_MAX 1024 +# endif +#endif + +static int __send_vectors(struct iovec vectors[], int cnt, + struct CommConnEntry *entry) +{ + ssize_t n; + int i; + + while (cnt > 0) + { + n = writev(entry->sockfd, vectors, cnt <= IOV_MAX ? cnt : IOV_MAX); + if (n < 0) + return errno == EAGAIN ? cnt : -1; + + for (i = 0; i < cnt; i++) + { + if ((size_t)n >= vectors[i].iov_len) + n -= vectors[i].iov_len; + else + { + vectors[i].iov_base = (char *)vectors[i].iov_base + n; + vectors[i].iov_len -= n; + break; + } + } + + vectors += i; + cnt -= i; + } + + return 0; +} + int CommTarget::init(const struct sockaddr *addr, socklen_t addrlen, int connect_timeout, int response_timeout) { @@ -304,6 +343,21 @@ CommSession::~CommSession() ((CommServiceTarget *)target)->decref(); } +class CommMessageOutEmpty : public CommMessageOut +{ +private: + virtual int encode(struct iovec *vectors, int max) + { + return 0; + } +}; + +CommMessageOut *CommChannel::message_out() +{ + static CommMessageOutEmpty empty; + return ∅ +} + inline int Communicator::first_timeout(CommSession *session) { int timeout = session->target->response_timeout; @@ -389,30 +443,10 @@ int Communicator::send_message_sync(struct iovec vectors[], int cnt, CommSession *session = entry->session; CommService *service; int timeout; - ssize_t n; - int i; - while (cnt > 0) - { - n = writev(entry->sockfd, vectors, cnt <= IOV_MAX ? cnt : IOV_MAX); - if (n < 0) - return errno == EAGAIN ? cnt : -1; - - for (i = 0; i < cnt; i++) - { - if ((size_t)n >= vectors[i].iov_len) - n -= vectors[i].iov_len; - else - { - vectors[i].iov_base = (char *)vectors[i].iov_base + n; - vectors[i].iov_len -= n; - break; - } - } - - vectors += i; - cnt -= i; - } + cnt = __send_vectors(vectors, cnt, entry); + if (cnt != 0) + return cnt; service = entry->service; if (service) @@ -529,7 +563,7 @@ int Communicator::send_message(struct CommConnEntry *entry) } end = vectors + cnt; - if (!entry->ssl) + if (!entry->ssl || cnt == 0) { cnt = this->send_message_sync(vectors, cnt, entry); if (cnt <= 0) @@ -837,12 +871,10 @@ struct CommConnEntry *Communicator::accept_conn(CommServiceTarget *target, CommService *service) { struct CommConnEntry *entry; - size_t size; if (__set_fd_nonblock(target->sockfd) >= 0) { - size = offsetof(struct CommConnEntry, mutex); - entry = (struct CommConnEntry *)malloc(size); + entry = (struct CommConnEntry *)malloc(sizeof (struct CommConnEntry)); if (entry) { entry->conn = service->new_connection(target->sockfd); @@ -855,6 +887,7 @@ struct CommConnEntry *Communicator::accept_conn(CommServiceTarget *target, entry->ssl = NULL; entry->sockfd = target->sockfd; entry->state = CONN_STATE_CONNECTED; + entry->channel_flag = 0; entry->ref = 1; return entry; } @@ -996,7 +1029,9 @@ void Communicator::handle_connect_result(struct poller_result *res) target->release(); session->handle(state, res->error); - this->release_conn(entry); + if (__sync_sub_and_fetch(&entry->ref, 1) == 0) + this->release_conn(entry); + break; } } @@ -1257,7 +1292,117 @@ int Communicator::partial_written(size_t n, void *context) void Communicator::callback(struct poller_result *res, void *context) { Communicator *comm = (Communicator *)context; - msgqueue_put(res, comm->queue); + struct CommConnEntry *entry; + CommSession *session; + int state; + + switch (res->data.operation) + { + case PD_OP_READ: + case PD_OP_WRITE: + case PD_OP_CONNECT: + case PD_OP_SSL_CONNECT: + entry = (struct CommConnEntry *)res->data.context; + if (entry->channel_flag) + break; + + default: + msgqueue_put(res, comm->queue); + return; + } + + session = entry->session; + if (entry->state == CONN_STATE_CONNECTING) + { + comm->handle_connect_result(res); + if (entry->ref == 1) + comm->release_conn(entry); + else if (entry->state == CONN_STATE_RECEIVING) + ((CommChannel *)session)->handle_established(); + } + else if (entry->state == CONN_STATE_SUCCESS) + { + ((CommChannel *)session)->handle_in(session->in); + session->in = NULL; + if (session->timeout == 0) + entry->state = CONN_STATE_CLOSING; + else + { + session->timeout = -1; + session->begin_time.tv_nsec = -1; + entry->state = CONN_STATE_RECEIVING; + } + } + else + { + if (res->data.operation == PD_OP_WRITE) + { + close(entry->sockfd); + free(entry->write_iov); + switch (res->state) + { + case PR_ST_FINISHED: + state = CS_STATE_SUCCESS; + break; + case PR_ST_ERROR: + state = CS_STATE_ERROR; + break; + case PR_ST_DELETED: + case PR_ST_STOPPED: + state = CS_STATE_STOPPED; + break; + } + + free(entry); + entry = ((TransSession *)session)->channel->entry; + session->handle(state, res->error); + session = entry->session; + } + else + { + switch (res->state) + { + case PR_ST_FINISHED: + if (session->in) + { + res->error = ECONNRESET; + case PR_ST_ERROR: + state = CS_STATE_ERROR; + } + else + state = CS_STATE_SUCCESS; + + break; + + case PR_ST_DELETED: + if (entry->state == CONN_STATE_CLOSING) + state = CS_STATE_SUCCESS; + else + case PR_ST_STOPPED: + state = CS_STATE_STOPPED; + + break; + } + + if (entry->channel_flag == 1) + { + ((CommChannel *)session)->handle_terminated(); + entry->state = state; + entry->error = res->error; + } + else + state = CS_STATE_SHUTDOWN; + } + + if (__sync_sub_and_fetch(&entry->ref, 1) == 0) + { + entry->target->release(); + session->handle(state, res->error); + comm->release_conn(entry); + } + } + + free(res); } void *Communicator::accept(const struct sockaddr *addr, socklen_t addrlen, @@ -1421,6 +1566,7 @@ struct CommConnEntry *Communicator::launch_conn(CommSession *session, entry->ssl = NULL; entry->sockfd = sockfd; entry->state = CONN_STATE_CONNECTING; + entry->channel_flag = 0; entry->ref = 1; return entry; } @@ -1657,6 +1803,111 @@ int Communicator::reply(CommSession *session) return 0; } +int Communicator::establish(CommChannel *channel, CommTarget *target) +{ + struct CommConnEntry *entry; + struct poller_data data; + + entry = this->launch_conn(channel, target); + if (entry) + { + entry->channel_flag = 1; + entry->ref++; + + channel->target = target; + channel->entry = entry; + channel->out = NULL; + channel->in = NULL; + channel->seq = 0; + + data.operation = PD_OP_CONNECT; + data.fd = entry->sockfd; + data.ssl = NULL; + data.context = entry; + if (mpoller_add(&data, target->connect_timeout, this->mpoller) >= 0) + return 0; + + this->release_conn(entry); + } + + return -1; +} + +int Communicator::send(TransSession *session, CommChannel *channel) +{ + struct CommConnEntry *entry = channel->entry; + struct iovec vectors[ENCODE_IOV_MAX]; + struct iovec *end; + int cnt; + + session->channel = channel; + session->target = channel->target; + session->seq = 0; + session->in = NULL; + session->out = session->message_out(); + if (!session->out) + return -1; + + cnt = session->out->encode(vectors, ENCODE_IOV_MAX); + if ((unsigned int)cnt > ENCODE_IOV_MAX) + { + if (cnt > ENCODE_IOV_MAX) + errno = EOVERFLOW; + return -1; + } + + end = vectors + cnt; + if (!entry->ssl || cnt == 0) + { + cnt = __send_vectors(vectors, cnt, entry); + if (cnt == 0) + return 1; + else if (cnt < 0) + return -1; + } + + entry = (struct CommConnEntry *)malloc(sizeof (struct CommConnEntry)); + if (entry) + { + entry->sockfd = dup(channel->entry->sockfd); + if (entry->sockfd >= 0) + { + entry->session = session; + entry->ssl = channel->entry->ssl; + entry->state = CONN_STATE_KEEPALIVE; + entry->channel_flag = 1; + if (this->send_message_async(end - cnt, cnt, entry) >= 0) + { + __sync_add_and_fetch(&channel->entry->ref, 1); + return 0; + } + + close(entry->sockfd); + } + + free(entry); + } + + return -1; +} + +void Communicator::shutdown(CommChannel *channel) +{ + struct CommConnEntry *entry = channel->entry; + int errno_bak = errno; + + entry->channel_flag = 2; + mpoller_del(entry->sockfd, this->mpoller); + if (__sync_sub_and_fetch(&entry->ref, 1) == 0) + { + entry->target->release(); + channel->handle(entry->state, entry->error); + this->release_conn(entry); + } + + errno = errno_bak; +} + int Communicator::sleep(SleepSession *session) { struct timespec value; diff --git a/src/kernel/Communicator.h b/src/kernel/Communicator.h index fa12597be6..178e97a773 100644 --- a/src/kernel/Communicator.h +++ b/src/kernel/Communicator.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -124,6 +125,7 @@ class CommMessageIn : private poller_message_t #define CS_STATE_ERROR 1 #define CS_STATE_STOPPED 2 #define CS_STATE_TOREPLY 3 /* for service session only. */ +#define CS_STATE_SHUTDOWN 4 /* for channel only */ class CommSession { @@ -238,6 +240,49 @@ class CommService friend class Communicator; }; +class CommChannel : public CommSession +{ +private: + virtual CommMessageIn *message_in() = 0; + virtual int keep_alive_timeout() { return -1; } + virtual int first_timeout() { return -1; } + virtual void handle_established() = 0; + virtual void handle_in(CommMessageIn *msg) = 0; + virtual void handle_terminated() { } + virtual void handle(int state, int error) = 0; + +private: + virtual CommMessageOut *message_out(); /* final */ + CommMessageOut *get_message_out() { return NULL; } /* deleted */ + +private: + struct CommConnEntry *entry; + friend class Communicator; +}; + +class TransSession : public CommSession +{ +private: + virtual CommMessageOut *message_out() + { + errno = ENOSYS; + return NULL; + } + + virtual CommMessageIn *message_in() + { + errno = ENOSYS; + return NULL; + } + +protected: + CommChannel *get_channel() const { return this->channel; } + +private: + CommChannel *channel; + friend class Communicator; +}; + #define SS_STATE_COMPLETE 0 #define SS_STATE_ERROR 1 #define SS_STATE_DISRUPTED 2 @@ -271,6 +316,10 @@ class Communicator int bind(CommService *service); void unbind(CommService *service); + int establish(CommChannel *channel, CommTarget *target); + int send(TransSession *session, CommChannel *channel); + void shutdown(CommChannel *channel); + int sleep(SleepSession *session); int io_bind(IOService *service); diff --git a/src/kernel/TransRequest.cc b/src/kernel/TransRequest.cc new file mode 100644 index 0000000000..950366b0cb --- /dev/null +++ b/src/kernel/TransRequest.cc @@ -0,0 +1,59 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Xie Han (xiehan@sogou-inc.com) +*/ + +#include +#include "CommScheduler.h" +#include "TransRequest.h" + +void ChanRequest::dispatch() +{ + if (!this->established) + { + if (this->scheduler->establish(this, this->object, this->wait_timeout, + &this->target) < 0) + { + this->state = CS_STATE_ERROR; + this->error = errno; + this->subtask_done(); + } + } + else + this->scheduler->shutdown(this); +} + +void TransRequest::dispatch() +{ + int ret = this->scheduler->send(this, this->channel); + + if (ret == 0) + return; + + if (ret > 0) + { + this->state = CS_STATE_SUCCESS; + this->error = 0; + } + else + { + this->state = CS_STATE_ERROR; + this->error = errno; + } + + this->subtask_done(); +} + diff --git a/src/kernel/TransRequest.h b/src/kernel/TransRequest.h new file mode 100644 index 0000000000..b83549c265 --- /dev/null +++ b/src/kernel/TransRequest.h @@ -0,0 +1,109 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Xie Han (xiehan@sogou-inc.com) +*/ + +#ifndef _TRANSREQUEST_H_ +#define _TRANSREQUEST_H_ + +#include "SubTask.h" +#include "Communicator.h" +#include "CommScheduler.h" + +class ChanRequest : public SubTask, public CommChannel +{ +public: + ChanRequest(CommSchedObject *object, CommScheduler *scheduler) + { + this->scheduler = scheduler; + this->object = object; + this->wait_timeout = 0; + this->established = 0; + } + + CommSchedObject *get_request_object() const { return this->object; } + void set_request_object(CommSchedObject *object) { this->object = object; } + int get_wait_timeout() const { return this->wait_timeout; } + void set_wait_timeout(int timeout) { this->wait_timeout = timeout; } + +public: + virtual void dispatch(); + +protected: + int state; + int error; + +protected: + CommTarget *target; + +protected: + int established; + int wait_timeout; + CommSchedObject *object; + CommScheduler *scheduler; + +protected: + virtual void handle_established() + { + this->state = CS_STATE_SUCCESS; + this->error = 0; + this->established = 1; + this->subtask_done(); + } + + virtual void handle(int state, int error) + { + this->state = state; + this->error = error; + this->established = 0; + this->subtask_done(); + } +}; + +class TransRequest : public SubTask, public TransSession +{ +public: + TransRequest(CommChannel *channel, CommScheduler *scheduler) + { + this->scheduler = scheduler; + this->channel = channel; + } + + CommChannel *get_request_channel() const { return this->channel; } + void set_request_channel(CommChannel *channel) { this->channel = channel; } + +public: + virtual void dispatch(); + +protected: + int state; + int error; + +protected: + CommChannel *channel; + CommScheduler *scheduler; + +protected: + virtual void handle(int state, int error) + { + this->state = state; + this->error = error; + this->subtask_done(); + } +}; + +#endif + diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index a60e031b5f..220b9ad381 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -4,6 +4,7 @@ project(manager) set(SRC DnsCache.cc RouteManager.cc + WFSemaphore.cc WFGlobal.cc ) diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc new file mode 100644 index 0000000000..12c5d62190 --- /dev/null +++ b/src/manager/WFCondition.cc @@ -0,0 +1,218 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" +#include "WFCondition.h" + +class WFTimedWaitTask; + +class WFWaitTask : public WFMailboxTask +{ +public: + void set_timer(WFTimedWaitTask *timer) { this->timer = timer; } + void clear_timer_waiter(); + + struct task_entry + { + struct list_head list; + WFWaitTask *ptr; + } entry; + +protected: + virtual void dispatch(); + virtual SubTask *done(); + +private: + WFTimedWaitTask *timer; + +public: + WFWaitTask(std::function&& cb) : + WFMailboxTask(&this->user_data, 1, std::move(cb)) + { + this->timer = NULL; + this->entry.list.next = NULL; + this->entry.ptr = this; + } +}; + +class WFTimedWaitTask : public __WFTimerTask +{ +public: + WFTimedWaitTask(WFWaitTask *wait_task, std::mutex *mutex, + const struct timespec *value, + CommScheduler *scheduler, + std::function cb) : + __WFTimerTask(value, scheduler, std::move(cb)) + { + this->mutex = mutex; + this->wait_task = wait_task; + } + + void clear_wait_task(); + +protected: + virtual SubTask *done(); + +private: + std::mutex *mutex; + WFWaitTask *wait_task; +}; + +void WFWaitTask::dispatch() +{ + if (this->timer) + timer->dispatch(); + + this->WFMailboxTask::count(); +} + +SubTask *WFWaitTask::done() +{ + SeriesWork *series = series_of(this); + + // TODO: data move + + WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, + [this](WFTimerTask *task) { + if (this->callback) + this->callback(this); + delete this; + }); + series->push_front(switch_task); + + return series->pop(); +} + +void WFWaitTask::clear_timer_waiter() +{ + if (this->timer) + timer->clear_wait_task(); +} + +SubTask *WFTimedWaitTask::done() +{ + WFWaitTask *tmp = NULL; + + this->mutex->lock(); + if (this->wait_task && this->wait_task->entry.list.next) + { + list_del(&this->wait_task->entry.list); + tmp = this->wait_task; + this->wait_task = NULL; + } + this->mutex->unlock(); + + if (tmp) + tmp->count(); + + SeriesWork *series = series_of(this); + + if (this->callback) + this->callback(this); + + delete this; + return series->pop(); +} + +void WFTimedWaitTask::clear_wait_task() +{ + this->mutex->lock(); + this->wait_task = NULL; + this->mutex->unlock(); +} + +WFMailboxTask *WFCondition::create_wait_task(mailbox_callback_t cb) +{ + WFWaitTask *task = new WFWaitTask(std::move(cb)); + + this->mutex.lock(); + list_add_tail(&task->entry.list, &this->waiter_list); + this->mutex.unlock(); + + return task; +} + +WFMailboxTask *WFCondition::create_timedwait_task(const struct timespec *abstime, + mailbox_callback_t cb) +{ + WFWaitTask *waiter = new WFWaitTask(std::move(cb)); + WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &this->mutex, abstime, + WFGlobal::get_scheduler(), + nullptr); + waiter->set_timer(task); + + this->mutex.lock(); + list_add_tail(&waiter->entry.list, &this->waiter_list); + this->mutex.unlock(); + + return waiter; +} + +void WFCondition::signal() +{ + WFWaitTask *task = NULL; + struct list_head *pos; + struct WFWaitTask::task_entry *entry; + + this->mutex.lock(); + + if (!list_empty(&this->waiter_list)) + { + pos = this->waiter_list.next; + entry = list_entry(pos, struct WFWaitTask::task_entry, list); + task = entry->ptr; + list_del(pos); + task->clear_timer_waiter(); + } + + this->mutex.unlock(); + + if (task) + task->count(); +} + +void WFCondition::broadcast() +{ + WFWaitTask *task; + struct list_head *pos, *tmp; + struct WFWaitTask::task_entry *entry; + LIST_HEAD(tmp_list); + + this->mutex.lock(); + if (!list_empty(&this->waiter_list)) + { + list_for_each_safe(pos, tmp, &this->waiter_list) + { + entry = list_entry(pos, struct WFWaitTask::task_entry, list); + list_move_tail(pos, &tmp_list); + } + } + this->mutex.unlock(); + + while (!list_empty(&tmp_list)) + { + entry = list_entry(tmp_list.next, struct WFWaitTask::task_entry, list); + task = entry->ptr; + list_del(&entry->list); + task->count(); + } +} + diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h new file mode 100644 index 0000000000..9c343f3158 --- /dev/null +++ b/src/manager/WFCondition.h @@ -0,0 +1,48 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WFCONDITION_H_ +#define _WFCONDITION_H_ + +#include +#include +#include +#include "list.h" +#include "WFTask.h" + +class WFCondition +{ +public: + WFMailboxTask *create_wait_task(std::function cb); + WFMailboxTask *create_timedwait_task(const struct timespec *abstime, + std::function cb); + void signal(); + void broadcast(); + +public: + WFCondition() { INIT_LIST_HEAD(&this->waiter_list); } + +public: + std::mutex mutex; + +private: + struct list_head waiter_list; +}; + +#endif + diff --git a/src/manager/WFGlobal.cc b/src/manager/WFGlobal.cc index 3987835477..d35c008944 100644 --- a/src/manager/WFGlobal.cc +++ b/src/manager/WFGlobal.cc @@ -156,17 +156,18 @@ __WFGlobal::__WFGlobal() : settings_(GLOBAL_SETTINGS_DEFAULT) static_scheme_port_["MySQL"] = "3306"; static_scheme_port_["MYSQL"] = "3306"; - static_scheme_port_["mysqls"] = "3306"; - static_scheme_port_["Mysqls"] = "3306"; - static_scheme_port_["MySqls"] = "3306"; - static_scheme_port_["MySQLs"] = "3306"; - static_scheme_port_["MYSQLs"] = "3306"; - static_scheme_port_["MYSQLS"] = "3306"; - static_scheme_port_["kafka"] = "9092"; static_scheme_port_["Kafka"] = "9092"; static_scheme_port_["KAFKA"] = "9092"; + static_scheme_port_["ws"] = "80"; + static_scheme_port_["Ws"] = "80"; + static_scheme_port_["WS"] = "80"; + + static_scheme_port_["wss"] = "443"; + static_scheme_port_["Wss"] = "443"; + static_scheme_port_["WSs"] = "443"; + sync_count_ = 0; sync_max_ = 0; } @@ -932,9 +933,6 @@ static inline const char *__get_task_error_string(int error) case WFT_ERR_MYSQL_QUERY_NOT_SET: return "MySQL Query Not Set"; - case WFT_ERR_MYSQL_SSL_NOT_SUPPORTED: - return "MySQL SSL Not Supported"; - case WFT_ERR_KAFKA_PARSE_RESPONSE_FAILED: return "Kafka parse response failed"; diff --git a/src/manager/WFSemaphore.cc b/src/manager/WFSemaphore.cc new file mode 100644 index 0000000000..9013dfec13 --- /dev/null +++ b/src/manager/WFSemaphore.cc @@ -0,0 +1,190 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) + Liu Kai (liukaidx@sogou-inc.com) +*/ + +#include +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFSemaphore.h" + +/////////////// Semaphore Impl /////////////// + +bool WFSemaphore::get(void **dest) +{ + this->mutex.lock(); + if (--this->concurrency >= 0) + { + *dest = this->resources[--this->index]; + this->mutex.unlock(); + return true; + } + + WFSemaphoreTask *task = new WFSemaphoreTask(nullptr); + + list_add_tail(&task->node.list, &this->get_list); + this->mutex.unlock(); + + return false; +} + +WFWaitTask *WFSemaphore::create_wait_task(std::function cb) +{ + WFSemaphoreTask *task = NULL; + struct list_head *pos; + struct WFSemaphoreTask::entry *node; + + this->mutex.lock(); + if (!list_empty(&this->get_list)) + { + pos = this->get_list.next; + list_move_tail(pos, &this->wait_list); + node = list_entry(pos, struct WFSemaphoreTask::entry, list); + task = node->ptr; + task->set_callback(std::move(cb)); + } + + this->mutex.unlock(); + return task; +} + +void WFSemaphore::post(void *src) +{ + WFSemaphoreTask *task = NULL; + struct list_head *pos; + struct WFSemaphoreTask::entry *node; + + this->mutex.lock(); + + if (++this->concurrency <= 0) + { + if (!list_empty(&this->wait_list)) + pos = this->wait_list.next; + else + pos = this->get_list.next; + node = list_entry(pos, struct WFSemaphoreTask::entry, list); + task = node->ptr; + list_del(pos); + } + + this->resources[this->index++] = src; + + this->mutex.unlock(); + if (task) + task->send(src); +} + +/////////////// Wait tasks Impl /////////////// + +void WFCondWaitTask::dispatch() +{ + if (this->timer) + timer->dispatch(); + + this->WFWaitTask::count(); +} + +SubTask *WFCondWaitTask::done() +{ + SeriesWork *series = series_of(this); + + WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, + [this](WFTimerTask *task) { + if (this->callback) + this->callback(this); + delete this; + }); + series->push_front(switch_task); + + return series->pop(); +} + +void WFCondWaitTask::clear_timer_waiter() +{ + if (this->timer) + timer->clear_wait_task(); +} + +SubTask *WFTimedWaitTask::done() +{ + this->mutex->lock(); + if (this->wait_task && this->wait_task->node.list.next) + { + list_del(&this->wait_task->node.list); + this->wait_task->set_error(ETIMEDOUT); + this->wait_task->count(); + this->wait_task = NULL; + } + this->mutex->unlock(); + + delete this; + return NULL; +} + +/////////////// Condition Impl /////////////// + +void WFCondition::signal(void *msg) +{ + WFCondWaitTask *task = NULL; + struct list_head *pos; + struct WFSemaphoreTask::entry *node; + + this->mutex.lock(); + if (!list_empty(&this->wait_list)) + { + pos = this->wait_list.next; + node = list_entry(pos, struct WFSemaphoreTask::entry, list); + task = (WFCondWaitTask *)node->ptr; + list_del(pos); + task->clear_timer_waiter(); + } + + this->mutex.unlock(); + if (task) + task->send(msg); +} + +void WFCondition::broadcast(void *msg) +{ + WFCondWaitTask *task; + struct list_head *pos, *tmp; + struct WFSemaphoreTask::entry *node; + LIST_HEAD(tmp_list); + + this->mutex.lock(); + if (!list_empty(&this->wait_list)) + { + list_for_each_safe(pos, tmp, &this->wait_list) + { + list_move_tail(pos, &tmp_list); + } + } + + this->mutex.unlock(); + while (!list_empty(&tmp_list)) + { + node = list_entry(tmp_list.next, struct WFSemaphoreTask::entry, list); + task = (WFCondWaitTask *)node->ptr; + list_del(&node->list); + task->send(msg); + } +} + diff --git a/src/manager/WFSemaphore.h b/src/manager/WFSemaphore.h new file mode 100644 index 0000000000..e5e8b9ffa7 --- /dev/null +++ b/src/manager/WFSemaphore.h @@ -0,0 +1,93 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) + Liu Kai (liukaidx@sogou-inc.com) +*/ + +#ifndef _WFSEMAPHORE_H_ +#define _WFSEMAPHORE_H_ + +#include +#include +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" + +using WFWaitTask = WFMailboxTask; +using wait_callback_t = mailbox_callback_t; + +class WFSemaphore +{ +public: + bool get(void **dest); + WFWaitTask *create_wait_task(std::function cb); + void post(void *src); + +public: + std::mutex mutex; + struct list_head get_list; + struct list_head wait_list; + +public: + WFSemaphore(int value, void **resources) + { + if (value <= 0) + value = 1; + + INIT_LIST_HEAD(&this->get_list); + INIT_LIST_HEAD(&this->wait_list); + this->concurrency = value; + this->total = value; + this->index = value; + this->resources = resources; +// this->resources = new void *[value]; + } + + virtual ~WFSemaphore() { /* delete []this->resources; */ } + +private: + std::atomic concurrency; + int total; + int index; + +protected: + void **resources; +}; + +class WFCondition : public WFSemaphore +{ +public: + void signal(void *msg); + void broadcast(void *msg); + +public: + WFCondition() : WFSemaphore(1, new void *[1]) { } + WFCondition(int value) : WFSemaphore(value, new void *[value]) { } + virtual ~WFCondition() { delete []this->resources;} + + bool get(void **dest) = delete; + WFWaitTask *create_wait_task(wait_callback_t cb) = delete; + void post(void *src) = delete; +}; + +#include "WFSemaphore.inl" + +#endif + diff --git a/src/manager/WFSemaphore.inl b/src/manager/WFSemaphore.inl new file mode 100644 index 0000000000..f6c4f89a53 --- /dev/null +++ b/src/manager/WFSemaphore.inl @@ -0,0 +1,112 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) + Liu Kai (liukaidx@sogou-inc.com) +*/ + +#include "list.h" +#include "WFTask.h" +#include "WFTaskFactory.h" + +class WFSemaphoreTask : public WFWaitTask +{ +public: + void start() + { + assert(!series_of(this)); + Workflow::start_series_work(this, nullptr); + } + + void dismiss() + { + assert(!series_of(this)); + delete this; + } + +public: + WFSemaphoreTask(std::function&& cb) : + WFWaitTask(&this->msg, 1, std::move(cb)) + { + this->node.list.next = NULL; + this->node.ptr = this; + } + + virtual ~WFSemaphoreTask() { } + +public: + struct entry + { + struct list_head list; + WFSemaphoreTask *ptr; + } node; + +private: + void *msg; +}; + +class WFTimedWaitTask; + +class WFCondWaitTask : public WFSemaphoreTask +{ +public: + void set_timer(WFTimedWaitTask *timer) { this->timer = timer; } + void clear_timer_waiter(); + void set_error(int error) { this->error = error; } + +protected: + void dispatch(); + SubTask *done(); + +private: + WFTimedWaitTask *timer; + +public: + WFCondWaitTask(std::function&& cb) : + WFSemaphoreTask(std::move(cb)) + { + this->timer = NULL; + } + + virtual ~WFCondWaitTask() { } +}; + +class WFTimedWaitTask : public __WFTimerTask +{ +public: + WFTimedWaitTask(WFCondWaitTask *wait_task, std::mutex *mutex, + const struct timespec *value, + CommScheduler *scheduler, + std::function cb) : + __WFTimerTask(value, scheduler, std::move(cb)) + { + this->mutex = mutex; + this->wait_task = wait_task; + } + + void clear_wait_task() // must called within this mutex + { + this->wait_task = NULL; + } + +protected: + virtual SubTask *done(); + +private: + std::mutex *mutex; + WFCondWaitTask *wait_task; +}; + diff --git a/src/protocol/CMakeLists.txt b/src/protocol/CMakeLists.txt index ae675375b1..0f5c8242df 100644 --- a/src/protocol/CMakeLists.txt +++ b/src/protocol/CMakeLists.txt @@ -9,6 +9,8 @@ set(SRC http_parser.c HttpMessage.cc HttpUtil.cc + websocket_parser.c + WebSocketMessage.cc ) if (MYSQL STREQUAL "y") diff --git a/src/protocol/WebSocketMessage.cc b/src/protocol/WebSocketMessage.cc new file mode 100644 index 0000000000..fe11f10cd2 --- /dev/null +++ b/src/protocol/WebSocketMessage.cc @@ -0,0 +1,299 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include +#include "WebSocketMessage.h" + +namespace protocol +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + +static inline void int2store(unsigned char *T, uint16_t A) +{ + memcpy(T, &A, sizeof(A)); +} + +static inline void int8store(unsigned char *T, uint64_t A) +{ + memcpy(T, &A, sizeof(A)); +} + +#elif __BYTE_ORDER == __BIG_ENDIAN + +static inline void int2store(unsigned char *T, uint16_t A) +{ + uint def_temp = A; + *(T) = (unsigned char)(def_temp); + *(T + 1) = (unsigned char)(def_temp >> 8); +} + +static inline void int4store(unsigned char *T, uint32_t A) +{ + *(T) = (unsigned char)(A); + *(T + 1) = (unsigned char)(A >> 8); + *(T + 2) = (unsigned char)(A >> 16); + *(T + 3) = (unsigned char)(A >> 24); +} + +static inline void int8store(unsigned char *T, uint64_t A) +{ + uint def_temp = (uint)A, def_temp2 = (uint)(A >> 32); + int4store(T, def_temp); + int4store(T + 4, def_temp2); +} + +#else +# error "unknown byte order" +#endif + +WebSocketFrame::WebSocketFrame(WebSocketFrame&& msg) : + ProtocolMessage(std::move(msg)) +{ + this->parser = msg.parser; + msg.parser = NULL; +} + +WebSocketFrame& WebSocketFrame::operator = (WebSocketFrame&& msg) +{ + if (&msg != this) + { + *(ProtocolMessage *)this = std::move(msg); + if (this->parser) + { + websocket_parser_deinit(this->parser); + delete this->parser; + } + + this->parser = msg.parser; + msg.parser = NULL; + } + + return *this; +} + +int WebSocketFrame::append(const void *buf, size_t *size) +{ + int ret = websocket_parser_append_message(buf, size, this->parser); + + if (this->parser->payload_length > this->size_limit) + { + this->parser->status_code = WSStatusCodeTooLarge; + return 1; // don`t need websocket_parser_parse() + } + + if (ret == 1) + websocket_parser_parse(this->parser); + + return ret; +} + +int WebSocketFrame::encode(struct iovec vectors[], int max) +{ + unsigned char *p = this->parser->header_buf; + int cnt = 0; + + if (this->parser->opcode == WebSocketFramePing || + this->parser->opcode == WebSocketFramePong || + this->parser->opcode == WebSocketFrameConnectionClose) + { + this->parser->fin = 1; + } + else if (!this->parser->fin) + { + this->parser->opcode = WebSocketFrameContinuation; + } + + *p = (this->parser->fin << 7) | this->parser->opcode; + p++; + + if (this->parser->payload_length < 126) + { + *p = (unsigned char)this->parser->payload_length; + p++; + } + else if (this->parser->payload_length < 65536) + { + *p = 126; + p++; + int2store(p, this->parser->payload_length); + p += 2; + } + else + { + *p = 127; + p++; + int8store(p, this->parser->payload_length); + p += 8; + } + + vectors[cnt].iov_base = this->parser->header_buf; + vectors[cnt].iov_len = p - this->parser->header_buf; + cnt++; + + p = this->parser->header_buf + 1; + *p = *p | (this->parser->mask << 7); + + if (!this->parser->is_server) + { + vectors[cnt].iov_base = this->parser->masking_key; + vectors[cnt].iov_len = WS_MASKING_KEY_LENGTH; + cnt++; + } + + if (this->parser->payload_length) + { + websocket_parser_mask_data(this->parser); + vectors[cnt].iov_base = this->parser->payload_data; + vectors[cnt].iov_len = this->parser->payload_length; + cnt++; + } + + return cnt; +} + +bool WebSocketFrame::set_opcode(int opcode) +{ + if (opcode < WebSocketFrameContinuation || opcode > WebSocketFramePong) + return false; + + this->parser->opcode = opcode; + return true; +} + +int WebSocketFrame::get_opcode() const +{ + return this->parser->opcode; +} + +void WebSocketFrame::set_masking_key(uint32_t masking_key) +{ + this->parser->mask = 1; + memcpy(this->parser->masking_key, &masking_key, WS_MASKING_KEY_LENGTH); +} + +uint32_t WebSocketFrame::get_masking_key() const +{ + if (!this->parser->mask) + return atoi((char *)this->parser->masking_key); + + return 0; +} + +bool WebSocketFrame::set_binary_data(const char *data, size_t size) +{ + return this->set_binary_data(data, size, true); +} + +bool WebSocketFrame::set_binary_data(const char *data, size_t size, bool fin) +{ + bool ret = true; + + this->parser->opcode = WebSocketFrameBinary; + this->parser->fin = fin; + + if (this->parser->payload_length && this->parser->payload_data) + { + ret = false; + free(this->parser->payload_data); + } + + this->parser->payload_data = (char *)malloc(size); + memcpy(this->parser->payload_data, data, size); + this->parser->payload_length = size; + + return ret; +} + +bool WebSocketFrame::set_text_data(const char *data) +{ + return set_text_data(data, strlen(data), true); +} + +bool WebSocketFrame::set_text_data(const char *data, size_t size, bool fin) +{ + bool ret = true; + + this->parser->opcode = WebSocketFrameText; + this->parser->fin = fin; + + if (this->parser->payload_length && this->parser->payload_data) + { + ret = false; + free(this->parser->payload_data); + } + + this->parser->payload_data = (char *)malloc(size); + memcpy(this->parser->payload_data, data, size); + this->parser->payload_length = size; + + return ret; +} + +bool WebSocketFrame::set_data(const websocket_parser_t *parser) +{ + bool ret = true; + unsigned char *p; + + if (this->parser->payload_length && this->parser->payload_data) + { + ret = false; + free(this->parser->payload_data); + } + +// this->parser->status_code = parser->status_code; + this->parser->payload_length = parser->payload_length; + + if (this->parser->opcode == WebSocketFrameConnectionClose && + parser->status_code != WSStatusCodeUndefined) + { + this->parser->payload_length += 2; + } + + this->parser->payload_data = malloc(this->parser->payload_length); + p = (unsigned char *)this->parser->payload_data; + + if (this->parser->opcode == WebSocketFrameConnectionClose && + parser->status_code != WSStatusCodeUndefined) + { + int2store(p, parser->status_code); + p += 2; + } + + memcpy(p, parser->payload_data, parser->payload_length); + + return ret; +} + +bool WebSocketFrame::get_data(const char **data, size_t *size) const +{ + if (!this->parser->payload_length || !this->parser->payload_data) + return false; + + *data = (char *)this->parser->payload_data; + *size = this->parser->payload_length; + return true; +} + +bool WebSocketFrame::finished() const +{ + return this->parser->fin; +} + +} // end namespace protocol + diff --git a/src/protocol/WebSocketMessage.h b/src/protocol/WebSocketMessage.h new file mode 100644 index 0000000000..e834be2a48 --- /dev/null +++ b/src/protocol/WebSocketMessage.h @@ -0,0 +1,84 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WEBSOCKETMESSAGE_H_ +#define _WEBSOCKETMESSAGE_H_ + +#include +#include +#include +#include "ProtocolMessage.h" +#include "websocket_parser.h" + +namespace protocol +{ + +#define WS_HANDSHAKE_TIMEOUT 10 * 1000 + +class WebSocketFrame : public ProtocolMessage +{ +public: + bool set_opcode(int opcode); + int get_opcode() const; + + void set_masking_key(uint32_t masking_key); + + bool set_text_data(const char *data); + bool set_text_data(const char *data, size_t size, bool fin); + + bool set_binary_data(const char *data, size_t size); + bool set_binary_data(const char *data, size_t size, bool fin); + + bool get_data(const char **data, size_t *size) const; + + bool finished() const; + +public: + void set_client() { this->parser->is_server = 0; } + void set_server() { this->parser->is_server = 1; } + const websocket_parser_t *get_parser() { return this->parser; } + bool set_data(const websocket_parser_t *parser); + uint32_t get_masking_key() const; + +private: + websocket_parser_t *parser; + +protected: + virtual int encode(struct iovec vectors[], int max); + virtual int append(const void *buf, size_t *size); + +public: + WebSocketFrame() : parser(new websocket_parser_t) + { + websocket_parser_init(this->parser); + } + + virtual ~WebSocketFrame() + { + websocket_parser_deinit(this->parser); + delete this->parser; + } + + WebSocketFrame(WebSocketFrame&& msg); + WebSocketFrame& operator = (WebSocketFrame&& msg); +}; + +} // end namespace protocol + +#endif + diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c new file mode 100644 index 0000000000..25e1db6ad5 --- /dev/null +++ b/src/protocol/websocket_parser.c @@ -0,0 +1,239 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include +#include +#include +#include "websocket_parser.h" + +void websocket_parser_init(websocket_parser_t *parser) +{ + parser->fin = 0; + parser->mask = 0; + parser->opcode = -1; + parser->nleft = WS_MASKING_KEY_LENGTH; + parser->payload_length = 0; + parser->payload_data = NULL; + parser->nreceived = 0; + parser->is_server = 0; + parser->status_code = WSStatusCodeUndefined; + memset(parser->masking_key, 0, WS_MASKING_KEY_LENGTH); + memset(parser->header_buf, 0, WS_HEADER_LENGTH_MAX); +} + +void websocket_parser_deinit(websocket_parser_t *parser) +{ + if (parser->payload_length != 0) + free(parser->payload_data); +} + +// 4: FIN 0 0 0 4:opcode | 1: MASK 7:PAYLOAD_LENGTH | +// 0or16or64 : extend PAYLOAD_LENGTH | 0or32 : MASK_KEY | +// PAYLOAD_LENGTH: PAYLOAD_DATA +int websocket_parser_append_message(const void *buf, size_t *n, + websocket_parser_t *parser) +{ + const unsigned char *p = (const unsigned char *)buf; + const unsigned char *buf_end = (const unsigned char *)buf + *n; + + int header_length_min = parser->is_server ? WS_CLIENT_LENGTH_MIN : + WS_SERVER_LENGTH_MIN; + +// if (parser->payload_length == 0) // receiving header + if (parser->nreceived < header_length_min) + { + memcpy(parser->header_buf + parser->nreceived, p, + WS_HEADER_LENGTH_MAX - parser->nreceived); + + if (parser->nreceived + *n < header_length_min) + { + parser->nreceived += *n; + return 0; + } + + parser->fin = *p >> 7; + parser->opcode = *p & 0xF; + p++; + parser->mask = *p >> 7; + parser->payload_length = *p & 0x7F; + p++; + } + + if (parser->masking_key_offset == 0) // receive payload_length + { + if (parser->payload_length == 126 && + parser->nreceived + *n >= header_length_min + 2) + { + uint16_t *len_ptr = (uint16_t *)p; + parser->payload_length = ntohs(*len_ptr); + p += 2; + parser->masking_key_offset = 4; + } + else if (parser->payload_length == 127 && + parser->nreceived + *n >= header_length_min + 8) + { + uint64_t *len_ptr = (uint64_t *)p; + parser->payload_length = (((uint64_t) ntohl(*len_ptr)) << 32) + + ntohl(*len_ptr >> 32); + p += 8; + parser->masking_key_offset = 10; + } + else + { + parser->nreceived += *n; + parser->masking_key_offset = 2; + } + + if (parser->masking_key_offset == 0) + return 0; // continue for length + } + + if (!parser->payload_data) // receive masking_key if needed + { + if (parser->mask && parser->nleft) + { +// if (parser->masking_key_offset + 4 < buf_end) + if (buf_end - p < parser->nleft) + { + parser->nleft -= buf_end - p; + parser->nreceived += *n; + return 0; + } + + memcpy(parser->masking_key, + parser->header_buf + parser->masking_key_offset, + WS_MASKING_KEY_LENGTH); + p += parser->nleft; + parser->nreceived += parser->nleft; + } + + parser->nleft = parser->payload_length; + } + + parser->payload_data = malloc(parser->payload_length); + + if (buf_end - p < parser->nleft) + { + memcpy(parser->payload_data + parser->payload_length - parser->nleft, + p, buf_end - p); + parser->nleft -= buf_end - p; + return 0; + } + else + { + memcpy(parser->payload_data + parser->payload_length - parser->nleft, + p, parser->nleft); + p += parser->nleft; + *n -= buf_end - p; + return 1; + } +} + +int websocket_parser_parse(websocket_parser_t *parser) +{ + if (parser->opcode < WebSocketFrameContinuation || + (parser->opcode < WebSocketFrameConnectionClose && + parser->opcode > WebSocketFrameBinary) || + parser->opcode > WebSocketFramePong) + { + parser->status_code = WSStatusCodeProtocolError; + return -1; + } + + unsigned char *p = (unsigned char *)parser->payload_data; + + if (parser->opcode == WebSocketFrameConnectionClose) + { + uint16_t *ptr = (uint16_t *)p; + parser->status_code = ntohs(*ptr); + parser->payload_data = p + 2; + } + + websocket_parser_mask_data(parser); + + if (parser->opcode == WebSocketFrameText && + !utf8_check(p, parser->payload_length)) + { + parser->status_code = WSStatusCodeUnsupportedData; + return -1; + } + + return 0; +} + +void websocket_parser_mask_data(websocket_parser_t *parser) +{ + if (!parser->mask) + return; + + unsigned long long i; + unsigned char *p = (unsigned char *)parser->payload_data; + + for (i = 0; i < parser->payload_length; i++) + *p++ ^= parser->masking_key[i % 4]; + + return; +} + +//https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c +unsigned char *utf8_check(unsigned char *s, size_t len) +{ + unsigned char *end = s + len; + while (*s && s != end) { + if (*s < 0x80) + /* 0xxxxxxx */ + s++; + else if ((s[0] & 0xe0) == 0xc0) { + /* 110XXXXx 10xxxxxx */ + if ((s[1] & 0xc0) != 0x80 || + (s[0] & 0xfe) == 0xc0) /* overlong? */ + return s; + else + s += 2; + } else if ((s[0] & 0xf0) == 0xe0) { + /* 1110XXXX 10Xxxxxx 10xxxxxx */ + if ((s[1] & 0xc0) != 0x80 || + (s[2] & 0xc0) != 0x80 || + (s[0] == 0xe0 && (s[1] & 0xe0) == 0x80) || /* overlong? */ + (s[0] == 0xed && (s[1] & 0xe0) == 0xa0) || /* surrogate? */ + (s[0] == 0xef && s[1] == 0xbf && + (s[2] & 0xfe) == 0xbe)) /* U+FFFE or U+FFFF? */ + return s; + else + s += 3; + } else if ((s[0] & 0xf8) == 0xf0) { + /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */ + if ((s[1] & 0xc0) != 0x80 || + (s[2] & 0xc0) != 0x80 || + (s[3] & 0xc0) != 0x80 || + (s[0] == 0xf0 && (s[1] & 0xf0) == 0x80) || /* overlong? */ + (s[0] == 0xf4 && s[1] > 0x8f) || s[0] > 0xf4) /* > U+10FFFF? */ + return s; + else + s += 4; + } else + return s; + } + + if (s == end) + return s; + + return NULL; +} + diff --git a/src/protocol/websocket_parser.h b/src/protocol/websocket_parser.h new file mode 100644 index 0000000000..7f9d9c17c9 --- /dev/null +++ b/src/protocol/websocket_parser.h @@ -0,0 +1,123 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#ifndef _WEBSOCKET_PARSER_H_ +#define _WEBSOCKET_PARSER_H_ + +#define WS_HEADER_LENGTH_MAX 14 +#define WS_SERVER_LENGTH_MIN 2 +#define WS_CLIENT_LENGTH_MIN 6 +#define WS_MASKING_KEY_LENGTH 4 + +#include +#include + +enum +{ + WebSocketFrameContinuation = 0, + WebSocketFrameText = 1, + WebSocketFrameBinary = 2, + WebSocketFrameConnectionClose = 8, + WebSocketFramePing = 9, + WebSocketFramePong = 10, +}; + +enum +{ + WSStatusCodeUndefined = 0, + + WSStatusCodeNormal = 1000, + WSStatusCodeGoingAway = 1001, + WSStatusCodeProtocolError = 1002, + WSStatusCodeUnsupported = 1003, + WSStatusCodeReserved = 1004, // reserved + WSStatusCodeNoStatus = 1005, // reserved + WSStatusCodeAbnomal = 1006, // reserved + WSStatusCodeUnsupportedData = 1007, + WSStatusCodePolicyViolation = 1008, + WSStatusCodeTooLarge = 1009, + WSStatusCodeMissExtention = 1010, + WSStatusCodeInternalError = 1011, +// WSStatusCodeServiceRestart = 1012, +// WSStatusCodeTryAgainLater = 1013, + WSStatusCodeTLSHandshake = 1015, // reserved + + WSStatusCodeProtocolMax = 2999, + + WSStatusCodeIANAMin = 3000, + WSStatusCodeIANAMax = 3999, + + WSStatusCodeUserMin = 4000, + WSStatusCodeUserMax = 4999, +}; + +typedef struct __websocket_parser +{ + char fin; + char mask; + int opcode; + unsigned char masking_key[WS_MASKING_KEY_LENGTH]; + unsigned long long payload_length; + unsigned char header_buf[WS_HEADER_LENGTH_MAX]; + void *payload_data; + unsigned long long nreceived; + int masking_key_offset; + int nleft; + int is_server; + int status_code; +} websocket_parser_t; + +#ifdef __cplusplus +extern "C" +{ +#endif + +void websocket_parser_init(websocket_parser_t *parser); +void websocket_parser_deinit(websocket_parser_t *parser); +int websocket_parser_append_message(const void *buf, size_t *n, + websocket_parser_t *parser); + +int websocket_parser_decode_payload_length(websocket_parser_t *parser); + +void websocket_parser_encode_payload_length(websocket_parser_t *parser); + +int websocket_parser_parse(websocket_parser_t *parser); + +void websocket_parser_mask_data(websocket_parser_t *parser); + +unsigned char *utf8_check(unsigned char *s, size_t len); + +#ifdef __cplusplus +} +#endif + +static inline void websocket_set_mask_key(unsigned int masking_key, + websocket_parser_t *parser) +{ + uint32_t *key = (uint32_t *)parser->masking_key; + *key = masking_key; + parser->mask = 1; +} + +static inline void websocket_set_opcode(int opcode, websocket_parser_t *parser) +{ + parser->opcode = opcode; +} + +#endif + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c673b168fb..1eb5fa8bab 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,6 +39,7 @@ set(TEST_LIST memory_unittest upstream_unittest dns_unittest + condition_unittest ) if (APPLE) diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc new file mode 100644 index 0000000000..5e32b986c0 --- /dev/null +++ b/test/condition_unittest.cc @@ -0,0 +1,167 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include +#include +#include +#include +#include +#include "workflow/WFTask.h" +#include "workflow/WFTaskFactory.h" +#include "workflow/WFSemaphore.h" +#include "workflow/WFSemTaskFactory.h" +#include "workflow/WFFacilities.h" + +TEST(condition_unittest, signal) +{ + WFCondition cond; + std::mutex mutex; + WFFacilities::WaitGroup wg(1); + int ret = 3; + int *ptr = &ret; + + auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&wg, &ptr](WFMailboxTask *) { + *ptr = 1; + wg.done(); + }); + + auto *task2 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + *ptr = 2; + }); + + SeriesWork *series1 = Workflow::create_series_work(task1, nullptr); + SeriesWork *series2 = Workflow::create_series_work(task2, nullptr); + + series1->start(); + series2->start(); + + mutex.lock(); + cond.signal(NULL); + mutex.unlock(); + wg.wait(); + EXPECT_EQ(ret, 1); + cond.signal(NULL); + usleep(1000); + EXPECT_EQ(ret, 2); +} + +TEST(condition_unittest, broadcast) +{ + WFCondition cond; + std::mutex mutex; + int ret = 0; + int *ptr = &ret; + + auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + (*ptr)++; + }); + SeriesWork *series1 = Workflow::create_series_work(task1, nullptr); + + auto *task2 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + (*ptr)++; + }); + SeriesWork *series2 = Workflow::create_series_work(task2, nullptr); + + series1->start(); + series2->start(); + + cond.broadcast(NULL); + usleep(1000); + EXPECT_EQ(ret, 2); +} + +TEST(condition_unittest, timedwait) +{ + WFFacilities::WaitGroup wait_group(2); + struct timespec ts; + ts.tv_sec = 1; + ts.tv_nsec = 0; + + auto *task1 = WFSemTaskFactory::create_timedwait_task("timedwait1", &ts, + [&wait_group](WFMailboxTask *task) { + EXPECT_EQ(task->get_error(), ETIMEDOUT); + wait_group.done(); + }); + + auto *task2 = WFSemTaskFactory::create_timedwait_task("timedwait2", &ts, + [&wait_group](WFMailboxTask *task) { + EXPECT_EQ(task->get_error(), 0); + void **msg; + size_t n; + msg = task->get_mailbox(&n); + EXPECT_EQ(n, 1); + EXPECT_TRUE(strcmp((char *)*msg, "wake up!!") == 0); + wait_group.done(); + }); + + Workflow::start_series_work(task1, nullptr); + Workflow::start_series_work(task2, nullptr); + + usleep(1000); + char msg[10] = "wake up!!"; + WFSemTaskFactory::signal_by_name("timedwait2", msg); + wait_group.wait(); +} + +TEST(condition_unittest, semaphore) +{ + int sem_concurrency = 1; + int task_concurrency = 3; + const char *words[3] = {"workflow", "srpc", "pyworkflow"}; + WFSemaphore sem(sem_concurrency, (void **)words); + WFFacilities::WaitGroup wg(task_concurrency); + + for (int i = 0; i < task_concurrency; i++) + { + auto *t = WFTaskFactory::create_timer_task(i * 1, [&sem, &wg](WFTimerTask *task) { + uint64_t id = (uint64_t)series_of(task)->get_context(); + void *dest_res; + + if (!sem.get(&dest_res)) + { + auto *waiter = sem.create_wait_task([&sem, &wg](WFWaitTask *task) { + uint64_t id = (uint64_t)series_of(task)->get_context(); + void **msg; + size_t n; + msg = task->get_mailbox(&n); + fprintf(stderr, "%lu after wait. get resource:%s\n", id, (char *)*msg); + usleep(10); + wg.done(); + sem.post(*msg); + //sem.post(reinterpret_cast(id)); + }); + series_of(task)->push_back(waiter); + } + else + { + fprintf(stderr, "%lu no wait. get resource: %s\n", id, (char *)dest_res); + usleep(10); + wg.done(); + //sem.post(reinterpret_cast(id)); + sem.post(dest_res); + } + }); + + SeriesWork *series = Workflow::create_series_work(t, nullptr); + series->set_context(reinterpret_cast(i)); + series->start(); + } + wg.wait(); +} + diff --git a/tutorial/CMakeLists.txt b/tutorial/CMakeLists.txt index 75f994bfc4..04d04c4b38 100644 --- a/tutorial/CMakeLists.txt +++ b/tutorial/CMakeLists.txt @@ -32,6 +32,8 @@ set(TUTORIAL_LIST tutorial-08-matrix_multiply tutorial-09-http_file_server tutorial-11-graph_task + tutorial-12-mysql_cli + tutorial-14-websocket_cli ) if (APPLE) diff --git a/tutorial/tutorial-14-websocket_cli.cc b/tutorial/tutorial-14-websocket_cli.cc new file mode 100644 index 0000000000..ba3bee7204 --- /dev/null +++ b/tutorial/tutorial-14-websocket_cli.cc @@ -0,0 +1,86 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include "workflow/WFFacilities.h" +#include "workflow/WFWebSocketClient.h" +#include "workflow/WebSocketMessage.h" + +#include +#include +#include +#include +#include "unistd.h" + +using namespace protocol; + +void process(WFWebSocketTask *task) +{ + const char *data; + size_t size; + + if (task->get_msg()->get_opcode() == WebSocketFrameText) + { + task->get_msg()->get_data(&data, &size); + fprintf(stderr, "get text message: [%.*s]\n", (int)size, data); + } + else + { + fprintf(stderr, "opcode=%d\n", task->get_msg()->get_opcode()); + } +} + +int main(int argc, char *argv[]) +{ + if (argc != 2) + { + fprintf(stderr, "USAGE: %s \n url format: ws://host:ip\n", argv[0]); + return 0; + } + + WebSocketClient client(process); + client.init(argv[1]); + + WFFacilities::WaitGroup wg(1); + auto *task = client.create_websocket_task([&wg, &client] (WFWebSocketTask *task) + { + fprintf(stderr, "send callback() state=%d error=%d\n", + task->get_state(), task->get_error()); + + if (task->get_state() != WFT_STATE_SUCCESS) + { + wg.done(); + return; + } + + auto *timer_task = WFTaskFactory::create_timer_task(3000000 /* 3s */, nullptr); + auto *close_task = client.create_close_task([&wg] (WFWebSocketTask *task) { + wg.done(); + }); + + series_of(task)->push_back(timer_task); + series_of(task)->push_back(close_task); + }); + + WebSocketFrame *msg = task->get_msg(); + msg->set_text_data("This is Workflow websocket client."); + task->start(); + + wg.wait(); + + return 0; +} From e08cabedb19c748992ecc38fe29e78251b6fc3e2 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 2 Jul 2021 20:38:05 +0800 Subject: [PATCH 26/75] seperate Semaphore and Condition; use WFConditional; --- CMakeLists.txt | 2 +- CMakeLists_Headers.txt | 6 +- src/factory/CMakeLists.txt | 2 +- src/factory/WFChannel.inl | 13 +- ...SemTaskFactory.cc => WFCondTaskFactory.cc} | 76 +++---- ...WFSemTaskFactory.h => WFCondTaskFactory.h} | 10 +- src/manager/CMakeLists.txt | 2 +- src/manager/WFCondition.cc | 181 ++++++----------- src/manager/WFCondition.h | 67 +++++- .../{WFSemaphore.inl => WFCondition.inl} | 47 +---- src/manager/WFSemaphore.cc | 190 ------------------ src/manager/WFSemaphore.h | 93 --------- test/condition_unittest.cc | 61 ++---- 13 files changed, 196 insertions(+), 554 deletions(-) rename src/factory/{WFSemTaskFactory.cc => WFCondTaskFactory.cc} (65%) rename src/factory/{WFSemTaskFactory.h => WFCondTaskFactory.h} (90%) rename src/manager/{WFSemaphore.inl => WFCondition.inl} (72%) delete mode 100644 src/manager/WFSemaphore.cc delete mode 100644 src/manager/WFSemaphore.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 217501dbda..f6aa89a30e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,7 +68,7 @@ if (WIN32) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP /wd4200 /std:c++14") else () set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -fPIC -pipe -std=gnu90") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fPIC -pipe -std=c++11 -fno-exceptions") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fPIC -pipe -std=c++11 -fno-exceptions -Wno-invalid-offsetof") endif () add_subdirectory(src) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index 3aabf79b4f..b47a8220ce 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -79,8 +79,8 @@ set(INCLUDE_HEADERS src/manager/WFFuture.h src/manager/WFFacilities.h src/manager/WFFacilities.inl - src/manager/WFSemaphore.h - src/manager/WFSemaphore.inl + src/manager/WFCondition.h + src/manager/WFCondition.inl src/util/EncodeStream.h src/util/LRUCache.h src/util/StringUtil.h @@ -95,7 +95,7 @@ set(INCLUDE_HEADERS src/factory/WFTaskFactory.inl src/factory/WFAlgoTaskFactory.h src/factory/WFAlgoTaskFactory.inl - src/factory/WFSemTaskFactory.h + src/factory/WFCondTaskFactory.h src/factory/Workflow.h src/factory/WFOperator.h src/factory/WFChannel.h diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index 2747102a89..9944f5babf 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -6,7 +6,7 @@ set(SRC DnsTaskImpl.cc Workflow.cc WFTaskFactory.cc - WFSemTaskFactory.cc + WFCondTaskFactory.cc HttpTaskImpl.cc WebSocketTaskImpl.cc ) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index d8a2c7de49..72843f12df 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -21,9 +21,8 @@ #include "TransRequest.h" #include "WFTask.h" #include "WFTaskFactory.h" -//#include "WFCondition.h" -#include "WFSemaphore.h" -#include "WFSemTaskFactory.h" +#include "WFCondition.h" +#include "WFCondTaskFactory.h" #include "WFNameService.h" #include "RouteManager.h" #include "WFGlobal.h" @@ -250,8 +249,8 @@ void WFComplexChannel::handle_terminated() this->sending = true; shutdown = true; } else { - waiter = WFSemTaskFactory::create_wait_task(&this->condition, - nullptr); + waiter = WFCondTaskFactory::create_wait_task(&this->condition, + nullptr); series_of(this)->push_front(this); series_of(this)->push_front(waiter); } @@ -354,7 +353,7 @@ void ComplexChannelOutTask::dispatch() } else { - waiter = WFSemTaskFactory::create_wait_task(&channel->condition, + waiter = WFCondTaskFactory::create_wait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); @@ -375,7 +374,7 @@ void ComplexChannelOutTask::dispatch() } else { - waiter = WFSemTaskFactory::create_wait_task(&channel->condition, + waiter = WFCondTaskFactory::create_wait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); diff --git a/src/factory/WFSemTaskFactory.cc b/src/factory/WFCondTaskFactory.cc similarity index 65% rename from src/factory/WFSemTaskFactory.cc rename to src/factory/WFCondTaskFactory.cc index bf8c5cd4a0..22e06396ab 100644 --- a/src/factory/WFSemTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -25,7 +25,7 @@ #include "WFTask.h" #include "WFTaskFactory.h" #include "WFGlobal.h" -#include "WFSemTaskFactory.h" +#include "WFCondTaskFactory.h" class __WFCondition : public WFCondition { @@ -33,16 +33,10 @@ class __WFCondition : public WFCondition __WFCondition(const std::string& str) : name(str) { - this->node.ptr = this; } public: - struct entry - { - struct rb_node rb; - __WFCondition *ptr; - } node; - + struct rb_node rb; std::string name; }; @@ -52,11 +46,10 @@ class __ConditionMap void signal(const std::string& name, void *msg); void broadcast(const std::string& name, void *msg); + WFWaitTask *create(const std::string& name, wait_callback_t&& cb); WFWaitTask *create(const std::string& name, - wait_callback_t&& cb); - WFWaitTask *create(const std::string& name, - const struct timespec *abstime, - wait_callback_t&& cb); + const struct timespec *abstime, + wait_callback_t&& cb); public: static __ConditionMap *get_instance() @@ -93,21 +86,21 @@ void __ConditionMap::broadcast(const std::string& name, void *msg) } WFWaitTask *__ConditionMap::create(const std::string& name, - wait_callback_t&& cb) + wait_callback_t&& cb) { __WFCondition *cond = this->find_condition(name); - return WFSemTaskFactory::create_wait_task(cond, std::move(cb)); + return WFCondTaskFactory::create_wait_task(cond, std::move(cb)); } WFWaitTask *__ConditionMap::create(const std::string& name, - const struct timespec *abstime, - wait_callback_t&& cb) + const struct timespec *abstime, + wait_callback_t&& cb) { __WFCondition *cond = this->find_condition(name); - return WFSemTaskFactory::create_timedwait_task(cond, abstime, - std::move(cb)); + return WFCondTaskFactory::create_timedwait_task(cond, abstime, + std::move(cb)); } __ConditionMap::~__ConditionMap() @@ -116,19 +109,15 @@ __ConditionMap::~__ConditionMap() WFCondWaitTask *task; struct list_head *pos; struct list_head *tmp; - struct WFSemaphoreTask::entry *node; - struct __WFCondition::entry *cond_node; while (this->condition_map.rb_node) { - cond_node = rb_entry(this->condition_map.rb_node, - struct __WFCondition::entry, rb); - cond = cond_node->ptr; + cond = rb_entry(this->condition_map.rb_node, + __WFCondition, rb); list_for_each_safe(pos, tmp, &cond->wait_list) { - node = list_entry(pos, struct WFSemaphoreTask::entry, list); - task = (WFCondWaitTask *)node->ptr; + task = list_entry(pos, WFCondWaitTask, list); list_del(pos); delete task; } @@ -141,16 +130,15 @@ __ConditionMap::~__ConditionMap() __WFCondition *__ConditionMap::find_condition(const std::string& name) { __WFCondition *cond; - struct __WFCondition::entry *cond_node; struct rb_node **p = &this->condition_map.rb_node; struct rb_node *parent = NULL; this->mutex.lock(); + while (*p) { parent = *p; - cond_node = rb_entry(*p, struct __WFCondition::entry, rb); - cond = cond_node->ptr; + cond = rb_entry(*p, __WFCondition, rb); if (name < cond->name) p = &(*p)->rb_left; @@ -163,8 +151,8 @@ __WFCondition *__ConditionMap::find_condition(const std::string& name) if (*p == NULL) { cond = new __WFCondition(name); - rb_link_node(&cond->node.rb, parent, p); - rb_insert_color(&cond->node.rb, &this->condition_map); + rb_link_node(&cond->rb, parent, p); + rb_insert_color(&cond->rb, &this->condition_map); } this->mutex.unlock(); @@ -174,45 +162,45 @@ __WFCondition *__ConditionMap::find_condition(const std::string& name) /////////////// factory api /////////////// -void WFSemTaskFactory::signal_by_name(const std::string& name, void *msg) +void WFCondTaskFactory::signal_by_name(const std::string& name, void *msg) { return __ConditionMap::get_instance()->signal(name, msg); } -void WFSemTaskFactory::broadcast_by_name(const std::string& name, void *msg) +void WFCondTaskFactory::broadcast_by_name(const std::string& name, void *msg) { return __ConditionMap::get_instance()->broadcast(name, msg); } -WFWaitTask *WFSemTaskFactory::create_wait_task(const std::string& name, - wait_callback_t callback) +WFWaitTask *WFCondTaskFactory::create_wait_task(const std::string& name, + wait_callback_t callback) { return __ConditionMap::get_instance()->create(name, std::move(callback)); } -WFWaitTask *WFSemTaskFactory::create_timedwait_task(const std::string& name, - const struct timespec *abstime, - wait_callback_t callback) +WFWaitTask *WFCondTaskFactory::create_timedwait_task(const std::string& name, + const struct timespec *abstime, + wait_callback_t callback) { return __ConditionMap::get_instance()->create(name, abstime, std::move(callback)); } -WFWaitTask *WFSemTaskFactory::create_wait_task(WFCondition *cond, - wait_callback_t callback) +WFWaitTask *WFCondTaskFactory::create_wait_task(WFCondition *cond, + wait_callback_t callback) { WFCondWaitTask *task = new WFCondWaitTask(std::move(callback)); cond->mutex.lock(); - list_add_tail(&task->node.list, &cond->wait_list); + list_add_tail(&task->list, &cond->wait_list); cond->mutex.unlock(); return task; } -WFWaitTask *WFSemTaskFactory::create_timedwait_task(WFCondition *cond, - const struct timespec *abstime, - wait_callback_t callback) +WFWaitTask *WFCondTaskFactory::create_timedwait_task(WFCondition *cond, + const struct timespec *abstime, + wait_callback_t callback) { WFCondWaitTask *waiter = new WFCondWaitTask(std::move(callback)); WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &cond->mutex, abstime, @@ -221,7 +209,7 @@ WFWaitTask *WFSemTaskFactory::create_timedwait_task(WFCondition *cond, waiter->set_timer(task); cond->mutex.lock(); - list_add_tail(&waiter->node.list, &cond->wait_list); + list_add_tail(&waiter->list, &cond->wait_list); cond->mutex.unlock(); return waiter; diff --git a/src/factory/WFSemTaskFactory.h b/src/factory/WFCondTaskFactory.h similarity index 90% rename from src/factory/WFSemTaskFactory.h rename to src/factory/WFCondTaskFactory.h index 403e0810b4..b828d9ad93 100644 --- a/src/factory/WFSemTaskFactory.h +++ b/src/factory/WFCondTaskFactory.h @@ -16,8 +16,8 @@ Author: Li Yingxin (liyingxin@sogou-inc.com) */ -#ifndef _WFSEMTASKFACTORY_H_ -#define _WFSEMTASKFACTORY_H_ +#ifndef _WFCONDTASKFACTORY_H_ +#define _WFCONDTASKFACTORY_H_ #include #include @@ -27,9 +27,9 @@ #include "WFTask.h" #include "WFTaskFactory.h" #include "WFGlobal.h" -#include "WFSemaphore.h" +#include "WFCondition.h" -class WFSemTaskFactory +class WFCondTaskFactory { public: // use condition by name @@ -50,7 +50,7 @@ class WFSemTaskFactory static WFWaitTask *create_timedwait_task(WFCondition *cond, const struct timespec *abstime, - wait_callback_t callback); + wait_callback_t callback); }; diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index 220b9ad381..b64a4b2850 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -4,7 +4,7 @@ project(manager) set(SRC DnsCache.cc RouteManager.cc - WFSemaphore.cc + WFCondition.cc WFGlobal.cc ) diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index 12c5d62190..6f84d48fa9 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -16,80 +16,75 @@ Author: Li Yingxin (liyingxin@sogou-inc.com) */ +#include #include +#include +#include "list.h" #include "WFTask.h" #include "WFTaskFactory.h" -#include "WFGlobal.h" #include "WFCondition.h" -class WFTimedWaitTask; +/////////////// Semaphore Impl /////////////// -class WFWaitTask : public WFMailboxTask +bool WFSemaphore::get(WFConditional *cond) { -public: - void set_timer(WFTimedWaitTask *timer) { this->timer = timer; } - void clear_timer_waiter(); - - struct task_entry + this->mutex.lock(); + if (--this->concurrency >= 0) { - struct list_head list; - WFWaitTask *ptr; - } entry; + cond->signal(this->resources[--this->index]); + this->mutex.unlock(); + return true; + } -protected: - virtual void dispatch(); - virtual SubTask *done(); + struct WFSemaphore::entry *entry; + entry = new WFSemaphore::entry; + entry->ptr = cond; + entry->list.next = NULL; -private: - WFTimedWaitTask *timer; + list_add_tail(&entry->list, &this->wait_list); + this->mutex.unlock(); -public: - WFWaitTask(std::function&& cb) : - WFMailboxTask(&this->user_data, 1, std::move(cb)) - { - this->timer = NULL; - this->entry.list.next = NULL; - this->entry.ptr = this; - } -}; + return false; +} -class WFTimedWaitTask : public __WFTimerTask +void WFSemaphore::post(void *msg) { -public: - WFTimedWaitTask(WFWaitTask *wait_task, std::mutex *mutex, - const struct timespec *value, - CommScheduler *scheduler, - std::function cb) : - __WFTimerTask(value, scheduler, std::move(cb)) + struct WFSemaphore::entry *entry; + WFConditional *cond = NULL; + struct list_head *pos; + + this->mutex.lock(); + + if (++this->concurrency <= 0) { - this->mutex = mutex; - this->wait_task = wait_task; + pos = this->wait_list.next; + entry = list_entry(pos, struct WFSemaphore::entry, list); + cond = entry->ptr; + list_del(pos); + delete entry; } + else + this->resources[this->index++] = msg; - void clear_wait_task(); - -protected: - virtual SubTask *done(); + this->mutex.unlock(); + if (cond) + cond->signal(msg); +} -private: - std::mutex *mutex; - WFWaitTask *wait_task; -}; +/////////////// Wait tasks Impl /////////////// -void WFWaitTask::dispatch() +void WFCondWaitTask::dispatch() { if (this->timer) timer->dispatch(); - - this->WFMailboxTask::count(); + + this->WFWaitTask::count(); } -SubTask *WFWaitTask::done() +SubTask *WFCondWaitTask::done() { SeriesWork *series = series_of(this); - // TODO: data move - WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, [this](WFTimerTask *task) { if (this->callback) @@ -101,7 +96,7 @@ SubTask *WFWaitTask::done() return series->pop(); } -void WFWaitTask::clear_timer_waiter() +void WFCondWaitTask::clear_timer_waiter() { if (this->timer) timer->clear_wait_task(); @@ -109,110 +104,62 @@ void WFWaitTask::clear_timer_waiter() SubTask *WFTimedWaitTask::done() { - WFWaitTask *tmp = NULL; - this->mutex->lock(); - if (this->wait_task && this->wait_task->entry.list.next) + if (this->wait_task && this->wait_task->list.next) { - list_del(&this->wait_task->entry.list); - tmp = this->wait_task; + list_del(&this->wait_task->list); + this->wait_task->set_error(ETIMEDOUT); + this->wait_task->count(); this->wait_task = NULL; } this->mutex->unlock(); - if (tmp) - tmp->count(); - - SeriesWork *series = series_of(this); - - if (this->callback) - this->callback(this); - delete this; - return series->pop(); + return NULL; } -void WFTimedWaitTask::clear_wait_task() -{ - this->mutex->lock(); - this->wait_task = NULL; - this->mutex->unlock(); -} - -WFMailboxTask *WFCondition::create_wait_task(mailbox_callback_t cb) -{ - WFWaitTask *task = new WFWaitTask(std::move(cb)); - - this->mutex.lock(); - list_add_tail(&task->entry.list, &this->waiter_list); - this->mutex.unlock(); - - return task; -} +/////////////// Condition Impl /////////////// -WFMailboxTask *WFCondition::create_timedwait_task(const struct timespec *abstime, - mailbox_callback_t cb) +void WFCondition::signal(void *msg) { - WFWaitTask *waiter = new WFWaitTask(std::move(cb)); - WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &this->mutex, abstime, - WFGlobal::get_scheduler(), - nullptr); - waiter->set_timer(task); - - this->mutex.lock(); - list_add_tail(&waiter->entry.list, &this->waiter_list); - this->mutex.unlock(); - - return waiter; -} - -void WFCondition::signal() -{ - WFWaitTask *task = NULL; + WFCondWaitTask *task = NULL; struct list_head *pos; - struct WFWaitTask::task_entry *entry; this->mutex.lock(); - - if (!list_empty(&this->waiter_list)) + if (!list_empty(&this->wait_list)) { - pos = this->waiter_list.next; - entry = list_entry(pos, struct WFWaitTask::task_entry, list); - task = entry->ptr; + pos = this->wait_list.next; + task = list_entry(pos, WFCondWaitTask, list); list_del(pos); task->clear_timer_waiter(); } this->mutex.unlock(); - if (task) - task->count(); + task->send(msg); } -void WFCondition::broadcast() +void WFCondition::broadcast(void *msg) { - WFWaitTask *task; + WFCondWaitTask *task; struct list_head *pos, *tmp; - struct WFWaitTask::task_entry *entry; LIST_HEAD(tmp_list); this->mutex.lock(); - if (!list_empty(&this->waiter_list)) + if (!list_empty(&this->wait_list)) { - list_for_each_safe(pos, tmp, &this->waiter_list) + list_for_each_safe(pos, tmp, &this->wait_list) { - entry = list_entry(pos, struct WFWaitTask::task_entry, list); list_move_tail(pos, &tmp_list); } } - this->mutex.unlock(); + this->mutex.unlock(); while (!list_empty(&tmp_list)) { - entry = list_entry(tmp_list.next, struct WFWaitTask::task_entry, list); - task = entry->ptr; - list_del(&entry->list); - task->count(); + task = list_entry(tmp_list.next, WFCondWaitTask, list); + list_del(&task->list); + task->send(msg); } } diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h index 9c343f3158..6a27d450b2 100644 --- a/src/manager/WFCondition.h +++ b/src/manager/WFCondition.h @@ -16,33 +16,78 @@ Author: Li Yingxin (liyingxin@sogou-inc.com) */ -#ifndef _WFCONDITION_H_ -#define _WFCONDITION_H_ +#ifndef _WFSEMAPHORE_H_ +#define _WFSEMAPHORE_H_ #include #include #include +#include #include "list.h" #include "WFTask.h" +#include "WFTaskFactory.h" +#include "WFGlobal.h" -class WFCondition +using WFWaitTask = WFMailboxTask; +using wait_callback_t = mailbox_callback_t; + +class WFSemaphore { public: - WFMailboxTask *create_wait_task(std::function cb); - WFMailboxTask *create_timedwait_task(const struct timespec *abstime, - std::function cb); - void signal(); - void broadcast(); + bool get(WFConditional *cond); + void post(void *msg); public: - WFCondition() { INIT_LIST_HEAD(&this->waiter_list); } + std::mutex mutex; + struct list_head wait_list; + +private: + struct entry + { + struct list_head list; + WFConditional *ptr; + }; public: - std::mutex mutex; + WFSemaphore(int value, void **resources) + { + if (value <= 0) + value = 1; + + INIT_LIST_HEAD(&this->wait_list); + this->concurrency = value; + this->total = value; + this->index = value; + this->resources = resources; + } + + virtual ~WFSemaphore() { } private: - struct list_head waiter_list; + std::atomic concurrency; + int total; + int index; + +protected: + void **resources; +}; + +class WFCondition +{ +public: + void signal(void *msg); + void broadcast(void *msg); + +public: + WFCondition() { INIT_LIST_HEAD(&this->wait_list); } + virtual ~WFCondition() { } + +public: + std::mutex mutex; + struct list_head wait_list; }; +#include "WFCondition.inl" + #endif diff --git a/src/manager/WFSemaphore.inl b/src/manager/WFCondition.inl similarity index 72% rename from src/manager/WFSemaphore.inl rename to src/manager/WFCondition.inl index f6c4f89a53..47ff09af7e 100644 --- a/src/manager/WFSemaphore.inl +++ b/src/manager/WFCondition.inl @@ -14,53 +14,15 @@ limitations under the License. Author: Li Yingxin (liyingxin@sogou-inc.com) - Xie Han (xiehan@sogou-inc.com) - Liu Kai (liukaidx@sogou-inc.com) */ #include "list.h" #include "WFTask.h" #include "WFTaskFactory.h" -class WFSemaphoreTask : public WFWaitTask -{ -public: - void start() - { - assert(!series_of(this)); - Workflow::start_series_work(this, nullptr); - } - - void dismiss() - { - assert(!series_of(this)); - delete this; - } - -public: - WFSemaphoreTask(std::function&& cb) : - WFWaitTask(&this->msg, 1, std::move(cb)) - { - this->node.list.next = NULL; - this->node.ptr = this; - } - - virtual ~WFSemaphoreTask() { } - -public: - struct entry - { - struct list_head list; - WFSemaphoreTask *ptr; - } node; - -private: - void *msg; -}; - class WFTimedWaitTask; -class WFCondWaitTask : public WFSemaphoreTask +class WFCondWaitTask : public WFWaitTask { public: void set_timer(WFTimedWaitTask *timer) { this->timer = timer; } @@ -73,15 +35,20 @@ protected: private: WFTimedWaitTask *timer; + void *msg; public: WFCondWaitTask(std::function&& cb) : - WFSemaphoreTask(std::move(cb)) + WFWaitTask(&this->msg, 1, std::move(cb)) { this->timer = NULL; + this->list.next = NULL; } virtual ~WFCondWaitTask() { } + +public: + struct list_head list; }; class WFTimedWaitTask : public __WFTimerTask diff --git a/src/manager/WFSemaphore.cc b/src/manager/WFSemaphore.cc deleted file mode 100644 index 9013dfec13..0000000000 --- a/src/manager/WFSemaphore.cc +++ /dev/null @@ -1,190 +0,0 @@ -/* - Copyright (c) 2021 Sogou, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Author: Li Yingxin (liyingxin@sogou-inc.com) - Xie Han (xiehan@sogou-inc.com) - Liu Kai (liukaidx@sogou-inc.com) -*/ - -#include -#include -#include -#include "list.h" -#include "WFTask.h" -#include "WFTaskFactory.h" -#include "WFSemaphore.h" - -/////////////// Semaphore Impl /////////////// - -bool WFSemaphore::get(void **dest) -{ - this->mutex.lock(); - if (--this->concurrency >= 0) - { - *dest = this->resources[--this->index]; - this->mutex.unlock(); - return true; - } - - WFSemaphoreTask *task = new WFSemaphoreTask(nullptr); - - list_add_tail(&task->node.list, &this->get_list); - this->mutex.unlock(); - - return false; -} - -WFWaitTask *WFSemaphore::create_wait_task(std::function cb) -{ - WFSemaphoreTask *task = NULL; - struct list_head *pos; - struct WFSemaphoreTask::entry *node; - - this->mutex.lock(); - if (!list_empty(&this->get_list)) - { - pos = this->get_list.next; - list_move_tail(pos, &this->wait_list); - node = list_entry(pos, struct WFSemaphoreTask::entry, list); - task = node->ptr; - task->set_callback(std::move(cb)); - } - - this->mutex.unlock(); - return task; -} - -void WFSemaphore::post(void *src) -{ - WFSemaphoreTask *task = NULL; - struct list_head *pos; - struct WFSemaphoreTask::entry *node; - - this->mutex.lock(); - - if (++this->concurrency <= 0) - { - if (!list_empty(&this->wait_list)) - pos = this->wait_list.next; - else - pos = this->get_list.next; - node = list_entry(pos, struct WFSemaphoreTask::entry, list); - task = node->ptr; - list_del(pos); - } - - this->resources[this->index++] = src; - - this->mutex.unlock(); - if (task) - task->send(src); -} - -/////////////// Wait tasks Impl /////////////// - -void WFCondWaitTask::dispatch() -{ - if (this->timer) - timer->dispatch(); - - this->WFWaitTask::count(); -} - -SubTask *WFCondWaitTask::done() -{ - SeriesWork *series = series_of(this); - - WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, - [this](WFTimerTask *task) { - if (this->callback) - this->callback(this); - delete this; - }); - series->push_front(switch_task); - - return series->pop(); -} - -void WFCondWaitTask::clear_timer_waiter() -{ - if (this->timer) - timer->clear_wait_task(); -} - -SubTask *WFTimedWaitTask::done() -{ - this->mutex->lock(); - if (this->wait_task && this->wait_task->node.list.next) - { - list_del(&this->wait_task->node.list); - this->wait_task->set_error(ETIMEDOUT); - this->wait_task->count(); - this->wait_task = NULL; - } - this->mutex->unlock(); - - delete this; - return NULL; -} - -/////////////// Condition Impl /////////////// - -void WFCondition::signal(void *msg) -{ - WFCondWaitTask *task = NULL; - struct list_head *pos; - struct WFSemaphoreTask::entry *node; - - this->mutex.lock(); - if (!list_empty(&this->wait_list)) - { - pos = this->wait_list.next; - node = list_entry(pos, struct WFSemaphoreTask::entry, list); - task = (WFCondWaitTask *)node->ptr; - list_del(pos); - task->clear_timer_waiter(); - } - - this->mutex.unlock(); - if (task) - task->send(msg); -} - -void WFCondition::broadcast(void *msg) -{ - WFCondWaitTask *task; - struct list_head *pos, *tmp; - struct WFSemaphoreTask::entry *node; - LIST_HEAD(tmp_list); - - this->mutex.lock(); - if (!list_empty(&this->wait_list)) - { - list_for_each_safe(pos, tmp, &this->wait_list) - { - list_move_tail(pos, &tmp_list); - } - } - - this->mutex.unlock(); - while (!list_empty(&tmp_list)) - { - node = list_entry(tmp_list.next, struct WFSemaphoreTask::entry, list); - task = (WFCondWaitTask *)node->ptr; - list_del(&node->list); - task->send(msg); - } -} - diff --git a/src/manager/WFSemaphore.h b/src/manager/WFSemaphore.h deleted file mode 100644 index e5e8b9ffa7..0000000000 --- a/src/manager/WFSemaphore.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - Copyright (c) 2021 Sogou, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Author: Li Yingxin (liyingxin@sogou-inc.com) - Xie Han (xiehan@sogou-inc.com) - Liu Kai (liukaidx@sogou-inc.com) -*/ - -#ifndef _WFSEMAPHORE_H_ -#define _WFSEMAPHORE_H_ - -#include -#include -#include -#include -#include "list.h" -#include "WFTask.h" -#include "WFTaskFactory.h" -#include "WFGlobal.h" - -using WFWaitTask = WFMailboxTask; -using wait_callback_t = mailbox_callback_t; - -class WFSemaphore -{ -public: - bool get(void **dest); - WFWaitTask *create_wait_task(std::function cb); - void post(void *src); - -public: - std::mutex mutex; - struct list_head get_list; - struct list_head wait_list; - -public: - WFSemaphore(int value, void **resources) - { - if (value <= 0) - value = 1; - - INIT_LIST_HEAD(&this->get_list); - INIT_LIST_HEAD(&this->wait_list); - this->concurrency = value; - this->total = value; - this->index = value; - this->resources = resources; -// this->resources = new void *[value]; - } - - virtual ~WFSemaphore() { /* delete []this->resources; */ } - -private: - std::atomic concurrency; - int total; - int index; - -protected: - void **resources; -}; - -class WFCondition : public WFSemaphore -{ -public: - void signal(void *msg); - void broadcast(void *msg); - -public: - WFCondition() : WFSemaphore(1, new void *[1]) { } - WFCondition(int value) : WFSemaphore(value, new void *[value]) { } - virtual ~WFCondition() { delete []this->resources;} - - bool get(void **dest) = delete; - WFWaitTask *create_wait_task(wait_callback_t cb) = delete; - void post(void *src) = delete; -}; - -#include "WFSemaphore.inl" - -#endif - diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index 5e32b986c0..f839ba37da 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -24,8 +24,8 @@ #include #include "workflow/WFTask.h" #include "workflow/WFTaskFactory.h" -#include "workflow/WFSemaphore.h" -#include "workflow/WFSemTaskFactory.h" +#include "workflow/WFCondition.h" +#include "workflow/WFCondTaskFactory.h" #include "workflow/WFFacilities.h" TEST(condition_unittest, signal) @@ -36,12 +36,12 @@ TEST(condition_unittest, signal) int ret = 3; int *ptr = &ret; - auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&wg, &ptr](WFMailboxTask *) { + auto *task1 = WFCondTaskFactory::create_wait_task(&cond, [&wg, &ptr](WFMailboxTask *) { *ptr = 1; wg.done(); }); - auto *task2 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + auto *task2 = WFCondTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { *ptr = 2; }); @@ -68,12 +68,12 @@ TEST(condition_unittest, broadcast) int ret = 0; int *ptr = &ret; - auto *task1 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + auto *task1 = WFCondTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { (*ptr)++; }); SeriesWork *series1 = Workflow::create_series_work(task1, nullptr); - auto *task2 = WFSemTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { + auto *task2 = WFCondTaskFactory::create_wait_task(&cond, [&ptr](WFMailboxTask *) { (*ptr)++; }); SeriesWork *series2 = Workflow::create_series_work(task2, nullptr); @@ -93,13 +93,13 @@ TEST(condition_unittest, timedwait) ts.tv_sec = 1; ts.tv_nsec = 0; - auto *task1 = WFSemTaskFactory::create_timedwait_task("timedwait1", &ts, + auto *task1 = WFCondTaskFactory::create_timedwait_task("timedwait1", &ts, [&wait_group](WFMailboxTask *task) { EXPECT_EQ(task->get_error(), ETIMEDOUT); wait_group.done(); }); - auto *task2 = WFSemTaskFactory::create_timedwait_task("timedwait2", &ts, + auto *task2 = WFCondTaskFactory::create_timedwait_task("timedwait2", &ts, [&wait_group](WFMailboxTask *task) { EXPECT_EQ(task->get_error(), 0); void **msg; @@ -115,53 +115,32 @@ TEST(condition_unittest, timedwait) usleep(1000); char msg[10] = "wake up!!"; - WFSemTaskFactory::signal_by_name("timedwait2", msg); + WFCondTaskFactory::signal_by_name("timedwait2", msg); wait_group.wait(); } TEST(condition_unittest, semaphore) { - int sem_concurrency = 1; - int task_concurrency = 3; + int sem_concurrency = 3; + int task_concurrency = 10; const char *words[3] = {"workflow", "srpc", "pyworkflow"}; WFSemaphore sem(sem_concurrency, (void **)words); WFFacilities::WaitGroup wg(task_concurrency); for (int i = 0; i < task_concurrency; i++) { - auto *t = WFTaskFactory::create_timer_task(i * 1, [&sem, &wg](WFTimerTask *task) { + auto *user_task = WFTaskFactory::create_timer_task(0, + [&wg, &sem](WFTimerTask *task) { uint64_t id = (uint64_t)series_of(task)->get_context(); - void *dest_res; - - if (!sem.get(&dest_res)) - { - auto *waiter = sem.create_wait_task([&sem, &wg](WFWaitTask *task) { - uint64_t id = (uint64_t)series_of(task)->get_context(); - void **msg; - size_t n; - msg = task->get_mailbox(&n); - fprintf(stderr, "%lu after wait. get resource:%s\n", id, (char *)*msg); - usleep(10); - wg.done(); - sem.post(*msg); - //sem.post(reinterpret_cast(id)); - }); - series_of(task)->push_back(waiter); - } - else - { - fprintf(stderr, "%lu no wait. get resource: %s\n", id, (char *)dest_res); - usleep(10); - wg.done(); - //sem.post(reinterpret_cast(id)); - sem.post(dest_res); - } + printf("task-%lu get [%s]\n", id, (char *)task->user_data); + sem.post(task->user_data); + wg.done(); }); - - SeriesWork *series = Workflow::create_series_work(t, nullptr); - series->set_context(reinterpret_cast(i)); - series->start(); + auto *cond = WFTaskFactory::create_conditional(user_task, &user_task->user_data); + sem.get(cond); + cond->start(); } + wg.wait(); } From 72bbda0fb543031cb6a9826369cccfc24df8f00d Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 2 Jul 2021 20:48:55 +0800 Subject: [PATCH 27/75] update CMakeLists.txt --- tutorial/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tutorial/CMakeLists.txt b/tutorial/CMakeLists.txt index 04d04c4b38..b3606823f7 100644 --- a/tutorial/CMakeLists.txt +++ b/tutorial/CMakeLists.txt @@ -32,7 +32,6 @@ set(TUTORIAL_LIST tutorial-08-matrix_multiply tutorial-09-http_file_server tutorial-11-graph_task - tutorial-12-mysql_cli tutorial-14-websocket_cli ) From a898586e24aecba14eff883d42824edd53b18f22 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 2 Jul 2021 20:55:55 +0800 Subject: [PATCH 28/75] update unittest --- test/condition_unittest.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index f839ba37da..2155afb2a5 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -138,7 +138,9 @@ TEST(condition_unittest, semaphore) }); auto *cond = WFTaskFactory::create_conditional(user_task, &user_task->user_data); sem.get(cond); - cond->start(); + SeriesWork *series = Workflow::create_series_work(cond, nullptr); + series->set_context(reinterpret_cast(i)); + series->start(); } wg.wait(); From 69238ab47f16d6d97128541e1c1dcf554a055c0a Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 8 Jul 2021 21:11:24 +0800 Subject: [PATCH 29/75] add WFSwitchTask and create_swait() --- src/factory/WFChannel.inl | 8 ++++---- src/factory/WFCondTaskFactory.cc | 29 +++++++++++++++++++++++++++++ src/factory/WFCondTaskFactory.h | 6 ++++++ src/manager/WFCondition.cc | 2 +- src/manager/WFCondition.inl | 12 +++++++++++- 5 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 72843f12df..6ffafac4cd 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -249,8 +249,8 @@ void WFComplexChannel::handle_terminated() this->sending = true; shutdown = true; } else { - waiter = WFCondTaskFactory::create_wait_task(&this->condition, - nullptr); + waiter = WFCondTaskFactory::create_swait_task(&this->condition, + nullptr); series_of(this)->push_front(this); series_of(this)->push_front(waiter); } @@ -353,7 +353,7 @@ void ComplexChannelOutTask::dispatch() } else { - waiter = WFCondTaskFactory::create_wait_task(&channel->condition, + waiter = WFCondTaskFactory::create_swait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); @@ -374,7 +374,7 @@ void ComplexChannelOutTask::dispatch() } else { - waiter = WFCondTaskFactory::create_wait_task(&channel->condition, + waiter = WFCondTaskFactory::create_swait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); diff --git a/src/factory/WFCondTaskFactory.cc b/src/factory/WFCondTaskFactory.cc index 22e06396ab..54236f88f4 100644 --- a/src/factory/WFCondTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -50,6 +50,8 @@ class __ConditionMap WFWaitTask *create(const std::string& name, const struct timespec *abstime, wait_callback_t&& cb); + WFWaitTask *create_switch(const std::string& name, + wait_callback_t&& cb); public: static __ConditionMap *get_instance() @@ -103,6 +105,14 @@ WFWaitTask *__ConditionMap::create(const std::string& name, std::move(cb)); } +WFWaitTask *__ConditionMap::create_switch(const std::string& name, + wait_callback_t&& cb) +{ + __WFCondition *cond = this->find_condition(name); + + return WFCondTaskFactory::create_swait_task(cond, std::move(cb)); +} + __ConditionMap::~__ConditionMap() { __WFCondition *cond; @@ -178,6 +188,13 @@ WFWaitTask *WFCondTaskFactory::create_wait_task(const std::string& name, return __ConditionMap::get_instance()->create(name, std::move(callback)); } +WFWaitTask *WFCondTaskFactory::create_swait_task(const std::string& name, + wait_callback_t callback) +{ + return __ConditionMap::get_instance()->create_switch(name, + std::move(callback)); +} + WFWaitTask *WFCondTaskFactory::create_timedwait_task(const std::string& name, const struct timespec *abstime, wait_callback_t callback) @@ -215,3 +232,15 @@ WFWaitTask *WFCondTaskFactory::create_timedwait_task(WFCondition *cond, return waiter; } +WFWaitTask *WFCondTaskFactory::create_swait_task(WFCondition *cond, + wait_callback_t callback) +{ + WFSwitchWaitTask *task = new WFSwitchWaitTask(std::move(callback)); + + cond->mutex.lock(); + list_add_tail(&task->list, &cond->wait_list); + cond->mutex.unlock(); + + return task; +} + diff --git a/src/factory/WFCondTaskFactory.h b/src/factory/WFCondTaskFactory.h index b828d9ad93..20e4f49b58 100644 --- a/src/factory/WFCondTaskFactory.h +++ b/src/factory/WFCondTaskFactory.h @@ -44,6 +44,9 @@ class WFCondTaskFactory const struct timespec *abstime, wait_callback_t callback); + static WFWaitTask *create_swait_task(const std::string& name, + wait_callback_t callback); + // use condition by ptr static WFWaitTask *create_wait_task(WFCondition *cond, wait_callback_t callback); @@ -52,6 +55,9 @@ class WFCondTaskFactory const struct timespec *abstime, wait_callback_t callback); + static WFWaitTask *create_swait_task(WFCondition *cond, + wait_callback_t callback); + }; #endif diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index 6f84d48fa9..d0cc5cbc0b 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -81,7 +81,7 @@ void WFCondWaitTask::dispatch() this->WFWaitTask::count(); } -SubTask *WFCondWaitTask::done() +SubTask *WFSwitchWaitTask::done() { SeriesWork *series = series_of(this); diff --git a/src/manager/WFCondition.inl b/src/manager/WFCondition.inl index 47ff09af7e..c28681d5d1 100644 --- a/src/manager/WFCondition.inl +++ b/src/manager/WFCondition.inl @@ -31,7 +31,6 @@ public: protected: void dispatch(); - SubTask *done(); private: WFTimedWaitTask *timer; @@ -77,3 +76,14 @@ private: WFCondWaitTask *wait_task; }; +class WFSwitchWaitTask : public WFCondWaitTask +{ +public: + WFSwitchWaitTask(std::function&& cb) : + WFCondWaitTask(std::move(cb)) + { } + +protected: + SubTask *done(); +}; + From f557a46489e0df2dea994027e22e879dc14136e3 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 9 Jul 2021 20:51:15 +0800 Subject: [PATCH 30/75] update WFTimedWaitTask --- src/factory/WFCondTaskFactory.cc | 19 ++++++------ src/factory/WFCondTaskFactory.h | 4 +-- src/manager/WFCondition.cc | 35 ++++++++++++++++----- src/manager/WFCondition.inl | 52 +++++++++++++++++++++----------- 4 files changed, 73 insertions(+), 37 deletions(-) diff --git a/src/factory/WFCondTaskFactory.cc b/src/factory/WFCondTaskFactory.cc index 54236f88f4..45a0377f12 100644 --- a/src/factory/WFCondTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -48,7 +48,7 @@ class __ConditionMap WFWaitTask *create(const std::string& name, wait_callback_t&& cb); WFWaitTask *create(const std::string& name, - const struct timespec *abstime, + const struct timespec *value, wait_callback_t&& cb); WFWaitTask *create_switch(const std::string& name, wait_callback_t&& cb); @@ -96,12 +96,12 @@ WFWaitTask *__ConditionMap::create(const std::string& name, } WFWaitTask *__ConditionMap::create(const std::string& name, - const struct timespec *abstime, + const struct timespec *value, wait_callback_t&& cb) { __WFCondition *cond = this->find_condition(name); - return WFCondTaskFactory::create_timedwait_task(cond, abstime, + return WFCondTaskFactory::create_timedwait_task(cond, value, std::move(cb)); } @@ -196,10 +196,10 @@ WFWaitTask *WFCondTaskFactory::create_swait_task(const std::string& name, } WFWaitTask *WFCondTaskFactory::create_timedwait_task(const std::string& name, - const struct timespec *abstime, + const struct timespec *value, wait_callback_t callback) { - return __ConditionMap::get_instance()->create(name, abstime, + return __ConditionMap::get_instance()->create(name, value, std::move(callback)); } @@ -216,13 +216,12 @@ WFWaitTask *WFCondTaskFactory::create_wait_task(WFCondition *cond, } WFWaitTask *WFCondTaskFactory::create_timedwait_task(WFCondition *cond, - const struct timespec *abstime, + const struct timespec *value, wait_callback_t callback) { - WFCondWaitTask *waiter = new WFCondWaitTask(std::move(callback)); - WFTimedWaitTask *task = new WFTimedWaitTask(waiter, &cond->mutex, abstime, - WFGlobal::get_scheduler(), - nullptr); + WFTimedWaitTask *waiter = new WFTimedWaitTask(std::move(callback)); + __WFWaitTimerTask *task = new __WFWaitTimerTask(waiter, &cond->mutex, value, + WFGlobal::get_scheduler()); waiter->set_timer(task); cond->mutex.lock(); diff --git a/src/factory/WFCondTaskFactory.h b/src/factory/WFCondTaskFactory.h index 20e4f49b58..7075405ae6 100644 --- a/src/factory/WFCondTaskFactory.h +++ b/src/factory/WFCondTaskFactory.h @@ -41,7 +41,7 @@ class WFCondTaskFactory wait_callback_t callback); static WFWaitTask *create_timedwait_task(const std::string& name, - const struct timespec *abstime, + const struct timespec *value, wait_callback_t callback); static WFWaitTask *create_swait_task(const std::string& name, @@ -52,7 +52,7 @@ class WFCondTaskFactory wait_callback_t callback); static WFWaitTask *create_timedwait_task(WFCondition *cond, - const struct timespec *abstime, + const struct timespec *value, wait_callback_t callback); static WFWaitTask *create_swait_task(WFCondition *cond, diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index d0cc5cbc0b..f71c0018b0 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -73,7 +73,17 @@ void WFSemaphore::post(void *msg) /////////////// Wait tasks Impl /////////////// -void WFCondWaitTask::dispatch() +void WFCondWaitTask::count() +{ + if (--this->value == 0) + { + if (this->state == WFT_STATE_UNDEFINED) + this->state = WFT_STATE_SUCCESS; + this->subtask_done(); + } +} + +void WFTimedWaitTask::dispatch() { if (this->timer) timer->dispatch(); @@ -96,24 +106,28 @@ SubTask *WFSwitchWaitTask::done() return series->pop(); } -void WFCondWaitTask::clear_timer_waiter() +void WFTimedWaitTask::clear_timer_waiter() { if (this->timer) timer->clear_wait_task(); } -SubTask *WFTimedWaitTask::done() +SubTask *__WFWaitTimerTask::done() { + WFTimedWaitTask *waiter = NULL; + this->mutex->lock(); - if (this->wait_task && this->wait_task->list.next) + if (this->wait_task) { list_del(&this->wait_task->list); + this->wait_task->set_state(WFT_STATE_SYS_ERROR); this->wait_task->set_error(ETIMEDOUT); - this->wait_task->count(); - this->wait_task = NULL; + waiter = this->wait_task; } this->mutex->unlock(); + if (waiter) + waiter->count(); delete this; return NULL; } @@ -123,6 +137,7 @@ SubTask *WFTimedWaitTask::done() void WFCondition::signal(void *msg) { WFCondWaitTask *task = NULL; + WFTimedWaitTask *waiter; struct list_head *pos; this->mutex.lock(); @@ -131,7 +146,9 @@ void WFCondition::signal(void *msg) pos = this->wait_list.next; task = list_entry(pos, WFCondWaitTask, list); list_del(pos); - task->clear_timer_waiter(); + waiter = dynamic_cast(task); + if (waiter) + waiter->clear_timer_waiter(); } this->mutex.unlock(); @@ -142,6 +159,7 @@ void WFCondition::signal(void *msg) void WFCondition::broadcast(void *msg) { WFCondWaitTask *task; + WFTimedWaitTask *waiter; struct list_head *pos, *tmp; LIST_HEAD(tmp_list); @@ -159,6 +177,9 @@ void WFCondition::broadcast(void *msg) { task = list_entry(tmp_list.next, WFCondWaitTask, list); list_del(&task->list); + waiter = dynamic_cast(task); + if (waiter) + waiter->clear_timer_waiter(); task->send(msg); } } diff --git a/src/manager/WFCondition.inl b/src/manager/WFCondition.inl index c28681d5d1..38f9e1d823 100644 --- a/src/manager/WFCondition.inl +++ b/src/manager/WFCondition.inl @@ -20,27 +20,20 @@ #include "WFTask.h" #include "WFTaskFactory.h" -class WFTimedWaitTask; - class WFCondWaitTask : public WFWaitTask { public: - void set_timer(WFTimedWaitTask *timer) { this->timer = timer; } - void clear_timer_waiter(); + void set_state(int state) { this->state = state; } void set_error(int error) { this->error = error; } - -protected: - void dispatch(); + virtual void count(); private: - WFTimedWaitTask *timer; void *msg; public: - WFCondWaitTask(std::function&& cb) : + WFCondWaitTask(mailbox_callback_t&& cb) : WFWaitTask(&this->msg, 1, std::move(cb)) { - this->timer = NULL; this->list.next = NULL; } @@ -50,14 +43,37 @@ public: struct list_head list; }; -class WFTimedWaitTask : public __WFTimerTask +class __WFWaitTimerTask; + +class WFTimedWaitTask : public WFCondWaitTask +{ +public: + void set_timer(__WFWaitTimerTask *timer) { this->timer = timer; } + void clear_timer_waiter(); + +protected: + virtual void dispatch(); + +private: + __WFWaitTimerTask *timer; + +public: + WFTimedWaitTask(mailbox_callback_t&& cb) : + WFCondWaitTask(std::move(cb)) + { + this->timer = NULL; + } + + virtual ~WFTimedWaitTask() { } +}; + +class __WFWaitTimerTask : public __WFTimerTask { public: - WFTimedWaitTask(WFCondWaitTask *wait_task, std::mutex *mutex, - const struct timespec *value, - CommScheduler *scheduler, - std::function cb) : - __WFTimerTask(value, scheduler, std::move(cb)) + __WFWaitTimerTask(WFTimedWaitTask *wait_task, std::mutex *mutex, + const struct timespec *value, + CommScheduler *scheduler) : + __WFTimerTask(value, scheduler, nullptr) { this->mutex = mutex; this->wait_task = wait_task; @@ -73,13 +89,13 @@ protected: private: std::mutex *mutex; - WFCondWaitTask *wait_task; + WFTimedWaitTask *wait_task; }; class WFSwitchWaitTask : public WFCondWaitTask { public: - WFSwitchWaitTask(std::function&& cb) : + WFSwitchWaitTask(mailbox_callback_t&& cb) : WFCondWaitTask(std::move(cb)) { } From 0829e58f800e576a0485aa1fec251b34bd97c9c2 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Mon, 12 Jul 2021 16:00:18 +0800 Subject: [PATCH 31/75] make cond->mutex as a ref; move to new file WFCondTask; --- CMakeLists_Headers.txt | 2 +- src/factory/CMakeLists.txt | 1 + src/factory/WFCondTask.cc | 100 +++++++++++++++++ .../WFCondition.inl => factory/WFCondTask.h} | 25 +++-- src/factory/WFCondTaskFactory.cc | 18 ++-- src/manager/WFCondition.cc | 101 +++++------------- src/manager/WFCondition.h | 14 ++- 7 files changed, 163 insertions(+), 98 deletions(-) create mode 100644 src/factory/WFCondTask.cc rename src/{manager/WFCondition.inl => factory/WFCondTask.h} (83%) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index b47a8220ce..35a15778b3 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -80,7 +80,6 @@ set(INCLUDE_HEADERS src/manager/WFFacilities.h src/manager/WFFacilities.inl src/manager/WFCondition.h - src/manager/WFCondition.inl src/util/EncodeStream.h src/util/LRUCache.h src/util/StringUtil.h @@ -90,6 +89,7 @@ set(INCLUDE_HEADERS src/factory/WFTask.h src/factory/WFTask.inl src/factory/WFGraphTask.h + src/factory/WFCondTask.h src/factory/WFTaskError.h src/factory/WFTaskFactory.h src/factory/WFTaskFactory.inl diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index 5a5827352c..1e41900752 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -7,6 +7,7 @@ set(SRC Workflow.cc WFTaskFactory.cc WFCondTaskFactory.cc + WFCondTask.cc HttpTaskImpl.cc WebSocketTaskImpl.cc ) diff --git a/src/factory/WFCondTask.cc b/src/factory/WFCondTask.cc new file mode 100644 index 0000000000..55f96f0c4f --- /dev/null +++ b/src/factory/WFCondTask.cc @@ -0,0 +1,100 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFCondTask.h" +#include "WFTaskFactory.h" + +void WFCondWaitTask::count() +{ + if (--this->value == 0) + { + if (this->state == WFT_STATE_UNDEFINED) + this->state = WFT_STATE_SUCCESS; + this->subtask_done(); + } +} + +void WFTimedWaitTask::dispatch() +{ + if (this->timer) + timer->dispatch(); + + this->WFMailboxTask::count(); +} + +WFTimedWaitTask::~WFTimedWaitTask() +{ + delete this->timer; +} + +SubTask *WFSwitchWaitTask::done() +{ + SeriesWork *series = series_of(this); + + WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, + [this](WFTimerTask *task) { + if (this->callback) + this->callback(this); + delete this; + }); + series->push_front(switch_task); + + return series->pop(); +} + +void WFTimedWaitTask::clear_timer_waiter() +{ + if (this->timer) + timer->clear_wait_task(); +} + +SubTask *__WFWaitTimerTask::done() +{ + WFTimedWaitTask *waiter = NULL; + + this->mutex->lock(); + if (this->wait_task) + { + list_del(&this->wait_task->list); + this->wait_task->set_state(WFT_STATE_SYS_ERROR); + this->wait_task->set_error(ETIMEDOUT); + waiter = this->wait_task; + waiter->set_timer(NULL); + } + this->mutex->unlock(); + + if (waiter) + waiter->count(); + delete this; + return NULL; +} + +__WFWaitTimerTask::~__WFWaitTimerTask() +{ + if (--*this->ref == 0) + { + delete this->mutex; + delete this->ref; + } +} + diff --git a/src/manager/WFCondition.inl b/src/factory/WFCondTask.h similarity index 83% rename from src/manager/WFCondition.inl rename to src/factory/WFCondTask.h index 38f9e1d823..27949fc7b9 100644 --- a/src/manager/WFCondition.inl +++ b/src/factory/WFCondTask.h @@ -16,11 +16,13 @@ Author: Li Yingxin (liyingxin@sogou-inc.com) */ +#include +#include #include "list.h" #include "WFTask.h" #include "WFTaskFactory.h" -class WFCondWaitTask : public WFWaitTask +class WFCondWaitTask : public WFMailboxTask { public: void set_state(int state) { this->state = state; } @@ -32,7 +34,7 @@ class WFCondWaitTask : public WFWaitTask public: WFCondWaitTask(mailbox_callback_t&& cb) : - WFWaitTask(&this->msg, 1, std::move(cb)) + WFMailboxTask(&this->msg, 1, std::move(cb)) { this->list.next = NULL; } @@ -64,31 +66,36 @@ class WFTimedWaitTask : public WFCondWaitTask this->timer = NULL; } - virtual ~WFTimedWaitTask() { } + virtual ~WFTimedWaitTask(); }; class __WFWaitTimerTask : public __WFTimerTask { public: - __WFWaitTimerTask(WFTimedWaitTask *wait_task, std::mutex *mutex, - const struct timespec *value, + void clear_wait_task() // must called within this mutex + { + this->wait_task = NULL; + } + + __WFWaitTimerTask(WFTimedWaitTask *wait_task, const struct timespec *value, + std::mutex *mutex, std::atomic *ref, CommScheduler *scheduler) : __WFTimerTask(value, scheduler, nullptr) { + this->ref = ref; + ++*this->ref; this->mutex = mutex; this->wait_task = wait_task; } - void clear_wait_task() // must called within this mutex - { - this->wait_task = NULL; - } + virtual ~__WFWaitTimerTask(); protected: virtual SubTask *done(); private: std::mutex *mutex; + std::atomic *ref; WFTimedWaitTask *wait_task; }; diff --git a/src/factory/WFCondTaskFactory.cc b/src/factory/WFCondTaskFactory.cc index 45a0377f12..e352101a1d 100644 --- a/src/factory/WFCondTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -23,9 +23,10 @@ #include "list.h" #include "rbtree.h" #include "WFTask.h" +#include "WFCondTask.h" #include "WFTaskFactory.h" -#include "WFGlobal.h" #include "WFCondTaskFactory.h" +#include "WFGlobal.h" class __WFCondition : public WFCondition { @@ -208,9 +209,9 @@ WFWaitTask *WFCondTaskFactory::create_wait_task(WFCondition *cond, { WFCondWaitTask *task = new WFCondWaitTask(std::move(callback)); - cond->mutex.lock(); + cond->mutex->lock(); list_add_tail(&task->list, &cond->wait_list); - cond->mutex.unlock(); + cond->mutex->unlock(); return task; } @@ -220,13 +221,14 @@ WFWaitTask *WFCondTaskFactory::create_timedwait_task(WFCondition *cond, wait_callback_t callback) { WFTimedWaitTask *waiter = new WFTimedWaitTask(std::move(callback)); - __WFWaitTimerTask *task = new __WFWaitTimerTask(waiter, &cond->mutex, value, + __WFWaitTimerTask *task = new __WFWaitTimerTask(waiter, value, + cond->mutex, cond->ref, WFGlobal::get_scheduler()); waiter->set_timer(task); - cond->mutex.lock(); + cond->mutex->lock(); list_add_tail(&waiter->list, &cond->wait_list); - cond->mutex.unlock(); + cond->mutex->unlock(); return waiter; } @@ -236,9 +238,9 @@ WFWaitTask *WFCondTaskFactory::create_swait_task(WFCondition *cond, { WFSwitchWaitTask *task = new WFSwitchWaitTask(std::move(callback)); - cond->mutex.lock(); + cond->mutex->lock(); list_add_tail(&task->list, &cond->wait_list); - cond->mutex.unlock(); + cond->mutex->unlock(); return task; } diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index f71c0018b0..bbe466dfd7 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -17,15 +17,11 @@ */ #include -#include -#include #include "list.h" #include "WFTask.h" -#include "WFTaskFactory.h" +#include "WFCondTask.h" #include "WFCondition.h" -/////////////// Semaphore Impl /////////////// - bool WFSemaphore::get(WFConditional *cond) { this->mutex.lock(); @@ -71,76 +67,13 @@ void WFSemaphore::post(void *msg) cond->signal(msg); } -/////////////// Wait tasks Impl /////////////// - -void WFCondWaitTask::count() -{ - if (--this->value == 0) - { - if (this->state == WFT_STATE_UNDEFINED) - this->state = WFT_STATE_SUCCESS; - this->subtask_done(); - } -} - -void WFTimedWaitTask::dispatch() -{ - if (this->timer) - timer->dispatch(); - - this->WFWaitTask::count(); -} - -SubTask *WFSwitchWaitTask::done() -{ - SeriesWork *series = series_of(this); - - WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, - [this](WFTimerTask *task) { - if (this->callback) - this->callback(this); - delete this; - }); - series->push_front(switch_task); - - return series->pop(); -} - -void WFTimedWaitTask::clear_timer_waiter() -{ - if (this->timer) - timer->clear_wait_task(); -} - -SubTask *__WFWaitTimerTask::done() -{ - WFTimedWaitTask *waiter = NULL; - - this->mutex->lock(); - if (this->wait_task) - { - list_del(&this->wait_task->list); - this->wait_task->set_state(WFT_STATE_SYS_ERROR); - this->wait_task->set_error(ETIMEDOUT); - waiter = this->wait_task; - } - this->mutex->unlock(); - - if (waiter) - waiter->count(); - delete this; - return NULL; -} - -/////////////// Condition Impl /////////////// - void WFCondition::signal(void *msg) { WFCondWaitTask *task = NULL; WFTimedWaitTask *waiter; struct list_head *pos; - this->mutex.lock(); + this->mutex->lock(); if (!list_empty(&this->wait_list)) { pos = this->wait_list.next; @@ -148,10 +81,13 @@ void WFCondition::signal(void *msg) list_del(pos); waiter = dynamic_cast(task); if (waiter) + { waiter->clear_timer_waiter(); + waiter->set_timer(NULL); + } } - this->mutex.unlock(); + this->mutex->unlock(); if (task) task->send(msg); } @@ -163,24 +99,39 @@ void WFCondition::broadcast(void *msg) struct list_head *pos, *tmp; LIST_HEAD(tmp_list); - this->mutex.lock(); + this->mutex->lock(); if (!list_empty(&this->wait_list)) { list_for_each_safe(pos, tmp, &this->wait_list) { list_move_tail(pos, &tmp_list); + task = list_entry(pos, WFCondWaitTask, list); + waiter = dynamic_cast(task); + if (waiter) + { + waiter->clear_timer_waiter(); + waiter->set_timer(NULL); + } } } - this->mutex.unlock(); + this->mutex->unlock(); while (!list_empty(&tmp_list)) { task = list_entry(tmp_list.next, WFCondWaitTask, list); list_del(&task->list); - waiter = dynamic_cast(task); - if (waiter) - waiter->clear_timer_waiter(); task->send(msg); } } +WFCondition::~WFCondition() +{ + this->broadcast(NULL); + + if (--*this->ref == 0) + { + delete this->mutex; + delete this->ref; + } +} + diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h index 6a27d450b2..0edd0a1ccd 100644 --- a/src/manager/WFCondition.h +++ b/src/manager/WFCondition.h @@ -79,15 +79,19 @@ class WFCondition void broadcast(void *msg); public: - WFCondition() { INIT_LIST_HEAD(&this->wait_list); } - virtual ~WFCondition() { } + WFCondition() + { + this->mutex = new std::mutex; + this->ref = new std::atomic(1); + INIT_LIST_HEAD(&this->wait_list); + } + virtual ~WFCondition(); public: - std::mutex mutex; + std::atomic *ref; + std::mutex *mutex; struct list_head wait_list; }; -#include "WFCondition.inl" - #endif From 7ee92d4ff30708cbbc0cf2302a123e7052f967a3 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Mon, 12 Jul 2021 19:29:16 +0800 Subject: [PATCH 32/75] mv WFCondTask impl into WFCondTaskFactory.cc and override send() --- src/factory/CMakeLists.txt | 1 - src/factory/WFCondTask.cc | 100 ------------------------ src/factory/WFCondTask.h | 80 +++---------------- src/factory/WFCondTaskFactory.cc | 129 +++++++++++++++++++++++++++++++ src/manager/WFCondition.cc | 17 ---- 5 files changed, 141 insertions(+), 186 deletions(-) delete mode 100644 src/factory/WFCondTask.cc diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index 1e41900752..5a5827352c 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -7,7 +7,6 @@ set(SRC Workflow.cc WFTaskFactory.cc WFCondTaskFactory.cc - WFCondTask.cc HttpTaskImpl.cc WebSocketTaskImpl.cc ) diff --git a/src/factory/WFCondTask.cc b/src/factory/WFCondTask.cc deleted file mode 100644 index 55f96f0c4f..0000000000 --- a/src/factory/WFCondTask.cc +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright (c) 2021 Sogou, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Author: Li Yingxin (liyingxin@sogou-inc.com) -*/ - -#include -#include -#include -#include "list.h" -#include "WFTask.h" -#include "WFCondTask.h" -#include "WFTaskFactory.h" - -void WFCondWaitTask::count() -{ - if (--this->value == 0) - { - if (this->state == WFT_STATE_UNDEFINED) - this->state = WFT_STATE_SUCCESS; - this->subtask_done(); - } -} - -void WFTimedWaitTask::dispatch() -{ - if (this->timer) - timer->dispatch(); - - this->WFMailboxTask::count(); -} - -WFTimedWaitTask::~WFTimedWaitTask() -{ - delete this->timer; -} - -SubTask *WFSwitchWaitTask::done() -{ - SeriesWork *series = series_of(this); - - WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, - [this](WFTimerTask *task) { - if (this->callback) - this->callback(this); - delete this; - }); - series->push_front(switch_task); - - return series->pop(); -} - -void WFTimedWaitTask::clear_timer_waiter() -{ - if (this->timer) - timer->clear_wait_task(); -} - -SubTask *__WFWaitTimerTask::done() -{ - WFTimedWaitTask *waiter = NULL; - - this->mutex->lock(); - if (this->wait_task) - { - list_del(&this->wait_task->list); - this->wait_task->set_state(WFT_STATE_SYS_ERROR); - this->wait_task->set_error(ETIMEDOUT); - waiter = this->wait_task; - waiter->set_timer(NULL); - } - this->mutex->unlock(); - - if (waiter) - waiter->count(); - delete this; - return NULL; -} - -__WFWaitTimerTask::~__WFWaitTimerTask() -{ - if (--*this->ref == 0) - { - delete this->mutex; - delete this->ref; - } -} - diff --git a/src/factory/WFCondTask.h b/src/factory/WFCondTask.h index 27949fc7b9..1b83e911d0 100644 --- a/src/factory/WFCondTask.h +++ b/src/factory/WFCondTask.h @@ -22,12 +22,20 @@ #include "WFTask.h" #include "WFTaskFactory.h" +class __WFWaitTimerTask; + class WFCondWaitTask : public WFMailboxTask { public: - void set_state(int state) { this->state = state; } - void set_error(int error) { this->error = error; } - virtual void count(); + virtual void count() + { + if (--this->value == 0) + { + if (this->state == WFT_STATE_UNDEFINED) + this->state = WFT_STATE_SUCCESS; + this->subtask_done(); + } + } private: void *msg; @@ -43,70 +51,6 @@ class WFCondWaitTask : public WFMailboxTask public: struct list_head list; -}; - -class __WFWaitTimerTask; - -class WFTimedWaitTask : public WFCondWaitTask -{ -public: - void set_timer(__WFWaitTimerTask *timer) { this->timer = timer; } - void clear_timer_waiter(); - -protected: - virtual void dispatch(); - -private: - __WFWaitTimerTask *timer; - -public: - WFTimedWaitTask(mailbox_callback_t&& cb) : - WFCondWaitTask(std::move(cb)) - { - this->timer = NULL; - } - - virtual ~WFTimedWaitTask(); -}; - -class __WFWaitTimerTask : public __WFTimerTask -{ -public: - void clear_wait_task() // must called within this mutex - { - this->wait_task = NULL; - } - - __WFWaitTimerTask(WFTimedWaitTask *wait_task, const struct timespec *value, - std::mutex *mutex, std::atomic *ref, - CommScheduler *scheduler) : - __WFTimerTask(value, scheduler, nullptr) - { - this->ref = ref; - ++*this->ref; - this->mutex = mutex; - this->wait_task = wait_task; - } - - virtual ~__WFWaitTimerTask(); - -protected: - virtual SubTask *done(); - -private: - std::mutex *mutex; - std::atomic *ref; - WFTimedWaitTask *wait_task; -}; - -class WFSwitchWaitTask : public WFCondWaitTask -{ -public: - WFSwitchWaitTask(mailbox_callback_t&& cb) : - WFCondWaitTask(std::move(cb)) - { } - -protected: - SubTask *done(); + friend class __WFWaitTimerTask; }; diff --git a/src/factory/WFCondTaskFactory.cc b/src/factory/WFCondTaskFactory.cc index e352101a1d..842a50cec0 100644 --- a/src/factory/WFCondTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -171,6 +171,135 @@ __WFCondition *__ConditionMap::find_condition(const std::string& name) return cond; } +class WFTimedWaitTask : public WFCondWaitTask +{ +public: + void set_timer(__WFWaitTimerTask *timer) { this->timer = timer; } + virtual void send(void *msg); + +protected: + virtual void dispatch(); + +private: + __WFWaitTimerTask *timer; + +public: + WFTimedWaitTask(mailbox_callback_t&& cb) : + WFCondWaitTask(std::move(cb)) + { + this->timer = NULL; + } + + virtual ~WFTimedWaitTask(); +}; + +class __WFWaitTimerTask : public __WFTimerTask +{ +public: + void clear_wait_task() // must called within this mutex + { + this->wait_task = NULL; + } + + __WFWaitTimerTask(WFTimedWaitTask *wait_task, const struct timespec *value, + std::mutex *mutex, std::atomic *ref, + CommScheduler *scheduler) : + __WFTimerTask(value, scheduler, nullptr) + { + this->ref = ref; + ++*this->ref; + this->mutex = mutex; + this->wait_task = wait_task; + } + + virtual ~__WFWaitTimerTask(); + +protected: + virtual SubTask *done(); + +private: + std::mutex *mutex; + std::atomic *ref; + WFTimedWaitTask *wait_task; +}; + +class WFSwitchWaitTask : public WFCondWaitTask +{ +public: + WFSwitchWaitTask(mailbox_callback_t&& cb) : + WFCondWaitTask(std::move(cb)) + { } + +protected: + SubTask *done(); +}; + +void WFTimedWaitTask::send(void *msg) +{ + this->timer->clear_wait_task(); + this->timer = NULL; + this->WFCondWaitTask::send(msg); +} + +void WFTimedWaitTask::dispatch() +{ + if (this->timer) + timer->dispatch(); + + this->WFMailboxTask::count(); +} + +WFTimedWaitTask::~WFTimedWaitTask() +{ + delete this->timer; +} + +SubTask *WFSwitchWaitTask::done() +{ + SeriesWork *series = series_of(this); + + WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, + [this](WFTimerTask *task) { + if (this->callback) + this->callback(this); + delete this; + }); + series->push_front(switch_task); + + return series->pop(); +} + +SubTask *__WFWaitTimerTask::done() +{ + WFTimedWaitTask *waiter = NULL; + + this->mutex->lock(); + if (this->wait_task) + { + list_del(&this->wait_task->list); + this->wait_task->state = WFT_STATE_SYS_ERROR; + this->wait_task->error = ETIMEDOUT; + waiter = this->wait_task; + waiter->set_timer(NULL); + } + this->mutex->unlock(); + + if (waiter) + waiter->count(); + delete this; + return NULL; +} + +__WFWaitTimerTask::~__WFWaitTimerTask() +{ + if (--*this->ref == 0) + { + delete this->mutex; + delete this->ref; + } +} + + /////////////// factory api /////////////// void WFCondTaskFactory::signal_by_name(const std::string& name, void *msg) diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index bbe466dfd7..6174469526 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -70,7 +70,6 @@ void WFSemaphore::post(void *msg) void WFCondition::signal(void *msg) { WFCondWaitTask *task = NULL; - WFTimedWaitTask *waiter; struct list_head *pos; this->mutex->lock(); @@ -79,12 +78,6 @@ void WFCondition::signal(void *msg) pos = this->wait_list.next; task = list_entry(pos, WFCondWaitTask, list); list_del(pos); - waiter = dynamic_cast(task); - if (waiter) - { - waiter->clear_timer_waiter(); - waiter->set_timer(NULL); - } } this->mutex->unlock(); @@ -95,7 +88,6 @@ void WFCondition::signal(void *msg) void WFCondition::broadcast(void *msg) { WFCondWaitTask *task; - WFTimedWaitTask *waiter; struct list_head *pos, *tmp; LIST_HEAD(tmp_list); @@ -103,16 +95,7 @@ void WFCondition::broadcast(void *msg) if (!list_empty(&this->wait_list)) { list_for_each_safe(pos, tmp, &this->wait_list) - { list_move_tail(pos, &tmp_list); - task = list_entry(pos, WFCondWaitTask, list); - waiter = dynamic_cast(task); - if (waiter) - { - waiter->clear_timer_waiter(); - waiter->set_timer(NULL); - } - } } this->mutex->unlock(); From 6026e9c2d660770561ceaad9c79a67434ae15eab Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Tue, 13 Jul 2021 03:12:33 +0800 Subject: [PATCH 33/75] 1. update WFSemaphore::get(); 2. add WFSemaphoreTask; 3. add clear_locked() for timedwait; 4. change WFSemaphore value outside the mutex; 5. move codes; --- src/factory/WFCondTask.h | 46 +++++++++++------ src/factory/WFCondTaskFactory.cc | 85 +++++++++++++++++++++++++------- src/factory/WFCondTaskFactory.h | 39 ++++++++++++++- src/manager/WFCondition.cc | 56 +++------------------ src/manager/WFCondition.h | 45 +---------------- test/condition_unittest.cc | 7 +-- 6 files changed, 148 insertions(+), 130 deletions(-) diff --git a/src/factory/WFCondTask.h b/src/factory/WFCondTask.h index 1b83e911d0..02db6c917f 100644 --- a/src/factory/WFCondTask.h +++ b/src/factory/WFCondTask.h @@ -16,41 +16,57 @@ Author: Li Yingxin (liyingxin@sogou-inc.com) */ +#ifndef _WFCONDTASK_H_ +#define _WFCONDTASK_H_ + #include #include #include "list.h" #include "WFTask.h" +#include "WFCondition.h" #include "WFTaskFactory.h" +#include "WFCondTaskFactory.h" class __WFWaitTimerTask; class WFCondWaitTask : public WFMailboxTask { public: - virtual void count() - { - if (--this->value == 0) - { - if (this->state == WFT_STATE_UNDEFINED) - this->state = WFT_STATE_SUCCESS; - this->subtask_done(); - } - } + virtual void clear_locked() { } + +public: + struct list_head list; private: void *msg; public: - WFCondWaitTask(mailbox_callback_t&& cb) : + WFCondWaitTask(wait_callback_t&& cb) : WFMailboxTask(&this->msg, 1, std::move(cb)) - { - this->list.next = NULL; - } + { } virtual ~WFCondWaitTask() { } -public: - struct list_head list; friend class __WFWaitTimerTask; +/* + friend class WFCondition; + friend class WFCondTaskFactory; + friend class __WFCondition; +*/ }; +class WFSemaphoreTask : public WFConditional +{ +private: + struct list_head list; + +public: + WFSemaphoreTask(SubTask *task, void **psem) : + WFConditional(task, psem) + { } + + friend class WFSemaphore; +}; + +#endif + diff --git a/src/factory/WFCondTaskFactory.cc b/src/factory/WFCondTaskFactory.cc index 842a50cec0..6b27abcd94 100644 --- a/src/factory/WFCondTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -33,8 +33,7 @@ class __WFCondition : public WFCondition public: __WFCondition(const std::string& str) : name(str) - { - } + { } public: struct rb_node rb; @@ -49,7 +48,7 @@ class __ConditionMap WFWaitTask *create(const std::string& name, wait_callback_t&& cb); WFWaitTask *create(const std::string& name, - const struct timespec *value, + const struct timespec *timeout, wait_callback_t&& cb); WFWaitTask *create_switch(const std::string& name, wait_callback_t&& cb); @@ -97,12 +96,12 @@ WFWaitTask *__ConditionMap::create(const std::string& name, } WFWaitTask *__ConditionMap::create(const std::string& name, - const struct timespec *value, + const struct timespec *timeout, wait_callback_t&& cb) { __WFCondition *cond = this->find_condition(name); - return WFCondTaskFactory::create_timedwait_task(cond, value, + return WFCondTaskFactory::create_timedwait_task(cond, timeout, std::move(cb)); } @@ -175,7 +174,8 @@ class WFTimedWaitTask : public WFCondWaitTask { public: void set_timer(__WFWaitTimerTask *timer) { this->timer = timer; } - virtual void send(void *msg); + virtual void count(); + virtual void clear_locked(); protected: virtual void dispatch(); @@ -184,7 +184,7 @@ class WFTimedWaitTask : public WFCondWaitTask __WFWaitTimerTask *timer; public: - WFTimedWaitTask(mailbox_callback_t&& cb) : + WFTimedWaitTask(wait_callback_t&& cb) : WFCondWaitTask(std::move(cb)) { this->timer = NULL; @@ -201,10 +201,10 @@ class __WFWaitTimerTask : public __WFTimerTask this->wait_task = NULL; } - __WFWaitTimerTask(WFTimedWaitTask *wait_task, const struct timespec *value, + __WFWaitTimerTask(WFTimedWaitTask *wait_task, const struct timespec *timeout, std::mutex *mutex, std::atomic *ref, CommScheduler *scheduler) : - __WFTimerTask(value, scheduler, nullptr) + __WFTimerTask(timeout, scheduler, nullptr) { this->ref = ref; ++*this->ref; @@ -226,7 +226,7 @@ class __WFWaitTimerTask : public __WFTimerTask class WFSwitchWaitTask : public WFCondWaitTask { public: - WFSwitchWaitTask(mailbox_callback_t&& cb) : + WFSwitchWaitTask(wait_callback_t&& cb) : WFCondWaitTask(std::move(cb)) { } @@ -234,11 +234,20 @@ class WFSwitchWaitTask : public WFCondWaitTask SubTask *done(); }; -void WFTimedWaitTask::send(void *msg) +void WFTimedWaitTask::clear_locked() { this->timer->clear_wait_task(); this->timer = NULL; - this->WFCondWaitTask::send(msg); +} + +void WFTimedWaitTask::count() +{ + if (--this->value == 0) + { + if (this->state == WFT_STATE_UNDEFINED) + this->state = WFT_STATE_SUCCESS; + this->subtask_done(); + } } void WFTimedWaitTask::dispatch() @@ -251,7 +260,8 @@ void WFTimedWaitTask::dispatch() WFTimedWaitTask::~WFTimedWaitTask() { - delete this->timer; + if (this->state != WFT_STATE_SUCCESS) + delete this->timer; } SubTask *WFSwitchWaitTask::done() @@ -299,8 +309,47 @@ __WFWaitTimerTask::~__WFWaitTimerTask() } } +/////////////// WFSemaphore impl /////////////// + +WFConditional *WFSemaphore::get(SubTask *task, void **psem) +{ + WFConditional *cond; + WFSemaphoreTask *sem_task; + + if (--this->value >= 0) + { + cond = new WFConditional(task, psem); + cond->signal(this->sems[--this->index]); + return cond; + } + + sem_task = new WFSemaphoreTask(task, psem); + this->mutex.lock(); + list_add_tail(&sem_task->list, &this->wait_list); + this->mutex.unlock(); + + return sem_task; +} + +void WFSemaphore::post(void *sem) +{ + WFSemaphoreTask *task = NULL; + + if (++this->value <= 0) + { + this->mutex.lock(); + task = list_entry(this->wait_list.next, WFSemaphoreTask, list); + list_del(&task->list); + this->mutex.unlock(); + } + else + this->sems[this->index++] = sem; + + if (task) + task->signal(sem); +} -/////////////// factory api /////////////// +/////////////// factory impl /////////////// void WFCondTaskFactory::signal_by_name(const std::string& name, void *msg) { @@ -326,10 +375,10 @@ WFWaitTask *WFCondTaskFactory::create_swait_task(const std::string& name, } WFWaitTask *WFCondTaskFactory::create_timedwait_task(const std::string& name, - const struct timespec *value, + const struct timespec *timeout, wait_callback_t callback) { - return __ConditionMap::get_instance()->create(name, value, + return __ConditionMap::get_instance()->create(name, timeout, std::move(callback)); } @@ -346,11 +395,11 @@ WFWaitTask *WFCondTaskFactory::create_wait_task(WFCondition *cond, } WFWaitTask *WFCondTaskFactory::create_timedwait_task(WFCondition *cond, - const struct timespec *value, + const struct timespec *timeout, wait_callback_t callback) { WFTimedWaitTask *waiter = new WFTimedWaitTask(std::move(callback)); - __WFWaitTimerTask *task = new __WFWaitTimerTask(waiter, value, + __WFWaitTimerTask *task = new __WFWaitTimerTask(waiter, timeout, cond->mutex, cond->ref, WFGlobal::get_scheduler()); waiter->set_timer(task); diff --git a/src/factory/WFCondTaskFactory.h b/src/factory/WFCondTaskFactory.h index 7075405ae6..5f7199a1aa 100644 --- a/src/factory/WFCondTaskFactory.h +++ b/src/factory/WFCondTaskFactory.h @@ -29,6 +29,41 @@ #include "WFGlobal.h" #include "WFCondition.h" +class WFSemaphore +{ +public: + WFConditional *get(SubTask *task, void **psem); + void post(void *sem); + +private: + std::mutex mutex; + struct list_head wait_list; + + struct entry + { + struct list_head list; + WFConditional *ptr; + }; + +public: + WFSemaphore(void **sems, int n) + { + INIT_LIST_HEAD(&this->wait_list); + this->value = n; + this->index = n; + this->sems = sems; + } + + virtual ~WFSemaphore() { } + +private: + std::atomic value; + std::atomic index; + +protected: + void **sems; +}; + class WFCondTaskFactory { public: @@ -41,7 +76,7 @@ class WFCondTaskFactory wait_callback_t callback); static WFWaitTask *create_timedwait_task(const std::string& name, - const struct timespec *value, + const struct timespec *timeout, wait_callback_t callback); static WFWaitTask *create_swait_task(const std::string& name, @@ -52,7 +87,7 @@ class WFCondTaskFactory wait_callback_t callback); static WFWaitTask *create_timedwait_task(WFCondition *cond, - const struct timespec *value, + const struct timespec *timeout, wait_callback_t callback); static WFWaitTask *create_swait_task(WFCondition *cond, diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index 6174469526..6fa3333ba7 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -22,62 +22,16 @@ #include "WFCondTask.h" #include "WFCondition.h" -bool WFSemaphore::get(WFConditional *cond) -{ - this->mutex.lock(); - if (--this->concurrency >= 0) - { - cond->signal(this->resources[--this->index]); - this->mutex.unlock(); - return true; - } - - struct WFSemaphore::entry *entry; - entry = new WFSemaphore::entry; - entry->ptr = cond; - entry->list.next = NULL; - - list_add_tail(&entry->list, &this->wait_list); - this->mutex.unlock(); - - return false; -} - -void WFSemaphore::post(void *msg) -{ - struct WFSemaphore::entry *entry; - WFConditional *cond = NULL; - struct list_head *pos; - - this->mutex.lock(); - - if (++this->concurrency <= 0) - { - pos = this->wait_list.next; - entry = list_entry(pos, struct WFSemaphore::entry, list); - cond = entry->ptr; - list_del(pos); - delete entry; - } - else - this->resources[this->index++] = msg; - - this->mutex.unlock(); - if (cond) - cond->signal(msg); -} - void WFCondition::signal(void *msg) { WFCondWaitTask *task = NULL; - struct list_head *pos; this->mutex->lock(); if (!list_empty(&this->wait_list)) { - pos = this->wait_list.next; - task = list_entry(pos, WFCondWaitTask, list); - list_del(pos); + task = list_entry(this->wait_list.next, WFCondWaitTask, list); + list_del(&task->list); + task->clear_locked(); } this->mutex->unlock(); @@ -95,7 +49,11 @@ void WFCondition::broadcast(void *msg) if (!list_empty(&this->wait_list)) { list_for_each_safe(pos, tmp, &this->wait_list) + { list_move_tail(pos, &tmp_list); + task = list_entry(pos, WFCondWaitTask, list); + task->clear_locked(); + } } this->mutex->unlock(); diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h index 0edd0a1ccd..c972557038 100644 --- a/src/manager/WFCondition.h +++ b/src/manager/WFCondition.h @@ -16,8 +16,8 @@ Author: Li Yingxin (liyingxin@sogou-inc.com) */ -#ifndef _WFSEMAPHORE_H_ -#define _WFSEMAPHORE_H_ +#ifndef _WFCONDITION_H_ +#define _WFCONDITION_H_ #include #include @@ -31,47 +31,6 @@ using WFWaitTask = WFMailboxTask; using wait_callback_t = mailbox_callback_t; -class WFSemaphore -{ -public: - bool get(WFConditional *cond); - void post(void *msg); - -public: - std::mutex mutex; - struct list_head wait_list; - -private: - struct entry - { - struct list_head list; - WFConditional *ptr; - }; - -public: - WFSemaphore(int value, void **resources) - { - if (value <= 0) - value = 1; - - INIT_LIST_HEAD(&this->wait_list); - this->concurrency = value; - this->total = value; - this->index = value; - this->resources = resources; - } - - virtual ~WFSemaphore() { } - -private: - std::atomic concurrency; - int total; - int index; - -protected: - void **resources; -}; - class WFCondition { public: diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index 2155afb2a5..bc2e56d01b 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -124,7 +124,7 @@ TEST(condition_unittest, semaphore) int sem_concurrency = 3; int task_concurrency = 10; const char *words[3] = {"workflow", "srpc", "pyworkflow"}; - WFSemaphore sem(sem_concurrency, (void **)words); + WFSemaphore sem((void **)words, sem_concurrency); WFFacilities::WaitGroup wg(task_concurrency); for (int i = 0; i < task_concurrency; i++) @@ -136,8 +136,9 @@ TEST(condition_unittest, semaphore) sem.post(task->user_data); wg.done(); }); - auto *cond = WFTaskFactory::create_conditional(user_task, &user_task->user_data); - sem.get(cond); + + auto *cond = sem.get(user_task, &user_task->user_data); + SeriesWork *series = Workflow::create_series_work(cond, nullptr); series->set_context(reinterpret_cast(i)); series->start(); From f356da56495750c27bc2c596a16d4bf682db8d22 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 15 Jul 2021 21:41:23 +0800 Subject: [PATCH 34/75] seperate WFSemaphore; override dispatch() and signal(); --- CMakeLists.txt | 2 +- CMakeLists_Headers.txt | 1 + src/factory/CMakeLists.txt | 1 + src/factory/WFChannel.inl | 2 +- src/factory/WFCondTask.h | 19 +------- src/factory/WFCondTaskFactory.cc | 48 ++---------------- src/factory/WFCondTaskFactory.h | 35 -------------- src/factory/WFSemaphore.cc | 83 ++++++++++++++++++++++++++++++++ src/factory/WFSemaphore.h | 59 +++++++++++++++++++++++ src/factory/WFTask.h | 2 +- 10 files changed, 153 insertions(+), 99 deletions(-) create mode 100644 src/factory/WFSemaphore.cc create mode 100644 src/factory/WFSemaphore.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f6aa89a30e..dde67fbe9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,7 +68,7 @@ if (WIN32) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP /wd4200 /std:c++14") else () set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -fPIC -pipe -std=gnu90") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fPIC -pipe -std=c++11 -fno-exceptions -Wno-invalid-offsetof") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fPIC -pipe -std=c++11 -fno-exceptions -Wno-invalid-offsetof") endif () add_subdirectory(src) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index 35a15778b3..bb1a3ec7f5 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -96,6 +96,7 @@ set(INCLUDE_HEADERS src/factory/WFAlgoTaskFactory.h src/factory/WFAlgoTaskFactory.inl src/factory/WFCondTaskFactory.h + src/factory/WFSemaphore.h src/factory/Workflow.h src/factory/WFOperator.h src/factory/WFChannel.h diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index 5a5827352c..28690e906d 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -7,6 +7,7 @@ set(SRC Workflow.cc WFTaskFactory.cc WFCondTaskFactory.cc + WFSemaphore.cc HttpTaskImpl.cc WebSocketTaskImpl.cc ) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 6ffafac4cd..89082d51c5 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -429,7 +429,7 @@ SubTask *ComplexChannelOutTask::done() template SubTask *ComplexChannelOutTask::upgrade() { - WFCounterTask *counter = new WFCounterTask(0, [this](WFCounterTask *task) + WFCounterTask *counter = new WFCounterTask(0, [this](WFCounterTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); diff --git a/src/factory/WFCondTask.h b/src/factory/WFCondTask.h index 02db6c917f..06fc09144a 100644 --- a/src/factory/WFCondTask.h +++ b/src/factory/WFCondTask.h @@ -34,7 +34,7 @@ class WFCondWaitTask : public WFMailboxTask public: virtual void clear_locked() { } -public: +private: struct list_head list; private: @@ -48,24 +48,9 @@ class WFCondWaitTask : public WFMailboxTask virtual ~WFCondWaitTask() { } friend class __WFWaitTimerTask; -/* friend class WFCondition; friend class WFCondTaskFactory; - friend class __WFCondition; -*/ -}; - -class WFSemaphoreTask : public WFConditional -{ -private: - struct list_head list; - -public: - WFSemaphoreTask(SubTask *task, void **psem) : - WFConditional(task, psem) - { } - - friend class WFSemaphore; + friend class __ConditionMap; }; #endif diff --git a/src/factory/WFCondTaskFactory.cc b/src/factory/WFCondTaskFactory.cc index 6b27abcd94..9e8cca2ac5 100644 --- a/src/factory/WFCondTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -309,46 +309,6 @@ __WFWaitTimerTask::~__WFWaitTimerTask() } } -/////////////// WFSemaphore impl /////////////// - -WFConditional *WFSemaphore::get(SubTask *task, void **psem) -{ - WFConditional *cond; - WFSemaphoreTask *sem_task; - - if (--this->value >= 0) - { - cond = new WFConditional(task, psem); - cond->signal(this->sems[--this->index]); - return cond; - } - - sem_task = new WFSemaphoreTask(task, psem); - this->mutex.lock(); - list_add_tail(&sem_task->list, &this->wait_list); - this->mutex.unlock(); - - return sem_task; -} - -void WFSemaphore::post(void *sem) -{ - WFSemaphoreTask *task = NULL; - - if (++this->value <= 0) - { - this->mutex.lock(); - task = list_entry(this->wait_list.next, WFSemaphoreTask, list); - list_del(&task->list); - this->mutex.unlock(); - } - else - this->sems[this->index++] = sem; - - if (task) - task->signal(sem); -} - /////////////// factory impl /////////////// void WFCondTaskFactory::signal_by_name(const std::string& name, void *msg) @@ -399,10 +359,10 @@ WFWaitTask *WFCondTaskFactory::create_timedwait_task(WFCondition *cond, wait_callback_t callback) { WFTimedWaitTask *waiter = new WFTimedWaitTask(std::move(callback)); - __WFWaitTimerTask *task = new __WFWaitTimerTask(waiter, timeout, - cond->mutex, cond->ref, - WFGlobal::get_scheduler()); - waiter->set_timer(task); + __WFWaitTimerTask *timer = new __WFWaitTimerTask(waiter, timeout, + cond->mutex, cond->ref, + WFGlobal::get_scheduler()); + waiter->set_timer(timer); cond->mutex->lock(); list_add_tail(&waiter->list, &cond->wait_list); diff --git a/src/factory/WFCondTaskFactory.h b/src/factory/WFCondTaskFactory.h index 5f7199a1aa..09e2c36399 100644 --- a/src/factory/WFCondTaskFactory.h +++ b/src/factory/WFCondTaskFactory.h @@ -29,41 +29,6 @@ #include "WFGlobal.h" #include "WFCondition.h" -class WFSemaphore -{ -public: - WFConditional *get(SubTask *task, void **psem); - void post(void *sem); - -private: - std::mutex mutex; - struct list_head wait_list; - - struct entry - { - struct list_head list; - WFConditional *ptr; - }; - -public: - WFSemaphore(void **sems, int n) - { - INIT_LIST_HEAD(&this->wait_list); - this->value = n; - this->index = n; - this->sems = sems; - } - - virtual ~WFSemaphore() { } - -private: - std::atomic value; - std::atomic index; - -protected: - void **sems; -}; - class WFCondTaskFactory { public: diff --git a/src/factory/WFSemaphore.cc b/src/factory/WFSemaphore.cc new file mode 100644 index 0000000000..973d07ea3e --- /dev/null +++ b/src/factory/WFSemaphore.cc @@ -0,0 +1,83 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) +*/ + +#include "list.h" +#include "WFTask.h" +#include "WFSemaphore.h" + +class __WFConditional : public WFConditional +{ +public: + struct list_head list; + struct WFSemaphore::Data *data; + +public: + virtual void dispatch(); + virtual void signal(void *msg) { } + +public: + __WFConditional(SubTask *task, void **pmsg, + struct WFSemaphore::Data *data) : + WFConditional(task, pmsg) + { + this->data = data; + } +}; + +void __WFConditional::dispatch() +{ + struct WFSemaphore::Data *data = this->data; + + data->mutex.lock(); + if (--data->value >= 0) + this->WFConditional::signal(data->sembuf[data->index++]); + else + list_add_tail(&this->list, &data->wait_list); + + data->mutex.unlock(); + this->WFConditional::dispatch(); +} + +WFConditional *WFSemaphore::get(SubTask *task, void **pmsg) +{ + return new __WFConditional(task, pmsg, &this->data); +} + +void WFSemaphore::post(void *msg) +{ + struct WFSemaphore::Data *data = &this->data; + WFConditional *cond; + + data->mutex.lock(); + if (++data->value <= 0) + { + cond = list_entry(data->wait_list.next, __WFConditional, list); + list_del(data->wait_list.next); + } + else + { + cond = NULL; + data->sembuf[--data->index] = msg; + } + + data->mutex.unlock(); + if (cond) + cond->WFConditional::signal(msg); +} + diff --git a/src/factory/WFSemaphore.h b/src/factory/WFSemaphore.h new file mode 100644 index 0000000000..a9177b1471 --- /dev/null +++ b/src/factory/WFSemaphore.h @@ -0,0 +1,59 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) + Xie Han (xiehan@sogou-inc.com) +*/ + +#ifndef _WFSEMAPHORE_H_ +#define _WFSEMAPHORE_H_ + +#include +#include +#include "list.h" +#include "WFTask.h" +#include "WFCondition.h" + +class WFSemaphore +{ +public: + WFConditional *get(SubTask *task, void **pmsg); + void post(void *msg); + +public: + struct Data + { + void **sembuf; + std::atomic value; + std::atomic index; + struct list_head wait_list; + std::mutex mutex; + }; + +private: + struct Data data; + +public: + WFSemaphore(void **sembuf, int nsems) + { + this->data.sembuf = sembuf; + this->data.value = nsems; + this->data.index = 0; + INIT_LIST_HEAD(&this->data.wait_list); + } +}; + +#endif + diff --git a/src/factory/WFTask.h b/src/factory/WFTask.h index 5dc337df2b..123f11d541 100644 --- a/src/factory/WFTask.h +++ b/src/factory/WFTask.h @@ -634,7 +634,7 @@ class WFMailboxTask : public WFGenericTask class WFConditional : public WFGenericTask { public: - void signal(void *msg) + virtual void signal(void *msg) { *this->msgbuf = msg; if (this->flag.exchange(true)) From e8b7f23091a76ffe972328f45efc68d13a66cf0f Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 16 Jul 2021 19:57:50 +0800 Subject: [PATCH 35/75] add WFResourcePool --- CMakeLists_Headers.txt | 2 +- src/factory/CMakeLists.txt | 2 +- src/factory/WFCondTaskFactory.cc | 34 +++++++++----- .../{WFSemaphore.cc => WFResourcePool.cc} | 28 ++++++------ .../{WFSemaphore.h => WFResourcePool.h} | 45 ++++++++++++++----- test/condition_unittest.cc | 13 +++--- 6 files changed, 81 insertions(+), 43 deletions(-) rename src/factory/{WFSemaphore.cc => WFResourcePool.cc} (69%) rename src/factory/{WFSemaphore.h => WFResourcePool.h} (59%) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index bb1a3ec7f5..515bdf3f6f 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -96,7 +96,7 @@ set(INCLUDE_HEADERS src/factory/WFAlgoTaskFactory.h src/factory/WFAlgoTaskFactory.inl src/factory/WFCondTaskFactory.h - src/factory/WFSemaphore.h + src/factory/WFResourcePool.h src/factory/Workflow.h src/factory/WFOperator.h src/factory/WFChannel.h diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index 28690e906d..9b5b03c553 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -7,7 +7,7 @@ set(SRC Workflow.cc WFTaskFactory.cc WFCondTaskFactory.cc - WFSemaphore.cc + WFResourcePool.cc HttpTaskImpl.cc WebSocketTaskImpl.cc ) diff --git a/src/factory/WFCondTaskFactory.cc b/src/factory/WFCondTaskFactory.cc index 9e8cca2ac5..ac24c6d934 100644 --- a/src/factory/WFCondTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -193,7 +193,7 @@ class WFTimedWaitTask : public WFCondWaitTask virtual ~WFTimedWaitTask(); }; -class __WFWaitTimerTask : public __WFTimerTask +class __WFWaitTimerTask : public WFTimerTask { public: void clear_wait_task() // must called within this mutex @@ -201,26 +201,38 @@ class __WFWaitTimerTask : public __WFTimerTask this->wait_task = NULL; } - __WFWaitTimerTask(WFTimedWaitTask *wait_task, const struct timespec *timeout, - std::mutex *mutex, std::atomic *ref, - CommScheduler *scheduler) : - __WFTimerTask(timeout, scheduler, nullptr) +protected: + virtual int duration(struct timespec *value) { - this->ref = ref; - ++*this->ref; - this->mutex = mutex; - this->wait_task = wait_task; + *value = this->timeout; + return 0; } - virtual ~__WFWaitTimerTask(); + virtual SubTask *done(); protected: - virtual SubTask *done(); + struct timespec timeout; private: std::mutex *mutex; std::atomic *ref; WFTimedWaitTask *wait_task; + +public: + __WFWaitTimerTask(WFTimedWaitTask *wait_task, + const struct timespec *timeout, + std::mutex *mutex, std::atomic *ref, + CommScheduler *scheduler) : + WFTimerTask(scheduler, nullptr) + { + this->timeout = *timeout; + this->ref = ref; + ++*this->ref; + this->mutex = mutex; + this->wait_task = wait_task; + } + + virtual ~__WFWaitTimerTask(); }; class WFSwitchWaitTask : public WFCondWaitTask diff --git a/src/factory/WFSemaphore.cc b/src/factory/WFResourcePool.cc similarity index 69% rename from src/factory/WFSemaphore.cc rename to src/factory/WFResourcePool.cc index 973d07ea3e..38deaf4aae 100644 --- a/src/factory/WFSemaphore.cc +++ b/src/factory/WFResourcePool.cc @@ -19,22 +19,22 @@ #include "list.h" #include "WFTask.h" -#include "WFSemaphore.h" +#include "WFResourcePool.h" class __WFConditional : public WFConditional { public: struct list_head list; - struct WFSemaphore::Data *data; + struct WFResourcePool::Data *data; public: virtual void dispatch(); - virtual void signal(void *msg) { } + virtual void signal(void *res) { } public: - __WFConditional(SubTask *task, void **pmsg, - struct WFSemaphore::Data *data) : - WFConditional(task, pmsg) + __WFConditional(SubTask *task, void **pres, + struct WFResourcePool::Data *data) : + WFConditional(task, pres) { this->data = data; } @@ -42,11 +42,11 @@ class __WFConditional : public WFConditional void __WFConditional::dispatch() { - struct WFSemaphore::Data *data = this->data; + struct WFResourcePool::Data *data = this->data; data->mutex.lock(); if (--data->value >= 0) - this->WFConditional::signal(data->sembuf[data->index++]); + this->WFConditional::signal(data->pop()); else list_add_tail(&this->list, &data->wait_list); @@ -54,14 +54,14 @@ void __WFConditional::dispatch() this->WFConditional::dispatch(); } -WFConditional *WFSemaphore::get(SubTask *task, void **pmsg) +WFConditional *WFResourcePool::get(SubTask *task, void **pres) { - return new __WFConditional(task, pmsg, &this->data); + return new __WFConditional(task, pres, &this->data); } -void WFSemaphore::post(void *msg) +void WFResourcePool::post(void *res) { - struct WFSemaphore::Data *data = &this->data; + struct WFResourcePool::Data *data = &this->data; WFConditional *cond; data->mutex.lock(); @@ -73,11 +73,11 @@ void WFSemaphore::post(void *msg) else { cond = NULL; - data->sembuf[--data->index] = msg; + data->push(res); } data->mutex.unlock(); if (cond) - cond->WFConditional::signal(msg); + cond->WFConditional::signal(res); } diff --git a/src/factory/WFSemaphore.h b/src/factory/WFResourcePool.h similarity index 59% rename from src/factory/WFSemaphore.h rename to src/factory/WFResourcePool.h index a9177b1471..fa3e5996f4 100644 --- a/src/factory/WFSemaphore.h +++ b/src/factory/WFResourcePool.h @@ -17,42 +17,67 @@ Xie Han (xiehan@sogou-inc.com) */ -#ifndef _WFSEMAPHORE_H_ -#define _WFSEMAPHORE_H_ +#ifndef _WFRESOURCEPOOL_H_ +#define _WFRESOURCEPOOL_H_ #include #include #include "list.h" #include "WFTask.h" -#include "WFCondition.h" -class WFSemaphore +class WFResourcePool { public: - WFConditional *get(SubTask *task, void **pmsg); - void post(void *msg); + WFConditional *get(SubTask *task, void **pres); + void post(void *res); public: struct Data { - void **sembuf; + void **res; std::atomic value; std::atomic index; struct list_head wait_list; std::mutex mutex; + WFResourcePool *ptr; + + virtual void *pop() + { + return this->ptr->pop(); + } + + virtual void push(void *res) + { + ptr->push(res); + } }; +private: + virtual void *pop() + { + return this->data.res[this->data.index++]; + } + + virtual void push(void *res) + { + this->data.res[--this->data.index] = res; + } + private: struct Data data; public: - WFSemaphore(void **sembuf, int nsems) + WFResourcePool(void **res, int n) { - this->data.sembuf = sembuf; - this->data.value = nsems; + this->data.ptr = this; + this->data.res = new void *[n]; + memcpy(this->data.res, res, n * sizeof(void *)); + this->data.value = n; this->data.index = 0; INIT_LIST_HEAD(&this->data.wait_list); } + + virtual ~WFResourcePool() { delete[] this->data.res; } }; #endif diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index bc2e56d01b..f71fbfe57f 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -24,6 +24,7 @@ #include #include "workflow/WFTask.h" #include "workflow/WFTaskFactory.h" +#include "workflow/WFResourcePool.h" #include "workflow/WFCondition.h" #include "workflow/WFCondTaskFactory.h" #include "workflow/WFFacilities.h" @@ -119,25 +120,25 @@ TEST(condition_unittest, timedwait) wait_group.wait(); } -TEST(condition_unittest, semaphore) +TEST(condition_unittest, resource_pool) { - int sem_concurrency = 3; + int res__concurrency = 3; int task_concurrency = 10; const char *words[3] = {"workflow", "srpc", "pyworkflow"}; - WFSemaphore sem((void **)words, sem_concurrency); + WFResourcePool res_pool((void **)words, res__concurrency); WFFacilities::WaitGroup wg(task_concurrency); for (int i = 0; i < task_concurrency; i++) { auto *user_task = WFTaskFactory::create_timer_task(0, - [&wg, &sem](WFTimerTask *task) { + [&wg, &res_pool](WFTimerTask *task) { uint64_t id = (uint64_t)series_of(task)->get_context(); printf("task-%lu get [%s]\n", id, (char *)task->user_data); - sem.post(task->user_data); + res_pool.post(task->user_data); wg.done(); }); - auto *cond = sem.get(user_task, &user_task->user_data); + auto *cond = res_pool.get(user_task, &user_task->user_data); SeriesWork *series = Workflow::create_series_work(cond, nullptr); series->set_context(reinterpret_cast(i)); From 8d2ab2821146d6b492a0d70ec0062e8506cfd925 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Mon, 19 Jul 2021 22:26:17 +0800 Subject: [PATCH 36/75] add BaseResource for WFCondition --- src/manager/WFCondition.cc | 62 ++++++++++++++++++++++++++- src/manager/WFCondition.h | 45 +++++++++++++++++--- test/condition_unittest.cc | 87 +++++++++++++++++++++++++++++++++++++- 3 files changed, 186 insertions(+), 8 deletions(-) diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index 6fa3333ba7..44597c2313 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -22,14 +22,64 @@ #include "WFCondTask.h" #include "WFCondition.h" +int WFCondition::get(void **pmsg) +{ +// if (this->res == NULL) +// return -2; + int ret; + + this->mutex->lock(); + if (this->empty == 1) + { + *pmsg = this->res->get(); + ret = 1; + } + else if (--this->empty == 0) + { + ret = 0; + } + else + { + WFCondWaitTask *task = new WFCondWaitTask(nullptr); + list_add_tail(&task->list, &this->get_list); + ret = -1; + } + this->mutex->unlock(); + + return ret; +} + +WFWaitTask *WFCondition::get_wait_task(wait_callback_t callback) +{ + WFCondWaitTask *task = NULL; + struct list_head *pos; + + this->mutex->lock(); + if (!list_empty(&this->get_list)) + { + pos = this->get_list.next; + list_move_tail(pos, &this->wait_list); + task = list_entry(pos, WFCondWaitTask, list); + task->set_callback(std::move(callback)); + } + + this->mutex->unlock(); + return task; +} + void WFCondition::signal(void *msg) { WFCondWaitTask *task = NULL; this->mutex->lock(); + if (!list_empty(&this->wait_list)) - { task = list_entry(this->wait_list.next, WFCondWaitTask, list); + else if (!list_empty(&this->get_list)) + task = list_entry(this->get_list.next, WFCondWaitTask, list); + + if (task) + { list_del(&task->list); task->clear_locked(); } @@ -56,6 +106,16 @@ void WFCondition::broadcast(void *msg) } } + if (!list_empty(&this->get_list)) + { + list_for_each_safe(pos, tmp, &this->get_list) + { + list_move_tail(pos, &tmp_list); + task = list_entry(pos, WFCondWaitTask, list); + task->clear_locked(); + } + } + this->mutex->unlock(); while (!list_empty(&tmp_list)) { diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h index c972557038..037d042d8d 100644 --- a/src/manager/WFCondition.h +++ b/src/manager/WFCondition.h @@ -36,20 +36,55 @@ class WFCondition public: void signal(void *msg); void broadcast(void *msg); + // 1: existed; 0: should get; -1: should wait; + int get(void **pmsg); + WFWaitTask *get_wait_task(wait_callback_t callback); public: + std::atomic *ref; + std::mutex *mutex; + struct list_head get_list; + struct list_head wait_list; + +public: + class BaseResource + { + public: + virtual void *get() = 0; + virtual void set() { this->ptr->empty = 0; } + void clear() { this->ptr->empty = 1; } + + private: + WFCondition *ptr; + friend WFCondition; + }; + +private: + BaseResource *res; + int empty; + +public: + WFCondition(BaseResource *res) + { + this->empty = 1; + this->res = res; + this->res->ptr = this; + this->mutex = new std::mutex; + this->ref = new std::atomic(1); + INIT_LIST_HEAD(&this->get_list); + INIT_LIST_HEAD(&this->wait_list); + } + WFCondition() { + this->empty = 0; + this->res = NULL; this->mutex = new std::mutex; this->ref = new std::atomic(1); + INIT_LIST_HEAD(&this->get_list); INIT_LIST_HEAD(&this->wait_list); } virtual ~WFCondition(); - -public: - std::atomic *ref; - std::mutex *mutex; - struct list_head wait_list; }; #endif diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index f71fbfe57f..4dff2bfc07 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -122,10 +122,10 @@ TEST(condition_unittest, timedwait) TEST(condition_unittest, resource_pool) { - int res__concurrency = 3; + int res_concurrency = 3; int task_concurrency = 10; const char *words[3] = {"workflow", "srpc", "pyworkflow"}; - WFResourcePool res_pool((void **)words, res__concurrency); + WFResourcePool res_pool((void **)words, res_concurrency); WFFacilities::WaitGroup wg(task_concurrency); for (int i = 0; i < task_concurrency; i++) @@ -148,3 +148,86 @@ TEST(condition_unittest, resource_pool) wg.wait(); } +class TestResource : public WFCondition::BaseResource +{ +public: + TestResource(std::string msg) : msg(msg) { } + void *get() { return &this->msg; } + void set(std::string&& msg) { this->msg = std::move(msg);} + +private: + std::string msg; +}; + +TestResource res(""); + +void wait_callback(WFWaitTask *task) +{ + void **msg; + size_t n; + msg = task->get_mailbox(&n); + std::string *str = *msg; + fprintf(stderr, "waiter get msg:%s\n", str->c_str()); + series->set_context(*msg); +} + +void work(std::string *msg, WFFacilities::WaitGroup &wg) +{ + fprintf(stderr, "working on message:%s\n", msg->c_str()); + wg.done(); +} + +SubTask *create_router_task(WFCondition *cond, std::function cb) +{ + auto *router = WFTaskFactory::create_timer_task(100000, std::move(cb)); + router->user_data = cond; +} + +void router_callback(WFTimerTask *task) +{ + fprintf(stderr, "finish routing and should broadcast every one else\n"); + + ((WFCondition *)(task->user_data))->broadcast(); +} + +TEST(condition_unittest, res_condition) +{ + int task_concurrency = 10; + WFFacilities::WaitGroup wg(task_concurrency); + WFCondition cond((WFCondition::BaseResource *)&res); + + for (int i = 0; i < task_concurrency; i++) + { + auto *timer = WFTaskFactory::create_timer_task(i * 100000, + [&cond](WFTimerTask *task) { + std::string *msg; + int ret = cond.get(&msg); + if (ret == -1) + { + auto *waiter = cond.create_wait_task(wait_callback); + series_of(task)->push_front(waiter); + fprintf(stderr, "task-%lu waiting\n", id); + } + else if (ret == 0) + { + auto *router = create_router_task(route_callback); + series_of(task)->push_front(router); + fprintf(stderr, "task-%lu routing\n", id); + } + else + { + fprintf(stderr, "task-%lu get [%s]\n", id, (*msg).c_str()); + } + }); + + SeriesWork *series = Workflow::create_series_work(timer, nullptr); + auto *worker = WFTaskFactory::create_go_task(work, + (std::string *)series->get_context(), + &wg); + series->push_back(worker); + series->start(); + } + + wg.wait(); +} + From f1cf95c379dfef74bf1bafbdeff267cd24bc4cc4 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Mon, 26 Jul 2021 18:44:31 +0800 Subject: [PATCH 37/75] remove WFSwitchWaitTask --- src/factory/WFChannel.inl | 8 +-- src/factory/WFCondTask.h | 3 ++ src/factory/WFCondTaskFactory.cc | 64 +++++------------------- src/factory/WFCondTaskFactory.h | 10 ++-- src/factory/WFResourcePool.cc | 21 ++++++-- src/factory/WFResourcePool.h | 35 ++++--------- src/manager/WFCondition.h | 1 + test/condition_unittest.cc | 85 +------------------------------- 8 files changed, 50 insertions(+), 177 deletions(-) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 89082d51c5..816bc43a4e 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -249,8 +249,8 @@ void WFComplexChannel::handle_terminated() this->sending = true; shutdown = true; } else { - waiter = WFCondTaskFactory::create_swait_task(&this->condition, - nullptr); + waiter = WFCondTaskFactory::create_wait_task(&this->condition, + nullptr); series_of(this)->push_front(this); series_of(this)->push_front(waiter); } @@ -353,7 +353,7 @@ void ComplexChannelOutTask::dispatch() } else { - waiter = WFCondTaskFactory::create_swait_task(&channel->condition, + waiter = WFCondTaskFactory::create_wait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); @@ -374,7 +374,7 @@ void ComplexChannelOutTask::dispatch() } else { - waiter = WFCondTaskFactory::create_swait_task(&channel->condition, + waiter = WFCondTaskFactory::create_wait_task(&channel->condition, [this](WFMailboxTask *task) { auto *channel = (WFComplexChannel *)this->get_request_channel(); diff --git a/src/factory/WFCondTask.h b/src/factory/WFCondTask.h index 06fc09144a..ee3d3e72de 100644 --- a/src/factory/WFCondTask.h +++ b/src/factory/WFCondTask.h @@ -34,6 +34,9 @@ class WFCondWaitTask : public WFMailboxTask public: virtual void clear_locked() { } +protected: + virtual SubTask *done(); + private: struct list_head list; diff --git a/src/factory/WFCondTaskFactory.cc b/src/factory/WFCondTaskFactory.cc index ac24c6d934..0fa983cfa3 100644 --- a/src/factory/WFCondTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -50,8 +50,6 @@ class __ConditionMap WFWaitTask *create(const std::string& name, const struct timespec *timeout, wait_callback_t&& cb); - WFWaitTask *create_switch(const std::string& name, - wait_callback_t&& cb); public: static __ConditionMap *get_instance() @@ -105,14 +103,6 @@ WFWaitTask *__ConditionMap::create(const std::string& name, std::move(cb)); } -WFWaitTask *__ConditionMap::create_switch(const std::string& name, - wait_callback_t&& cb) -{ - __WFCondition *cond = this->find_condition(name); - - return WFCondTaskFactory::create_swait_task(cond, std::move(cb)); -} - __ConditionMap::~__ConditionMap() { __WFCondition *cond; @@ -235,16 +225,20 @@ class __WFWaitTimerTask : public WFTimerTask virtual ~__WFWaitTimerTask(); }; -class WFSwitchWaitTask : public WFCondWaitTask +SubTask *WFCondWaitTask::done() { -public: - WFSwitchWaitTask(wait_callback_t&& cb) : - WFCondWaitTask(std::move(cb)) - { } + SeriesWork *series = series_of(this); -protected: - SubTask *done(); -}; + WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, + [this](WFTimerTask *task) { + if (this->callback) + this->callback(this); + delete this; + }); + series->push_front(switch_task); + + return series->pop(); +} void WFTimedWaitTask::clear_locked() { @@ -276,21 +270,6 @@ WFTimedWaitTask::~WFTimedWaitTask() delete this->timer; } -SubTask *WFSwitchWaitTask::done() -{ - SeriesWork *series = series_of(this); - - WFTimerTask *switch_task = WFTaskFactory::create_timer_task(0, - [this](WFTimerTask *task) { - if (this->callback) - this->callback(this); - delete this; - }); - series->push_front(switch_task); - - return series->pop(); -} - SubTask *__WFWaitTimerTask::done() { WFTimedWaitTask *waiter = NULL; @@ -339,13 +318,6 @@ WFWaitTask *WFCondTaskFactory::create_wait_task(const std::string& name, return __ConditionMap::get_instance()->create(name, std::move(callback)); } -WFWaitTask *WFCondTaskFactory::create_swait_task(const std::string& name, - wait_callback_t callback) -{ - return __ConditionMap::get_instance()->create_switch(name, - std::move(callback)); -} - WFWaitTask *WFCondTaskFactory::create_timedwait_task(const std::string& name, const struct timespec *timeout, wait_callback_t callback) @@ -383,15 +355,3 @@ WFWaitTask *WFCondTaskFactory::create_timedwait_task(WFCondition *cond, return waiter; } -WFWaitTask *WFCondTaskFactory::create_swait_task(WFCondition *cond, - wait_callback_t callback) -{ - WFSwitchWaitTask *task = new WFSwitchWaitTask(std::move(callback)); - - cond->mutex->lock(); - list_add_tail(&task->list, &cond->wait_list); - cond->mutex->unlock(); - - return task; -} - diff --git a/src/factory/WFCondTaskFactory.h b/src/factory/WFCondTaskFactory.h index 09e2c36399..8a4b376434 100644 --- a/src/factory/WFCondTaskFactory.h +++ b/src/factory/WFCondTaskFactory.h @@ -32,6 +32,10 @@ class WFCondTaskFactory { public: + // simple timedwait task +// static WFWaitTask *create_timedwait_task(const struct timespec *timeout, +// wait_callback_t callback); + // use condition by name static void signal_by_name(const std::string& name, void *msg); @@ -44,9 +48,6 @@ class WFCondTaskFactory const struct timespec *timeout, wait_callback_t callback); - static WFWaitTask *create_swait_task(const std::string& name, - wait_callback_t callback); - // use condition by ptr static WFWaitTask *create_wait_task(WFCondition *cond, wait_callback_t callback); @@ -55,9 +56,6 @@ class WFCondTaskFactory const struct timespec *timeout, wait_callback_t callback); - static WFWaitTask *create_swait_task(WFCondition *cond, - wait_callback_t callback); - }; #endif diff --git a/src/factory/WFResourcePool.cc b/src/factory/WFResourcePool.cc index 38deaf4aae..f52bb8101f 100644 --- a/src/factory/WFResourcePool.cc +++ b/src/factory/WFResourcePool.cc @@ -17,6 +17,7 @@ Xie Han (xiehan@sogou-inc.com) */ +#include #include "list.h" #include "WFTask.h" #include "WFResourcePool.h" @@ -32,9 +33,9 @@ class __WFConditional : public WFConditional virtual void signal(void *res) { } public: - __WFConditional(SubTask *task, void **pres, + __WFConditional(SubTask *task, void **resbuf, struct WFResourcePool::Data *data) : - WFConditional(task, pres) + WFConditional(task, resbuf) { this->data = data; } @@ -54,9 +55,19 @@ void __WFConditional::dispatch() this->WFConditional::dispatch(); } -WFConditional *WFResourcePool::get(SubTask *task, void **pres) +WFConditional *WFResourcePool::get(SubTask *task, void **resbuf) { - return new __WFConditional(task, pres, &this->data); + return new __WFConditional(task, resbuf, &this->data); +} + +WFResourcePool::WFResourcePool(void *const *res, size_t n) +{ + this->data.res = new void *[n]; + memcpy(this->data.res, res, n * sizeof (void *)); + this->data.value = n; + this->data.index = 0; + INIT_LIST_HEAD(&this->data.wait_list); + this->data.pool = this; } void WFResourcePool::post(void *res) @@ -73,7 +84,7 @@ void WFResourcePool::post(void *res) else { cond = NULL; - data->push(res); + this->push(res); } data->mutex.unlock(); diff --git a/src/factory/WFResourcePool.h b/src/factory/WFResourcePool.h index fa3e5996f4..38f44797a8 100644 --- a/src/factory/WFResourcePool.h +++ b/src/factory/WFResourcePool.h @@ -21,35 +21,27 @@ #define _WFRESOURCEPOOL_H_ #include -#include #include "list.h" #include "WFTask.h" class WFResourcePool { public: - WFConditional *get(SubTask *task, void **pres); + WFConditional *get(SubTask *task, void **resbuf); void post(void *res); public: struct Data { + void *pop() { return this->pool->pop(); } + void push(void *res) { this->pool->push(res); } + void **res; - std::atomic value; - std::atomic index; + long value; + size_t index; struct list_head wait_list; std::mutex mutex; - WFResourcePool *ptr; - - virtual void *pop() - { - return this->ptr->pop(); - } - - virtual void push(void *res) - { - ptr->push(res); - } + WFResourcePool *pool; }; private: @@ -67,17 +59,8 @@ class WFResourcePool struct Data data; public: - WFResourcePool(void **res, int n) - { - this->data.ptr = this; - this->data.res = new void *[n]; - memcpy(this->data.res, res, n * sizeof(void *)); - this->data.value = n; - this->data.index = 0; - INIT_LIST_HEAD(&this->data.wait_list); - } - - virtual ~WFResourcePool() { delete[] this->data.res; } + WFResourcePool(void *const *res, size_t n); + virtual ~WFResourcePool() { delete []this->data.res; } }; #endif diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h index 037d042d8d..dfb47085af 100644 --- a/src/manager/WFCondition.h +++ b/src/manager/WFCondition.h @@ -84,6 +84,7 @@ class WFCondition INIT_LIST_HEAD(&this->get_list); INIT_LIST_HEAD(&this->wait_list); } + virtual ~WFCondition(); }; diff --git a/test/condition_unittest.cc b/test/condition_unittest.cc index 4dff2bfc07..69d26a3d95 100644 --- a/test/condition_unittest.cc +++ b/test/condition_unittest.cc @@ -125,7 +125,7 @@ TEST(condition_unittest, resource_pool) int res_concurrency = 3; int task_concurrency = 10; const char *words[3] = {"workflow", "srpc", "pyworkflow"}; - WFResourcePool res_pool((void **)words, res_concurrency); + WFResourcePool res_pool((void *const *)words, res_concurrency); WFFacilities::WaitGroup wg(task_concurrency); for (int i = 0; i < task_concurrency; i++) @@ -148,86 +148,3 @@ TEST(condition_unittest, resource_pool) wg.wait(); } -class TestResource : public WFCondition::BaseResource -{ -public: - TestResource(std::string msg) : msg(msg) { } - void *get() { return &this->msg; } - void set(std::string&& msg) { this->msg = std::move(msg);} - -private: - std::string msg; -}; - -TestResource res(""); - -void wait_callback(WFWaitTask *task) -{ - void **msg; - size_t n; - msg = task->get_mailbox(&n); - std::string *str = *msg; - fprintf(stderr, "waiter get msg:%s\n", str->c_str()); - series->set_context(*msg); -} - -void work(std::string *msg, WFFacilities::WaitGroup &wg) -{ - fprintf(stderr, "working on message:%s\n", msg->c_str()); - wg.done(); -} - -SubTask *create_router_task(WFCondition *cond, std::function cb) -{ - auto *router = WFTaskFactory::create_timer_task(100000, std::move(cb)); - router->user_data = cond; -} - -void router_callback(WFTimerTask *task) -{ - fprintf(stderr, "finish routing and should broadcast every one else\n"); - - ((WFCondition *)(task->user_data))->broadcast(); -} - -TEST(condition_unittest, res_condition) -{ - int task_concurrency = 10; - WFFacilities::WaitGroup wg(task_concurrency); - WFCondition cond((WFCondition::BaseResource *)&res); - - for (int i = 0; i < task_concurrency; i++) - { - auto *timer = WFTaskFactory::create_timer_task(i * 100000, - [&cond](WFTimerTask *task) { - std::string *msg; - int ret = cond.get(&msg); - if (ret == -1) - { - auto *waiter = cond.create_wait_task(wait_callback); - series_of(task)->push_front(waiter); - fprintf(stderr, "task-%lu waiting\n", id); - } - else if (ret == 0) - { - auto *router = create_router_task(route_callback); - series_of(task)->push_front(router); - fprintf(stderr, "task-%lu routing\n", id); - } - else - { - fprintf(stderr, "task-%lu get [%s]\n", id, (*msg).c_str()); - } - }); - - SeriesWork *series = Workflow::create_series_work(timer, nullptr); - auto *worker = WFTaskFactory::create_go_task(work, - (std::string *)series->get_context(), - &wg); - series->push_back(worker); - series->start(); - } - - wg.wait(); -} - From 3c1026c9e9ace2b1e4d97a400d125e9675cbfd76 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Wed, 28 Jul 2021 01:49:30 +0800 Subject: [PATCH 38/75] update WFCondition with resource --- src/factory/WFCondTaskFactory.h | 4 -- src/manager/WFCondition.cc | 13 ++-- src/manager/WFCondition.h | 22 +++---- test/resource_unittest.cc | 108 ++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 23 deletions(-) diff --git a/src/factory/WFCondTaskFactory.h b/src/factory/WFCondTaskFactory.h index 8a4b376434..c6c0c95f92 100644 --- a/src/factory/WFCondTaskFactory.h +++ b/src/factory/WFCondTaskFactory.h @@ -32,10 +32,6 @@ class WFCondTaskFactory { public: - // simple timedwait task -// static WFWaitTask *create_timedwait_task(const struct timespec *timeout, -// wait_callback_t callback); - // use condition by name static void signal_by_name(const std::string& name, void *msg); diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index 44597c2313..64886ffbab 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -29,14 +29,15 @@ int WFCondition::get(void **pmsg) int ret; this->mutex->lock(); - if (this->empty == 1) + if (this->res->empty() == false) { *pmsg = this->res->get(); - ret = 1; + ret = 0; } - else if (--this->empty == 0) + else if (this->flag == false) { - ret = 0; + this->flag = true; + ret = 1; } else { @@ -49,7 +50,7 @@ int WFCondition::get(void **pmsg) return ret; } -WFWaitTask *WFCondition::get_wait_task(wait_callback_t callback) +WFWaitTask *WFCondition::create_wait_task(wait_callback_t callback) { WFCondWaitTask *task = NULL; struct list_head *pos; @@ -96,6 +97,8 @@ void WFCondition::broadcast(void *msg) LIST_HEAD(tmp_list); this->mutex->lock(); + this->flag = false; + if (!list_empty(&this->wait_list)) { list_for_each_safe(pos, tmp, &this->wait_list) diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h index dfb47085af..c38c5dad51 100644 --- a/src/manager/WFCondition.h +++ b/src/manager/WFCondition.h @@ -38,9 +38,10 @@ class WFCondition void broadcast(void *msg); // 1: existed; 0: should get; -1: should wait; int get(void **pmsg); - WFWaitTask *get_wait_task(wait_callback_t callback); + WFWaitTask *create_wait_task(wait_callback_t callback); public: + bool flag; std::atomic *ref; std::mutex *mutex; struct list_head get_list; @@ -50,25 +51,18 @@ class WFCondition class BaseResource { public: - virtual void *get() = 0; - virtual void set() { this->ptr->empty = 0; } - void clear() { this->ptr->empty = 1; } - - private: - WFCondition *ptr; - friend WFCondition; + virtual void *get() const = 0; + virtual bool empty() const = 0; }; private: - BaseResource *res; - int empty; + const BaseResource *res; public: - WFCondition(BaseResource *res) + WFCondition(const BaseResource *res) { - this->empty = 1; this->res = res; - this->res->ptr = this; + this->flag = false; this->mutex = new std::mutex; this->ref = new std::atomic(1); INIT_LIST_HEAD(&this->get_list); @@ -77,8 +71,8 @@ class WFCondition WFCondition() { - this->empty = 0; this->res = NULL; + this->flag = false; this->mutex = new std::mutex; this->ref = new std::atomic(1); INIT_LIST_HEAD(&this->get_list); diff --git a/test/resource_unittest.cc b/test/resource_unittest.cc index ed8e0d9205..5ef81750ae 100644 --- a/test/resource_unittest.cc +++ b/test/resource_unittest.cc @@ -146,3 +146,111 @@ TEST(condition_unittest, timedwait) wait_group.wait(); } +///////////// condition with resource ///////////// + +class TestResource : public WFCondition::BaseResource +{ +public: + TestResource() { this->res = NULL; } + void *get() const override { return (void *)this->res; } + bool empty() const override { return this->res == NULL; } + void set(const char *res) { this->res = res; } + +private: + const char *res; +}; + +struct series_ctx_t +{ + int id; + char *info; + TestResource *res; + WFCondition *cond; +}; + +char global_info[50] = "ResourceConditionInfo"; + +SubTask *create_router_task(std::function cb) +{ + return WFTaskFactory::create_timer_task(500000, std::move(cb)); +} + +void router_callback(WFTimerTask *task) +{ + struct series_ctx_t *ctx; + ctx = (struct series_ctx_t *)series_of(task)->get_context(); + ctx->res->set(global_info); + fprintf(stderr, "task-%d finish routing and broadcast.\n", ctx->id); + ctx->cond->broadcast(global_info); + ctx->info = global_info; +} + +void wait_callback(WFWaitTask *task) +{ + void **msg; + size_t n; + struct series_ctx_t *ctx; + ctx = (struct series_ctx_t *)series_of(task)->get_context(); + + msg = task->get_mailbox(&n); + EXPECT_EQ(n, 1); + fprintf(stderr, "task-%d waiter get msg:%s\n", ctx->id, (char *)*msg); + ctx->info = (char *)*msg; +} + +void work(struct series_ctx_t *ctx, WFFacilities::WaitGroup *wg) +{ + EXPECT_TRUE(strcmp(ctx->info, global_info) == 0); + wg->done(); +} + +#define task_concurrency 10 + +TEST(condition_unittest, res_condition) +{ + WFFacilities::WaitGroup wg(task_concurrency); + TestResource res; + WFCondition cond((WFCondition::BaseResource *)&res); + struct series_ctx_t ctx[task_concurrency]; + + for (int i = 0; i < task_concurrency; i++) + { + auto *timer = WFTaskFactory::create_timer_task(i * 100000, + [](WFTimerTask *task) { + struct series_ctx_t *ctx; + ctx = (struct series_ctx_t *)series_of(task)->get_context(); + char *info; + int ret = ctx->cond->get((void **)&info); + + if (ret == -1) + { + auto *waiter = ctx->cond->create_wait_task(wait_callback); + series_of(task)->push_front(waiter); + fprintf(stderr, "task-%d waiting\n", ctx->id); + } + else if (ret == 1) + { + auto *router = create_router_task(router_callback); + series_of(task)->push_front(router); + fprintf(stderr, "task-%d routing\n", ctx->id); + } + else + { + fprintf(stderr, "task-%d get [%s]\n", ctx->id, info); + ctx->info = info; + } + }); + + SeriesWork *series = Workflow::create_series_work(timer, nullptr); + auto *worker = WFTaskFactory::create_go_task("w", work, &ctx[i], &wg); + ctx[i].id = i; + ctx[i].cond = &cond; + ctx[i].res = &res; + series->set_context(&ctx[i]); + series->push_back(worker); + series->start(); + } + + wg.wait(); +} + From c5a5f09e4cbacf060f1dbdb89c91f7a10debf853 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Tue, 3 Aug 2021 21:13:32 +0800 Subject: [PATCH 39/75] add WFChannelFactory api for self-define protocol channel; simplify WFCondition; --- src/factory/WFChannel.h | 16 ++++++ src/factory/WFChannel.inl | 19 +++++++ src/manager/WFCondition.cc | 65 +--------------------- src/manager/WFCondition.h | 40 ++------------ test/resource_unittest.cc | 108 ------------------------------------- 5 files changed, 41 insertions(+), 207 deletions(-) diff --git a/src/factory/WFChannel.h b/src/factory/WFChannel.h index c54e834e07..9af9b39ba2 100644 --- a/src/factory/WFChannel.h +++ b/src/factory/WFChannel.h @@ -91,6 +91,22 @@ using WFWebSocketTask = WFChannelTask; using websocket_callback_t = std::function; using websocket_process_t = std::function; +template +class WFComplexChannel; + +template +class WFChannelFactory +{ +private: + using CH = WFComplexChannel; + using T = WFChannelTask; + +public: + static CH *create_channel(std::function process); + static T *create_out_task(CH *channel, + std::function callback); +}; + #include "WFChannel.inl" #endif diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 816bc43a4e..2735b6d08d 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -487,3 +487,22 @@ public: } }; +/**********WFChannelFactory impl**********/ + +template +WFComplexChannel * +WFChannelFactory::create_channel(std::function *)> process) +{ + return new WFComplexChannel(NULL, WFGlobal::get_scheduler(), + std::move(process)); +} + +template +WFChannelTask * +WFChannelFactory::create_out_task(WFComplexChannel *channel, + std::function *)> cb) +{ + return new ComplexChannelOutTask(channel, WFGlobal::get_scheduler(), + std::move(cb)); +} + diff --git a/src/manager/WFCondition.cc b/src/manager/WFCondition.cc index 64886ffbab..6fa3333ba7 100644 --- a/src/manager/WFCondition.cc +++ b/src/manager/WFCondition.cc @@ -22,65 +22,14 @@ #include "WFCondTask.h" #include "WFCondition.h" -int WFCondition::get(void **pmsg) -{ -// if (this->res == NULL) -// return -2; - int ret; - - this->mutex->lock(); - if (this->res->empty() == false) - { - *pmsg = this->res->get(); - ret = 0; - } - else if (this->flag == false) - { - this->flag = true; - ret = 1; - } - else - { - WFCondWaitTask *task = new WFCondWaitTask(nullptr); - list_add_tail(&task->list, &this->get_list); - ret = -1; - } - this->mutex->unlock(); - - return ret; -} - -WFWaitTask *WFCondition::create_wait_task(wait_callback_t callback) -{ - WFCondWaitTask *task = NULL; - struct list_head *pos; - - this->mutex->lock(); - if (!list_empty(&this->get_list)) - { - pos = this->get_list.next; - list_move_tail(pos, &this->wait_list); - task = list_entry(pos, WFCondWaitTask, list); - task->set_callback(std::move(callback)); - } - - this->mutex->unlock(); - return task; -} - void WFCondition::signal(void *msg) { WFCondWaitTask *task = NULL; this->mutex->lock(); - if (!list_empty(&this->wait_list)) - task = list_entry(this->wait_list.next, WFCondWaitTask, list); - else if (!list_empty(&this->get_list)) - task = list_entry(this->get_list.next, WFCondWaitTask, list); - - if (task) { + task = list_entry(this->wait_list.next, WFCondWaitTask, list); list_del(&task->list); task->clear_locked(); } @@ -97,8 +46,6 @@ void WFCondition::broadcast(void *msg) LIST_HEAD(tmp_list); this->mutex->lock(); - this->flag = false; - if (!list_empty(&this->wait_list)) { list_for_each_safe(pos, tmp, &this->wait_list) @@ -109,16 +56,6 @@ void WFCondition::broadcast(void *msg) } } - if (!list_empty(&this->get_list)) - { - list_for_each_safe(pos, tmp, &this->get_list) - { - list_move_tail(pos, &tmp_list); - task = list_entry(pos, WFCondWaitTask, list); - task->clear_locked(); - } - } - this->mutex->unlock(); while (!list_empty(&tmp_list)) { diff --git a/src/manager/WFCondition.h b/src/manager/WFCondition.h index c38c5dad51..c972557038 100644 --- a/src/manager/WFCondition.h +++ b/src/manager/WFCondition.h @@ -36,50 +36,20 @@ class WFCondition public: void signal(void *msg); void broadcast(void *msg); - // 1: existed; 0: should get; -1: should wait; - int get(void **pmsg); - WFWaitTask *create_wait_task(wait_callback_t callback); public: - bool flag; - std::atomic *ref; - std::mutex *mutex; - struct list_head get_list; - struct list_head wait_list; - -public: - class BaseResource - { - public: - virtual void *get() const = 0; - virtual bool empty() const = 0; - }; - -private: - const BaseResource *res; - -public: - WFCondition(const BaseResource *res) - { - this->res = res; - this->flag = false; - this->mutex = new std::mutex; - this->ref = new std::atomic(1); - INIT_LIST_HEAD(&this->get_list); - INIT_LIST_HEAD(&this->wait_list); - } - WFCondition() { - this->res = NULL; - this->flag = false; this->mutex = new std::mutex; this->ref = new std::atomic(1); - INIT_LIST_HEAD(&this->get_list); INIT_LIST_HEAD(&this->wait_list); } - virtual ~WFCondition(); + +public: + std::atomic *ref; + std::mutex *mutex; + struct list_head wait_list; }; #endif diff --git a/test/resource_unittest.cc b/test/resource_unittest.cc index 5ef81750ae..ed8e0d9205 100644 --- a/test/resource_unittest.cc +++ b/test/resource_unittest.cc @@ -146,111 +146,3 @@ TEST(condition_unittest, timedwait) wait_group.wait(); } -///////////// condition with resource ///////////// - -class TestResource : public WFCondition::BaseResource -{ -public: - TestResource() { this->res = NULL; } - void *get() const override { return (void *)this->res; } - bool empty() const override { return this->res == NULL; } - void set(const char *res) { this->res = res; } - -private: - const char *res; -}; - -struct series_ctx_t -{ - int id; - char *info; - TestResource *res; - WFCondition *cond; -}; - -char global_info[50] = "ResourceConditionInfo"; - -SubTask *create_router_task(std::function cb) -{ - return WFTaskFactory::create_timer_task(500000, std::move(cb)); -} - -void router_callback(WFTimerTask *task) -{ - struct series_ctx_t *ctx; - ctx = (struct series_ctx_t *)series_of(task)->get_context(); - ctx->res->set(global_info); - fprintf(stderr, "task-%d finish routing and broadcast.\n", ctx->id); - ctx->cond->broadcast(global_info); - ctx->info = global_info; -} - -void wait_callback(WFWaitTask *task) -{ - void **msg; - size_t n; - struct series_ctx_t *ctx; - ctx = (struct series_ctx_t *)series_of(task)->get_context(); - - msg = task->get_mailbox(&n); - EXPECT_EQ(n, 1); - fprintf(stderr, "task-%d waiter get msg:%s\n", ctx->id, (char *)*msg); - ctx->info = (char *)*msg; -} - -void work(struct series_ctx_t *ctx, WFFacilities::WaitGroup *wg) -{ - EXPECT_TRUE(strcmp(ctx->info, global_info) == 0); - wg->done(); -} - -#define task_concurrency 10 - -TEST(condition_unittest, res_condition) -{ - WFFacilities::WaitGroup wg(task_concurrency); - TestResource res; - WFCondition cond((WFCondition::BaseResource *)&res); - struct series_ctx_t ctx[task_concurrency]; - - for (int i = 0; i < task_concurrency; i++) - { - auto *timer = WFTaskFactory::create_timer_task(i * 100000, - [](WFTimerTask *task) { - struct series_ctx_t *ctx; - ctx = (struct series_ctx_t *)series_of(task)->get_context(); - char *info; - int ret = ctx->cond->get((void **)&info); - - if (ret == -1) - { - auto *waiter = ctx->cond->create_wait_task(wait_callback); - series_of(task)->push_front(waiter); - fprintf(stderr, "task-%d waiting\n", ctx->id); - } - else if (ret == 1) - { - auto *router = create_router_task(router_callback); - series_of(task)->push_front(router); - fprintf(stderr, "task-%d routing\n", ctx->id); - } - else - { - fprintf(stderr, "task-%d get [%s]\n", ctx->id, info); - ctx->info = info; - } - }); - - SeriesWork *series = Workflow::create_series_work(timer, nullptr); - auto *worker = WFTaskFactory::create_go_task("w", work, &ctx[i], &wg); - ctx[i].id = i; - ctx[i].cond = &cond; - ctx[i].res = &res; - series->set_context(&ctx[i]); - series->push_back(worker); - series->start(); - } - - wg.wait(); -} - From ea11a674fcfb68ca4c02d55596ace6a74b4bfc6b Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Sun, 31 Oct 2021 13:33:55 +0800 Subject: [PATCH 40/75] fix set_request_uri for websocket --- src/factory/WebSocketTaskImpl.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc index 4b278ddb23..4ff1ba076e 100644 --- a/src/factory/WebSocketTaskImpl.cc +++ b/src/factory/WebSocketTaskImpl.cc @@ -139,6 +139,8 @@ SubTask *ComplexWebSocketOutTask::done() SubTask *ComplexWebSocketOutTask::upgrade() { auto *channel = (ComplexWebSocketChannel *)this->get_request_channel(); + const ParsedURI *uri = channel->get_uri(); + std::string request_uri; auto *http_task = new WFChannelOutTask(this->channel, WFGlobal::get_scheduler(), @@ -152,10 +154,16 @@ SubTask *ComplexWebSocketOutTask::upgrade() this->ready = true; }); + + if (uri->path && uri->path[0]) + request_uri = uri->path; + else + request_uri = "/"; + HttpRequest *req = http_task->get_msg(); req->set_method(HttpMethodGet); req->set_http_version("HTTP/1.1"); - req->set_request_uri("/"); + req->set_request_uri(request_uri); req->add_header_pair("Host", channel->get_uri()->host); req->add_header_pair("Upgrade", "websocket"); req->add_header_pair("Connection", "Upgrade"); From 8d0274ed972203884b5442dc9e9fb8ca776a131c Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Sun, 31 Oct 2021 16:08:19 +0800 Subject: [PATCH 41/75] make channel as a member of WFWebsocketClient --- src/client/WFWebSocketClient.h | 31 +++++++++++++++---------------- src/factory/WFChannel.inl | 4 ++-- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h index 0cc778d543..3598514448 100644 --- a/src/client/WFWebSocketClient.h +++ b/src/client/WFWebSocketClient.h @@ -52,7 +52,7 @@ class WebSocketClient WFWebSocketTask *create_close_task(websocket_callback_t cb); private: - ComplexWebSocketChannel *channel; + ComplexWebSocketChannel channel; struct WFWebSocketParams params; public: @@ -65,7 +65,7 @@ class WebSocketClient inline WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) { - return new ComplexWebSocketOutTask(this->channel, WFGlobal::get_scheduler(), + return new ComplexWebSocketOutTask(&this->channel, WFGlobal::get_scheduler(), std::move(cb)); } @@ -75,14 +75,14 @@ inline int WebSocketClient::init(const std::string& url) if (URIParser::parse(url, uri) != 0) return -1; - this->channel->set_uri(uri); + this->channel.set_uri(uri); return 0; } inline WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t cb) { ComplexWebSocketOutTask *ping_task; - ping_task = new ComplexWebSocketOutTask(this->channel, + ping_task = new ComplexWebSocketOutTask(&this->channel, WFGlobal::get_scheduler(), std::move(cb)); @@ -95,7 +95,7 @@ inline WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t c inline WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) { ComplexWebSocketOutTask *close_task; - close_task = new ComplexWebSocketOutTask(this->channel, + close_task = new ComplexWebSocketOutTask(&this->channel, WFGlobal::get_scheduler(), std::move(cb)); @@ -106,23 +106,22 @@ inline WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t } WebSocketClient::WebSocketClient(const struct WFWebSocketParams *params, - websocket_process_t process) + websocket_process_t process) : + channel(NULL, WFGlobal::get_scheduler(), std::move(process)) { this->params = *params; - this->channel = new ComplexWebSocketChannel(NULL, WFGlobal::get_scheduler(), - std::move(process)); - this->channel->set_idle_timeout(this->params.idle_timeout); - this->channel->set_size_limit(this->params.size_limit); + this->channel.set_idle_timeout(this->params.idle_timeout); + this->channel.set_size_limit(this->params.size_limit); - this->channel->set_callback([this](WFChannel *channel) + this->channel.set_callback([this](WFChannel *channel) { - pthread_mutex_lock(&this->channel->mutex); - if (this->channel->is_established() == 0) + pthread_mutex_lock(&this->channel.mutex); + if (this->channel.is_established() == 0) { - this->channel->set_state(WFT_STATE_SYS_ERROR); - this->channel->set_sending(false); + this->channel.set_state(WFT_STATE_SYS_ERROR); + this->channel.set_sending(false); } - pthread_mutex_unlock(&this->channel->mutex); + pthread_mutex_unlock(&this->channel.mutex); }); } diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 2735b6d08d..eefdc05d24 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -229,8 +229,8 @@ SubTask *WFComplexChannel::done() if (this->state == WFT_STATE_SUCCESS) this->state = WFT_STATE_UNDEFINED; - if (this->established == 0 && this->state == WFT_STATE_SYS_ERROR) // sending == false - delete this; +// if (this->established == 0 && this->state == WFT_STATE_SYS_ERROR) // sending == false +// delete this; return series->pop(); } From 7915b5bf1d02ce34c612fe0db2b486140d8abcd1 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Sun, 31 Oct 2021 16:27:45 +0800 Subject: [PATCH 42/75] update channel state after close --- src/factory/WFChannel.inl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index eefdc05d24..4408b78aa3 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -229,7 +229,8 @@ SubTask *WFComplexChannel::done() if (this->state == WFT_STATE_SUCCESS) this->state = WFT_STATE_UNDEFINED; -// if (this->established == 0 && this->state == WFT_STATE_SYS_ERROR) // sending == false + if (this->established == 0 && this->state == WFT_STATE_SYS_ERROR) // sending == false + this->state = WFT_STATE_UNDEFINED; // delete this; return series->pop(); From bd19d50d43622454f9b18b737693d1a2804067fe Mon Sep 17 00:00:00 2001 From: holmesyxli Date: Mon, 1 Nov 2021 20:27:46 +0800 Subject: [PATCH 43/75] add WFWebSocketClient::deinit(), waiting for channel close --- src/client/CMakeLists.txt | 1 + src/client/WFWebSocketClient.cc | 106 ++++++++++++++++++++++++++ src/client/WFWebSocketClient.h | 67 +--------------- src/factory/WFChannel.inl | 9 +-- src/factory/WebSocketTaskImpl.cc | 14 ++-- tutorial/tutorial-14-websocket_cli.cc | 1 + 6 files changed, 122 insertions(+), 76 deletions(-) create mode 100644 src/client/WFWebSocketClient.cc diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index ba54d7bc46..23b77cbd46 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -3,6 +3,7 @@ project(client) set(SRC WFDnsClient.cc + WFWebsocketClient.cc ) if (NOT MYSQL STREQUAL "n") diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc new file mode 100644 index 0000000000..526363d855 --- /dev/null +++ b/src/client/WFWebSocketClient.cc @@ -0,0 +1,106 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include "WFChannel.h" +#include "HttpMessage.h" +#include "WebSocketMessage.h" +#include "WFWebSocketClient.h" + +WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) +{ + return new ComplexWebSocketOutTask(&this->channel, WFGlobal::get_scheduler(), + std::move(cb)); +} + +int WebSocketClient::init(const std::string& url) +{ + ParsedURI uri; + + if (pthread_cond_init(&this->shutdown_cond, NULL) == 0) + { + if (URIParser::parse(url, uri) == 0) + { + this->channel.set_uri(uri); + return 0; + } + + pthread_cond_destroy(&this->shutdown_cond); + } + + return -1; +} + +void WebSocketClient::deinit() +{ + pthread_mutex_lock(&this->channel.mutex); + + while (this->channel.get_sending()) + pthread_cond_wait(&this->shutdown_cond, &this->channel.mutex); + + pthread_mutex_unlock(&this->channel.mutex); + pthread_cond_destroy(&this->shutdown_cond); +} + +WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t cb) +{ + ComplexWebSocketOutTask *ping_task; + ping_task = new ComplexWebSocketOutTask(&this->channel, + WFGlobal::get_scheduler(), + std::move(cb)); + + protocol::WebSocketFrame *msg = ping_task->get_msg(); + msg->set_opcode(WebSocketFramePing); + + return ping_task; +} + +WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) +{ + ComplexWebSocketOutTask *close_task; + close_task = new ComplexWebSocketOutTask(&this->channel, + WFGlobal::get_scheduler(), + std::move(cb)); + + protocol::WebSocketFrame *msg = close_task->get_msg(); + msg->set_opcode(WebSocketFrameConnectionClose); + + return close_task; +} + +WebSocketClient::WebSocketClient(const struct WFWebSocketParams *params, + websocket_process_t process) : + channel(NULL, WFGlobal::get_scheduler(), std::move(process)) +{ + this->params = *params; + this->channel.set_idle_timeout(this->params.idle_timeout); + this->channel.set_size_limit(this->params.size_limit); + + this->channel.set_callback([this](WFChannel *channel) + { + pthread_mutex_lock(&this->channel.mutex); + + if (this->channel.is_established() == 0) + this->channel.set_sending(false); + + this->channel.condition.signal(NULL); + pthread_cond_signal(&this->shutdown_cond); + + pthread_mutex_unlock(&this->channel.mutex); + }); +} + diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h index 3598514448..3d29c32fc5 100644 --- a/src/client/WFWebSocketClient.h +++ b/src/client/WFWebSocketClient.h @@ -20,10 +20,9 @@ #define _WFWEBSOCKETCLIENT_H_ #include +#include #include -#include "WFGlobal.h" #include "HttpUtil.h" -#include "HttpMessage.h" #include "WFChannel.h" #include "WebSocketMessage.h" @@ -50,10 +49,12 @@ class WebSocketClient WFWebSocketTask *create_websocket_task(websocket_callback_t cb); WFWebSocketTask *create_ping_task(websocket_callback_t cb); WFWebSocketTask *create_close_task(websocket_callback_t cb); + void deinit(); private: ComplexWebSocketChannel channel; struct WFWebSocketParams params; + pthread_cond_t shutdown_cond; public: WebSocketClient(const struct WFWebSocketParams *params, @@ -63,67 +64,5 @@ class WebSocketClient { } }; -inline WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) -{ - return new ComplexWebSocketOutTask(&this->channel, WFGlobal::get_scheduler(), - std::move(cb)); -} - -inline int WebSocketClient::init(const std::string& url) -{ - ParsedURI uri; - if (URIParser::parse(url, uri) != 0) - return -1; - - this->channel.set_uri(uri); - return 0; -} - -inline WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t cb) -{ - ComplexWebSocketOutTask *ping_task; - ping_task = new ComplexWebSocketOutTask(&this->channel, - WFGlobal::get_scheduler(), - std::move(cb)); - - protocol::WebSocketFrame *msg = ping_task->get_msg(); - msg->set_opcode(WebSocketFramePing); - - return ping_task; -} - -inline WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) -{ - ComplexWebSocketOutTask *close_task; - close_task = new ComplexWebSocketOutTask(&this->channel, - WFGlobal::get_scheduler(), - std::move(cb)); - - protocol::WebSocketFrame *msg = close_task->get_msg(); - msg->set_opcode(WebSocketFrameConnectionClose); - - return close_task; -} - -WebSocketClient::WebSocketClient(const struct WFWebSocketParams *params, - websocket_process_t process) : - channel(NULL, WFGlobal::get_scheduler(), std::move(process)) -{ - this->params = *params; - this->channel.set_idle_timeout(this->params.idle_timeout); - this->channel.set_size_limit(this->params.size_limit); - - this->channel.set_callback([this](WFChannel *channel) - { - pthread_mutex_lock(&this->channel.mutex); - if (this->channel.is_established() == 0) - { - this->channel.set_state(WFT_STATE_SYS_ERROR); - this->channel.set_sending(false); - } - pthread_mutex_unlock(&this->channel.mutex); - }); -} - #endif diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 4408b78aa3..569445afd9 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -226,12 +226,11 @@ SubTask *WFComplexChannel::done() if (this->callback) this->callback(this); - if (this->state == WFT_STATE_SUCCESS) - this->state = WFT_STATE_UNDEFINED; - - if (this->established == 0 && this->state == WFT_STATE_SYS_ERROR) // sending == false + if (this->state == WFT_STATE_SUCCESS || // after upgrade + this->established == 0) // after close + { this->state = WFT_STATE_UNDEFINED; -// delete this; + } return series->pop(); } diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc index 4ff1ba076e..00087e77f5 100644 --- a/src/factory/WebSocketTaskImpl.cc +++ b/src/factory/WebSocketTaskImpl.cc @@ -123,15 +123,15 @@ SubTask *ComplexWebSocketOutTask::done() this->get_state() == WFT_STATE_SUCCESS && channel->is_established()) { - series->push_front(this); series->push_front(channel); - return series->pop(); } - - pthread_mutex_lock(&channel->mutex); - channel->set_sending(false); - channel->condition.signal(NULL); - pthread_mutex_unlock(&channel->mutex); + else + { + pthread_mutex_lock(&channel->mutex); + channel->set_sending(false); + channel->condition.signal(NULL); + pthread_mutex_unlock(&channel->mutex); + } return WFChannelOutTask::done(); } diff --git a/tutorial/tutorial-14-websocket_cli.cc b/tutorial/tutorial-14-websocket_cli.cc index ba3bee7204..adfa11884f 100644 --- a/tutorial/tutorial-14-websocket_cli.cc +++ b/tutorial/tutorial-14-websocket_cli.cc @@ -81,6 +81,7 @@ int main(int argc, char *argv[]) task->start(); wg.wait(); + client.deinit(); return 0; } From 95cd70fa9db332295fcd08f3484daa4a287887ab Mon Sep 17 00:00:00 2001 From: holmesyxli Date: Mon, 1 Nov 2021 20:56:21 +0800 Subject: [PATCH 44/75] update doc and fix cmake --- docs/tutorial-14-websocket_cli.md | 7 ++++--- src/client/CMakeLists.txt | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md index 080ba20c97..e701e76a1a 100644 --- a/docs/tutorial-14-websocket_cli.md +++ b/docs/tutorial-14-websocket_cli.md @@ -85,7 +85,7 @@ public: #### 2. masking_key -**WebSocket**协议的文本和二进制数据都需要经过一个掩码加密，用户可以不填，也可以通过``void set_masking_key(uint32_t masking_key)``手动指定； +**WebSocket**协议的文本和二进制数据都需要经过一个掩码加密，可以通过``void set_masking_key(uint32_t masking_key)``手动指定，框架自动随机生成的功能正在开发中，很快可以投入使用。 #### 3. data @@ -165,9 +165,10 @@ task->start(); wait_group.wait(); ``` -这里发起了一个close任务，由于close是异步的，因此在``task->start()``之后当前线程会退出，我们在当前线程结合一个了``wait_group``进行不占线程的阻塞，并在close任务的回调函数里唤醒，然后当前线程就可以安全删除client实例和退出了。 +这里发起了一个close任务，由于close是异步的，因此在``task->start()``之后当前线程会退出，我们在当前线程结合一个了``wait_group``进行不占线程的阻塞，并在close任务的回调函数里唤醒，然后当前线程就可以安全调用``client.deinit()``、删除client实例以及退出了。 -需要注意的是，如果不主动发起close任务，直接删除client实例，那么底层使用的那个网络连接还会存在，直到超时或其他原因断开。 +需要注意的是，如果不主动发起close任务，直接删除client实例，那么底层使用的那个网络连接还会存在，直到超时或其他原因断开； +而``client.deinit()``是个等待内部网络资源完全释放的同步接口，需要手动调用，以保证程序退出前client的所有资源安全释放。 # websocket_cli的参数 diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 23b77cbd46..1a4b93655b 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -3,7 +3,7 @@ project(client) set(SRC WFDnsClient.cc - WFWebsocketClient.cc + WFWebSocketClient.cc ) if (NOT MYSQL STREQUAL "n") From df4a75ea7bf42555c4ffb3aee0fa8a16298a9efd Mon Sep 17 00:00:00 2001 From: holmesyxli Date: Tue, 2 Nov 2021 23:30:54 +0800 Subject: [PATCH 45/75] delete channel in WFWebSocketClient::deinit() --- src/client/WFWebSocketClient.cc | 81 +++++++++++++++++++------------- src/client/WFWebSocketClient.h | 6 +-- src/factory/WFChannel.inl | 8 ++-- src/factory/WebSocketTaskImpl.cc | 20 ++------ 4 files changed, 59 insertions(+), 56 deletions(-) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index 526363d855..53b747bb6e 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -16,50 +16,80 @@ Author: Li Yingxin (liyingxin@sogou-inc.com) */ -#include "WFChannel.h" #include "HttpMessage.h" +#include "WFFacilities.h" +#include "WFChannel.h" #include "WebSocketMessage.h" #include "WFWebSocketClient.h" +struct websocket_series_ctx_t +{ + WebSocketClient *client; + WFFacilities::WaitGroup *wait_group; +}; + WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) { - return new ComplexWebSocketOutTask(&this->channel, WFGlobal::get_scheduler(), + return new ComplexWebSocketOutTask(this->channel, + WFGlobal::get_scheduler(), std::move(cb)); } int WebSocketClient::init(const std::string& url) { ParsedURI uri; + if (URIParser::parse(url, uri) != 0) + return -1; + + this->channel = new ComplexWebSocketChannel(NULL, + WFGlobal::get_scheduler(), + std::move(process)); + this->channel->set_idle_timeout(this->params.idle_timeout); + this->channel->set_size_limit(this->params.size_limit); + this->channel->set_uri(uri); - if (pthread_cond_init(&this->shutdown_cond, NULL) == 0) + this->channel->set_callback([this](WFChannel *channel) { - if (URIParser::parse(url, uri) == 0) + pthread_mutex_lock(&this->channel->mutex); + if (this->channel->is_established() == 0) { - this->channel.set_uri(uri); - return 0; + this->channel->set_state(WFT_STATE_SYS_ERROR); + this->channel->set_sending(false); } + pthread_mutex_unlock(&this->channel->mutex); + }); - pthread_cond_destroy(&this->shutdown_cond); - } - - return -1; + return 0; } void WebSocketClient::deinit() { - pthread_mutex_lock(&this->channel.mutex); + WFFacilities::WaitGroup wait_group(1); - while (this->channel.get_sending()) - pthread_cond_wait(&this->shutdown_cond, &this->channel.mutex); + struct websocket_series_ctx_t ctx = { + .client = this, + .wait_group = &wait_group, + }; - pthread_mutex_unlock(&this->channel.mutex); - pthread_cond_destroy(&this->shutdown_cond); + SeriesWork *series = Workflow::create_series_work(this->channel, + [](const SeriesWork *series) + { + struct websocket_series_ctx_t *ctx; + ctx = (struct websocket_series_ctx_t *)series->get_context(); + delete ctx->client->channel; + ctx->wait_group->done(); + }); + + series->set_context(&ctx); + series->start(); + + wait_group.wait(); } WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t cb) { ComplexWebSocketOutTask *ping_task; - ping_task = new ComplexWebSocketOutTask(&this->channel, + ping_task = new ComplexWebSocketOutTask(this->channel, WFGlobal::get_scheduler(), std::move(cb)); @@ -72,7 +102,7 @@ WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t cb) WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) { ComplexWebSocketOutTask *close_task; - close_task = new ComplexWebSocketOutTask(&this->channel, + close_task = new ComplexWebSocketOutTask(this->channel, WFGlobal::get_scheduler(), std::move(cb)); @@ -84,23 +114,8 @@ WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) WebSocketClient::WebSocketClient(const struct WFWebSocketParams *params, websocket_process_t process) : - channel(NULL, WFGlobal::get_scheduler(), std::move(process)) + process(std::move(process)) { this->params = *params; - this->channel.set_idle_timeout(this->params.idle_timeout); - this->channel.set_size_limit(this->params.size_limit); - - this->channel.set_callback([this](WFChannel *channel) - { - pthread_mutex_lock(&this->channel.mutex); - - if (this->channel.is_established() == 0) - this->channel.set_sending(false); - - this->channel.condition.signal(NULL); - pthread_cond_signal(&this->shutdown_cond); - - pthread_mutex_unlock(&this->channel.mutex); - }); } diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h index 3d29c32fc5..e04135726c 100644 --- a/src/client/WFWebSocketClient.h +++ b/src/client/WFWebSocketClient.h @@ -20,9 +20,9 @@ #define _WFWEBSOCKETCLIENT_H_ #include -#include #include #include "HttpUtil.h" +#include "HttpMessage.h" #include "WFChannel.h" #include "WebSocketMessage.h" @@ -52,9 +52,9 @@ class WebSocketClient void deinit(); private: - ComplexWebSocketChannel channel; + ComplexWebSocketChannel *channel; struct WFWebSocketParams params; - pthread_cond_t shutdown_cond; + websocket_process_t process; public: WebSocketClient(const struct WFWebSocketParams *params, diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 569445afd9..eefdc05d24 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -226,11 +226,11 @@ SubTask *WFComplexChannel::done() if (this->callback) this->callback(this); - if (this->state == WFT_STATE_SUCCESS || // after upgrade - this->established == 0) // after close - { + if (this->state == WFT_STATE_SUCCESS) this->state = WFT_STATE_UNDEFINED; - } + +// if (this->established == 0 && this->state == WFT_STATE_SYS_ERROR) // sending == false +// delete this; return series->pop(); } diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc index 00087e77f5..72b4f231d5 100644 --- a/src/factory/WebSocketTaskImpl.cc +++ b/src/factory/WebSocketTaskImpl.cc @@ -102,7 +102,6 @@ SubTask *ComplexWebSocketInTask::done() SubTask *ComplexWebSocketOutTask::done() { - SeriesWork *series = series_of(this); auto *channel = (ComplexWebSocketChannel *)this->get_request_channel(); if (channel->get_state() == WFT_STATE_UNDEFINED || @@ -117,21 +116,10 @@ SubTask *ComplexWebSocketOutTask::done() this->error = channel->get_error(); } - const websocket_parser_t *parser = this->get_msg()->get_parser(); - - if (parser->opcode == WebSocketFrameConnectionClose && - this->get_state() == WFT_STATE_SUCCESS && - channel->is_established()) - { - series->push_front(channel); - } - else - { - pthread_mutex_lock(&channel->mutex); - channel->set_sending(false); - channel->condition.signal(NULL); - pthread_mutex_unlock(&channel->mutex); - } + pthread_mutex_lock(&channel->mutex); + channel->set_sending(false); + channel->condition.signal(NULL); + pthread_mutex_unlock(&channel->mutex); return WFChannelOutTask::done(); } From a2e0a9c17437279de74c6d5c575a7e76d77581d1 Mon Sep 17 00:00:00 2001 From: holmesyxli Date: Fri, 5 Nov 2021 20:13:05 +0800 Subject: [PATCH 46/75] update WFWebSocketClient deinit --- src/client/WFWebSocketClient.cc | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index 53b747bb6e..288a315896 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -17,17 +17,10 @@ */ #include "HttpMessage.h" -#include "WFFacilities.h" #include "WFChannel.h" #include "WebSocketMessage.h" #include "WFWebSocketClient.h" -struct websocket_series_ctx_t -{ - WebSocketClient *client; - WFFacilities::WaitGroup *wait_group; -}; - WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) { return new ComplexWebSocketOutTask(this->channel, @@ -38,7 +31,7 @@ WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) int WebSocketClient::init(const std::string& url) { ParsedURI uri; - if (URIParser::parse(url, uri) != 0) + if (URIParser::parse(url, uri) < 0) return -1; this->channel = new ComplexWebSocketChannel(NULL, @@ -64,26 +57,16 @@ int WebSocketClient::init(const std::string& url) void WebSocketClient::deinit() { - WFFacilities::WaitGroup wait_group(1); - - struct websocket_series_ctx_t ctx = { - .client = this, - .wait_group = &wait_group, - }; - - SeriesWork *series = Workflow::create_series_work(this->channel, - [](const SeriesWork *series) - { - struct websocket_series_ctx_t *ctx; - ctx = (struct websocket_series_ctx_t *)series->get_context(); - delete ctx->client->channel; - ctx->wait_group->done(); + SeriesWork *series; + series = Workflow::create_series_work(this->channel, + [](const SeriesWork *series){ + ComplexWebSocketChannel *channel; + channel = (ComplexWebSocketChannel *)series->get_context(); + delete channel; }); - series->set_context(&ctx); + series->set_context(this->channel); series->start(); - - wait_group.wait(); } WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t cb) From 9ab2dcc83601bbcba9dbbb602c4671dc24828eb4 Mon Sep 17 00:00:00 2001 From: holmesyxli Date: Sun, 7 Nov 2021 20:08:13 +0800 Subject: [PATCH 47/75] support random masking_key --- src/client/WFWebSocketClient.cc | 12 +++++++++--- src/client/WFWebSocketClient.h | 2 +- src/factory/WFChannel.inl | 21 ++++++++++++++++++--- src/factory/WebSocketTaskImpl.cc | 2 ++ 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index 288a315896..ad431b96d1 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -23,9 +23,12 @@ WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) { - return new ComplexWebSocketOutTask(this->channel, - WFGlobal::get_scheduler(), - std::move(cb)); + WFWebSocketTask *task = new ComplexWebSocketOutTask(this->channel, + WFGlobal::get_scheduler(), + std::move(cb)); + + task->get_msg()->set_masking_key(this->channel->gen_masking_key()); + return task; } int WebSocketClient::init(const std::string& url) @@ -36,6 +39,7 @@ int WebSocketClient::init(const std::string& url) this->channel = new ComplexWebSocketChannel(NULL, WFGlobal::get_scheduler(), + this->params.random_masking_key, std::move(process)); this->channel->set_idle_timeout(this->params.idle_timeout); this->channel->set_size_limit(this->params.size_limit); @@ -78,6 +82,7 @@ WFWebSocketTask *WebSocketClient::create_ping_task(websocket_callback_t cb) protocol::WebSocketFrame *msg = ping_task->get_msg(); msg->set_opcode(WebSocketFramePing); + msg->set_masking_key(this->channel->gen_masking_key()); return ping_task; } @@ -91,6 +96,7 @@ WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) protocol::WebSocketFrame *msg = close_task->get_msg(); msg->set_opcode(WebSocketFrameConnectionClose); + msg->set_masking_key(this->channel->gen_masking_key()); return close_task; } diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h index e04135726c..9962003b93 100644 --- a/src/client/WFWebSocketClient.h +++ b/src/client/WFWebSocketClient.h @@ -39,7 +39,7 @@ static constexpr struct WFWebSocketParams WEBSOCKET_PARAMS_DEFAULT = .idle_timeout = WS_HANDSHAKE_TIMEOUT, .ping_interval = -1, .size_limit = (size_t)-1, - .random_masking_key = false, + .random_masking_key = true, }; class WebSocketClient diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index eefdc05d24..e4cbdbb433 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -17,6 +17,7 @@ Xie Han (xiehan@sogou-inc.com) */ +#include #include #include "TransRequest.h" #include "WFTask.h" @@ -451,6 +452,12 @@ class ComplexWebSocketChannel : public WFComplexChannelidle_timeout = timeout; } void set_size_limit(size_t size_limit) { this->size_limit = size_limit; } + uint32_t gen_masking_key() + { + if (random_masking_key == false) + return 0; + return this->gen(); + } protected: virtual CommMessageIn *message_in(); @@ -461,13 +468,20 @@ protected: private: int idle_timeout; size_t size_limit; + bool random_masking_key; + std::random_device rd; + std::mt19937 gen; public: - ComplexWebSocketChannel(CommSchedObject *object, CommScheduler *scheduler, + ComplexWebSocketChannel(CommSchedObject *object, + CommScheduler *scheduler, + bool random_masking_key, websocket_process_t&& process) : WFComplexChannel(object, scheduler, - std::move(process)) + std::move(process)), + gen(rd()) { + this->random_masking_key = random_masking_key; } }; @@ -478,7 +492,8 @@ protected: virtual SubTask *done(); public: - ComplexWebSocketOutTask(ComplexWebSocketChannel *channel, CommScheduler *scheduler, + ComplexWebSocketOutTask(ComplexWebSocketChannel *channel, + CommScheduler *scheduler, websocket_callback_t&& cb) : ComplexChannelOutTask(channel, scheduler, diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc index 72b4f231d5..dbb5293038 100644 --- a/src/factory/WebSocketTaskImpl.cc +++ b/src/factory/WebSocketTaskImpl.cc @@ -80,6 +80,7 @@ SubTask *ComplexWebSocketInTask::done() WebSocketFrame *msg = close_task->get_msg(); msg->set_opcode(WebSocketFrameConnectionClose); msg->set_data(parser); + msg->set_masking_key(channel->gen_masking_key()); series->push_front(close_task); } else if (parser->opcode == WebSocketFramePing) @@ -90,6 +91,7 @@ SubTask *ComplexWebSocketInTask::done() WebSocketFrame *msg = pong_task->get_msg(); msg->set_opcode(WebSocketFramePong); msg->set_data(parser); + msg->set_masking_key(channel->gen_masking_key()); series->push_front(pong_task); } From 33b4afa4d8163e78d77db3a3f2451d554cbdcc55 Mon Sep 17 00:00:00 2001 From: holmesyxli Date: Wed, 17 Nov 2021 19:32:16 +0800 Subject: [PATCH 48/75] fix problems about Sec-Websocket-Protocol and Sec-Websocket-Version --- src/client/WFWebSocketClient.cc | 9 ++++++++- src/client/WFWebSocketClient.h | 12 ++++++++++-- src/factory/WFChannel.inl | 18 ++++++++++++++---- src/factory/WebSocketTaskImpl.cc | 9 +++++---- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index ad431b96d1..0a26c91b67 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -41,9 +41,11 @@ int WebSocketClient::init(const std::string& url) WFGlobal::get_scheduler(), this->params.random_masking_key, std::move(process)); + this->channel->set_uri(uri); this->channel->set_idle_timeout(this->params.idle_timeout); this->channel->set_size_limit(this->params.size_limit); - this->channel->set_uri(uri); + this->channel->set_sec_protocol(this->sec_protocol); + this->channel->set_sec_version(this->sec_version); this->channel->set_callback([this](WFChannel *channel) { @@ -106,5 +108,10 @@ WebSocketClient::WebSocketClient(const struct WFWebSocketParams *params, process(std::move(process)) { this->params = *params; + + if (params->sec_protocol) + this->sec_protocol = params->sec_protocol; + if (params->sec_version) + this->sec_version = params->sec_version; } diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h index 9962003b93..73ff35a3a2 100644 --- a/src/client/WFWebSocketClient.h +++ b/src/client/WFWebSocketClient.h @@ -32,6 +32,8 @@ struct WFWebSocketParams int ping_interval; size_t size_limit; bool random_masking_key; + const char *sec_protocol; + const char *sec_version; }; static constexpr struct WFWebSocketParams WEBSOCKET_PARAMS_DEFAULT = @@ -40,22 +42,28 @@ static constexpr struct WFWebSocketParams WEBSOCKET_PARAMS_DEFAULT = .ping_interval = -1, .size_limit = (size_t)-1, .random_masking_key = true, + .sec_protocol = NULL, + .sec_version = NULL, }; class WebSocketClient { public: int init(const std::string& url); + void deinit(); + WFWebSocketTask *create_websocket_task(websocket_callback_t cb); WFWebSocketTask *create_ping_task(websocket_callback_t cb); WFWebSocketTask *create_close_task(websocket_callback_t cb); - void deinit(); private: ComplexWebSocketChannel *channel; - struct WFWebSocketParams params; websocket_process_t process; + struct WFWebSocketParams params; + std::string sec_protocol; + std::string sec_version; + public: WebSocketClient(const struct WFWebSocketParams *params, websocket_process_t process); diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index e4cbdbb433..f578b9f237 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -452,10 +452,18 @@ class ComplexWebSocketChannel : public WFComplexChannelidle_timeout = timeout; } void set_size_limit(size_t size_limit) { this->size_limit = size_limit; } + + void set_sec_protocol(std::string& protocol) { this->sec_protocol = protocol; } + void set_sec_version(std::string& version) { this->sec_version = version; } + + const char *get_sec_protocol() const { return this->sec_protocol.c_str(); } + const char *get_sec_version() const { return this->sec_version.c_str(); } + uint32_t gen_masking_key() { - if (random_masking_key == false) + if (this->auto_gen_mkey == false) return 0; + return this->gen(); } @@ -468,20 +476,22 @@ protected: private: int idle_timeout; size_t size_limit; - bool random_masking_key; + bool auto_gen_mkey; // random Masking-Key std::random_device rd; std::mt19937 gen; + std::string sec_protocol; // Sec-WebSocket-Protocol + std::string sec_version; // Sec-WebSocket-Version public: ComplexWebSocketChannel(CommSchedObject *object, CommScheduler *scheduler, - bool random_masking_key, + bool auto_gen_mkey, websocket_process_t&& process) : WFComplexChannel(object, scheduler, std::move(process)), gen(rd()) { - this->random_masking_key = random_masking_key; + this->auto_gen_mkey = auto_gen_mkey; } }; diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc index dbb5293038..bf97a46cbe 100644 --- a/src/factory/WebSocketTaskImpl.cc +++ b/src/factory/WebSocketTaskImpl.cc @@ -23,9 +23,7 @@ #define WS_HTTP_SEC_KEY_K "Sec-WebSocket-Key" #define WS_HTTP_SEC_KEY_V "dGhlIHNhbXBsZSBub25jZQ==" #define WS_HTTP_SEC_PROTOCOL_K "Sec-WebSocket-Protocol" -#define WS_HTTP_SEC_PROTOCOL_V "chat" #define WS_HTTP_SEC_VERSION_K "Sec-WebSocket-Version" -#define WS_HTTP_SEC_VERSION_V "13" using namespace protocol; @@ -158,8 +156,11 @@ SubTask *ComplexWebSocketOutTask::upgrade() req->add_header_pair("Upgrade", "websocket"); req->add_header_pair("Connection", "Upgrade"); req->add_header_pair(WS_HTTP_SEC_KEY_K, WS_HTTP_SEC_KEY_V); - req->add_header_pair(WS_HTTP_SEC_PROTOCOL_K, WS_HTTP_SEC_PROTOCOL_V); - req->add_header_pair(WS_HTTP_SEC_VERSION_K, WS_HTTP_SEC_VERSION_V); + + if (channel->get_sec_protocol()) + req->add_header_pair(WS_HTTP_SEC_PROTOCOL_K, channel->get_sec_protocol()); + if (channel->get_sec_version()) + req->add_header_pair(WS_HTTP_SEC_VERSION_K, channel->get_sec_version()); return http_task; } From a5e4237d4817e69ca9f39bcbb83f7386e1cd3eaf Mon Sep 17 00:00:00 2001 From: holmesyxli Date: Wed, 17 Nov 2021 22:02:30 +0800 Subject: [PATCH 49/75] move params to init() --- src/client/WFWebSocketClient.cc | 34 ++++++++++++++++----------------- src/client/WFWebSocketClient.h | 11 ++++------- src/factory/WFChannel.inl | 4 ++-- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index 0a26c91b67..aa052be232 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -32,20 +32,30 @@ WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) } int WebSocketClient::init(const std::string& url) +{ + struct WFWebSocketParams params = WEBSOCKET_PARAMS_DEFAULT; + params.url = url.c_str(); + return this->init(¶ms); +} + +int WebSocketClient::init(const struct WFWebSocketParams *params) { ParsedURI uri; - if (URIParser::parse(url, uri) < 0) + if (URIParser::parse(params->url, uri) < 0) return -1; this->channel = new ComplexWebSocketChannel(NULL, WFGlobal::get_scheduler(), - this->params.random_masking_key, + params->random_masking_key, std::move(process)); this->channel->set_uri(uri); - this->channel->set_idle_timeout(this->params.idle_timeout); - this->channel->set_size_limit(this->params.size_limit); - this->channel->set_sec_protocol(this->sec_protocol); - this->channel->set_sec_version(this->sec_version); + this->channel->set_idle_timeout(params->idle_timeout); + this->channel->set_size_limit(params->size_limit); + + if (params->sec_protocol) + this->channel->set_sec_protocol(params->sec_protocol); + if (params->sec_version) + this->channel->set_sec_version(params->sec_version); this->channel->set_callback([this](WFChannel *channel) { @@ -103,15 +113,3 @@ WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) return close_task; } -WebSocketClient::WebSocketClient(const struct WFWebSocketParams *params, - websocket_process_t process) : - process(std::move(process)) -{ - this->params = *params; - - if (params->sec_protocol) - this->sec_protocol = params->sec_protocol; - if (params->sec_version) - this->sec_version = params->sec_version; -} - diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h index 73ff35a3a2..815da07bef 100644 --- a/src/client/WFWebSocketClient.h +++ b/src/client/WFWebSocketClient.h @@ -28,6 +28,7 @@ struct WFWebSocketParams { + const char *url; int idle_timeout; int ping_interval; size_t size_limit; @@ -38,6 +39,7 @@ struct WFWebSocketParams static constexpr struct WFWebSocketParams WEBSOCKET_PARAMS_DEFAULT = { + .url = NULL, .idle_timeout = WS_HANDSHAKE_TIMEOUT, .ping_interval = -1, .size_limit = (size_t)-1, @@ -50,6 +52,7 @@ class WebSocketClient { public: int init(const std::string& url); + int init(const struct WFWebSocketParams *params); void deinit(); WFWebSocketTask *create_websocket_task(websocket_callback_t cb); @@ -60,15 +63,9 @@ class WebSocketClient ComplexWebSocketChannel *channel; websocket_process_t process; - struct WFWebSocketParams params; - std::string sec_protocol; - std::string sec_version; - public: - WebSocketClient(const struct WFWebSocketParams *params, - websocket_process_t process); WebSocketClient(websocket_process_t process) : - WebSocketClient(&WEBSOCKET_PARAMS_DEFAULT, std::move(process)) + process(std::move(process)) { } }; diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index f578b9f237..3c39c9cde1 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -453,8 +453,8 @@ public: void set_idle_timeout(int timeout) { this->idle_timeout = timeout; } void set_size_limit(size_t size_limit) { this->size_limit = size_limit; } - void set_sec_protocol(std::string& protocol) { this->sec_protocol = protocol; } - void set_sec_version(std::string& version) { this->sec_version = version; } + void set_sec_protocol(const char *protocol) { this->sec_protocol = protocol; } + void set_sec_version(const char *version) { this->sec_version = version; } const char *get_sec_protocol() const { return this->sec_protocol.c_str(); } const char *get_sec_version() const { return this->sec_version.c_str(); } From 139be596941af9df64bb0ad00793f4150e0848f0 Mon Sep 17 00:00:00 2001 From: liyingxin Date: Wed, 17 Nov 2021 22:20:24 +0800 Subject: [PATCH 50/75] Update tutorial-14-websocket_cli.md --- docs/tutorial-14-websocket_cli.md | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md index e701e76a1a..cc6cfd2018 100644 --- a/docs/tutorial-14-websocket_cli.md +++ b/docs/tutorial-14-websocket_cli.md @@ -85,7 +85,7 @@ public: #### 2. masking_key -**WebSocket**协议的文本和二进制数据都需要经过一个掩码加密，可以通过``void set_masking_key(uint32_t masking_key)``手动指定，框架自动随机生成的功能正在开发中，很快可以投入使用。 +**WebSocket**协议的文本和二进制数据都需要经过一个掩码加密，默认框架会随机生成，也可以通过``void set_masking_key(uint32_t masking_key)``手动指定。 #### 3. data @@ -172,31 +172,44 @@ wait_group.wait(); # websocket_cli的参数 -``WebSocketClient``的构造函数有两个，除了刚才介绍的传入``process()``函数的接口以外，还可以传入client的参数： +``WebSocketClient``的init函数有两个，除了刚才介绍的传入URL函数的接口以外，还可以传入client的参数： ```cpp class WebSocketClient { public: - WebSocketClient(const struct WFWebSocketParams *params, - websocket_process_t process); - WebSocketClient(websocket_process_t process); + int init(const std::string& url); + int init(const struct WFWebSocketParams *params); ... ``` +使用时记得用默认参数初始化一下： +```cpp +struct WFWebSocketParams params = WEBSOCKET_PARAMS_DEFAULT; +params.url = "ws://127.0.0.1:80"; +params.sec_protocol = "chat"; + +WebSocketClient client(process); +client.init(¶ms); +... +``` + 其中，参数的定义如下： ```cpp struct WFWebSocketParams { + const char *url; // 目标URL int idle_timeout; // client保持长连接的空闲时间，超过idle_timeout没有数据过来会自动断开。默认：10s int ping_interval; // client自动发ping的时间间隔，用于做心跳，保持与远端的连接。默认：-1，不自动发ping(功能开发中) size_t size_limit; // 每个数据包的大小限制，超过的话会拿到错误码1009(WSStatusCodeTooLarge)。默认：不限制 - bool random_masking_key; // WebSocket协议中数据包的掩码，框架帮每次自动随机生成一个。默认：不自动生成(功能开发中) + bool random_masking_key; // WebSocket协议中数据包的掩码，框架帮每次自动随机生成一个。默认：自动生成 + const char *sec_protocol;// 应用层的Sec-Websocket-Protocol字段 + const char *sec_version; // 应用层的Sec-Websocket-Version字段 }; ``` -如果不传入参数，会使用默认参数来构造client。 +如果只使用URL调用init()，则会使用默参数。 # 进阶版：注意事项！ From ce0589e305ffcc4879f9811464e3cefe0604db0d Mon Sep 17 00:00:00 2001 From: holmesyxli Date: Thu, 18 Nov 2021 22:22:41 +0800 Subject: [PATCH 51/75] make wait after upgrade out task --- src/factory/WebSocketTaskImpl.cc | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc index bf97a46cbe..c9bf9b50de 100644 --- a/src/factory/WebSocketTaskImpl.cc +++ b/src/factory/WebSocketTaskImpl.cc @@ -132,15 +132,25 @@ SubTask *ComplexWebSocketOutTask::upgrade() auto *http_task = new WFChannelOutTask(this->channel, WFGlobal::get_scheduler(), - [this](WFChannelTask *task) + [this](WFChannelTask *upgrade_task) { - if (task->get_state() == WFT_STATE_SYS_ERROR) + WFMailboxTask *waiter; + ComplexWebSocketChannel *channel; + channel = (ComplexWebSocketChannel *)this->get_request_channel(); + + if (upgrade_task->get_state() == WFT_STATE_SUCCESS) { - this->state = WFT_STATE_SYS_ERROR; - this->error = task->get_error(); + waiter = WFCondTaskFactory::create_wait_task(&channel->condition, + nullptr); + series_of(upgrade_task)->push_front(waiter); + this->ready = true; + } + else + { + channel->set_state(WFT_STATE_SYS_ERROR); + this->state = upgrade_task->get_state(); + this->error = upgrade_task->get_error(); } - - this->ready = true; }); if (uri->path && uri->path[0]) From d1d84a30131da2d9e7882f3727a2df24aa36e2c8 Mon Sep 17 00:00:00 2001 From: holmesyxli Date: Fri, 19 Nov 2021 19:50:18 +0800 Subject: [PATCH 52/75] fix get_sec_protocol/version() --- src/factory/WFChannel.inl | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 3c39c9cde1..c8c5c40d61 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -456,8 +456,21 @@ public: void set_sec_protocol(const char *protocol) { this->sec_protocol = protocol; } void set_sec_version(const char *version) { this->sec_version = version; } - const char *get_sec_protocol() const { return this->sec_protocol.c_str(); } - const char *get_sec_version() const { return this->sec_version.c_str(); } + const char *get_sec_protocol() const + { + if (!this->sec_protocol.empty()) + return this->sec_protocol.c_str(); + + return NULL; + } + + const char *get_sec_version() const + { + if (!this->sec_version.empty()) + return this->sec_version.c_str(); + + return NULL; + } uint32_t gen_masking_key() { From ddfa7045dec1735b3a4770492f4c3305a40303f4 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 10 Dec 2021 13:40:56 +0800 Subject: [PATCH 53/75] fix compile error --- src/kernel/Communicator.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/kernel/Communicator.cc b/src/kernel/Communicator.cc index be1dba5738..d46cf0f9c7 100644 --- a/src/kernel/Communicator.cc +++ b/src/kernel/Communicator.cc @@ -1907,7 +1907,8 @@ void Communicator::shutdown(CommChannel *channel) } errno = errno_bak; -======= +} + int Communicator::push(const void *buf, size_t size, CommSession *session) { CommTarget *target = session->target; From 9621036c1908a5be603b3fe8b071c372f877387c Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 8 Apr 2022 17:22:26 +0800 Subject: [PATCH 54/75] update CMakeLists_Headers.txt --- CMakeLists_Headers.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index b973697c98..3c4b054d37 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -7,6 +7,7 @@ set(COMMON_KERNEL_HEADERS src/kernel/SleepRequest.h src/kernel/ExecRequest.h src/kernel/IORequest.h + src/kernel/TransRequest.h src/kernel/Executor.h src/kernel/list.h src/kernel/mpoller.h From 0f3c6dd87aa36804191dc63f52c5f4a5ac132a67 Mon Sep 17 00:00:00 2001 From: liyingxin Date: Thu, 5 May 2022 03:40:55 +0800 Subject: [PATCH 55/75] add Sec-WebSocket-Protocol/Version and add check_handshake() --- src/factory/WFChannel.inl | 3 +++ src/factory/WebSocketTaskImpl.cc | 42 +++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index c8c5c40d61..5413a62ec7 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -506,6 +506,9 @@ public: { this->auto_gen_mkey = auto_gen_mkey; } + +private: + bool check_handshake(const protocol::HttpResponse *resp); }; class ComplexWebSocketOutTask : public ComplexChannelOutTask diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc index c9bf9b50de..b8abc10bb2 100644 --- a/src/factory/WebSocketTaskImpl.cc +++ b/src/factory/WebSocketTaskImpl.cc @@ -22,8 +22,15 @@ #define WS_HTTP_SEC_KEY_K "Sec-WebSocket-Key" #define WS_HTTP_SEC_KEY_V "dGhlIHNhbXBsZSBub25jZQ==" + #define WS_HTTP_SEC_PROTOCOL_K "Sec-WebSocket-Protocol" +#define WS_HTTP_SEC_PROTOCOL_V "chat" + #define WS_HTTP_SEC_VERSION_K "Sec-WebSocket-Version" +#define WS_HTTP_SEC_VERSION_V "13" + +#define WS_HTTP_SEC_ACCEPT_K "Sec-WebSocket-Accept" +#define WS_HTTP_SEC_ACCEPT_V "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" using namespace protocol; @@ -166,6 +173,8 @@ SubTask *ComplexWebSocketOutTask::upgrade() req->add_header_pair("Upgrade", "websocket"); req->add_header_pair("Connection", "Upgrade"); req->add_header_pair(WS_HTTP_SEC_KEY_K, WS_HTTP_SEC_KEY_V); + req->add_header_pair(WS_HTTP_SEC_PROTOCOL_K, WS_HTTP_SEC_PROTOCOL_V); + req->add_header_pair(WS_HTTP_SEC_VERSION_K, WS_HTTP_SEC_VERSION_V); if (channel->get_sec_protocol()) req->add_header_pair(WS_HTTP_SEC_PROTOCOL_K, channel->get_sec_protocol()); @@ -193,7 +202,7 @@ void ComplexWebSocketChannel::handle_in(CommMessageIn *in) { HttpResponse *resp = static_cast(in); - if (strcmp(resp->get_status_code(), "101") == 0) + if (this->check_handshake(resp)) this->state = WFT_STATE_SUCCESS; else this->state = WFT_STATE_TASK_ERROR; @@ -232,3 +241,34 @@ WFWebSocketTask *ComplexWebSocketChannel::new_session() return task; } +bool ComplexWebSocketChannel::check_handshake(const HttpResponse *resp) +{ + if (strcmp(resp->get_status_code(), "101")) + return false; + + std::string name; + std::string value; + HttpHeaderCursor resp_cursor(resp); + int flag = 0; + + while (resp_cursor.next(name, value) && flag != 7) + { + if (name.compare("Upgrade") == 0 && value.compare("websocket") == 0) + { + flag |= 1; + } + else if (name.compare("Connection") == 0 && + value.compare("Upgrade") == 0) + { + flag |= (1 << 1); + } + else if (name.compare(WS_HTTP_SEC_ACCEPT_K) == 0 && + value.compare(WS_HTTP_SEC_ACCEPT_V) == 0) + { + flag |= (1 << 2); + } + } + + return flag == 7; +} + From cf6726338db2b8adfa0d7e956f04e9f014d7b019 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 24 Nov 2022 21:10:10 +0800 Subject: [PATCH 56/75] init parser->masking_key_offset --- src/protocol/websocket_parser.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c index 25e1db6ad5..aeb0feba5e 100644 --- a/src/protocol/websocket_parser.c +++ b/src/protocol/websocket_parser.c @@ -33,6 +33,7 @@ void websocket_parser_init(websocket_parser_t *parser) parser->nreceived = 0; parser->is_server = 0; parser->status_code = WSStatusCodeUndefined; + parser->masking_key_offset = 0; memset(parser->masking_key, 0, WS_MASKING_KEY_LENGTH); memset(parser->header_buf, 0, WS_HEADER_LENGTH_MAX); } From cec6a3a6d3d86d02b4ea272de0980fc6797ca628 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Mon, 28 Nov 2022 16:23:51 +0800 Subject: [PATCH 57/75] fix websocket length encode --- src/protocol/WebSocketMessage.cc | 46 +++++++++++++++----------------- src/protocol/websocket_parser.h | 13 --------- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/src/protocol/WebSocketMessage.cc b/src/protocol/WebSocketMessage.cc index fe11f10cd2..f18667a7f8 100644 --- a/src/protocol/WebSocketMessage.cc +++ b/src/protocol/WebSocketMessage.cc @@ -18,50 +18,44 @@ #include #include +#include #include "WebSocketMessage.h" namespace protocol { + +#ifndef htonll + #if __BYTE_ORDER == __LITTLE_ENDIAN -static inline void int2store(unsigned char *T, uint16_t A) +static inline uint64_t htonll(uint64_t x) { - memcpy(T, &A, sizeof(A)); + return ((uint64_t)htonl(x & 0xFFFFFFFF) << 32) + htonl(x >> 32); } -static inline void int8store(unsigned char *T, uint64_t A) +static inline uint64_t ntohll(uint64_t x) { - memcpy(T, &A, sizeof(A)); + return ((uint64_t)ntohl(x & 0xFFFFFFFF) << 32) + ntohl(x >> 32); } #elif __BYTE_ORDER == __BIG_ENDIAN -static inline void int2store(unsigned char *T, uint16_t A) -{ - uint def_temp = A; - *(T) = (unsigned char)(def_temp); - *(T + 1) = (unsigned char)(def_temp >> 8); -} - -static inline void int4store(unsigned char *T, uint32_t A) +static inline uint64_t htonll(uint64_t x) { - *(T) = (unsigned char)(A); - *(T + 1) = (unsigned char)(A >> 8); - *(T + 2) = (unsigned char)(A >> 16); - *(T + 3) = (unsigned char)(A >> 24); + return x; } -static inline void int8store(unsigned char *T, uint64_t A) +static inline uint64_t ntohll(uint64_t x) { - uint def_temp = (uint)A, def_temp2 = (uint)(A >> 32); - int4store(T, def_temp); - int4store(T + 4, def_temp2); + return x; } #else # error "unknown byte order" #endif +#endif + WebSocketFrame::WebSocketFrame(WebSocketFrame&& msg) : ProtocolMessage(std::move(msg)) { @@ -131,14 +125,17 @@ int WebSocketFrame::encode(struct iovec vectors[], int max) { *p = 126; p++; - int2store(p, this->parser->payload_length); + uint16_t tmp = htons(this->parser->payload_length); + memcpy(p, &tmp, sizeof(tmp)); p += 2; + } else { *p = 127; p++; - int8store(p, this->parser->payload_length); + uint64_t tmp = htonll(this->parser->payload_length); + memcpy(p, &tmp, sizeof(tmp)); p += 8; } @@ -149,7 +146,7 @@ int WebSocketFrame::encode(struct iovec vectors[], int max) p = this->parser->header_buf + 1; *p = *p | (this->parser->mask << 7); - if (!this->parser->is_server) + if (!this->parser->is_server || this->parser->mask) { vectors[cnt].iov_base = this->parser->masking_key; vectors[cnt].iov_len = WS_MASKING_KEY_LENGTH; @@ -271,7 +268,8 @@ bool WebSocketFrame::set_data(const websocket_parser_t *parser) if (this->parser->opcode == WebSocketFrameConnectionClose && parser->status_code != WSStatusCodeUndefined) { - int2store(p, parser->status_code); + uint16_t tmp = htons(parser->status_code); + memcpy(p, &tmp, sizeof(tmp)); p += 2; } diff --git a/src/protocol/websocket_parser.h b/src/protocol/websocket_parser.h index 7f9d9c17c9..ebda0e41d5 100644 --- a/src/protocol/websocket_parser.h +++ b/src/protocol/websocket_parser.h @@ -106,18 +106,5 @@ unsigned char *utf8_check(unsigned char *s, size_t len); } #endif -static inline void websocket_set_mask_key(unsigned int masking_key, - websocket_parser_t *parser) -{ - uint32_t *key = (uint32_t *)parser->masking_key; - *key = masking_key; - parser->mask = 1; -} - -static inline void websocket_set_opcode(int opcode, websocket_parser_t *parser) -{ - parser->opcode = opcode; -} - #endif From 114fdb6a6d741194e157ed06f4ed35fb22cef9d2 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Tue, 29 Nov 2022 03:32:43 +0800 Subject: [PATCH 58/75] fix websocket_parser_append_message --- src/protocol/websocket_parser.c | 117 +++++++++++--------------- src/protocol/websocket_parser.h | 1 - tutorial/tutorial-14-websocket_cli.cc | 5 +- 3 files changed, 53 insertions(+), 70 deletions(-) diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c index aeb0feba5e..dbf211fada 100644 --- a/src/protocol/websocket_parser.c +++ b/src/protocol/websocket_parser.c @@ -27,13 +27,12 @@ void websocket_parser_init(websocket_parser_t *parser) parser->fin = 0; parser->mask = 0; parser->opcode = -1; - parser->nleft = WS_MASKING_KEY_LENGTH; + parser->nleft = 0; parser->payload_length = 0; parser->payload_data = NULL; parser->nreceived = 0; parser->is_server = 0; parser->status_code = WSStatusCodeUndefined; - parser->masking_key_offset = 0; memset(parser->masking_key, 0, WS_MASKING_KEY_LENGTH); memset(parser->header_buf, 0, WS_HEADER_LENGTH_MAX); } @@ -51,97 +50,81 @@ int websocket_parser_append_message(const void *buf, size_t *n, websocket_parser_t *parser) { const unsigned char *p = (const unsigned char *)buf; - const unsigned char *buf_end = (const unsigned char *)buf + *n; + size_t len = *n; - int header_length_min = parser->is_server ? WS_CLIENT_LENGTH_MIN : - WS_SERVER_LENGTH_MIN; + if (parser->nreceived == 0) + parser->nleft = WS_SERVER_LENGTH_MIN; -// if (parser->payload_length == 0) // receiving header - if (parser->nreceived < header_length_min) + if (parser->payload_data == NULL) { - memcpy(parser->header_buf + parser->nreceived, p, - WS_HEADER_LENGTH_MAX - parser->nreceived); + size_t header_consumed = *n < parser->nleft ? *n : parser->nleft; + if (header_consumed) + memcpy(parser->header_buf + parser->nreceived, p, header_consumed); - if (parser->nreceived + *n < header_length_min) + if (*n < parser->nleft) { + parser->nleft -= *n; parser->nreceived += *n; return 0; } - parser->fin = *p >> 7; - parser->opcode = *p & 0xF; - p++; - parser->mask = *p >> 7; - parser->payload_length = *p & 0x7F; - p++; - } - - if (parser->masking_key_offset == 0) // receive payload_length - { - if (parser->payload_length == 126 && - parser->nreceived + *n >= header_length_min + 2) + if (parser->payload_length == 0) { - uint16_t *len_ptr = (uint16_t *)p; - parser->payload_length = ntohs(*len_ptr); - p += 2; - parser->masking_key_offset = 4; + parser->nleft = 0; + parser->fin = parser->header_buf[0] >> 7; + parser->opcode = parser->header_buf[0] & 0xF; + parser->mask = parser->header_buf[1] >> 7; + if (parser->mask == 1) + parser->nleft += 4; + parser->payload_length = parser->header_buf[1] & 0x7F; + if (parser->payload_length == 126) + parser->nleft += 2; + else if (parser->payload_length == 127) + parser->nleft += 4; + + if (parser->payload_length == 0) + return 1; + parser->nreceived += header_consumed; + *n = header_consumed; + + return 0; } - else if (parser->payload_length == 127 && - parser->nreceived + *n >= header_length_min + 8) + + unsigned char *ptr = &parser->header_buf[2]; + + if (parser->payload_length == 126) { - uint64_t *len_ptr = (uint64_t *)p; - parser->payload_length = (((uint64_t) ntohl(*len_ptr)) << 32) + - ntohl(*len_ptr >> 32); - p += 8; - parser->masking_key_offset = 10; + parser->payload_length = ntohs(*((uint16_t *)ptr)); + ptr += 2; } - else + else if (parser->payload_length == 127) { - parser->nreceived += *n; - parser->masking_key_offset = 2; + parser->payload_length = ntohll(*((uint64_t *)ptr)); + ptr += 4; } - if (parser->masking_key_offset == 0) - return 0; // continue for length - } + if (parser->mask == 1) + memcpy(parser->masking_key, ptr, WS_MASKING_KEY_LENGTH); - if (!parser->payload_data) // receive masking_key if needed - { - if (parser->mask && parser->nleft) - { -// if (parser->masking_key_offset + 4 < buf_end) - if (buf_end - p < parser->nleft) - { - parser->nleft -= buf_end - p; - parser->nreceived += *n; - return 0; - } - - memcpy(parser->masking_key, - parser->header_buf + parser->masking_key_offset, - WS_MASKING_KEY_LENGTH); - p += parser->nleft; - parser->nreceived += parser->nleft; - } + parser->payload_data = malloc(parser->payload_length); + if (!parser->payload_data) + return -1; parser->nleft = parser->payload_length; + len = *n - header_consumed; } - parser->payload_data = malloc(parser->payload_length); - - if (buf_end - p < parser->nleft) + if (len < parser->nleft) { memcpy(parser->payload_data + parser->payload_length - parser->nleft, - p, buf_end - p); - parser->nleft -= buf_end - p; + buf, len); + parser->nleft -= len; return 0; } else { memcpy(parser->payload_data + parser->payload_length - parser->nleft, - p, parser->nleft); - p += parser->nleft; - *n -= buf_end - p; + buf, parser->nleft); return 1; } } @@ -159,8 +142,8 @@ int websocket_parser_parse(websocket_parser_t *parser) unsigned char *p = (unsigned char *)parser->payload_data; - if (parser->opcode == WebSocketFrameConnectionClose) - { + if (parser->opcode == WebSocketFrameConnectionClose && p != NULL) + { uint16_t *ptr = (uint16_t *)p; parser->status_code = ntohs(*ptr); parser->payload_data = p + 2; diff --git a/src/protocol/websocket_parser.h b/src/protocol/websocket_parser.h index ebda0e41d5..433cdb6c78 100644 --- a/src/protocol/websocket_parser.h +++ b/src/protocol/websocket_parser.h @@ -76,7 +76,6 @@ typedef struct __websocket_parser unsigned char header_buf[WS_HEADER_LENGTH_MAX]; void *payload_data; unsigned long long nreceived; - int masking_key_offset; int nleft; int is_server; int status_code; diff --git a/tutorial/tutorial-14-websocket_cli.cc b/tutorial/tutorial-14-websocket_cli.cc index adfa11884f..2b025514ec 100644 --- a/tutorial/tutorial-14-websocket_cli.cc +++ b/tutorial/tutorial-14-websocket_cli.cc @@ -40,7 +40,7 @@ void process(WFWebSocketTask *task) } else { - fprintf(stderr, "opcode=%d\n", task->get_msg()->get_opcode()); + fprintf(stderr, "process opcode=%d\n", task->get_msg()->get_opcode()); } } @@ -66,12 +66,13 @@ int main(int argc, char *argv[]) wg.done(); return; } - + auto *ping_task = client.create_ping_task(nullptr); auto *timer_task = WFTaskFactory::create_timer_task(3000000 /* 3s */, nullptr); auto *close_task = client.create_close_task([&wg] (WFWebSocketTask *task) { wg.done(); }); + series_of(task)->push_back(ping_task); series_of(task)->push_back(timer_task); series_of(task)->push_back(close_task); }); From 18128be7f4bc74b9cc8e7b695c54ec2fc1512fc2 Mon Sep 17 00:00:00 2001 From: liyingxin Date: Tue, 29 Nov 2022 03:41:48 +0800 Subject: [PATCH 59/75] update codes --- src/protocol/WebSocketMessage.cc | 10 ---------- src/protocol/websocket_parser.c | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/protocol/WebSocketMessage.cc b/src/protocol/WebSocketMessage.cc index f18667a7f8..c502f3dd22 100644 --- a/src/protocol/WebSocketMessage.cc +++ b/src/protocol/WebSocketMessage.cc @@ -33,11 +33,6 @@ static inline uint64_t htonll(uint64_t x) return ((uint64_t)htonl(x & 0xFFFFFFFF) << 32) + htonl(x >> 32); } -static inline uint64_t ntohll(uint64_t x) -{ - return ((uint64_t)ntohl(x & 0xFFFFFFFF) << 32) + ntohl(x >> 32); -} - #elif __BYTE_ORDER == __BIG_ENDIAN static inline uint64_t htonll(uint64_t x) @@ -45,11 +40,6 @@ static inline uint64_t htonll(uint64_t x) return x; } -static inline uint64_t ntohll(uint64_t x) -{ - return x; -} - #else # error "unknown byte order" #endif diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c index dbf211fada..6e01e6f76c 100644 --- a/src/protocol/websocket_parser.c +++ b/src/protocol/websocket_parser.c @@ -19,9 +19,32 @@ #include #include #include +#include #include #include "websocket_parser.h" +#ifndef ntohll + +#if __BYTE_ORDER == __LITTLE_ENDIAN + +static inline uint64_t ntohll(uint64_t x) +{ + return ((uint64_t)ntohl(x & 0xFFFFFFFF) << 32) + ntohl(x >> 32); +} + +#elif __BYTE_ORDER == __BIG_ENDIAN + +static inline uint64_t ntohll(uint64_t x) +{ + return x; +} + +#else +# error "unknown byte order" +#endif + +#endif + void websocket_parser_init(websocket_parser_t *parser) { parser->fin = 0; From 6e84b5a32a3d2ef68cdbc8e6d685929cc1212abe Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Tue, 29 Nov 2022 19:13:32 +0800 Subject: [PATCH 60/75] update websocket append message --- src/protocol/websocket_parser.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c index 6e01e6f76c..ac26204c0f 100644 --- a/src/protocol/websocket_parser.c +++ b/src/protocol/websocket_parser.c @@ -74,13 +74,14 @@ int websocket_parser_append_message(const void *buf, size_t *n, { const unsigned char *p = (const unsigned char *)buf; size_t len = *n; + size_t header_consumed = 0; if (parser->nreceived == 0) parser->nleft = WS_SERVER_LENGTH_MIN; if (parser->payload_data == NULL) { - size_t header_consumed = *n < parser->nleft ? *n : parser->nleft; + header_consumed = *n < parser->nleft ? *n : parser->nleft; if (header_consumed) memcpy(parser->header_buf + parser->nreceived, p, header_consumed); @@ -107,27 +108,27 @@ int websocket_parser_append_message(const void *buf, size_t *n, if (parser->payload_length == 0) return 1; + parser->nreceived += header_consumed; *n = header_consumed; - return 0; } - unsigned char *ptr = &parser->header_buf[2]; + p = &parser->header_buf[2]; if (parser->payload_length == 126) { - parser->payload_length = ntohs(*((uint16_t *)ptr)); - ptr += 2; + parser->payload_length = ntohs(*((uint16_t *)p)); + p += 2; } else if (parser->payload_length == 127) { - parser->payload_length = ntohll(*((uint64_t *)ptr)); - ptr += 4; + parser->payload_length = ntohll(*((uint64_t *)p)); + p += 4; } if (parser->mask == 1) - memcpy(parser->masking_key, ptr, WS_MASKING_KEY_LENGTH); + memcpy(parser->masking_key, p, WS_MASKING_KEY_LENGTH); parser->payload_data = malloc(parser->payload_length); if (!parser->payload_data) @@ -137,17 +138,19 @@ int websocket_parser_append_message(const void *buf, size_t *n, len = *n - header_consumed; } + p = (const unsigned char *)buf + header_consumed; if (len < parser->nleft) { memcpy(parser->payload_data + parser->payload_length - parser->nleft, - buf, len); + p, len); parser->nleft -= len; return 0; } else { memcpy(parser->payload_data + parser->payload_length - parser->nleft, - buf, parser->nleft); + p, parser->nleft); + *n = header_consumed + parser->nleft; return 1; } } From 5314fef8560bf85fff92f0bc48cf2f559a5293b7 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Tue, 29 Nov 2022 22:33:27 +0800 Subject: [PATCH 61/75] fix websocket CloseFrame process payload_data --- src/protocol/websocket_parser.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c index ac26204c0f..5bb8d2adda 100644 --- a/src/protocol/websocket_parser.c +++ b/src/protocol/websocket_parser.c @@ -170,9 +170,13 @@ int websocket_parser_parse(websocket_parser_t *parser) if (parser->opcode == WebSocketFrameConnectionClose && p != NULL) { - uint16_t *ptr = (uint16_t *)p; - parser->status_code = ntohs(*ptr); - parser->payload_data = p + 2; + parser->status_code = ntohs(*((uint16_t*)p)); + p = malloc(parser->payload_length - 2); + memcpy(p, (unsigned char *)parser->payload_data + 2, + parser->payload_length - 2); + free(parser->payload_data); + parser->payload_data = p; + parser->payload_length -= 2; } websocket_parser_mask_data(parser); From 440cf2a68c26df3abcaa8d7dfef824a469a9a370 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 8 Dec 2022 23:39:40 +0800 Subject: [PATCH 62/75] fix parsing payload_length --- src/protocol/WebSocketMessage.cc | 2 +- src/protocol/WebSocketMessage.h | 2 -- src/protocol/websocket_parser.c | 5 ++--- src/protocol/websocket_parser.h | 1 - 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/protocol/WebSocketMessage.cc b/src/protocol/WebSocketMessage.cc index c502f3dd22..7daaed384e 100644 --- a/src/protocol/WebSocketMessage.cc +++ b/src/protocol/WebSocketMessage.cc @@ -136,7 +136,7 @@ int WebSocketFrame::encode(struct iovec vectors[], int max) p = this->parser->header_buf + 1; *p = *p | (this->parser->mask << 7); - if (!this->parser->is_server || this->parser->mask) + if (this->parser->mask) { vectors[cnt].iov_base = this->parser->masking_key; vectors[cnt].iov_len = WS_MASKING_KEY_LENGTH; diff --git a/src/protocol/WebSocketMessage.h b/src/protocol/WebSocketMessage.h index e834be2a48..a539c0f0f5 100644 --- a/src/protocol/WebSocketMessage.h +++ b/src/protocol/WebSocketMessage.h @@ -49,8 +49,6 @@ class WebSocketFrame : public ProtocolMessage bool finished() const; public: - void set_client() { this->parser->is_server = 0; } - void set_server() { this->parser->is_server = 1; } const websocket_parser_t *get_parser() { return this->parser; } bool set_data(const websocket_parser_t *parser); uint32_t get_masking_key() const; diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c index 5bb8d2adda..a3e6d54377 100644 --- a/src/protocol/websocket_parser.c +++ b/src/protocol/websocket_parser.c @@ -54,7 +54,6 @@ void websocket_parser_init(websocket_parser_t *parser) parser->payload_length = 0; parser->payload_data = NULL; parser->nreceived = 0; - parser->is_server = 0; parser->status_code = WSStatusCodeUndefined; memset(parser->masking_key, 0, WS_MASKING_KEY_LENGTH); memset(parser->header_buf, 0, WS_HEADER_LENGTH_MAX); @@ -104,7 +103,7 @@ int websocket_parser_append_message(const void *buf, size_t *n, if (parser->payload_length == 126) parser->nleft += 2; else if (parser->payload_length == 127) - parser->nleft += 4; + parser->nleft += 8; if (parser->payload_length == 0) return 1; @@ -124,7 +123,7 @@ int websocket_parser_append_message(const void *buf, size_t *n, else if (parser->payload_length == 127) { parser->payload_length = ntohll(*((uint64_t *)p)); - p += 4; + p += 8; } if (parser->mask == 1) diff --git a/src/protocol/websocket_parser.h b/src/protocol/websocket_parser.h index 433cdb6c78..70c8b352d5 100644 --- a/src/protocol/websocket_parser.h +++ b/src/protocol/websocket_parser.h @@ -77,7 +77,6 @@ typedef struct __websocket_parser void *payload_data; unsigned long long nreceived; int nleft; - int is_server; int status_code; } websocket_parser_t; From f3dfa39e54a45c76d3748ebb46590a6b81ed9c04 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 9 Dec 2022 03:11:55 +0800 Subject: [PATCH 63/75] fix websocket channel callback --- src/client/WFWebSocketClient.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index aa052be232..5ec6480f89 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -57,15 +57,17 @@ int WebSocketClient::init(const struct WFWebSocketParams *params) if (params->sec_version) this->channel->set_sec_version(params->sec_version); - this->channel->set_callback([this](WFChannel *channel) + this->channel->set_callback([](WFChannel *channel) { - pthread_mutex_lock(&this->channel->mutex); - if (this->channel->is_established() == 0) + ComplexWebSocketChannel *ch = (ComplexWebSocketChannel *)channel; + + pthread_mutex_lock(&ch->mutex); + if (ch->is_established() == 0) { - this->channel->set_state(WFT_STATE_SYS_ERROR); - this->channel->set_sending(false); + ch->set_state(WFT_STATE_SYS_ERROR); + ch->set_sending(false); } - pthread_mutex_unlock(&this->channel->mutex); + pthread_mutex_unlock(&ch->mutex); }); return 0; From d225800b9616a4b51721ad58c0e38cfd194a5c53 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Sat, 10 Dec 2022 03:07:16 +0800 Subject: [PATCH 64/75] websocket: skip parse payload_data if NULL; update get_data() and utf8_check() --- src/protocol/WebSocketMessage.cc | 5 +- src/protocol/websocket_parser.c | 99 +++++++++++++++++--------------- 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/src/protocol/WebSocketMessage.cc b/src/protocol/WebSocketMessage.cc index 7daaed384e..ee289db6d5 100644 --- a/src/protocol/WebSocketMessage.cc +++ b/src/protocol/WebSocketMessage.cc @@ -270,8 +270,11 @@ bool WebSocketFrame::set_data(const websocket_parser_t *parser) bool WebSocketFrame::get_data(const char **data, size_t *size) const { - if (!this->parser->payload_length || !this->parser->payload_data) + if (this->parser->status_code == WSStatusCodeUnsupportedData || + this->parser->status_code == WSStatusCodeProtocolError) + { return false; + } *data = (char *)this->parser->payload_data; *size = this->parser->payload_length; diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c index a3e6d54377..616f2b869b 100644 --- a/src/protocol/websocket_parser.c +++ b/src/protocol/websocket_parser.c @@ -61,8 +61,7 @@ void websocket_parser_init(websocket_parser_t *parser) void websocket_parser_deinit(websocket_parser_t *parser) { - if (parser->payload_length != 0) - free(parser->payload_data); + free(parser->payload_data); } // 4: FIN 0 0 0 4:opcode | 1: MASK 7:PAYLOAD_LENGTH | @@ -156,6 +155,8 @@ int websocket_parser_append_message(const void *buf, size_t *n, int websocket_parser_parse(websocket_parser_t *parser) { + unsigned char *p; + if (parser->opcode < WebSocketFrameContinuation || (parser->opcode < WebSocketFrameConnectionClose && parser->opcode > WebSocketFrameBinary) || @@ -165,9 +166,12 @@ int websocket_parser_parse(websocket_parser_t *parser) return -1; } - unsigned char *p = (unsigned char *)parser->payload_data; + if (parser->payload_data == NULL) + return 0; + + p = (unsigned char *)parser->payload_data; - if (parser->opcode == WebSocketFrameConnectionClose && p != NULL) + if (parser->opcode == WebSocketFrameConnectionClose) { parser->status_code = ntohs(*((uint16_t*)p)); p = malloc(parser->payload_length - 2); @@ -207,46 +211,51 @@ void websocket_parser_mask_data(websocket_parser_t *parser) //https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c unsigned char *utf8_check(unsigned char *s, size_t len) { - unsigned char *end = s + len; - while (*s && s != end) { - if (*s < 0x80) - /* 0xxxxxxx */ - s++; - else if ((s[0] & 0xe0) == 0xc0) { - /* 110XXXXx 10xxxxxx */ - if ((s[1] & 0xc0) != 0x80 || - (s[0] & 0xfe) == 0xc0) /* overlong? */ - return s; - else - s += 2; - } else if ((s[0] & 0xf0) == 0xe0) { - /* 1110XXXX 10Xxxxxx 10xxxxxx */ - if ((s[1] & 0xc0) != 0x80 || - (s[2] & 0xc0) != 0x80 || - (s[0] == 0xe0 && (s[1] & 0xe0) == 0x80) || /* overlong? */ - (s[0] == 0xed && (s[1] & 0xe0) == 0xa0) || /* surrogate? */ - (s[0] == 0xef && s[1] == 0xbf && - (s[2] & 0xfe) == 0xbe)) /* U+FFFE or U+FFFF? */ - return s; - else - s += 3; - } else if ((s[0] & 0xf8) == 0xf0) { - /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */ - if ((s[1] & 0xc0) != 0x80 || - (s[2] & 0xc0) != 0x80 || - (s[3] & 0xc0) != 0x80 || - (s[0] == 0xf0 && (s[1] & 0xf0) == 0x80) || /* overlong? */ - (s[0] == 0xf4 && s[1] > 0x8f) || s[0] > 0xf4) /* > U+10FFFF? */ - return s; - else - s += 4; - } else - return s; - } - - if (s == end) - return s; - - return NULL; + unsigned char *end = s + len; + while (*s && s != end) + { + if (*s < 0x80) /* 0xxxxxxx */ + s++; + else if ((s[0] & 0xe0) == 0xc0) /* 110XXXXx 10xxxxxx */ + { + if ((s[1] & 0xc0) != 0x80 || (s[0] & 0xfe) == 0xc0) /* overlong? */ + return s; + else + s += 2; + } + else if ((s[0] & 0xf0) == 0xe0) /* 1110XXXX 10Xxxxxx 10xxxxxx */ + { + if ((s[1] & 0xc0) != 0x80 || (s[2] & 0xc0) != 0x80 || + (s[0] == 0xe0 && (s[1] & 0xe0) == 0x80) || /* overlong? */ + (s[0] == 0xed && (s[1] & 0xe0) == 0xa0) || /* surrogate? */ + (s[0] == 0xef && s[1] == 0xbf && + (s[2] & 0xfe) == 0xbe)) /* U+FFFE or U+FFFF? */ + { + return s; + } + else + s += 3; + } + else if ((s[0] & 0xf8) == 0xf0) /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */ + { + if ((s[1] & 0xc0) != 0x80 || + (s[2] & 0xc0) != 0x80 || + (s[3] & 0xc0) != 0x80 || + (s[0] == 0xf0 && (s[1] & 0xf0) == 0x80) || /* overlong? */ + (s[0] == 0xf4 && s[1] > 0x8f) || s[0] > 0xf4) /* > U+10FFFF? */ + { + return s; + } + else + s += 4; + } + else + return s; + } + + if (s == end) + return s; + + return NULL; } From 6ca114b6dc654897366e9c6f48f44eb9920ff14e Mon Sep 17 00:00:00 2001 From: liyingxin Date: Fri, 16 Dec 2022 17:11:22 +0800 Subject: [PATCH 65/75] channel set_pointer(NULL) when WFWebSocketClient deinit --- src/client/WFWebSocketClient.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index 5ec6480f89..4f05a20a84 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -76,6 +76,7 @@ int WebSocketClient::init(const struct WFWebSocketParams *params) void WebSocketClient::deinit() { SeriesWork *series; + this->channel->set_pointer(NULL); series = Workflow::create_series_work(this->channel, [](const SeriesWork *series){ ComplexWebSocketChannel *channel; From 3895f7c116803dcfce6d6edd77d68cc2a65a4836 Mon Sep 17 00:00:00 2001 From: liyingxin Date: Mon, 20 Mar 2023 18:41:29 +0800 Subject: [PATCH 66/75] websocket: support wss --- src/client/WFWebSocketClient.cc | 2 ++ tutorial/tutorial-14-websocket_cli.cc | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index 4f05a20a84..4146c15524 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -51,6 +51,8 @@ int WebSocketClient::init(const struct WFWebSocketParams *params) this->channel->set_uri(uri); this->channel->set_idle_timeout(params->idle_timeout); this->channel->set_size_limit(params->size_limit); + if (uri.scheme && strcasecmp(uri.scheme, "wss") == 0) + this->channel->set_transport_type(TT_TCP_SSL); if (params->sec_protocol) this->channel->set_sec_protocol(params->sec_protocol); diff --git a/tutorial/tutorial-14-websocket_cli.cc b/tutorial/tutorial-14-websocket_cli.cc index 2b025514ec..cabb842139 100644 --- a/tutorial/tutorial-14-websocket_cli.cc +++ b/tutorial/tutorial-14-websocket_cli.cc @@ -48,7 +48,9 @@ int main(int argc, char *argv[]) { if (argc != 2) { - fprintf(stderr, "USAGE: %s \n url format: ws://host:ip\n", argv[0]); + fprintf(stderr, "USAGE: %s \n" + " url format: ws://host:ip\n" + " wss://host:ip\n", argv[0]); return 0; } From 6af758027d0c4de2a8acb4ed8a0d1d9d935fa23b Mon Sep 17 00:00:00 2001 From: liyingxin Date: Mon, 20 Mar 2023 21:32:11 +0800 Subject: [PATCH 67/75] websocket: support wss --- src/client/WFWebSocketClient.cc | 2 ++ src/factory/WFChannel.inl | 7 ++++++- tutorial/tutorial-14-websocket_cli.cc | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index 4f05a20a84..4146c15524 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -51,6 +51,8 @@ int WebSocketClient::init(const struct WFWebSocketParams *params) this->channel->set_uri(uri); this->channel->set_idle_timeout(params->idle_timeout); this->channel->set_size_limit(params->size_limit); + if (uri.scheme && strcasecmp(uri.scheme, "wss") == 0) + this->channel->set_transport_type(TT_TCP_SSL); if (params->sec_protocol) this->channel->set_sec_protocol(params->sec_protocol); diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 5413a62ec7..444710edde 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -156,6 +156,9 @@ public: void set_sending(bool sending) { this->sending = sending; } bool get_sending() const { return this->sending; } + void set_transport_type(TransportType type) { this->type = type; } + TransportType get_transport_type() const { return this->type; } + protected: virtual void dispatch(); virtual SubTask *done(); @@ -170,6 +173,7 @@ public: protected: bool sending; WFRouterTask *router_task; + TransportType type; ParsedURI uri; WFNSPolicy *ns_policy; RouteManager::RouteResult route_result; @@ -183,6 +187,7 @@ public: this->state = WFT_STATE_UNDEFINED; this->error = 0; this->sending = false; + this->type = TT_TCP; } protected: @@ -267,7 +272,7 @@ WFRouterTask *WFComplexChannel::route() auto&& cb = std::bind(&WFComplexChannel::router_callback, this, std::placeholders::_1); struct WFNSParams params = { - .type = TT_TCP, + .type = this->type, .uri = this->uri, .info = "", .fixed_addr = true, diff --git a/tutorial/tutorial-14-websocket_cli.cc b/tutorial/tutorial-14-websocket_cli.cc index 2b025514ec..cabb842139 100644 --- a/tutorial/tutorial-14-websocket_cli.cc +++ b/tutorial/tutorial-14-websocket_cli.cc @@ -48,7 +48,9 @@ int main(int argc, char *argv[]) { if (argc != 2) { - fprintf(stderr, "USAGE: %s \n url format: ws://host:ip\n", argv[0]); + fprintf(stderr, "USAGE: %s \n" + " url format: ws://host:ip\n" + " wss://host:ip\n", argv[0]); return 0; } From 7f4a58546db2747c97b9cb2bc4518cbd052e485e Mon Sep 17 00:00:00 2001 From: liyingxin Date: Thu, 30 Mar 2023 00:49:27 +0800 Subject: [PATCH 68/75] websocket : add websocket_close_t for WFChannel closed without WebSocketFrameConnectionClose --- docs/tutorial-14-websocket_cli.md | 16 ++++++++++++--- src/client/WFWebSocketClient.cc | 33 +++++++++++++++++++------------ src/client/WFWebSocketClient.h | 17 +++++++++++++++- src/factory/WFChannel.inl | 2 +- 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md index cc6cfd2018..2aaac2c54f 100644 --- a/docs/tutorial-14-websocket_cli.md +++ b/docs/tutorial-14-websocket_cli.md @@ -136,8 +136,7 @@ void process(WFWebSocketTask *task) ``process()``函数里拿到的参数``WFWebSocketTask *task``，与callback回调函数里拿到的类型是一样的，因此用法也非常类似： - 可以通过``get_msg()``拿到对应的数据，也就是上述的``WebSocketFrame``； -- 可以通过msg上的接口``get_opcode()``判断是什么类型的数据包，``process()``可能收到的数据包类型包括：**WebSocketFrameText**、**WebSocketFrameBinary**、**WebSocketFramePong**； - +- 可以通过msg上的接口``get_opcode()``判断是什么类型的数据包，``process()``可能收到的数据包类型包括：**WebSocketFrameText**、**WebSocketFrameBinary**、**WebSocketFramePong**、**WebSocketFrameConnectionClose**； #### 2. data 无论是**文本**还是**二进制**，都由``bool get_data(const char **data, size_t *size) const``拿收到的数据。 @@ -200,7 +199,7 @@ client.init(¶ms); struct WFWebSocketParams { const char *url; // 目标URL - int idle_timeout; // client保持长连接的空闲时间，超过idle_timeout没有数据过来会自动断开。默认：10s + int idle_timeout; // client保持长连接的空闲时间，超过idle_timeout没有数据过来会自动断开。默认：不断开 int ping_interval; // client自动发ping的时间间隔，用于做心跳，保持与远端的连接。默认：-1，不自动发ping(功能开发中) size_t size_limit; // 每个数据包的大小限制，超过的话会拿到错误码1009(WSStatusCodeTooLarge)。默认：不限制 bool random_masking_key; // WebSocket协议中数据包的掩码，框架帮每次自动随机生成一个。默认：自动生成 @@ -229,6 +228,17 @@ struct WFWebSocketParams 前面提到，需要发起**CLOSE** task关闭连接，回到该回调函数时则表示连接已关闭。 +如果连接被意外关闭，对方没有发**WebSocketFrameConnectionClose**，那么我们可以在一个``close()``函数上获得这个事件。我们可以在构造WebSocketClient时通过第二个函数传递进去，如下： + +``` +void connection_close() +{ + // connection is closed +} + +WebSocketClient client(process, connection_close); +``` + #### 3. 时序性保证 [**发消息**] diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index 4146c15524..96a93342d2 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -47,7 +47,7 @@ int WebSocketClient::init(const struct WFWebSocketParams *params) this->channel = new ComplexWebSocketChannel(NULL, WFGlobal::get_scheduler(), params->random_masking_key, - std::move(process)); + this->process); this->channel->set_uri(uri); this->channel->set_idle_timeout(params->idle_timeout); this->channel->set_size_limit(params->size_limit); @@ -59,18 +59,9 @@ int WebSocketClient::init(const struct WFWebSocketParams *params) if (params->sec_version) this->channel->set_sec_version(params->sec_version); - this->channel->set_callback([](WFChannel *channel) - { - ComplexWebSocketChannel *ch = (ComplexWebSocketChannel *)channel; - - pthread_mutex_lock(&ch->mutex); - if (ch->is_established() == 0) - { - ch->set_state(WFT_STATE_SYS_ERROR); - ch->set_sending(false); - } - pthread_mutex_unlock(&ch->mutex); - }); + auto&& cb = std::bind(&WebSocketClient::channel_callback, this, + std::placeholders::_1, this->close); + this->channel->set_callback(cb); return 0; } @@ -118,3 +109,19 @@ WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) return close_task; } +void WebSocketClient::channel_callback(WFChannel *ch, + std::function close) +{ + ComplexWebSocketChannel *channel = (ComplexWebSocketChannel *)ch; + + pthread_mutex_lock(&channel->mutex); + if (channel->is_established() == 0) + { + channel->set_state(WFT_STATE_SYS_ERROR); + channel->set_sending(false); + if (close != nullptr) + close(); + } + pthread_mutex_unlock(&channel->mutex); +} + diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h index 815da07bef..499b87b1c0 100644 --- a/src/client/WFWebSocketClient.h +++ b/src/client/WFWebSocketClient.h @@ -51,6 +51,8 @@ static constexpr struct WFWebSocketParams WEBSOCKET_PARAMS_DEFAULT = class WebSocketClient { public: + using websocket_close_t = std::function; + int init(const std::string& url); int init(const struct WFWebSocketParams *params); void deinit(); @@ -59,14 +61,27 @@ class WebSocketClient WFWebSocketTask *create_ping_task(websocket_callback_t cb); WFWebSocketTask *create_close_task(websocket_callback_t cb); +private: + void channel_callback(WFChannel *channel, + websocket_close_t close); + private: ComplexWebSocketChannel *channel; websocket_process_t process; + websocket_close_t close; public: WebSocketClient(websocket_process_t process) : - process(std::move(process)) + process(std::move(process)), + close(nullptr) { } + + WebSocketClient(websocket_process_t process, + websocket_close_t close) : + process(std::move(process)), + close(std::move(close)) + { } + }; #endif diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 444710edde..b1b16a9b2c 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -504,7 +504,7 @@ public: ComplexWebSocketChannel(CommSchedObject *object, CommScheduler *scheduler, bool auto_gen_mkey, - websocket_process_t&& process) : + websocket_process_t process) : WFComplexChannel(object, scheduler, std::move(process)), gen(rd()) From 2a1f026cdaba6acded2ad762bfc4672336da751c Mon Sep 17 00:00:00 2001 From: liyingxin Date: Thu, 13 Apr 2023 13:13:57 +0800 Subject: [PATCH 69/75] websocket : update handle_terminate() and client.deinit() codes --- src/client/WFWebSocketClient.cc | 58 ++++++++++++++++++++--- src/factory/WFChannel.inl | 66 +++++++++++++++++++++------ tutorial/tutorial-14-websocket_cli.cc | 7 ++- 3 files changed, 110 insertions(+), 21 deletions(-) diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index 96a93342d2..dc8209d2bb 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -21,6 +21,32 @@ #include "WebSocketMessage.h" #include "WFWebSocketClient.h" +static void deinit_wait_callback(WFMailboxTask *task) +{ + ComplexWebSocketChannel *channel; + channel = (ComplexWebSocketChannel *)series_of(task)->get_context(); + SubTask *next; + + if (channel->is_established() == false) + return; + + pthread_mutex_lock(&channel->mutex); + if (channel->get_sending() == false) + { + channel->set_sending(true); + channel->set_pointer(NULL); + next = channel; + } + else // if out_task started after deinit(), which is not recommanded + { + next = WFCondTaskFactory::create_wait_task(&channel->condition, + deinit_wait_callback); + } + pthread_mutex_unlock(&channel->mutex); + + series_of(task)->push_back(next); +} + WFWebSocketTask *WebSocketClient::create_websocket_task(websocket_callback_t cb) { WFWebSocketTask *task = new ComplexWebSocketOutTask(this->channel, @@ -69,9 +95,29 @@ int WebSocketClient::init(const struct WFWebSocketParams *params) void WebSocketClient::deinit() { SeriesWork *series; - this->channel->set_pointer(NULL); - series = Workflow::create_series_work(this->channel, - [](const SeriesWork *series){ + SubTask *first; + + if (this->channel->is_established() == false) + { + delete this->channel; + return; + } + + pthread_mutex_lock(&this->channel->mutex); + if (this->channel->get_sending() == false) + { + this->channel->set_sending(true); + this->channel->set_pointer(NULL); + first = this->channel; + } + else + { + first = WFCondTaskFactory::create_wait_task(&this->channel->condition, + deinit_wait_callback); + } + pthread_mutex_unlock(&this->channel->mutex); + + series = Workflow::create_series_work(first, [](const SeriesWork *series) { ComplexWebSocketChannel *channel; channel = (ComplexWebSocketChannel *)series->get_context(); delete channel; @@ -115,10 +161,10 @@ void WebSocketClient::channel_callback(WFChannel *ch, ComplexWebSocketChannel *channel = (ComplexWebSocketChannel *)ch; pthread_mutex_lock(&channel->mutex); - if (channel->is_established() == 0) + if (channel->is_established() == false) { - channel->set_state(WFT_STATE_SYS_ERROR); - channel->set_sending(false); +// channel->set_state(WFT_STATE_SYS_ERROR); +// channel->set_sending(false); if (close != nullptr) close(); } diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index b1b16a9b2c..3fb929a468 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -165,6 +165,7 @@ protected: virtual void handle_terminated(); virtual WFRouterTask *route(); void router_callback(WFRouterTask *task); + void wait_callback(WFMailboxTask *task); public: pthread_mutex_t mutex; @@ -229,41 +230,78 @@ SubTask *WFComplexChannel::done() return series->pop(); } + // dispacth() by handle_terminate() or deinit() + if (this->established == 0 && + (WFComplexChannel *)series->get_context() == this) + { + pthread_mutex_lock(&this->mutex); + this->sending = false; + this->condition.signal(NULL); + pthread_mutex_unlock(&this->mutex); + } + if (this->callback) this->callback(this); if (this->state == WFT_STATE_SUCCESS) this->state = WFT_STATE_UNDEFINED; -// if (this->established == 0 && this->state == WFT_STATE_SYS_ERROR) // sending == false -// delete this; - return series->pop(); } template void WFComplexChannel::handle_terminated() { - WFMailboxTask *waiter; - bool shutdown = false; + SeriesWork *series; + SubTask *first; + + if (this->established == 0) + return; pthread_mutex_lock(&this->mutex); - Workflow::create_series_work(this, nullptr); + if (this->sending == false) + { + this->sending = true; + this->set_pointer(NULL); + first = this; + } + else + { + first = WFCondTaskFactory::create_wait_task(&this->condition, + std::bind(&WFComplexChannel::wait_callback, + this, std::placeholders::_1)); + } + pthread_mutex_unlock(&this->mutex); + + series = Workflow::create_series_work(first, nullptr); + series->set_context(this); + series->start(); +} + +template +void WFComplexChannel::wait_callback(WFMailboxTask *task) +{ + SubTask *next; + if (this->established == 0) + return; + + pthread_mutex_lock(&this->mutex); if (this->sending == false) { this->sending = true; - shutdown = true; - } else { - waiter = WFCondTaskFactory::create_wait_task(&this->condition, - nullptr); - series_of(this)->push_front(this); - series_of(this)->push_front(waiter); + this->set_pointer(NULL); + next = this; + } + else + { + next = WFCondTaskFactory::create_wait_task(&this->condition, + std::bind(&WFComplexChannel::wait_callback, + this, std::placeholders::_1)); } pthread_mutex_unlock(&this->mutex); - if (shutdown == true) - this->dispatch(); + series_of(task)->push_back(next); } template diff --git a/tutorial/tutorial-14-websocket_cli.cc b/tutorial/tutorial-14-websocket_cli.cc index cabb842139..9e277358f0 100644 --- a/tutorial/tutorial-14-websocket_cli.cc +++ b/tutorial/tutorial-14-websocket_cli.cc @@ -44,6 +44,11 @@ void process(WFWebSocketTask *task) } } +void closed() +{ + fprintf(stderr, "connection is close by system\n"); +} + int main(int argc, char *argv[]) { if (argc != 2) @@ -54,7 +59,7 @@ int main(int argc, char *argv[]) return 0; } - WebSocketClient client(process); + WebSocketClient client(process, closed); client.init(argv[1]); WFFacilities::WaitGroup wg(1); From bc3fdf7d349190319998d53734b4cf2e07bf6a62 Mon Sep 17 00:00:00 2001 From: liyingxin Date: Sat, 15 Apr 2023 23:46:00 +0800 Subject: [PATCH 70/75] websocket : fix initialize state and add swith timer to avoid recursive calls --- src/factory/WFChannel.inl | 23 ++++++++++++++++++++++- src/factory/WebSocketTaskImpl.cc | 9 ++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 3fb929a468..9595da7144 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -357,9 +357,14 @@ public: std::function *)>&& cb) : WFChannelOutTask(channel, scheduler, std::move(cb)) { + this->state = WFT_STATE_UNDEFINED; + this->error = 0; + this->user_data = NULL; this->ready = true; } + void switch_callback(void *t); + protected: virtual ~ComplexChannelOutTask() { } }; @@ -445,6 +450,15 @@ void ComplexChannelOutTask::dispatch() return this->subtask_done(); } +template +void ComplexChannelOutTask::switch_callback(void *t) +{ + if (this->callback) + this->callback(this); + + delete this; +} + template SubTask *ComplexChannelOutTask::done() { @@ -467,7 +481,14 @@ SubTask *ComplexChannelOutTask::done() channel->condition.signal(NULL); pthread_mutex_unlock(&channel->mutex); - return WFChannelOutTask::done(); + auto&& cb = std::bind(&ComplexChannelOutTask::switch_callback, + this, + std::placeholders::_1); + + WFTimerTask *timer = WFTaskFactory::create_timer_task(0, 0, std::move(cb)); + series_of(this)->push_front(timer); + + return series_of(this)->pop(); } template diff --git a/src/factory/WebSocketTaskImpl.cc b/src/factory/WebSocketTaskImpl.cc index b8abc10bb2..12ba717668 100644 --- a/src/factory/WebSocketTaskImpl.cc +++ b/src/factory/WebSocketTaskImpl.cc @@ -128,7 +128,14 @@ SubTask *ComplexWebSocketOutTask::done() channel->condition.signal(NULL); pthread_mutex_unlock(&channel->mutex); - return WFChannelOutTask::done(); + auto&& cb = std::bind(&ComplexChannelOutTask::switch_callback, + this, + std::placeholders::_1); + + WFTimerTask *timer = WFTaskFactory::create_timer_task(0, 0, std::move(cb)); + series_of(this)->push_front(timer); + + return series_of(this)->pop(); } SubTask *ComplexWebSocketOutTask::upgrade() From ae011222a954d2599e35b293bbe69947ff0bc82f Mon Sep 17 00:00:00 2001 From: liyingxin Date: Fri, 28 Apr 2023 09:47:31 +0000 Subject: [PATCH 71/75] Update tutorial-14-websocket_cli.md --- docs/tutorial-14-websocket_cli.md | 45 ++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md index 2aaac2c54f..41eb36f5f8 100644 --- a/docs/tutorial-14-websocket_cli.md +++ b/docs/tutorial-14-websocket_cli.md @@ -10,7 +10,7 @@ 运行方式：**./websocket_cli \** -URL格式：**ws://host:port** +URL格式：**ws://host:port** 或 **wss://host:port** - port缺省值为80； - 如果是ssl，URL格式为：wss://host:port @@ -147,7 +147,9 @@ void process(WFWebSocketTask *task) 更多接口细节可以查看[websocket_parser.h ](/src/protocol/websocket_parser.h) -# 关闭client +# 连接关闭 + +#### 1. 主动关闭 根据**WebSocket**协议，用户需要发起一个**close**包已告诉对方以示断开连接。 @@ -169,6 +171,25 @@ wait_group.wait(); 需要注意的是，如果不主动发起close任务，直接删除client实例，那么底层使用的那个网络连接还会存在，直到超时或其他原因断开； 而``client.deinit()``是个等待内部网络资源完全释放的同步接口，需要手动调用，以保证程序退出前client的所有资源安全释放。 +#### 2. 被动关闭 + +如果连接被意外关闭，对方没有发**WebSocketFrameConnectionClose**，那么我们可以在一个``close()``函数上获得这个事件。 + +我们可以在构造WebSocketClient时通过第二个函数传递进去，如下： + +``` +void close() +{ + // connection is closed +} + +WebSocketClient client(process, close); +``` + +注意：无论是**主动关闭**还是**被动关闭**，只要构造client时传递了``close()``函数，那么连接关闭时``close()``都会被调用，也就是说，用户主动调用client.deinit()之后也会调用``close()``。 + +当然，如果连接被动关闭，那么**下一个任务发出时，内部依然会自动重建连接**，无需用户感知。所以``close()``函数只是用于让用户知道被断了，如果服务正常只是连接被断，用户无需做任何干预。 + # websocket_cli的参数 ``WebSocketClient``的init函数有两个，除了刚才介绍的传入URL函数的接口以外，还可以传入client的参数： @@ -226,17 +247,21 @@ struct WFWebSocketParams 只有在第一个任务发出的时候，连接才会真正被建立。因此如果只希望监听server而没有写消息需求的用户依然需要手动发一个**PING**，让内部建立连接。可以通过client的``create_ping_task()``接口创建一个**PING** task，该回调函数里可以通过**state**判断连接是否可用，如果等于**WFT_STATE_SUCCESS**,则表示``process()``里已经随时可以接收server来的消息了。 -前面提到，需要发起**CLOSE** task关闭连接，回到该回调函数时则表示连接已关闭。 +如果连接内部被断开，内部会伴随下一个请求自动重连，所以``close()``被调起时并不需要我们做什么。 -如果连接被意外关闭，对方没有发**WebSocketFrameConnectionClose**，那么我们可以在一个``close()``函数上获得这个事件。我们可以在构造WebSocketClient时通过第二个函数传递进去，如下： +client调用``deinit()``之后，可以重新调用``init()``继续使用。一般来说只有程序退出时才需要调用client的``deinit()``。 -``` -void connection_close() -{ - // connection is closed -} +以下是一个连接在task1发出时才真正建立，而task2被断开后被调用到用的close，而之后继续发task3内部回重建连接，最后client主动deinit()去关闭连接的生命周期图示： -WebSocketClient client(process, connection_close); +``` +[client.init()]------->------------------->---------[client.deinit()] + [task1]-[task2] |(主动关) + [conn]---->----[close] | + ^ | + | [task3] V + | [conn]-->--[close] + | + (被对方关) ``` #### 3. 时序性保证 From 022369b8dfde6aaa5406182694fdb108b1bb2660 Mon Sep 17 00:00:00 2001 From: liyingxin Date: Mon, 3 Jul 2023 20:22:07 +0800 Subject: [PATCH 72/75] websocket: support keep_alive_timeout --- docs/tutorial-14-websocket_cli.md | 5 +++-- src/client/WFWebSocketClient.cc | 4 +++- src/client/WFWebSocketClient.h | 3 +++ src/factory/WFChannel.inl | 3 +++ src/protocol/websocket_parser.c | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md index 41eb36f5f8..18ed0be456 100644 --- a/docs/tutorial-14-websocket_cli.md +++ b/docs/tutorial-14-websocket_cli.md @@ -173,7 +173,7 @@ wait_group.wait(); #### 2. 被动关闭 -如果连接被意外关闭，对方没有发**WebSocketFrameConnectionClose**，那么我们可以在一个``close()``函数上获得这个事件。 +如果连接被意外关闭(比如设置了keep_alive_timeout，连接已经断了我们并不会给对方发**WebSocketFrameConnectionClose**)，或由于其他意外连接被断开，对方没有发**WebSocketFrameConnectionClose**，那么我们可以在一个``close()``函数上获得这个事件。 我们可以在构造WebSocketClient时通过第二个函数传递进去，如下： @@ -220,7 +220,8 @@ client.init(¶ms); struct WFWebSocketParams { const char *url; // 目标URL - int idle_timeout; // client保持长连接的空闲时间，超过idle_timeout没有数据过来会自动断开。默认：不断开 + int idle_timeout; // client第一次连接上之后可接受的空闲时间，超过idle_timeout没有数据过来会自动断开。默认：不断开 + int keep_alive_timeout; // client保持长连接的时间，超过keep_alive_timeout没有数据过来会自动断开。默认：不断开 int ping_interval; // client自动发ping的时间间隔，用于做心跳，保持与远端的连接。默认：-1，不自动发ping(功能开发中) size_t size_limit; // 每个数据包的大小限制，超过的话会拿到错误码1009(WSStatusCodeTooLarge)。默认：不限制 bool random_masking_key; // WebSocket协议中数据包的掩码，框架帮每次自动随机生成一个。默认：自动生成 diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index dc8209d2bb..eb1671929d 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -76,7 +76,9 @@ int WebSocketClient::init(const struct WFWebSocketParams *params) this->process); this->channel->set_uri(uri); this->channel->set_idle_timeout(params->idle_timeout); + this->channel->set_keep_alive(params->keep_alive_timeout); this->channel->set_size_limit(params->size_limit); + if (uri.scheme && strcasecmp(uri.scheme, "wss") == 0) this->channel->set_transport_type(TT_TCP_SSL); @@ -87,7 +89,7 @@ int WebSocketClient::init(const struct WFWebSocketParams *params) auto&& cb = std::bind(&WebSocketClient::channel_callback, this, std::placeholders::_1, this->close); - this->channel->set_callback(cb); + this->channel->set_callback(std::move(cb)); return 0; } diff --git a/src/client/WFWebSocketClient.h b/src/client/WFWebSocketClient.h index 499b87b1c0..846a3f7ffd 100644 --- a/src/client/WFWebSocketClient.h +++ b/src/client/WFWebSocketClient.h @@ -30,6 +30,7 @@ struct WFWebSocketParams { const char *url; int idle_timeout; + int keep_alive_timeout; int ping_interval; size_t size_limit; bool random_masking_key; @@ -41,6 +42,7 @@ static constexpr struct WFWebSocketParams WEBSOCKET_PARAMS_DEFAULT = { .url = NULL, .idle_timeout = WS_HANDSHAKE_TIMEOUT, + .keep_alive_timeout = -1, .ping_interval = -1, .size_limit = (size_t)-1, .random_masking_key = true, @@ -82,6 +84,7 @@ class WebSocketClient close(std::move(close)) { } + virtual ~WebSocketClient() { } }; #endif diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 9595da7144..20e521927c 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -515,6 +515,7 @@ class ComplexWebSocketChannel : public WFComplexChannelidle_timeout = timeout; } + void set_keep_alive(int timeout) { this->keep_alive_timeo = timeout; } void set_size_limit(size_t size_limit) { this->size_limit = size_limit; } void set_sec_protocol(const char *protocol) { this->sec_protocol = protocol; } @@ -549,9 +550,11 @@ protected: virtual void handle_in(CommMessageIn *in); virtual int first_timeout(); virtual WFWebSocketTask *new_session(); + virtual int keep_alive_timeout() { return this->keep_alive_timeo; } private: int idle_timeout; + int keep_alive_timeo; size_t size_limit; bool auto_gen_mkey; // random Masking-Key std::random_device rd; diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c index 616f2b869b..e593a9d824 100644 --- a/src/protocol/websocket_parser.c +++ b/src/protocol/websocket_parser.c @@ -212,7 +212,7 @@ void websocket_parser_mask_data(websocket_parser_t *parser) unsigned char *utf8_check(unsigned char *s, size_t len) { unsigned char *end = s + len; - while (*s && s != end) + while (s != end && *s) { if (*s < 0x80) /* 0xxxxxxx */ s++; From 5c7eb21883b162d822b2239d8d5d53589748ffae Mon Sep 17 00:00:00 2001 From: liyingxin Date: Mon, 3 Jul 2023 20:43:09 +0800 Subject: [PATCH 73/75] websocket: support send_timeout --- docs/tutorial-14-websocket_cli.md | 17 ++++++++++++++--- src/factory/WFChannel.h | 7 +++++++ src/factory/WFChannel.inl | 3 +++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md index 18ed0be456..0739e56127 100644 --- a/docs/tutorial-14-websocket_cli.md +++ b/docs/tutorial-14-websocket_cli.md @@ -137,6 +137,7 @@ void process(WFWebSocketTask *task) - 可以通过``get_msg()``拿到对应的数据，也就是上述的``WebSocketFrame``； - 可以通过msg上的接口``get_opcode()``判断是什么类型的数据包，``process()``可能收到的数据包类型包括：**WebSocketFrameText**、**WebSocketFrameBinary**、**WebSocketFramePong**、**WebSocketFrameConnectionClose**； + #### 2. data 无论是**文本**还是**二进制**，都由``bool get_data(const char **data, size_t *size) const``拿收到的数据。 @@ -190,7 +191,7 @@ WebSocketClient client(process, close); 当然，如果连接被动关闭，那么**下一个任务发出时，内部依然会自动重建连接**，无需用户感知。所以``close()``函数只是用于让用户知道被断了，如果服务正常只是连接被断，用户无需做任何干预。 -# websocket_cli的参数 +# WebSocketClient的参数 ``WebSocketClient``的init函数有两个，除了刚才介绍的传入URL函数的接口以外，还可以传入client的参数： @@ -220,8 +221,8 @@ client.init(¶ms); struct WFWebSocketParams { const char *url; // 目标URL - int idle_timeout; // client第一次连接上之后可接受的空闲时间，超过idle_timeout没有数据过来会自动断开。默认：不断开 - int keep_alive_timeout; // client保持长连接的时间，超过keep_alive_timeout没有数据过来会自动断开。默认：不断开 + int idle_timeout; // client第一次连接上之后可接受的空闲时间，超过idle_timeout没有数据过来会自动断开。默认：不断开。单位：毫秒 + int keep_alive_timeout; // client保持长连接的时间，超过keep_alive_timeout没有数据过来会自动断开。默认：不断开。单位：毫秒 int ping_interval; // client自动发ping的时间间隔，用于做心跳，保持与远端的连接。默认：-1，不自动发ping(功能开发中) size_t size_limit; // 每个数据包的大小限制，超过的话会拿到错误码1009(WSStatusCodeTooLarge)。默认：不限制 bool random_masking_key; // WebSocket协议中数据包的掩码，框架帮每次自动随机生成一个。默认：自动生成 @@ -232,6 +233,16 @@ struct WFWebSocketParams 如果只使用URL调用init()，则会使用默参数。 +# WFWebSocketTask的参数 + +task除了上述常用的``get_msg()``接口以外，可以设置一些参数，比如发送超时send_timeout： + +``` + WFWebSocketTask *task = client.create_websocket_task(process); + task->set_send_timeout(5000); // 单位：毫秒 + task->start(); +``` + # 进阶版：注意事项！ websocket_connect_close websocket_read_write diff --git a/src/factory/WFChannel.h b/src/factory/WFChannel.h index 9af9b39ba2..7f79b8e1a1 100644 --- a/src/factory/WFChannel.h +++ b/src/factory/WFChannel.h @@ -59,6 +59,9 @@ class WFChannelTask : public TransRequest this->callback = std::move(cb); } + /* In milliseconds. timeout == -1 for unlimited. */ + void set_send_timeout(int timeout) { this->send_timeo = timeout; } + protected: virtual SubTask *done() { @@ -71,7 +74,10 @@ class WFChannelTask : public TransRequest return series->pop(); } + virtual int send_timeout() { return this->send_timeo; } + protected: + int send_timeo; MSG msg; std::function *)> callback; @@ -81,6 +87,7 @@ class WFChannelTask : public TransRequest TransRequest(channel, scheduler), callback(std::move(cb)) { + this->send_timeo = -1; } protected: diff --git a/src/factory/WFChannel.inl b/src/factory/WFChannel.inl index 20e521927c..0a0f485ee7 100644 --- a/src/factory/WFChannel.inl +++ b/src/factory/WFChannel.inl @@ -571,6 +571,9 @@ public: std::move(process)), gen(rd()) { + this->idle_timeout = -1; + this->keep_alive_timeo = -1; + this->size_limit = (size_t)-1; this->auto_gen_mkey = auto_gen_mkey; } From 59144544b38b19e2e70328d6564d8c3a969bde61 Mon Sep 17 00:00:00 2001 From: liyingxin Date: Tue, 18 Jul 2023 17:38:03 +0800 Subject: [PATCH 74/75] websocket : update codes for WFMailboxTask and kernel --- src/factory/WFCondTask.h | 2 +- src/factory/WFCondTaskFactory.cc | 17 +++-------------- src/kernel/CommScheduler.cc | 6 +++--- src/kernel/CommScheduler.h | 6 +++--- src/kernel/Communicator.cc | 10 +++++----- src/kernel/Communicator.h | 4 +++- 6 files changed, 18 insertions(+), 27 deletions(-) diff --git a/src/factory/WFCondTask.h b/src/factory/WFCondTask.h index ee3d3e72de..8676e014c5 100644 --- a/src/factory/WFCondTask.h +++ b/src/factory/WFCondTask.h @@ -45,7 +45,7 @@ class WFCondWaitTask : public WFMailboxTask public: WFCondWaitTask(wait_callback_t&& cb) : - WFMailboxTask(&this->msg, 1, std::move(cb)) + WFMailboxTask(&this->msg, std::move(cb)) { } virtual ~WFCondWaitTask() { } diff --git a/src/factory/WFCondTaskFactory.cc b/src/factory/WFCondTaskFactory.cc index 0fa983cfa3..c5e44ac233 100644 --- a/src/factory/WFCondTaskFactory.cc +++ b/src/factory/WFCondTaskFactory.cc @@ -164,7 +164,6 @@ class WFTimedWaitTask : public WFCondWaitTask { public: void set_timer(__WFWaitTimerTask *timer) { this->timer = timer; } - virtual void count(); virtual void clear_locked(); protected: @@ -246,22 +245,12 @@ void WFTimedWaitTask::clear_locked() this->timer = NULL; } -void WFTimedWaitTask::count() -{ - if (--this->value == 0) - { - if (this->state == WFT_STATE_UNDEFINED) - this->state = WFT_STATE_SUCCESS; - this->subtask_done(); - } -} - void WFTimedWaitTask::dispatch() { if (this->timer) - timer->dispatch(); + this->timer->dispatch(); - this->WFMailboxTask::count(); + this->WFMailboxTask::dispatch(); } WFTimedWaitTask::~WFTimedWaitTask() @@ -286,7 +275,7 @@ SubTask *__WFWaitTimerTask::done() this->mutex->unlock(); if (waiter) - waiter->count(); + waiter->send(NULL); delete this; return NULL; } diff --git a/src/kernel/CommScheduler.cc b/src/kernel/CommScheduler.cc index 1558a09e27..2f668fc470 100644 --- a/src/kernel/CommScheduler.cc +++ b/src/kernel/CommScheduler.cc @@ -142,7 +142,7 @@ CommTarget *CommSchedTarget::acquire(int wait_timeout) return this; } -void CommSchedTarget::release(int keep_alive) +void CommSchedTarget::release() { pthread_mutex_t *mutex = &this->mutex; @@ -164,7 +164,7 @@ void CommSchedTarget::release(int keep_alive) if (this->wait_cnt == 0 && this->group->wait_cnt > 0) pthread_cond_signal(&this->group->cond); - this->group->heap_adjust(this->index, keep_alive); + this->group->heap_adjust(this->index, this->has_idle_conn()); } pthread_mutex_unlock(mutex); @@ -192,7 +192,7 @@ void CommSchedGroup::heap_adjust(int index, int swap_on_equal) while (index > 0) { parent = this->tg_heap[(index - 1) / 2]; - if (CommSchedGroup::target_cmp(target, parent) < !!swap_on_equal) + if (CommSchedGroup::target_cmp(target, parent) < swap_on_equal) { this->tg_heap[index] = parent; parent->index = index; diff --git a/src/kernel/CommScheduler.h b/src/kernel/CommScheduler.h index 319a1d2e59..eaf22aadbf 100644 --- a/src/kernel/CommScheduler.h +++ b/src/kernel/CommScheduler.h @@ -69,7 +69,7 @@ class CommSchedTarget : public CommSchedObject, public CommTarget private: virtual CommTarget *acquire(int wait_timeout); /* final */ - virtual void release(int keep_alive); /* final */ + virtual void release(); /* final */ private: CommSchedGroup *group; @@ -132,7 +132,7 @@ class CommScheduler { ret = this->comm.request(session, *target); if (ret < 0) - (*target)->release(0); + (*target)->release(); } return ret; @@ -187,7 +187,7 @@ class CommScheduler { ret = this->comm.establish(channel, *target); if (ret < 0) - (*target)->release(0); + (*target)->release(); } return ret; diff --git a/src/kernel/Communicator.cc b/src/kernel/Communicator.cc index 73e479f07e..9baa99cf60 100644 --- a/src/kernel/Communicator.cc +++ b/src/kernel/Communicator.cc @@ -771,7 +771,7 @@ void Communicator::handle_incoming_reply(struct poller_result *res) { if (session) { - target->release(entry->state == CONN_STATE_IDLE); + target->release(); session->handle(state, res->error); } @@ -895,7 +895,7 @@ void Communicator::handle_request_result(struct poller_result *res) case PR_ST_STOPPED: state = CS_STATE_STOPPED; - entry->target->release(0); + entry->target->release(); session->handle(state, res->error); pthread_mutex_lock(&entry->mutex); /* do nothing */ @@ -1019,7 +1019,7 @@ void Communicator::handle_connect_result(struct poller_result *res) case PR_ST_STOPPED: state = CS_STATE_STOPPED; - target->release(0); + target->release(); session->handle(state, res->error); if (__sync_sub_and_fetch(&entry->ref, 1) == 0) this->release_conn(entry); @@ -1478,7 +1478,7 @@ void Communicator::callback(struct poller_result *res, void *context) if (__sync_sub_and_fetch(&entry->ref, 1) == 0) { - entry->target->release(entry->state == CONN_STATE_IDLE); + entry->target->release(); session->handle(state, res->error); comm->release_conn(entry); } @@ -1994,7 +1994,7 @@ void Communicator::shutdown(CommChannel *channel) mpoller_del(entry->sockfd, this->mpoller); if (__sync_sub_and_fetch(&entry->ref, 1) == 0) { - entry->target->release(0); + entry->target->release(); channel->handle(entry->state, entry->error); this->release_conn(entry); } diff --git a/src/kernel/Communicator.h b/src/kernel/Communicator.h index 7da99e8c30..267ff17fa8 100644 --- a/src/kernel/Communicator.h +++ b/src/kernel/Communicator.h @@ -51,6 +51,8 @@ class CommTarget *addrlen = this->addrlen; } + int has_idle_conn() const { return !list_empty(&this->idle_list); } + protected: void set_ssl(SSL_CTX *ssl_ctx, int ssl_connect_timeout) { @@ -74,7 +76,7 @@ class CommTarget virtual int init_ssl(SSL *ssl) { return 0; } public: - virtual void release(int keep_alive) { } + virtual void release() { } private: struct sockaddr *addr; From f19516f1479050ae800ebbecab3d13aac3d84ceb Mon Sep 17 00:00:00 2001 From: liyingxin Date: Wed, 16 Aug 2023 23:27:54 +0800 Subject: [PATCH 75/75] websocket : 1. check payload_length in close; 2. add get_status_code() and set_close_message(); 3. optimize the malloc codes; --- docs/tutorial-14-websocket_cli.md | 10 +- src/client/WFWebSocketClient.cc | 1 + src/protocol/WebSocketMessage.cc | 135 +++++++++++++++++--------- src/protocol/WebSocketMessage.h | 7 +- src/protocol/websocket_parser.c | 5 +- tutorial/tutorial-14-websocket_cli.cc | 10 +- 6 files changed, 116 insertions(+), 52 deletions(-) diff --git a/docs/tutorial-14-websocket_cli.md b/docs/tutorial-14-websocket_cli.md index 0739e56127..1f32452da0 100644 --- a/docs/tutorial-14-websocket_cli.md +++ b/docs/tutorial-14-websocket_cli.md @@ -93,9 +93,9 @@ public: - **文本**，通过``set_text_data()``这类接口设置； - **二进制**，通过``set_binary_data()``这类接口设置； -注意这些均为**非拷贝接口**，消息在发出之前需要用户来保证data在内存的生命周期； +注意这些均为**拷贝接口**，消息会在task里拷贝一份； -这两类接口都有一个带``bool fin``参数的接口，表示本消息是否finish。因为**WebSocket**协议的数据包允许分段传输，如果你要发送一个完整的消息想分多次发送，则可以使用带``bool fin``的接口，并且把``fin``值设置为``false``。 +这两类接口都有一个带``bool fin``参数的接口，表示本消息是否finish。因为**WebSocket**协议的数据包允许分段传输，如果你要发送一个完整的消息想分多次发送，则可以使用带``bool fin``的接口，并且把``fin``值设置为``false``，默认值是``true``。 #### 4. callback @@ -169,6 +169,12 @@ wait_group.wait(); 这里发起了一个close任务，由于close是异步的，因此在``task->start()``之后当前线程会退出，我们在当前线程结合一个了``wait_group``进行不占线程的阻塞，并在close任务的回调函数里唤醒，然后当前线程就可以安全调用``client.deinit()``、删除client实例以及退出了。 +开发者可以对close任务设置status_code和close_reason，以表示主动关闭的原因。默认status_code为`WSStatusCodeNormal`，如需设置，接口参考： + +```cpp +bool WebSocketFrame::set_close_message(uint16_t status_code, const char *data, size_t size); +``` + 需要注意的是，如果不主动发起close任务，直接删除client实例，那么底层使用的那个网络连接还会存在，直到超时或其他原因断开； 而``client.deinit()``是个等待内部网络资源完全释放的同步接口，需要手动调用，以保证程序退出前client的所有资源安全释放。 diff --git a/src/client/WFWebSocketClient.cc b/src/client/WFWebSocketClient.cc index eb1671929d..3ef94bcf34 100644 --- a/src/client/WFWebSocketClient.cc +++ b/src/client/WFWebSocketClient.cc @@ -153,6 +153,7 @@ WFWebSocketTask *WebSocketClient::create_close_task(websocket_callback_t cb) protocol::WebSocketFrame *msg = close_task->get_msg(); msg->set_opcode(WebSocketFrameConnectionClose); msg->set_masking_key(this->channel->gen_masking_key()); + msg->set_close_message(WSStatusCodeNormal, ""); return close_task; } diff --git a/src/protocol/WebSocketMessage.cc b/src/protocol/WebSocketMessage.cc index ee289db6d5..bd27bc727b 100644 --- a/src/protocol/WebSocketMessage.cc +++ b/src/protocol/WebSocketMessage.cc @@ -118,7 +118,6 @@ int WebSocketFrame::encode(struct iovec vectors[], int max) uint16_t tmp = htons(this->parser->payload_length); memcpy(p, &tmp, sizeof(tmp)); p += 2; - } else { @@ -189,96 +188,133 @@ bool WebSocketFrame::set_binary_data(const char *data, size_t size) bool WebSocketFrame::set_binary_data(const char *data, size_t size, bool fin) { - bool ret = true; + // -1/0/text/bin. Cannot set into close/ping/pong. + if (this->parser->opcode > WebSocketFrameBinary) + return false; - this->parser->opcode = WebSocketFrameBinary; - this->parser->fin = fin; + void *payload_data = this->parser->payload_data; - if (this->parser->payload_length && this->parser->payload_data) - { - ret = false; - free(this->parser->payload_data); - } + if (!payload_data) + payload_data = malloc(size); + else if (this->parser->payload_length < size) + payload_data = realloc(payload_data, size); + + if (!payload_data) + return false; - this->parser->payload_data = (char *)malloc(size); + this->parser->payload_data = payload_data; memcpy(this->parser->payload_data, data, size); this->parser->payload_length = size; - return ret; + this->parser->opcode = WebSocketFrameBinary; + this->parser->fin = fin; + + return true; } bool WebSocketFrame::set_text_data(const char *data) { - return set_text_data(data, strlen(data), true); + return this->set_text_data(data, strlen(data), true); } bool WebSocketFrame::set_text_data(const char *data, size_t size, bool fin) { - bool ret = true; + if (this->parser->opcode > WebSocketFrameBinary) + return false; - this->parser->opcode = WebSocketFrameText; - this->parser->fin = fin; + void *payload_data = this->parser->payload_data; - if (this->parser->payload_length && this->parser->payload_data) - { - ret = false; - free(this->parser->payload_data); - } + if (!payload_data) + payload_data = malloc(size); + else if (this->parser->payload_length < size) + payload_data = realloc(payload_data, size); - this->parser->payload_data = (char *)malloc(size); + if (!payload_data) + return false; + + this->parser->payload_data = payload_data; memcpy(this->parser->payload_data, data, size); this->parser->payload_length = size; - return ret; + this->parser->opcode = WebSocketFrameText; + this->parser->fin = fin; + + return true; } -bool WebSocketFrame::set_data(const websocket_parser_t *parser) +bool WebSocketFrame::set_close_message(uint16_t status_code, const char *data) { - bool ret = true; - unsigned char *p; + return this->set_close_message(status_code, data, strlen(data)); +} - if (this->parser->payload_length && this->parser->payload_data) - { - ret = false; - free(this->parser->payload_data); - } +bool WebSocketFrame::set_close_message(uint16_t status_code, + const char *data, size_t size) +{ + if (this->parser->opcode != WebSocketFrameConnectionClose) + return false; + + size_t payload_length = size + sizeof(uint16_t); + void *payload_data = this->parser->payload_data; + + if (!payload_data) + payload_data = malloc(payload_length); + else if (this->parser->payload_length < payload_length) + payload_data = realloc(payload_data, payload_length); + + if (!payload_data) + return false; + + this->parser->payload_data = payload_data; + this->parser->payload_length = payload_length; +// this->parser->status_code = status_code; -// this->parser->status_code = parser->status_code; - this->parser->payload_length = parser->payload_length; + uint16_t tmp = htons(status_code); + memcpy(this->parser->payload_data, &tmp, sizeof(uint16_t)); + memcpy((char *)this->parser->payload_data + sizeof(uint16_t), data, size); + + return true; +} + +bool WebSocketFrame::set_data(const websocket_parser_t *parser) +{ + void *payload_data = this->parser->payload_data; + size_t payload_length = parser->payload_length; if (this->parser->opcode == WebSocketFrameConnectionClose && parser->status_code != WSStatusCodeUndefined) { - this->parser->payload_length += 2; + payload_length += sizeof(uint16_t); } - this->parser->payload_data = malloc(this->parser->payload_length); - p = (unsigned char *)this->parser->payload_data; + if (!payload_data) + payload_data = malloc(payload_length); + else if (this->parser->payload_length < payload_length) + payload_data = realloc(payload_data, payload_length); + + if (!payload_data) + return false; + + this->parser->payload_data = payload_data; + this->parser->payload_length = payload_length; + this->parser->status_code = parser->status_code; if (this->parser->opcode == WebSocketFrameConnectionClose && parser->status_code != WSStatusCodeUndefined) { uint16_t tmp = htons(parser->status_code); - memcpy(p, &tmp, sizeof(tmp)); - p += 2; + memcpy(this->parser->payload_data, &tmp, sizeof(uint16_t)); + payload_data = (char *)payload_data + sizeof(uint16_t); } - memcpy(p, parser->payload_data, parser->payload_length); + memcpy(payload_data, parser->payload_data, parser->payload_length); - return ret; + return true; } -bool WebSocketFrame::get_data(const char **data, size_t *size) const +void WebSocketFrame::get_data(const char **data, size_t *size) const { - if (this->parser->status_code == WSStatusCodeUnsupportedData || - this->parser->status_code == WSStatusCodeProtocolError) - { - return false; - } - *data = (char *)this->parser->payload_data; *size = this->parser->payload_length; - return true; } bool WebSocketFrame::finished() const @@ -286,5 +322,10 @@ bool WebSocketFrame::finished() const return this->parser->fin; } +uint16_t WebSocketFrame::get_status_code() const +{ + return this->parser->status_code; +} + } // end namespace protocol diff --git a/src/protocol/WebSocketMessage.h b/src/protocol/WebSocketMessage.h index a539c0f0f5..278242182b 100644 --- a/src/protocol/WebSocketMessage.h +++ b/src/protocol/WebSocketMessage.h @@ -44,7 +44,12 @@ class WebSocketFrame : public ProtocolMessage bool set_binary_data(const char *data, size_t size); bool set_binary_data(const char *data, size_t size, bool fin); - bool get_data(const char **data, size_t *size) const; + bool set_close_message(uint16_t code, const char *data); + bool set_close_message(uint16_t code, const char *data, size_t size); + + void get_data(const char **data, size_t *size) const; + + uint16_t get_status_code() const; bool finished() const; diff --git a/src/protocol/websocket_parser.c b/src/protocol/websocket_parser.c index e593a9d824..6984a3700a 100644 --- a/src/protocol/websocket_parser.c +++ b/src/protocol/websocket_parser.c @@ -171,10 +171,13 @@ int websocket_parser_parse(websocket_parser_t *parser) p = (unsigned char *)parser->payload_data; - if (parser->opcode == WebSocketFrameConnectionClose) + if (parser->opcode == WebSocketFrameConnectionClose && + parser->payload_length >= 2) { parser->status_code = ntohs(*((uint16_t*)p)); p = malloc(parser->payload_length - 2); + if (!p) + return -1; memcpy(p, (unsigned char *)parser->payload_data + 2, parser->payload_length - 2); free(parser->payload_data); diff --git a/tutorial/tutorial-14-websocket_cli.cc b/tutorial/tutorial-14-websocket_cli.cc index 9e277358f0..6069f0be1b 100644 --- a/tutorial/tutorial-14-websocket_cli.cc +++ b/tutorial/tutorial-14-websocket_cli.cc @@ -38,9 +38,15 @@ void process(WFWebSocketTask *task) task->get_msg()->get_data(&data, &size); fprintf(stderr, "get text message: [%.*s]\n", (int)size, data); } + else if (task->get_msg()->get_opcode() == WebSocketFrameConnectionClose) + { + task->get_msg()->get_data(&data, &size); + fprintf(stderr, "close message: [%.*s] status code: %u\n", + (int)size, data, task->get_msg()->get_status_code()); + } else { - fprintf(stderr, "process opcode=%d\n", task->get_msg()->get_opcode()); + fprintf(stderr, "process opcode: %d\n", task->get_msg()->get_opcode()); } } @@ -73,12 +79,14 @@ int main(int argc, char *argv[]) wg.done(); return; } + auto *ping_task = client.create_ping_task(nullptr); auto *timer_task = WFTaskFactory::create_timer_task(3000000 /* 3s */, nullptr); auto *close_task = client.create_close_task([&wg] (WFWebSocketTask *task) { wg.done(); }); + close_task->get_msg()->set_close_message(WSStatusCodeNormal, "close after 3 seconds"); series_of(task)->push_back(ping_task); series_of(task)->push_back(timer_task); series_of(task)->push_back(close_task);