diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index e64b8406..caeb4497 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -8,7 +8,8 @@ on: env: BUILD_TYPE: Release - CH_SERVER_VERSION: 21.3.17.2 + CLICKHOUSE_SERVER_IMAGE: "clickhouse/clickhouse-server:22.3" + jobs: build: runs-on: ubuntu-latest @@ -48,7 +49,16 @@ jobs: - uses: actions/checkout@v2 - name: Install dependencies - run: sudo apt-get install -y cmake ${{ matrix.INSTALL }} ${{ matrix.INSTALL_SSL }} + run: sudo apt-get install -y docker cmake ${{ matrix.INSTALL }} ${{ matrix.INSTALL_SSL }} + + - name: Install dependencies - Docker + run: | + sudo apt remove -y docker docker-engine docker.io containerd runc + sudo apt install -y apt-transport-https ca-certificates curl gnupg lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt update -q + sudo apt install docker-ce docker-ce-cli containerd.io - name: Configure CMake run: | @@ -62,17 +72,14 @@ jobs: - name: Build run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} --target all - - name: Start ClickHouse server + - name: Test - Start ClickHouse server in background run: | - sudo apt-get install apt-transport-https ca-certificates dirmngr - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 - echo "deb https://repo.clickhouse.com/deb/stable/ main/" | sudo tee /etc/apt/sources.list.d/clickhouse.list - sudo apt-get update - sudo apt-get install -y \ - clickhouse-server=${{env.CH_SERVER_VERSION}} \ - clickhouse-client=${{env.CH_SERVER_VERSION}} \ - clickhouse-common-static=${{env.CH_SERVER_VERSION}} - sudo service clickhouse-server start + docker pull ${CLICKHOUSE_SERVER_IMAGE} + docker run -d --name clickhouse -p 9000:9000 ${CLICKHOUSE_SERVER_IMAGE} + docker ps -a + docker stats -a --no-stream + ## Check and wait until CH is ready to accept connections + docker exec clickhouse bash -c 'for i in {1..10}; do echo checking if clickhouse server is started attempt \#$i; if ( grep -q " Application: Ready for connections." /var/log/clickhouse-server/clickhouse-server.log ); then echo seems like clickhouse server is started; exit 0; fi; sleep 1; done; exit -1' - name: Test working-directory: ${{github.workspace}}/build/ut diff --git a/CMakeLists.txt b/CMakeLists.txt index 31cd8a5a..c7635c79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,12 +23,10 @@ PROJECT (CLICKHOUSE-CLIENT) ENDIF () SET (CMAKE_EXE_LINKER_FLAGS, "${CMAKE_EXE_LINKER_FLAGS} -lpthread") # -Wpedantic makes int128 support somewhat harder and less performant (by not allowing builtin __int128) - SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror") + # -Wno-deprecated-declarations to produce less cluttered output when building library itself (`deprecated` attributes are for library users) + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wno-deprecated-declarations") ENDIF () - INCLUDE_DIRECTORIES (.) - INCLUDE_DIRECTORIES (contrib) - SUBDIRS ( clickhouse contrib/absl diff --git a/README.md b/README.md index ac64d979..acb4232a 100644 --- a/README.md +++ b/README.md @@ -34,48 +34,56 @@ $ make ## Example ```cpp +#include #include using namespace clickhouse; -/// Initialize client connection. -Client client(ClientOptions().SetHost("localhost")); +int main() +{ + /// Initialize client connection. + Client client(ClientOptions().SetHost("localhost")); -/// Create a table. -client.Execute("CREATE TABLE IF NOT EXISTS test.numbers (id UInt64, name String) ENGINE = Memory"); + /// Create a table. + client.Execute("CREATE TABLE IF NOT EXISTS default.numbers (id UInt64, name String) ENGINE = Memory"); -/// Insert some values. -{ - Block block; + /// Insert some values. + { + Block block; - auto id = std::make_shared(); - id->Append(1); - id->Append(7); + auto id = std::make_shared(); + id->Append(1); + id->Append(7); - auto name = std::make_shared(); - name->Append("one"); - name->Append("seven"); + auto name = std::make_shared(); + name->Append("one"); + name->Append("seven"); - block.AppendColumn("id" , id); - block.AppendColumn("name", name); + block.AppendColumn("id" , id); + block.AppendColumn("name", name); - client.Insert("test.numbers", block); -} + client.Insert("default.numbers", block); + } -/// Select values inserted in the previous step. -client.Select("SELECT id, name FROM test.numbers", [] (const Block& block) - { - for (size_t i = 0; i < block.GetRowCount(); ++i) { - std::cout << block[0]->As()->At(i) << " " - << block[1]->As()->At(i) << "\n"; + /// Select values inserted in the previous step. + client.Select("SELECT id, name FROM default.numbers", [] (const Block& block) + { + for (size_t i = 0; i < block.GetRowCount(); ++i) { + std::cout << block[0]->As()->At(i) << " " + << block[1]->As()->At(i) << "\n"; + } } - } -); + ); + + /// Delete table. + client.Execute("DROP TABLE default.numbers"); -/// Delete table. -client.Execute("DROP TABLE test.numbers"); + return 0; +} ``` -Please note that `Client` instance is NOT thread-safe. I.e. you must create a separate `Client` for each thread or utilize some synchronization techniques. + +## Thread-safety +⚠ Please note that `Client` instance is NOT thread-safe. I.e. you must create a separate `Client` for each thread or utilize some synchronization techniques. ⚠ ## Retries If you wish to implement some retry logic atop of `clickhouse::Client` there are few simple rules to make you life easier: diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index d96ff88a..db1c8692 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -43,6 +43,9 @@ TARGET_LINK_LIBRARIES (clickhouse-cpp-lib cityhash-lib lz4-lib ) +TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib + PUBLIC ${PROJECT_SOURCE_DIR} +) ADD_LIBRARY (clickhouse-cpp-lib-static STATIC ${clickhouse-cpp-lib-src}) TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static @@ -50,6 +53,9 @@ TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static cityhash-lib lz4-lib ) +TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib-static + PUBLIC ${PROJECT_SOURCE_DIR} +) IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") INCLUDE (CheckCXXSourceCompiles) @@ -92,12 +98,14 @@ INSTALL(FILES query.h DESTINATION include/clickhouse/) INSTALL(FILES base/buffer.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/compressed.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/input.h DESTINATION include/clickhouse/base/) +INSTALL(FILES base/open_telemetry.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/output.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/platform.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/singleton.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/socket.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/string_utils.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/string_view.h DESTINATION include/clickhouse/base/) +INSTALL(FILES base/uuid.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/wire_format.h DESTINATION include/clickhouse/base/) # columns diff --git a/clickhouse/base/compressed.h b/clickhouse/base/compressed.h index 0b40d0be..8ba01fa8 100644 --- a/clickhouse/base/compressed.h +++ b/clickhouse/base/compressed.h @@ -8,8 +8,8 @@ namespace clickhouse { class CompressedInput : public ZeroCopyInput { public: - CompressedInput(InputStream* input); - ~CompressedInput(); + explicit CompressedInput(InputStream* input); + ~CompressedInput() override; protected: size_t DoNext(const void** ptr, size_t len) override; @@ -25,8 +25,8 @@ class CompressedInput : public ZeroCopyInput { class CompressedOutput : public OutputStream { public: - CompressedOutput(OutputStream * destination, size_t max_compressed_chunk_size = 0); - ~CompressedOutput(); + explicit CompressedOutput(OutputStream * destination, size_t max_compressed_chunk_size = 0); + ~CompressedOutput() override; protected: size_t DoWrite(const void* data, size_t len) override; diff --git a/clickhouse/base/open_telemetry.h b/clickhouse/base/open_telemetry.h new file mode 100644 index 00000000..34f33113 --- /dev/null +++ b/clickhouse/base/open_telemetry.h @@ -0,0 +1,23 @@ +#pragma once + +#include "uuid.h" + +#include + +namespace clickhouse::open_telemetry { + +/// See https://www.w3.org/TR/trace-context/ for trace_flags definition +enum TraceFlags : uint8_t { + TRACE_FLAG_NONE = 0, + TRACE_FLAG_SAMPLED = 1, +}; + +/// The runtime info we need to create new OpenTelemetry spans. +struct TracingContext { + UUID trace_id{}; + uint64_t span_id = 0; + std::string tracestate; + uint8_t trace_flags = TRACE_FLAG_NONE; +}; + +} // namespace clickhouse::open_telemetry diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index c6dc920e..e62e90df 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -27,7 +27,7 @@ char const* windowsErrorCategory::name() const noexcept { std::string windowsErrorCategory::message(int c) const { char error[UINT8_MAX]; - auto len = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, nullptr, static_cast(c), 0, error, sizeof(error), nullptr); + auto len = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, nullptr, static_cast(c), 0, error, sizeof(error), nullptr); if (len == 0) { return "unknown"; } @@ -112,6 +112,30 @@ void SetNonBlock(SOCKET fd, bool value) { #endif } +void SetTimeout(SOCKET fd, const SocketTimeoutParams& timeout_params) { +#if defined(_unix_) + timeval recv_timeout{ timeout_params.recv_timeout.count() / 1000, static_cast(timeout_params.recv_timeout.count() % 1000 * 1000) }; + auto recv_ret = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &recv_timeout, sizeof(recv_timeout)); + + timeval send_timeout{ timeout_params.send_timeout.count() / 1000, static_cast(timeout_params.send_timeout.count() % 1000 * 1000) }; + auto send_ret = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &send_timeout, sizeof(send_timeout)); + + if (recv_ret == -1 || send_ret == -1) { + throw std::system_error(getSocketErrorCode(), getErrorCategory(), "fail to set socket timeout"); + } +#else + DWORD recv_timeout = static_cast(timeout_params.recv_timeout.count()); + auto recv_ret = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&recv_timeout, sizeof(DWORD)); + + DWORD send_timeout = static_cast(timeout_params.send_timeout.count()); + auto send_ret = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (const char*)&send_timeout, sizeof(DWORD)); + + if (recv_ret == SOCKET_ERROR || send_ret == SOCKET_ERROR) { + throw std::system_error(getSocketErrorCode(), getErrorCategory(), "fail to set socket timeout"); + } +#endif +}; + ssize_t Poll(struct pollfd* fds, int nfds, int timeout) noexcept { #if defined(_win_) return WSAPoll(fds, nfds, timeout); @@ -120,18 +144,53 @@ ssize_t Poll(struct pollfd* fds, int nfds, int timeout) noexcept { #endif } -SOCKET SocketConnect(const NetworkAddress& addr) { +#ifndef INVALID_SOCKET +const SOCKET INVALID_SOCKET = -1; +#endif + +void CloseSocket(SOCKET socket) { + if (socket == INVALID_SOCKET) + return; + +#if defined(_win_) + closesocket(socket); +#else + close(socket); +#endif +} + +struct SocketRAIIWrapper { + SOCKET socket = INVALID_SOCKET; + + ~SocketRAIIWrapper() { + CloseSocket(socket); + } + + SOCKET operator*() const { + return socket; + } + + SOCKET release() { + auto result = socket; + socket = INVALID_SOCKET; + + return result; + } +}; + +SOCKET SocketConnect(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params) { int last_err = 0; for (auto res = addr.Info(); res != nullptr; res = res->ai_next) { - SOCKET s(socket(res->ai_family, res->ai_socktype, res->ai_protocol)); + SocketRAIIWrapper s{socket(res->ai_family, res->ai_socktype, res->ai_protocol)}; - if (s == -1) { + if (*s == INVALID_SOCKET) { continue; } - SetNonBlock(s, true); + SetNonBlock(*s, true); + SetTimeout(*s, timeout_params); - if (connect(s, res->ai_addr, (int)res->ai_addrlen) != 0) { + if (connect(*s, res->ai_addr, (int)res->ai_addrlen) != 0) { int err = getSocketErrorCode(); if ( err == EINPROGRESS || err == EAGAIN || err == EWOULDBLOCK @@ -140,7 +199,7 @@ SOCKET SocketConnect(const NetworkAddress& addr) { #endif ) { pollfd fd; - fd.fd = s; + fd.fd = *s; fd.events = POLLOUT; fd.revents = 0; ssize_t rval = Poll(&fd, 1, 5000); @@ -150,18 +209,18 @@ SOCKET SocketConnect(const NetworkAddress& addr) { } if (rval > 0) { socklen_t len = sizeof(err); - getsockopt(s, SOL_SOCKET, SO_ERROR, (char*)&err, &len); + getsockopt(*s, SOL_SOCKET, SO_ERROR, (char*)&err, &len); if (!err) { - SetNonBlock(s, false); - return s; + SetNonBlock(*s, false); + return s.release(); } last_err = err; } } } else { - SetNonBlock(s, false); - return s; + SetNonBlock(*s, false); + return s.release(); } } if (last_err > 0) { @@ -213,6 +272,7 @@ NetworkAddress::~NetworkAddress() { const struct addrinfo* NetworkAddress::Info() const { return info_; } + const std::string & NetworkAddress::Host() const { return host_; } @@ -220,6 +280,7 @@ const std::string & NetworkAddress::Host() const { SocketBase::~SocketBase() = default; + SocketFactory::~SocketFactory() = default; void SocketFactory::sleepFor(const std::chrono::milliseconds& duration) { @@ -227,14 +288,18 @@ void SocketFactory::sleepFor(const std::chrono::milliseconds& duration) { } -Socket::Socket(const NetworkAddress& addr) - : handle_(SocketConnect(addr)) +Socket::Socket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params) + : handle_(SocketConnect(addr, timeout_params)) +{} + +Socket::Socket(const NetworkAddress & addr) + : handle_(SocketConnect(addr, SocketTimeoutParams{})) {} Socket::Socket(Socket&& other) noexcept : handle_(other.handle_) { - other.handle_ = -1; + other.handle_ = INVALID_SOCKET; } Socket& Socket::operator=(Socket&& other) noexcept { @@ -242,7 +307,7 @@ Socket& Socket::operator=(Socket&& other) noexcept { Close(); handle_ = other.handle_; - other.handle_ = -1; + other.handle_ = INVALID_SOCKET; } return *this; @@ -253,14 +318,8 @@ Socket::~Socket() { } void Socket::Close() { - if (handle_ != -1) { -#if defined(_win_) - closesocket(handle_); -#else - close(handle_); -#endif - handle_ = -1; - } + CloseSocket(handle_); + handle_ = INVALID_SOCKET; } void Socket::SetTcpKeepAlive(int idle, int intvl, int cnt) noexcept { @@ -300,19 +359,21 @@ std::unique_ptr Socket::makeOutputStream() const { return std::make_unique(handle_); } + NonSecureSocketFactory::~NonSecureSocketFactory() {} std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts) { const auto address = NetworkAddress(opts.host, std::to_string(opts.port)); - auto socket = doConnect(address); + auto socket = doConnect(address, opts); setSocketOptions(*socket, opts); return socket; } -std::unique_ptr NonSecureSocketFactory::doConnect(const NetworkAddress& address) { - return std::make_unique(address); +std::unique_ptr NonSecureSocketFactory::doConnect(const NetworkAddress& address, const ClientOptions& opts) { + SocketTimeoutParams timeout_params { opts.connection_recv_timeout, opts.connection_send_timeout }; + return std::make_unique(address, timeout_params); } void NonSecureSocketFactory::setSocketOptions(Socket &socket, const ClientOptions &opts) { @@ -327,6 +388,7 @@ void NonSecureSocketFactory::setSocketOptions(Socket &socket, const ClientOption } } + SocketInput::SocketInput(SOCKET s) : s_(s) { diff --git a/clickhouse/base/socket.h b/clickhouse/base/socket.h index e7cacc19..c68f250d 100644 --- a/clickhouse/base/socket.h +++ b/clickhouse/base/socket.h @@ -82,8 +82,14 @@ class SocketFactory { }; +struct SocketTimeoutParams { + std::chrono::milliseconds recv_timeout{ 0 }; + std::chrono::milliseconds send_timeout{ 0 }; +}; + class Socket : public SocketBase { public: + Socket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params); Socket(const NetworkAddress& addr); Socket(Socket&& other) noexcept; Socket& operator=(Socket&& other) noexcept; @@ -119,7 +125,7 @@ class NonSecureSocketFactory : public SocketFactory { std::unique_ptr connect(const ClientOptions& opts) override; protected: - virtual std::unique_ptr doConnect(const NetworkAddress& address); + virtual std::unique_ptr doConnect(const NetworkAddress& address, const ClientOptions& opts); void setSocketOptions(Socket& socket, const ClientOptions& opts); }; diff --git a/clickhouse/base/sslsocket.cpp b/clickhouse/base/sslsocket.cpp index 392c22fd..03b064b1 100644 --- a/clickhouse/base/sslsocket.cpp +++ b/clickhouse/base/sslsocket.cpp @@ -78,11 +78,11 @@ void configureSSL(const clickhouse::SSLParams::ConfigurationType & configuration else if (err == 1 && value_present) throw clickhouse::OpenSSLError("Failed to configure OpenSSL: command '" + kv.first + "' needs no value"); else if (err == -2) - throw clickhouse::OpenSSLError("Failed to cofigure OpenSSL: unknown command '" + kv.first + "'"); + throw clickhouse::OpenSSLError("Failed to configure OpenSSL: unknown command '" + kv.first + "'"); else if (err == -3) - throw clickhouse::OpenSSLError("Failed to cofigure OpenSSL: command '" + kv.first + "' requires a value"); + throw clickhouse::OpenSSLError("Failed to configure OpenSSL: command '" + kv.first + "' requires a value"); else - throw clickhouse::OpenSSLError("Failed to cofigure OpenSSL: command '" + kv.first + "' unknown error: " + std::to_string(err)); + throw clickhouse::OpenSSLError("Failed to configure OpenSSL: command '" + kv.first + "' unknown error: " + std::to_string(err)); } } @@ -198,9 +198,9 @@ SSL_CTX * SSLContext::getContext() { << "\n\t handshake state: " << SSL_get_state(ssl_) \ << std::endl */ -SSLSocket::SSLSocket(const NetworkAddress& addr, const SSLParams & ssl_params, - SSLContext& context) - : Socket(addr) +SSLSocket::SSLSocket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params, + const SSLParams & ssl_params, SSLContext& context) + : Socket(addr, timeout_params) , ssl_(SSL_new(context.getContext()), &SSL_free) { auto ssl = ssl_.get(); @@ -267,8 +267,9 @@ SSLSocketFactory::SSLSocketFactory(const ClientOptions& opts) SSLSocketFactory::~SSLSocketFactory() = default; -std::unique_ptr SSLSocketFactory::doConnect(const NetworkAddress& address) { - return std::make_unique(address, ssl_params_, *ssl_context_); +std::unique_ptr SSLSocketFactory::doConnect(const NetworkAddress& address, const ClientOptions& opts) { + SocketTimeoutParams timeout_params { opts.connection_recv_timeout, opts.connection_send_timeout }; + return std::make_unique(address, timeout_params, ssl_params_, *ssl_context_); } std::unique_ptr SSLSocket::makeInputStream() const { diff --git a/clickhouse/base/sslsocket.h b/clickhouse/base/sslsocket.h index f37e4a5a..945de86d 100644 --- a/clickhouse/base/sslsocket.h +++ b/clickhouse/base/sslsocket.h @@ -48,7 +48,9 @@ class SSLContext class SSLSocket : public Socket { public: - explicit SSLSocket(const NetworkAddress& addr, const SSLParams & ssl_params, SSLContext& context); + explicit SSLSocket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params, + const SSLParams& ssl_params, SSLContext& context); + SSLSocket(SSLSocket &&) = default; ~SSLSocket() override = default; @@ -69,7 +71,7 @@ class SSLSocketFactory : public NonSecureSocketFactory { ~SSLSocketFactory() override; protected: - std::unique_ptr doConnect(const NetworkAddress& address) override; + std::unique_ptr doConnect(const NetworkAddress& address, const ClientOptions& opts) override; private: const SSLParams ssl_params_; diff --git a/clickhouse/base/string_utils.h b/clickhouse/base/string_utils.h index f2e66ba7..4d19485d 100644 --- a/clickhouse/base/string_utils.h +++ b/clickhouse/base/string_utils.h @@ -8,6 +8,7 @@ namespace clickhouse { template +[[deprecated("Not used by clickhosue-cpp itself, and will be removed in next major release (3.0) ")]] inline T FromString(const std::string& s) { std::istringstream iss(s); T result; @@ -16,6 +17,7 @@ inline T FromString(const std::string& s) { } template +[[deprecated("Not used by clickhosue-cpp itself, and will be removed in next major release (3.0) ")]] inline T FromString(const StringView& s) { std::istringstream iss((std::string(s))); T result; diff --git a/clickhouse/base/string_view.h b/clickhouse/base/string_view.h index 46619ed5..ad71907e 100644 --- a/clickhouse/base/string_view.h +++ b/clickhouse/base/string_view.h @@ -11,7 +11,9 @@ template < typename TChar, typename TTraits = std::char_traits > -class StringViewImpl { +class +[[deprecated("Obsolete due to C++17's std::string_view. Will be removed in next major release (3.0) ")]] +StringViewImpl { public: using size_type = size_t; using traits_type = TTraits; diff --git a/clickhouse/base/uuid.h b/clickhouse/base/uuid.h new file mode 100644 index 00000000..d78a1866 --- /dev/null +++ b/clickhouse/base/uuid.h @@ -0,0 +1,12 @@ +#pragma once + +#include +#include + +namespace clickhouse { + +using UInt128 = std::pair; + +using UUID = UInt128; + +} diff --git a/clickhouse/block.cpp b/clickhouse/block.cpp index aca77c00..28f0ddc9 100644 --- a/clickhouse/block.cpp +++ b/clickhouse/block.cpp @@ -71,6 +71,11 @@ const BlockInfo& Block::Info() const { return info_; } +/// Set block info +void Block::SetInfo(BlockInfo info) { + info_ = std::move(info); +} + /// Count of rows in the block. size_t Block::GetRowCount() const { return rows_; diff --git a/clickhouse/block.h b/clickhouse/block.h index a647f12d..5b8f57da 100644 --- a/clickhouse/block.h +++ b/clickhouse/block.h @@ -73,6 +73,9 @@ class Block { const BlockInfo& Info() const; + /// Set block info + void SetInfo(BlockInfo info); + /// Count of rows in the block. size_t GetRowCount() const; diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index 36c1bcb3..e4b0c7ef 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -34,8 +34,16 @@ #define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372 #define DBMS_MIN_REVISION_WITH_VERSION_PATCH 54401 #define DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE 54405 +#define DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA 54410 +#define DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO 54420 +#define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429 +#define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441 +#define DBMS_MIN_REVISION_WITH_OPENTELEMETRY 54442 +#define DBMS_MIN_REVISION_WITH_DISTRIBUTED_DEPTH 54448 +#define DBMS_MIN_REVISION_WITH_INITIAL_QUERY_START_TIME 54449 +#define DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS 54451 -#define REVISION DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE +#define REVISION DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS namespace clickhouse { @@ -129,7 +137,7 @@ class Client::Impl { bool ReceivePacket(uint64_t* server_packet = nullptr); - void SendQuery(const std::string& query, const std::string& query_id); + void SendQuery(const Query& query); void SendData(const Block& block); @@ -228,7 +236,7 @@ void Client::Impl::ExecuteQuery(Query query) { RetryGuard([this]() { Ping(); }); } - SendQuery(query.GetText(), query.GetQueryID()); + SendQuery(query); while (ReceivePacket()) { ; @@ -270,7 +278,8 @@ void Client::Impl::Insert(const std::string& table_name, const std::string& quer } } - SendQuery("INSERT INTO " + table_name + " ( " + fields_section.str() + " ) VALUES", query_id); + Query query("INSERT INTO " + table_name + " ( " + fields_section.str() + " ) VALUES", query_id); + SendQuery(query); uint64_t server_packet; // Receive data packet. @@ -407,6 +416,15 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { return false; } } + if (REVISION >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO) + { + if (!WireFormat::ReadUInt64(*input_, &info.written_rows)) { + return false; + } + if (!WireFormat::ReadUInt64(*input_, &info.written_bytes)) { + return false; + } + } if (events_) { events_->OnProgress(info); @@ -430,6 +448,53 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { return false; } + case ServerCodes::Log: { + // log tag + if (!WireFormat::SkipString(*input_)) { + return false; + } + Block block; + + // Use uncompressed stream since log blocks usually contain only one row + if (!ReadBlock(*input_, &block)) { + return false; + } + + if (events_) { + events_->OnServerLog(block); + } + return true; + } + + case ServerCodes::TableColumns: { + // external table name + if (!WireFormat::SkipString(*input_)) { + return false; + } + + // columns metadata + if (!WireFormat::SkipString(*input_)) { + return false; + } + return true; + } + + case ServerCodes::ProfileEvents: { + if (!WireFormat::SkipString(*input_)) { + return false; + } + + Block block; + if (!ReadBlock(*input_, &block)) { + return false; + } + + if (events_) { + events_->OnProfileEvents(block); + } + return true; + } + default: throw UnimplementedError("unimplemented " + std::to_string((int)packet_type)); break; @@ -461,7 +526,7 @@ bool Client::Impl::ReadBlock(InputStream& input, Block* block) { return false; } - // TODO use data + block->SetInfo(std::move(info)); } uint64_t num_columns = 0; @@ -584,9 +649,9 @@ void Client::Impl::SendCancel() { output_->Flush(); } -void Client::Impl::SendQuery(const std::string& query, const std::string& query_id) { +void Client::Impl::SendQuery(const Query& query) { WireFormat::WriteUInt64(*output_, ClientCodes::Query); - WireFormat::WriteString(*output_, query_id); + WireFormat::WriteString(*output_, query.GetQueryID()); /// Client info. if (server_info_.revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) { @@ -603,6 +668,9 @@ void Client::Impl::SendQuery(const std::string& query, const std::string& query_ WireFormat::WriteString(*output_, info.initial_user); WireFormat::WriteString(*output_, info.initial_query_id); WireFormat::WriteString(*output_, info.initial_address); + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_INITIAL_QUERY_START_TIME) { + WireFormat::WriteFixed(*output_, 0); + } WireFormat::WriteFixed(*output_, info.iface_type); WireFormat::WriteString(*output_, info.os_user); @@ -614,20 +682,56 @@ void Client::Impl::SendQuery(const std::string& query, const std::string& query_ if (server_info_.revision >= DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO) WireFormat::WriteString(*output_, info.quota_key); + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_DISTRIBUTED_DEPTH) + WireFormat::WriteUInt64(*output_, 0u); if (server_info_.revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) { WireFormat::WriteUInt64(*output_, info.client_version_patch); } + + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) { + if (const auto& tracing_context = query.GetTracingContext()) { + // Have OpenTelemetry header. + WireFormat::WriteFixed(*output_, uint8_t(1)); + // No point writing these numbers with variable length, because they + // are random and will probably require the full length anyway. + WireFormat::WriteFixed(*output_, tracing_context->trace_id); + WireFormat::WriteFixed(*output_, tracing_context->span_id); + WireFormat::WriteString(*output_, tracing_context->tracestate); + WireFormat::WriteFixed(*output_, tracing_context->trace_flags); + } else { + // Don't have OpenTelemetry header. + WireFormat::WriteFixed(*output_, uint8_t(0)); + } + } else { + if (query.GetTracingContext()) { + // Current implementation works only for server version >= v20.11.2.1-stable + throw UnimplementedError(std::string("Can't send open telemetry tracing context to a server, server version is too old")); + } + } } - /// Per query settings. - //if (settings) - // settings->serialize(*out); - //else + /// Per query settings + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) { + for(const auto& [name, field] : query.GetQuerySettings()) { + WireFormat::WriteString(*output_, name); + WireFormat::WriteVarint64(*output_, field.flags); + WireFormat::WriteString(*output_, field.value); + } + } + else if (query.GetQuerySettings().size() > 0) { + // Current implementation works only for server version >= v20.1.2.4-stable, since we do not implement binary settings serialization. + throw UnimplementedError(std::string("Can't send query settings to a server, server version is too old")); + } + // Empty string signals end of serialized settings WireFormat::WriteString(*output_, std::string()); + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET) { + WireFormat::WriteString(*output_, ""); + } + WireFormat::WriteUInt64(*output_, Stages::Complete); WireFormat::WriteUInt64(*output_, compression_); - WireFormat::WriteString(*output_, query); + WireFormat::WriteString(*output_, query.GetText()); // Send empty block as marker of // end of data SendData(Block()); diff --git a/clickhouse/client.h b/clickhouse/client.h index 6de09b8a..452226d9 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -86,22 +86,24 @@ struct ClientOptions { // TCP options DECLARE_FIELD(tcp_nodelay, bool, TcpNoDelay, true); - // TODO deprecate setting + /// Connection socket timeout. If the timeout is set to zero then the operation will never timeout. + DECLARE_FIELD(connection_recv_timeout, std::chrono::seconds, SetConnectionRecvTimeout, std::chrono::seconds(0)); + DECLARE_FIELD(connection_send_timeout, std::chrono::seconds, SetConnectionSendTimeout, std::chrono::seconds(0)); + /** It helps to ease migration of the old codebases, which can't afford to switch * to using ColumnLowCardinalityT or ColumnLowCardinality directly, * but still want to benefit from smaller on-wire LowCardinality bandwidth footprint. * * @see LowCardinalitySerializationAdaptor, CreateColumnByType */ + [[deprecated("Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) ")]] DECLARE_FIELD(backward_compatibility_lowcardinality_as_wrapped_column, bool, SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn, true); /** Set max size data to compress if compression enabled. * - * Allows choosing tradeoff betwen RAM\CPU: + * Allows choosing tradeoff between RAM\CPU: * - Lower value reduces RAM usage, but slightly increases CPU usage. * - Higher value increases RAM usage but slightly decreases CPU usage. - * - * Default is 0, use natural implementation-defined chunk size. */ DECLARE_FIELD(max_compression_chunk_size, unsigned int, SetMaxCompressionChunkSize, 65535); @@ -129,7 +131,7 @@ struct ClientOptions { * If no CAs are configured, the server's identity can't be validated, and the Client would err. * See https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_default_verify_paths.html */ - /// Load deafult CA certificates from deafult locations. + /// Load default CA certificates from default locations. DECLARE_FIELD(use_default_ca_locations, bool, SetUseDefaultCALocations, true); /// Path to the CA files to verify server certificate, may be empty. DECLARE_FIELD(path_to_ca_files, std::vector, SetPathToCAFiles, {}); diff --git a/clickhouse/columns/array.h b/clickhouse/columns/array.h index 6144e430..1d3eb192 100644 --- a/clickhouse/columns/array.h +++ b/clickhouse/columns/array.h @@ -19,7 +19,7 @@ class ColumnArray : public Column { /** Create an array of given type. * - * `data` is used internaly (and modified) by ColumnArray. + * `data` is used internally (and modified) by ColumnArray. * Users are strongly advised against supplying non-empty columns and/or modifying * contents of `data` afterwards. */ @@ -35,7 +35,7 @@ class ColumnArray : public Column { /// Converts input column to array and appends as one row to the current column. void AppendAsColumn(ColumnRef array); - /// Convets array at pos n to column. + /// Converts array at pos n to column. /// Type of element of result column same as type of array element. ColumnRef GetAsColumn(size_t n) const; diff --git a/clickhouse/columns/date.h b/clickhouse/columns/date.h index 3518aa1e..2a240c90 100644 --- a/clickhouse/columns/date.h +++ b/clickhouse/columns/date.h @@ -145,8 +145,8 @@ class ColumnDateTime64 : public Column { /// Appends one element to the end of column. void Append(const Int64& value); - // It is a bit controversal: users might expect it to parse string of ISO8601 or some other human-friendly format, - // but current implemntation parses it as fractional integer with decimal point, e.g. "123.456". + // It is a bit controversial: users might expect it to parse string of ISO8601 or some other human-friendly format, + // but current implementation parses it as fractional integer with decimal point, e.g. "123.456". // void Append(const std::string& value); /// Returns element at given row number. diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index 47c3feeb..38b02e1d 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -137,12 +137,16 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti case TypeAst::Enum: { std::vector enum_items; + //ast.elements.size() minimum is 1. + if ((ast.elements.size() % 2) != 0) { + throw ValidationError(ast.name + " content is not correct"); + } enum_items.reserve(ast.elements.size() / 2); for (size_t i = 0; i < ast.elements.size(); i += 2) { enum_items.push_back( - Type::EnumItem{ast.elements[i].value_string, - (int16_t)ast.elements[i + 1].value}); + Type::EnumItem{ ast.elements[i].value_string, + (int16_t)ast.elements[i + 1].value }); } if (ast.code == Type::Enum8) { diff --git a/clickhouse/columns/itemview.cpp b/clickhouse/columns/itemview.cpp index 3a186531..a2cb69c2 100644 --- a/clickhouse/columns/itemview.cpp +++ b/clickhouse/columns/itemview.cpp @@ -90,7 +90,7 @@ void ItemView::ValidateData(Type::Code type, DataType data) { return AssertSize({4, 8, 16}); default: - throw UnimplementedError("Unknon type code:" + std::to_string(static_cast(type))); + throw UnimplementedError("Unknown type code:" + std::to_string(static_cast(type))); } } diff --git a/clickhouse/columns/string.cpp b/clickhouse/columns/string.cpp index cfd4c061..937e6035 100644 --- a/clickhouse/columns/string.cpp +++ b/clickhouse/columns/string.cpp @@ -4,11 +4,11 @@ #include "../base/wire_format.h" namespace { -const size_t DEFAULT_BLOCK_SIZE = 4096; + +constexpr size_t DEFAULT_BLOCK_SIZE = 4096; template -size_t ComputeTotalSize(const Container & strings, size_t begin = 0, size_t len = -1) -{ +size_t ComputeTotalSize(const Container & strings, size_t begin = 0, size_t len = -1) { size_t result = 0; if (begin < strings.size()) { len = std::min(len, strings.size() - begin); @@ -37,8 +37,7 @@ void ColumnFixedString::Append(std::string_view str) { + std::to_string(str.size()) + " bytes."); } - if (data_.capacity() - data_.size() < str.size()) - { + if (data_.capacity() - data_.size() < str.size()) { // round up to the next block size const auto new_size = (((data_.size() + string_size_) / DEFAULT_BLOCK_SIZE) + 1) * DEFAULT_BLOCK_SIZE; data_.reserve(new_size); @@ -64,8 +63,7 @@ std::string_view ColumnFixedString::operator [](size_t n) const { return std::string_view(&data_[pos], string_size_); } -size_t ColumnFixedString::FixedSize() const -{ +size_t ColumnFixedString::FixedSize() const { return string_size_; } @@ -126,17 +124,15 @@ struct ColumnString::Block explicit Block(size_t starting_capacity) : size(0), - capacity(starting_capacity), - data_(new CharT[capacity]) + capacity(starting_capacity), + data_(new CharT[capacity]) {} - inline auto GetAvailable() const - { + inline auto GetAvailable() const { return capacity - size; } - std::string_view AppendUnsafe(std::string_view str) - { + std::string_view AppendUnsafe(std::string_view str) { const auto pos = &data_[size]; memcpy(pos, str.data(), str.size()); @@ -145,13 +141,11 @@ struct ColumnString::Block return std::string_view(pos, str.size()); } - auto GetCurrentWritePos() - { + auto GetCurrentWritePos() { return &data_[size]; } - std::string_view ConsumeTailAsStringViewUnsafe(size_t len) - { + std::string_view ConsumeTailAsStringViewUnsafe(size_t len) { const auto start = &data_[size]; size += len; return std::string_view(start, len); @@ -167,38 +161,71 @@ ColumnString::ColumnString() { } -ColumnString::ColumnString(const std::vector & data) +ColumnString::ColumnString(size_t element_count) : Column(Type::CreateString()) +{ + items_.reserve(element_count); + // 100 is arbitrary number, assumption that string values are about ~40 bytes long. + blocks_.reserve(std::max(1, element_count / 100)); +} + +ColumnString::ColumnString(const std::vector& data) + : ColumnString() { items_.reserve(data.size()); blocks_.emplace_back(ComputeTotalSize(data)); - for (const auto & s : data) - { + for (const auto & s : data) { AppendUnsafe(s); } +}; + +ColumnString::ColumnString(std::vector&& data) + : ColumnString() +{ + items_.reserve(data.size()); + + for (auto&& d : data) { + append_data_.emplace_back(std::move(d)); + auto& last_data = append_data_.back(); + items_.emplace_back(std::string_view{ last_data.data(),last_data.length() }); + } } ColumnString::~ColumnString() {} void ColumnString::Append(std::string_view str) { - if (blocks_.size() == 0 || blocks_.back().GetAvailable() < str.length()) - { + if (blocks_.size() == 0 || blocks_.back().GetAvailable() < str.length()) { blocks_.emplace_back(std::max(DEFAULT_BLOCK_SIZE, str.size())); } items_.emplace_back(blocks_.back().AppendUnsafe(str)); } -void ColumnString::AppendUnsafe(std::string_view str) -{ +void ColumnString::Append(const char* str) { + Append(std::string_view(str, strlen(str))); +} + +void ColumnString::Append(std::string&& steal_value) { + append_data_.emplace_back(std::move(steal_value)); + auto& last_data = append_data_.back(); + items_.emplace_back(std::string_view{ last_data.data(),last_data.length() }); +} + +void ColumnString::AppendNoManagedLifetime(std::string_view str) { + items_.emplace_back(str); +} + +void ColumnString::AppendUnsafe(std::string_view str) { items_.emplace_back(blocks_.back().AppendUnsafe(str)); } void ColumnString::Clear() { items_.clear(); blocks_.clear(); + append_data_.clear(); + append_data_.shrink_to_fit(); } std::string_view ColumnString::At(size_t n) const { @@ -216,8 +243,8 @@ void ColumnString::Append(ColumnRef column) { // TODO: fill up existing block with some items and then add a new one for the rest of items if (blocks_.size() == 0 || blocks_.back().GetAvailable() < total_size) blocks_.emplace_back(std::max(DEFAULT_BLOCK_SIZE, total_size)); - items_.reserve(items_.size() + col->Size()); + // Intentionally not doing items_.reserve() since that cripples performance. for (size_t i = 0; i < column->Size(); ++i) { this->AppendUnsafe((*col)[i]); } @@ -264,10 +291,10 @@ ColumnRef ColumnString::Slice(size_t begin, size_t len) const { if (begin < items_.size()) { len = std::min(len, items_.size() - begin); + result->items_.reserve(len); result->blocks_.emplace_back(ComputeTotalSize(items_, begin, len)); - for (size_t i = begin; i < begin + len; ++i) - { + for (size_t i = begin; i < begin + len; ++i) { result->Append(items_[i]); } } @@ -283,6 +310,7 @@ void ColumnString::Swap(Column& other) { auto & col = dynamic_cast(other); items_.swap(col.items_); blocks_.swap(col.blocks_); + append_data_.swap(col.append_data_); } ItemView ColumnString::GetItem(size_t index) const { diff --git a/clickhouse/columns/string.h b/clickhouse/columns/string.h index d6defe50..9b83a088 100644 --- a/clickhouse/columns/string.h +++ b/clickhouse/columns/string.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace clickhouse { @@ -77,13 +78,25 @@ class ColumnString : public Column { ColumnString(); ~ColumnString(); + explicit ColumnString(size_t element_count); explicit ColumnString(const std::vector & data); + explicit ColumnString(std::vector&& data); ColumnString& operator=(const ColumnString&) = delete; ColumnString(const ColumnString&) = delete; /// Appends one element to the column. void Append(std::string_view str); + /// Appends one element to the column. + void Append(const char* str); + + /// Appends one element to the column. + void Append(std::string&& steal_value); + + /// Appends one element to the column. + /// If str lifetime is managed elsewhere and guaranteed to outlive the Block sent to the server + void AppendNoManagedLifetime(std::string_view str); + /// Returns element at given row number. std::string_view At(size_t n) const; @@ -120,6 +133,7 @@ class ColumnString : public Column { std::vector items_; std::vector blocks_; + std::deque append_data_; }; } diff --git a/clickhouse/columns/uuid.cpp b/clickhouse/columns/uuid.cpp index 8e89f7af..19e94761 100644 --- a/clickhouse/columns/uuid.cpp +++ b/clickhouse/columns/uuid.cpp @@ -21,7 +21,7 @@ ColumnUUID::ColumnUUID(ColumnRef data) } } -void ColumnUUID::Append(const UInt128& value) { +void ColumnUUID::Append(const UUID& value) { data_->Append(value.first); data_->Append(value.second); } @@ -30,12 +30,12 @@ void ColumnUUID::Clear() { data_->Clear(); } -const UInt128 ColumnUUID::At(size_t n) const { - return UInt128(data_->At(n * 2), data_->At(n * 2 + 1)); +const UUID ColumnUUID::At(size_t n) const { + return UUID(data_->At(n * 2), data_->At(n * 2 + 1)); } -const UInt128 ColumnUUID::operator [] (size_t n) const { - return UInt128((*data_)[n * 2], (*data_)[n * 2 + 1]); +const UUID ColumnUUID::operator [] (size_t n) const { + return UUID((*data_)[n * 2], (*data_)[n * 2 + 1]); } void ColumnUUID::Append(ColumnRef column) { @@ -78,4 +78,3 @@ ItemView ColumnUUID::GetItem(size_t index) const { } } - diff --git a/clickhouse/columns/uuid.h b/clickhouse/columns/uuid.h index 2b7b58de..dd7d0b9d 100644 --- a/clickhouse/columns/uuid.h +++ b/clickhouse/columns/uuid.h @@ -1,11 +1,11 @@ #pragma once +#include "../base/uuid.h" #include "column.h" #include "numeric.h" namespace clickhouse { -using UInt128 = std::pair; /** * Represents a UUID column. @@ -17,13 +17,13 @@ class ColumnUUID : public Column { explicit ColumnUUID(ColumnRef data); /// Appends one element to the end of column. - void Append(const UInt128& value); + void Append(const UUID& value); /// Returns element at given row number. - const UInt128 At(size_t n) const; + const UUID At(size_t n) const; /// Returns element at given row number. - const UInt128 operator [] (size_t n) const; + const UUID operator [] (size_t n) const; public: /// Appends content of given column to the end of current one. @@ -34,7 +34,7 @@ class ColumnUUID : public Column { /// Saves column data to output stream. void SaveBody(OutputStream* output) override; - + /// Clear column data . void Clear() override; diff --git a/clickhouse/protocol.h b/clickhouse/protocol.h index dc51f32a..8d361936 100644 --- a/clickhouse/protocol.h +++ b/clickhouse/protocol.h @@ -7,7 +7,7 @@ namespace clickhouse { enum { Hello = 0, /// Name, version, revision. Data = 1, /// `Block` of data, may be compressed. - Exception = 2, /// Exception that occured on server side during query execution. + Exception = 2, /// Exception that occurred on server side during query execution. Progress = 3, /// Query execcution progress: rows and bytes read. Pong = 4, /// response to Ping sent by client. EndOfStream = 5, /// All packets were sent. @@ -16,6 +16,12 @@ namespace clickhouse { Extremes = 8, /// Block of mins and maxs, may be compressed. TablesStatusResponse = 9, /// Response to TableStatus. Log = 10, /// Query execution log. + TableColumns = 11, /// Columns' description for default values calculation + PartUUIDs = 12, /// List of unique parts ids. + ReadTaskRequest = 13, /// String (UUID) describes a request for which next task is needed + /// This is such an inverted logic, where server sends requests + /// And client returns back response + ProfileEvents = 14, /// Packet with profile events from server. }; } @@ -23,7 +29,7 @@ namespace clickhouse { namespace ClientCodes { enum { Hello = 0, /// Name, version, default database name. - Query = 1, /** Query id, query settings, query processing stage, + Query = 1, /** Query id, query settings, query processing stage, * compression status, and query text (no INSERT data). */ Data = 2, /// Data `Block` (e.g. INSERT data), may be compressed. @@ -32,7 +38,7 @@ namespace clickhouse { }; } - /// Should we compress `Block`s of data + /// Should we compress `Block`s of data namespace CompressionState { enum { Disable = 0, diff --git a/clickhouse/query.h b/clickhouse/query.h index ae98690d..21d7231f 100644 --- a/clickhouse/query.h +++ b/clickhouse/query.h @@ -3,36 +3,28 @@ #include "block.h" #include "server_exception.h" +#include "base/open_telemetry.h" + #include #include #include +#include #include +#include namespace clickhouse { -/** - * Settings of individual query. - */ -struct QuerySettings { - /// Maximum thread to use on the server-side to process a query. Default - let the server choose. - int max_threads = 0; - /// Compute min and max values of the result. - bool extremes = false; - /// Silently skip unavailable shards. - bool skip_unavailable_shards = false; - /// Write statistics about read rows, bytes, time elapsed, etc. - bool output_format_write_statistics = true; - /// Use client timezone for interpreting DateTime string values, instead of adopting server timezone. - bool use_client_time_zone = false; - - // connect_timeout - // max_block_size - // distributed_group_by_no_merge = false - // strict_insert_defaults = 0 - // network_compression_method = LZ4 - // priority = 0 +struct QuerySettingsField { + enum Flags : uint64_t + { + IMPORTANT = 0x01, + CUSTOM = 0x02, + }; + std::string value; + uint64_t flags{0}; }; +using QuerySettings = std::unordered_map; struct Profile { uint64_t rows = 0; @@ -48,6 +40,8 @@ struct Progress { uint64_t rows = 0; uint64_t bytes = 0; uint64_t total_rows = 0; + uint64_t written_rows = 0; + uint64_t written_bytes = 0; }; @@ -66,6 +60,15 @@ class QueryEvents { virtual void OnProgress(const Progress& progress) = 0; + /** Handle query execution logs provided by server. + * Amount of logs regulated by `send_logs_level` setting. + * By-default only `fatal` log events are sent to the client side. + */ + virtual void OnServerLog(const Block& block) = 0; + + /// Handle query execution profile events. + virtual void OnProfileEvents(const Block& block) = 0; + virtual void OnFinish() = 0; }; @@ -74,6 +77,8 @@ using ExceptionCallback = std::function; using ProgressCallback = std::function; using SelectCallback = std::function; using SelectCancelableCallback = std::function; +using SelectServerLogCallback = std::function; +using ProfileEventsCallback = std::function; class Query : public QueryEvents { @@ -92,6 +97,32 @@ class Query : public QueryEvents { return query_id_; } + inline const QuerySettings& GetQuerySettings() const { + return query_settings_; + } + + /// Set per query settings + inline Query& SetQuerySettings(QuerySettings query_settings) { + query_settings_ = std::move(query_settings); + return *this; + } + + /// Set per query setting + inline Query& SetSetting(const std::string& key, const QuerySettingsField& value) { + query_settings_[key] = value; + return *this; + } + + inline const std::optional& GetTracingContext() const { + return tracing_context_; + } + + /// Set tracing context for open telemetry signals + inline Query& SetTracingContext(open_telemetry::TracingContext tracing_context) { + tracing_context_ = std::move(tracing_context); + return *this; + } + /// Set handler for receiving result data. inline Query& OnData(SelectCallback cb) { select_cb_ = std::move(cb); @@ -109,13 +140,24 @@ class Query : public QueryEvents { return *this; } - - /// Set handler for receiving a progress of query exceution. + /// Set handler for receiving a progress of query execution. inline Query& OnProgress(ProgressCallback cb) { progress_cb_ = std::move(cb); return *this; } + /// Set handler for receiving a server log of query exceution. + inline Query& OnServerLog(SelectServerLogCallback cb) { + select_server_log_cb_ = std::move(cb); + return *this; + } + + /// Set handler for receiving profile events. + inline Query& OnProfileEvents(ProfileEventsCallback cb) { + profile_events_callback_cb_ = std::move(cb); + return *this; + } + static const std::string default_query_id; private: @@ -149,16 +191,32 @@ class Query : public QueryEvents { } } + void OnServerLog(const Block& block) override { + if (select_server_log_cb_) { + select_server_log_cb_(block); + } + } + + void OnProfileEvents(const Block& block) override { + if (profile_events_callback_cb_) { + profile_events_callback_cb_(block); + } + } + void OnFinish() override { } private: const std::string query_; const std::string query_id_; + std::optional tracing_context_; + QuerySettings query_settings_; ExceptionCallback exception_cb_; ProgressCallback progress_cb_; SelectCallback select_cb_; SelectCancelableCallback select_cancelable_cb_; + SelectServerLogCallback select_server_log_cb_; + ProfileEventsCallback profile_events_callback_cb_; }; } diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index 36bd9271..37a049a0 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -1,5 +1,4 @@ #include "type_parser.h" -#include "../base/string_utils.h" #include #include diff --git a/contrib/absl/CMakeLists.txt b/contrib/absl/CMakeLists.txt index 87c8dff1..2cd0f2be 100644 --- a/contrib/absl/CMakeLists.txt +++ b/contrib/absl/CMakeLists.txt @@ -1,3 +1,6 @@ ADD_LIBRARY (absl-lib STATIC numeric/int128.cc ) +TARGET_INCLUDE_DIRECTORIES (absl-lib + PUBLIC ${PROJECT_SOURCE_DIR}/contrib +) diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index de8a21ac..51cd4980 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -283,14 +283,14 @@ TYPED_TEST(GenericColumnTest, RoundTrip) { // Date32 first appeared in v21.9.2.17-stable const auto server_info = client.GetServerInfo(); if (versionNumber(server_info) < versionNumber(21, 9)) { - GTEST_SKIP() << "Date32 is availble since v21.9.2.17-stable and can't be tested against server: " << server_info; + GTEST_SKIP() << "Date32 is available since v21.9.2.17-stable and can't be tested against server: " << server_info; } } if constexpr (std::is_same_v) { const auto server_info = client.GetServerInfo(); if (versionNumber(server_info) < versionNumber(21, 7)) { - GTEST_SKIP() << "ColumnInt128 is availble since v21.7.2.7-stable and can't be tested against server: " << server_info; + GTEST_SKIP() << "ColumnInt128 is available since v21.7.2.7-stable and can't be tested against server: " << server_info; } } diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 5cc1b81a..6a0af56b 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -43,6 +43,30 @@ class ClientCase : public testing::TestWithParam { return "SELECT " + column_name + " FROM " + table_name; } + void FlushLogs() { + try { + client_->Execute("SYSTEM FLUSH LOGS"); + } catch (const std::exception & e) { + std::cerr << "Got error while flushing logs: " << e.what() << std::endl; + const auto wait_for_flush = []() { + // Insufficient privileges, the only safe way is to wait long enough for system + // to flush the logs automaticaly. Usually it takes 7.5 seconds, so just in case, + // wait 3 times that to ensure that all previously executed queries are in the logs now. + const auto wait_duration = std::chrono::seconds(23); + std::cerr << "Now we wait " << wait_duration << "..." << std::endl; + std::this_thread::sleep_for(wait_duration); + }; + // DB::Exception: clickhouse_cpp_cicd: Not enough privileges. To execute this query it's necessary to have grant SYSTEM FLUSH LOGS ON + if (std::string(e.what()).find("To execute this query it's necessary to have grant SYSTEM FLUSH LOGS ON") != std::string::npos) { + wait_for_flush(); + } + // DB::Exception: clickhouse_cpp_cicd: Cannot execute query in readonly mode + if (std::string(e.what()).find("Cannot execute query in readonly mode") != std::string::npos) { + wait_for_flush(); + } + } + } + std::unique_ptr client_; const std::string table_name = "test_clickhouse_cpp_test_ut_table"; const std::string column_name = "test_column"; @@ -428,7 +452,7 @@ TEST_P(ClientCase, Cancellable) { "CREATE TEMPORARY TABLE IF NOT EXISTS test_clickhouse_cpp_cancel (x UInt64) "); /// Insert a few blocks. In order to make cancel have effect, we have to - /// insert a relative larget amount of data. + /// insert a relative larger amount of data. const int kBlock = 10; const int kRowEachBlock = 1000000; for (unsigned j = 0; j < kBlock; j++) { @@ -850,19 +874,7 @@ TEST_P(ClientCase, Query_ID) { client_->SelectCancelable("SELECT 'b', count(*) FROM " + table_name, query_id, [](const Block &) { return true; }); client_->Execute(Query("TRUNCATE TABLE " + table_name, query_id)); - try { - client_->Execute("SYSTEM FLUSH LOGS"); - } catch (const std::exception & e) { - // DB::Exception: clickhouse_cpp_cicd: Not enough privileges. To execute this query it's necessary to have grant SYSTEM FLUSH LOGS ON - if (std::string(e.what()).find("To execute this query it's necessary to have grant SYSTEM FLUSH LOGS ON") != std::string::npos) { - // Insufficient privileges, the only safe way is to wait long enough for system - // to flush the logs automaticaly. Usualy it takes 7.5 seconds, so just in case, - // wait 3 times that to ensure that all previously executed queries are in the logs now. - const auto wait_duration = std::chrono::seconds(23); - std::cerr << "Got error while flushing logs, now we wait " << wait_duration << "..." << std::endl; - std::this_thread::sleep_for(wait_duration); - } - } + FlushLogs(); size_t total_count = 0; client_->Select("SELECT type, query_kind, query_id, query " @@ -1010,6 +1022,155 @@ TEST_P(ClientCase, RoundtripArrayTString) { EXPECT_TRUE(CompareRecursive(*array, *result_typed)); } +TEST_P(ClientCase, OnProgress) { + Block block; + createTableWithOneColumn(block); + + std::optional received_progress; + Query query("INSERT INTO " + table_name + " (*) VALUES (\'Foo\'), (\'Bar\')" ); + query.OnProgress([&](const Progress& progress) { + received_progress = progress; + }); + client_->Execute(query); + + ASSERT_TRUE(received_progress.has_value()); + + EXPECT_GE(received_progress->rows, 0u); + EXPECT_LE(received_progress->rows, 2u); + + EXPECT_GE(received_progress->bytes, 0u); + EXPECT_LE(received_progress->bytes, 10000u); + + EXPECT_GE(received_progress->total_rows, 0u); + EXPECT_LE(received_progress->total_rows, 2u); + + EXPECT_GE(received_progress->written_rows, 0u); + EXPECT_LE(received_progress->written_rows, 2u); + + EXPECT_GE(received_progress->written_bytes, 0u); + EXPECT_LE(received_progress->written_bytes, 10000u); +} + +TEST_P(ClientCase, QuerySettings) { + client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_query_settings_table_1;"); + client_->Execute("CREATE TEMPORARY TABLE IF NOT EXISTS test_clickhouse_query_settings_table_1 ( id Int64 )"); + + client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_query_settings_table_2;"); + client_->Execute("CREATE TEMPORARY TABLE IF NOT EXISTS test_clickhouse_query_settings_table_2 ( id Int64, value Int64 )"); + + client_->Execute("INSERT INTO test_clickhouse_query_settings_table_1 (*) VALUES (1)"); + + Query query("SELECT value " + "FROM test_clickhouse_query_settings_table_1 " + "LEFT OUTER JOIN test_clickhouse_query_settings_table_2 " + "ON test_clickhouse_query_settings_table_1.id = test_clickhouse_query_settings_table_2.id"); + + + bool checked = false; + + query.SetSetting("join_use_nulls", {"1"}); + + query.OnData( + [&](const Block& block) { + if (block.GetRowCount() == 0) + return; + ASSERT_EQ(1U, block.GetColumnCount()); + ASSERT_EQ(1U, block.GetRowCount()); + ASSERT_TRUE(block[0]->GetType().IsEqual(Type::CreateNullable(Type::CreateSimple()))); + auto cl = block[0]->As(); + EXPECT_TRUE(cl->IsNull(0)); + checked = true; + }); + client_->Execute(query); + + EXPECT_TRUE(checked); + + query.SetSetting("join_use_nulls", {"0"}); + + query.OnData( + [&](const Block& block) { + if (block.GetRowCount() == 0) + return; + ASSERT_EQ(1U, block.GetColumnCount()); + ASSERT_EQ(1U, block.GetRowCount()); + ASSERT_TRUE(block[0]->GetType().IsEqual(Type::CreateSimple())); + auto cl = block[0]->As(); + EXPECT_EQ(cl->At(0), 0); + checked = true; + } + ); + checked = false; + client_->Execute(query); + + EXPECT_TRUE(checked); + + query.SetSetting("wrong_setting_name", {"0", QuerySettingsField::IMPORTANT}); + + EXPECT_THROW(client_->Execute(query), ServerException); +} + +TEST_P(ClientCase, ServerLogs) { + + Block block; + createTableWithOneColumn(block); + + size_t received_row_count = 0; + Query query("INSERT INTO " + table_name + " (*) VALUES (\'Foo\'), (\'Bar\')" ); + query.SetSetting("send_logs_level", {"trace"}); + query.OnServerLog([&](const Block& block) { + received_row_count += block.GetRowCount(); + return true; + }); + client_->Execute(query); + + EXPECT_GT(received_row_count, 0U); +} + +TEST_P(ClientCase, TracingContext) { + Block block; + createTableWithOneColumn(block); + + Query query("INSERT INTO " + table_name + " (*) VALUES (\'Foo\'), (\'Bar\')" ); + open_telemetry::TracingContext tracing_context; + std::srand(std::time(0)); + tracing_context.trace_id = {std::rand(), std::rand()}; + query.SetTracingContext(tracing_context); + client_->Execute(query); + + FlushLogs(); + + size_t received_rows = 0; + client_->Select("SELECT trace_id, toString(trace_id), operation_name " + "FROM system.opentelemetry_span_log " + "WHERE trace_id = toUUID(\'" + ToString(tracing_context.trace_id) + "\');", + [&](const Block& block) { + // std::cerr << PrettyPrintBlock{block} << std::endl; + received_rows += block.GetRowCount(); + }); + + EXPECT_GT(received_rows, 0u); +} + +TEST_P(ClientCase, OnProfileEvents) { + Block block; + createTableWithOneColumn(block); + + client_->Execute("INSERT INTO " + table_name + " (*) VALUES (\'Foo\'), (\'Bar\')"); + size_t received_row_count = 0; + Query query("SELECT * FROM " + table_name); + + query.OnProfileEvents([&](const Block& block) { + received_row_count += block.GetRowCount(); + return true; + }); + client_->Execute(query); + + const int DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS = 54451; + if (client_->GetServerInfo().revision >= DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS) { + EXPECT_GT(received_row_count, 0U); + } +} + const auto LocalHostEndpoint = ClientOptions() .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) diff --git a/ut/columns_ut.cpp b/ut/columns_ut.cpp index 9a806e39..604c94a4 100644 --- a/ut/columns_ut.cpp +++ b/ut/columns_ut.cpp @@ -111,6 +111,19 @@ TEST(ColumnsCase, StringInit) { ASSERT_EQ(col->At(3), "abcd"); } +TEST(ColumnsCase, StringAppend) { + auto col = std::make_shared(); + const char* expected = "ufiudhf3493fyiudferyer3yrifhdflkdjfeuroe"; + std::string data(expected); + col->Append(data); + col->Append(std::move(data)); + col->Append("11"); + + ASSERT_EQ(col->Size(), 3u); + ASSERT_EQ(col->At(0), expected); + ASSERT_EQ(col->At(1), expected); + ASSERT_EQ(col->At(2), "11"); +} TEST(ColumnsCase, TupleAppend){ auto tuple1 = std::make_shared(std::vector({ @@ -364,8 +377,8 @@ TEST(ColumnsCase, UUIDInit) { auto col = std::make_shared(std::make_shared(MakeUUID_data())); ASSERT_EQ(col->Size(), 3u); - ASSERT_EQ(col->At(0), UInt128(0xbb6a8c699ab2414cllu, 0x86697b7fd27f0825llu)); - ASSERT_EQ(col->At(2), UInt128(0x3507213c178649f9llu, 0x9faf035d662f60aellu)); + ASSERT_EQ(col->At(0), UUID(0xbb6a8c699ab2414cllu, 0x86697b7fd27f0825llu)); + ASSERT_EQ(col->At(2), UUID(0x3507213c178649f9llu, 0x9faf035d662f60aellu)); } TEST(ColumnsCase, UUIDSlice) { @@ -373,8 +386,8 @@ TEST(ColumnsCase, UUIDSlice) { auto sub = col->Slice(1, 2)->As(); ASSERT_EQ(sub->Size(), 2u); - ASSERT_EQ(sub->At(0), UInt128(0x84b9f24bc26b49c6llu, 0xa03b4ab723341951llu)); - ASSERT_EQ(sub->At(1), UInt128(0x3507213c178649f9llu, 0x9faf035d662f60aellu)); + ASSERT_EQ(sub->At(0), UUID(0x84b9f24bc26b49c6llu, 0xa03b4ab723341951llu)); + ASSERT_EQ(sub->At(1), UUID(0x3507213c178649f9llu, 0x9faf035d662f60aellu)); } TEST(ColumnsCase, Int128) { @@ -613,7 +626,7 @@ TEST(ColumnsCase, ColumnDecimal128_from_string_overflow) { EXPECT_ANY_THROW(col->Append("400000000000000000000000000000000000000")); #ifndef ABSL_HAVE_INTRINSIC_INT128 - // unfortunatelly std::numeric_limits::min() overflows when there is no __int128 intrinsic type. + // unfortunately std::numeric_limits::min() overflows when there is no __int128 intrinsic type. EXPECT_ANY_THROW(col->Append("-170141183460469231731687303715884105728")); #endif } @@ -669,7 +682,7 @@ TEST(ColumnsCase, ColumnLowCardinalityString_Load) { } } -// This is temporary diabled since we are not 100% compatitable with ClickHouse +// This is temporary disabled since we are not 100% compatitable with ClickHouse // on how we serailize LC columns, but we check interoperability in other tests (see client_ut.cpp) TEST(ColumnsCase, DISABLED_ColumnLowCardinalityString_Save) { const size_t items_count = 10; @@ -774,4 +787,3 @@ TEST(ColumnsCase, ColumnLowCardinalityString_WithEmptyString_3) { EXPECT_EQ(values[i], col.At(i)) << " at pos: " << i; } } - diff --git a/ut/socket_ut.cpp b/ut/socket_ut.cpp index 6f428428..5a263435 100644 --- a/ut/socket_ut.cpp +++ b/ut/socket_ut.cpp @@ -33,6 +33,36 @@ TEST(Socketcase, connecterror) { } } +TEST(Socketcase, timeoutrecv) { + using Seconds = std::chrono::seconds; + + int port = 19979; + NetworkAddress addr("localhost", std::to_string(port)); + LocalTcpServer server(port); + server.start(); + + std::this_thread::sleep_for(std::chrono::seconds(1)); + try { + Socket socket(addr, SocketTimeoutParams { Seconds(5), Seconds(5) }); + + std::unique_ptr ptr_input_stream = socket.makeInputStream(); + char buf[1024]; + ptr_input_stream->Read(buf, sizeof(buf)); + + } + catch (const std::system_error& e) { +#if defined(_unix_) + auto expected = EAGAIN; +#else + auto expected = WSAETIMEDOUT; +#endif + ASSERT_EQ(expected, e.code().value()); + } + + std::this_thread::sleep_for(std::chrono::seconds(1)); + server.stop(); +} + // Test to verify that reading from empty socket doesn't hangs. //TEST(Socketcase, ReadFromEmptySocket) { // const int port = 12345; diff --git a/ut/ssl_ut.cpp b/ut/ssl_ut.cpp index 2539ac93..f68db08c 100644 --- a/ut/ssl_ut.cpp +++ b/ut/ssl_ut.cpp @@ -75,7 +75,7 @@ INSTANTIATE_TEST_SUITE_P( } )); -// For some reasons doen't work on MacOS. +// For some reasons doesn't work on MacOS. // Looks like `VerifyCAPath` has no effect, while parsing and setting value works. // Also for some reason SetPathToCADirectory() + SSL_CTX_load_verify_locations() works. #if !defined(__APPLE__) @@ -141,7 +141,7 @@ INSTANTIATE_TEST_SUITE_P( ClientOptions(ClickHouseExplorerConfig) .SetSSLOptions(ClientOptions::SSLOptions() .SetUseDefaultCALocations(false) - .SetSkipVerification(true)), // No CA loaded, but verfication is skipped + .SetSkipVerification(true)), // No CA loaded, but verification is skipped {"SELECT 1;"} } )); diff --git a/ut/types_ut.cpp b/ut/types_ut.cpp index 8e355ee1..c5922d0e 100644 --- a/ut/types_ut.cpp +++ b/ut/types_ut.cpp @@ -86,8 +86,6 @@ TEST(TypesCase, IsEqual) { "DateTime64(3, 'UTC')", "Decimal(9,3)", "Decimal(18,3)", - "Enum8()", - "Enum16()", "Enum8('ONE' = 1)", "Enum8('ONE' = 1, 'TWO' = 2)", "Enum16('ONE' = 1, 'TWO' = 2, 'THREE' = 3, 'FOUR' = 4)", @@ -127,3 +125,17 @@ TEST(TypesCase, IsEqual) { } } } + +TEST(TypesCase, ErrorEnumContent) { + const std::string type_names[] = { + "Enum8()", + "Enum8('ONE')", + "Enum8('ONE'=1,'TWO')", + "Enum16('TWO'=,'TWO')", + }; + + for (const auto& type_name : type_names) { + SCOPED_TRACE(type_name); + EXPECT_THROW(clickhouse::CreateColumnByType(type_name)->Type(), ValidationError); + } +} \ No newline at end of file diff --git a/ut/utils.cpp b/ut/utils.cpp index 07ae5174..e624f45c 100644 --- a/ut/utils.cpp +++ b/ut/utils.cpp @@ -12,9 +12,11 @@ #include #include #include +#include #include // for ipv4-ipv6 platform-specific stuff +#include #include #include @@ -114,6 +116,15 @@ bool doPrintValue(const ColumnRef & c, const size_t row, std: return false; } +template <> +bool doPrintValue(const ColumnRef & c, const size_t row, std::ostream & ostr) { + if (const auto & uuid_col = c->As()) { + ostr << ToString(uuid_col->At(row)); + return true; + } + return false; +} + std::ostream & printColumnValue(const ColumnRef& c, const size_t row, std::ostream & ostr) { const auto r = false @@ -138,7 +149,8 @@ std::ostream & printColumnValue(const ColumnRef& c, const size_t row, std::ostre || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) - || doPrintValue(c, row, ostr); + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr); if (!r) ostr << "Unable to print value of type " << c->GetType().GetName(); @@ -271,3 +283,14 @@ std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info) { uint64_t versionNumber(const ServerInfo & server_info) { return versionNumber(server_info.version_major, server_info.version_minor, server_info.version_patch, server_info.revision); } + +std::string ToString(const clickhouse::UUID& v) { + std::string result(36, 0); + // ffff ff ff ss ssssss + const int count = std::snprintf(result.data(), result.size() + 1, "%.8" PRIx64 "-%.4" PRIx64 "-%.4" PRIx64 "-%.4" PRIx64 "-%.12" PRIx64, + v.first >> 32, (v.first >> 16) & 0xffff, v.first & 0xffff, v.second >> 48, v.second & 0xffffffffffff); + if (count != 36) { + throw std::runtime_error("Error while converting UUID to string"); + } + return result; +} diff --git a/ut/utils.h b/ut/utils.h index f0a6194f..621119fb 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "utils_meta.h" #include "utils_comparison.h" @@ -157,3 +158,5 @@ inline uint64_t versionNumber( } uint64_t versionNumber(const clickhouse::ServerInfo & server_info); + +std::string ToString(const clickhouse::UUID& v); diff --git a/ut/utils_ut.cpp b/ut/utils_ut.cpp index a600ada0..0f67c043 100644 --- a/ut/utils_ut.cpp +++ b/ut/utils_ut.cpp @@ -33,3 +33,9 @@ TEST(TestCompareContainer, CompareNested) { EXPECT_FALSE(CompareRecursive(std::vector>{{1, 2, 3}, {4, 5, 6}}, std::vector>{{1, 2, 3}, {}})); EXPECT_FALSE(CompareRecursive(std::vector>{{1, 2, 3}, {4, 5, 6}}, std::vector>{{}})); } + +TEST(StringUtils, UUID) { + const clickhouse::UUID& uuid{0x0102030405060708, 0x090a0b0c0d0e0f10}; + const std::string uuid_string = "01020304-0506-0708-090a-0b0c0d0e0f10"; + EXPECT_EQ(ToString(uuid), uuid_string); +} diff --git a/ut/value_generators.cpp b/ut/value_generators.cpp index 43b9dff5..41e36a61 100644 --- a/ut/value_generators.cpp +++ b/ut/value_generators.cpp @@ -29,12 +29,12 @@ std::vector MakeStrings() { return {"a", "ab", "abc", "abcd"}; } -std::vector MakeUUIDs() { +std::vector MakeUUIDs() { return { - UInt128(0llu, 0llu), - UInt128(0xbb6a8c699ab2414cllu, 0x86697b7fd27f0825llu), - UInt128(0x84b9f24bc26b49c6llu, 0xa03b4ab723341951llu), - UInt128(0x3507213c178649f9llu, 0x9faf035d662f60aellu) + UUID(0llu, 0llu), + UUID(0xbb6a8c699ab2414cllu, 0x86697b7fd27f0825llu), + UUID(0x84b9f24bc26b49c6llu, 0xa03b4ab723341951llu), + UUID(0x3507213c178649f9llu, 0x9faf035d662f60aellu) }; } diff --git a/ut/value_generators.h b/ut/value_generators.h index a3004102..3632ca9e 100644 --- a/ut/value_generators.h +++ b/ut/value_generators.h @@ -38,7 +38,7 @@ std::vector MakeDates32(); std::vector MakeDateTimes(); std::vector MakeIPv4s(); std::vector MakeIPv6s(); -std::vector MakeUUIDs(); +std::vector MakeUUIDs(); std::vector MakeInt128s(); std::vector MakeDecimals(size_t precision, size_t scale);