From 1350915fbebb62ba0bdf808fefa64c4364b167f0 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 26 Apr 2023 13:15:45 +0200 Subject: [PATCH 01/53] added DEBUG_DEPENDENCIES option to simplify debugging dependencies issues --- CMakeLists.txt | 34 ++++++++++++++++++++++++++++++++++ cmake/Findlz4.cmake | 1 + 2 files changed, 35 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index aed0d85c..3152b552 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,7 @@ OPTION (WITH_OPENSSL "Use OpenSSL for TLS connections" OFF) OPTION (WITH_SYSTEM_ABSEIL "Use system ABSEIL" OFF) OPTION (WITH_SYSTEM_LZ4 "Use system LZ4" OFF) OPTION (WITH_SYSTEM_CITYHASH "Use system cityhash" OFF) +OPTION (DEBUG_DEPENDENCIES "Print debug info about dependencies duting build" ON) PROJECT (CLICKHOUSE-CLIENT) @@ -99,3 +100,36 @@ PROJECT (CLICKHOUSE-CLIENT) ut ) ENDIF (BUILD_TESTS) + + if(DEBUG_DEPENDENCIES) + function(print_target_properties target) + MESSAGE("${target} properties:") + set(properties "${ARGN}") + foreach(property_name ${properties}) + get_target_property(PROPERTY ${target} ${property_name}) + MESSAGE(NOTICE "\t${property_name} : ${PROPERTY}") + endforeach() + + # Can't get path to the target file at configure time, + # so have to create a target to fetch that info at generate time. + string(REPLACE ":" "_" target_plain_name ${target}) + add_custom_target(${target_plain_name}_print_debug_info COMMAND ${CMAKE_COMMAND} -E echo "${target} : $") + add_dependencies(clickhouse-cpp-lib ${target_plain_name}_print_debug_info) + endfunction() + + function(print_target_debug_info target) + print_target_properties(${target} + INCLUDE_DIRECTORIES + BINARY_DIR + INTERFACE_INCLUDE_DIRECTORIES + INTERFACE_LINK_LIBRARIES + LINK_LIBRARIES + LINK_LIBRARIES_ONLY_TARGETS + IMPORTED_LOCATION + ) + endfunction() + + print_target_debug_info(absl::int128) + print_target_debug_info(cityhash::cityhash) + print_target_debug_info(lz4::lz4) + endif() diff --git a/cmake/Findlz4.cmake b/cmake/Findlz4.cmake index ef8a366e..20917b84 100644 --- a/cmake/Findlz4.cmake +++ b/cmake/Findlz4.cmake @@ -32,6 +32,7 @@ if (lz4_FOUND) if (NOT TARGET lz4::lz4) add_library(lz4::lz4 UNKNOWN IMPORTED) set_target_properties(lz4::lz4 PROPERTIES + INCLUDE_DIRECTORIES ${lz4_INCLUDE_DIRS} IMPORTED_LOCATION "${lz4_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}") endif () From bb8545991d900a842808243255afb6317a67405f Mon Sep 17 00:00:00 2001 From: Ruoyu Zhong Date: Sat, 22 Apr 2023 02:15:39 +0800 Subject: [PATCH 02/53] Fix system cityhash setup cityhash does not provide any CMake packages. Add a `Findcityhash.cmake` script to help find cityhash. Signed-off-by: Ruoyu Zhong --- cmake/Findcityhash.cmake | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 cmake/Findcityhash.cmake diff --git a/cmake/Findcityhash.cmake b/cmake/Findcityhash.cmake new file mode 100644 index 00000000..5f71dcc4 --- /dev/null +++ b/cmake/Findcityhash.cmake @@ -0,0 +1,26 @@ +find_path(cityhash_INCLUDE_DIR + NAMES city.h + DOC "cityhash include directory") +mark_as_advanced(cityhash_INCLUDE_DIR) +find_library(cityhash_LIBRARY + NAMES cityhash libcityhash + DOC "cityhash library") +mark_as_advanced(cityhash_LIBRARY) + +# Unlike lz4, cityhash's version information does not seem to be available. + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(cityhash + REQUIRED_VARS cityhash_LIBRARY cityhash_INCLUDE_DIR) + +if (cityhash_FOUND) + set(cityhash_INCLUDE_DIRS "${cityhash_INCLUDE_DIR}") + set(cityhash_LIBRARIES "${cityhash_LIBRARY}") + + if (NOT TARGET cityhash::cityhash) + add_library(cityhash::cityhash UNKNOWN IMPORTED) + set_target_properties(cityhash::cityhash PROPERTIES + IMPORTED_LOCATION "${cityhash_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${cityhash_INCLUDE_DIR}") + endif () +endif () From 551ae094900d4f76185df10a90c54029fc8e370b Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 24 May 2023 08:00:11 +0000 Subject: [PATCH 03/53] Draft --- clickhouse/CMakeLists.txt | 1 + clickhouse/base/hosts_iterator.cpp | 46 +++++++++++++++++++++++ clickhouse/base/hosts_iterator.h | 44 ++++++++++++++++++++++ clickhouse/base/socket.cpp | 6 +-- clickhouse/base/socket.h | 5 ++- clickhouse/client.cpp | 60 ++++++++++++++++++++++++------ clickhouse/client.h | 3 ++ tests/simple/main.cpp | 14 ++++++- 8 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 clickhouse/base/hosts_iterator.cpp create mode 100644 clickhouse/base/hosts_iterator.h diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 67663ec5..a9fbdb9d 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -5,6 +5,7 @@ SET ( clickhouse-cpp-lib-src base/platform.cpp base/socket.cpp base/wire_format.cpp + base/hosts_iterator.cpp columns/array.cpp columns/column.cpp diff --git a/clickhouse/base/hosts_iterator.cpp b/clickhouse/base/hosts_iterator.cpp new file mode 100644 index 00000000..033669f4 --- /dev/null +++ b/clickhouse/base/hosts_iterator.cpp @@ -0,0 +1,46 @@ +#include "hosts_iterator.h" +#include "../client.h" + +namespace clickhouse { + +RoundRobinHostsIterator::RoundRobinHostsIterator(const ClientOptions& opts) : + hosts (opts.hosts) + , ports (opts.ports) + , current_index (0) + , reseted (true) + , iteration_counter(0) +{ + +} + +const std::string RoundRobinHostsIterator::getHostAddr() const +{ + return hosts[current_index]; +} + +unsigned int RoundRobinHostsIterator::getPort() const +{ + return ports[current_index]; +} + +void RoundRobinHostsIterator::ResetIterations() +{ + reseted = true; + iteration_counter = 0; +} + +void RoundRobinHostsIterator::next() +{ + current_index = (current_index + 1) % hosts.size(); + iteration_counter++; +} + +bool RoundRobinHostsIterator::nextIsExist() const +{ + return iteration_counter < hosts.size(); +} + +RoundRobinHostsIterator::~RoundRobinHostsIterator() = default; + + +} \ No newline at end of file diff --git a/clickhouse/base/hosts_iterator.h b/clickhouse/base/hosts_iterator.h new file mode 100644 index 00000000..57941278 --- /dev/null +++ b/clickhouse/base/hosts_iterator.h @@ -0,0 +1,44 @@ +#pragma once + +#include "../client.h" +#include + +namespace clickhouse { + +struct ClientOptions; + +class HostsIteratorBase +{ + public: + virtual ~HostsIteratorBase() = default; + + virtual void next() = 0; + virtual const std::string getHostAddr() const = 0; + virtual unsigned int getPort() const = 0; + virtual void ResetIterations() = 0; + virtual bool nextIsExist() const = 0; +}; + + +class RoundRobinHostsIterator : public HostsIteratorBase +{ + public: + RoundRobinHostsIterator(const ClientOptions& opts); + const std::string getHostAddr() const override; + unsigned int getPort() const override; + void ResetIterations() override; + bool nextIsExist() const override; + void next() override; + + ~RoundRobinHostsIterator() override; + + private: + + const std::vector& hosts; + const std::vector& ports; + int current_index; + bool reseted; + size_t iteration_counter; +}; + +} \ No newline at end of file diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index 48e90c73..36fb8e74 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -390,9 +390,9 @@ std::unique_ptr Socket::makeOutputStream() const { NonSecureSocketFactory::~NonSecureSocketFactory() {} -std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts) { - const auto address = NetworkAddress(opts.host, std::to_string(opts.port)); - +std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts, const std::shared_ptr hosts_iterator) { + + const auto address = NetworkAddress(hosts_iterator->getHostAddr(), std::to_string(hosts_iterator->getPort())); auto socket = doConnect(address, opts); setSocketOptions(*socket, opts); diff --git a/clickhouse/base/socket.h b/clickhouse/base/socket.h index 694d0d69..631da7d1 100644 --- a/clickhouse/base/socket.h +++ b/clickhouse/base/socket.h @@ -3,6 +3,7 @@ #include "platform.h" #include "input.h" #include "output.h" +#include "hosts_iterator.h" #include #include @@ -88,7 +89,7 @@ class SocketFactory { // TODO: move connection-related options to ConnectionOptions structure. - virtual std::unique_ptr connect(const ClientOptions& opts) = 0; + virtual std::unique_ptr connect(const ClientOptions& opts, const std::shared_ptr hosts_iterator) = 0; virtual void sleepFor(const std::chrono::milliseconds& duration); }; @@ -135,7 +136,7 @@ class NonSecureSocketFactory : public SocketFactory { public: ~NonSecureSocketFactory() override; - std::unique_ptr connect(const ClientOptions& opts) override; + std::unique_ptr connect(const ClientOptions& opts, const std::shared_ptr hosts_iterator) override; protected: virtual std::unique_ptr doConnect(const NetworkAddress& address, const ClientOptions& opts); diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index e4b0c7ef..b9ca4a5d 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -14,7 +14,7 @@ #include #include #include - +#include #if defined(WITH_OPENSSL) #include "base/sslsocket.h" #endif @@ -161,6 +161,8 @@ class Client::Impl { /// In case of network errors tries to reconnect to server and /// call fuc several times. void RetryGuard(std::function func); + + void RetryToConstEndpoint(std::function func); private: class EnsureNull { @@ -194,30 +196,52 @@ class Client::Impl { std::unique_ptr input_; std::unique_ptr output_; std::unique_ptr socket_; + std::shared_ptr hosts_iterator; ServerInfo server_info_; }; +ClientOptions modifyClientOptions(ClientOptions opts) +{ + if (!opts.host.empty()) + opts.hosts.insert(opts.hosts.begin(), opts.host); + + opts.ports.insert(opts.ports.begin(), opts.port); + return opts; +} Client::Impl::Impl(const ClientOptions& opts) : Impl(opts, GetSocketFactory(opts)) {} Client::Impl::Impl(const ClientOptions& opts, std::unique_ptr socket_factory) - : options_(opts) + : options_(modifyClientOptions(opts)) , events_(nullptr) , socket_factory_(std::move(socket_factory)) + , hosts_iterator(new RoundRobinHostsIterator(options_)) { - for (unsigned int i = 0; ; ) { - try { - ResetConnection(); - break; + auto init_connection_with_host = [&](){ + for (unsigned int i = 0; ; ) { + try { + ResetConnection(); + break; + } catch (const std::system_error&) { + if (++i > options_.send_retries) { + throw; + } + socket_factory_->sleepFor(options_.retry_timeout); + } + } + }; + + for (; hosts_iterator->nextIsExist(); hosts_iterator->next()) + { + try + { + init_connection_with_host(); } catch (const std::system_error&) { - if (++i > options_.send_retries) { + if(!hosts_iterator->nextIsExist()) throw; - } - - socket_factory_->sleepFor(options_.retry_timeout); } } @@ -329,7 +353,7 @@ void Client::Impl::Ping() { } void Client::Impl::ResetConnection() { - InitializeStreams(socket_factory_->connect(options_)); + InitializeStreams(socket_factory_->connect(options_, hosts_iterator)); if (!Handshake()) { throw ProtocolError("fail to connect to " + options_.host); @@ -861,6 +885,20 @@ bool Client::Impl::ReceiveHello() { } void Client::Impl::RetryGuard(std::function func) { + for(hosts_iterator->ResetIterations(); ; hosts_iterator->next()) + { + try + { + RetryToConstEndpoint(func); + return; + } catch (const std::system_error&) { + if (!hosts_iterator->nextIsExist()) + throw; + } + } +} + +void Client::Impl::RetryToConstEndpoint(std::function func) { for (unsigned int i = 0; ; ++i) { try { func(); diff --git a/clickhouse/client.h b/clickhouse/client.h index 63177313..f5e1fc42 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -58,6 +58,9 @@ struct ClientOptions { /// Service port. DECLARE_FIELD(port, unsigned int, SetPort, 9000); + DECLARE_FIELD(hosts, std::vector, SetHosts, std::vector()); + DECLARE_FIELD(ports, std::vector, SetPorts, std::vector()); + /// Default database. DECLARE_FIELD(default_database, std::string, SetDefaultDatabase, "default"); /// User name. diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index 51340a86..4379250e 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -496,8 +496,18 @@ static void RunTests(Client& client) { int main() { try { const auto localHostEndpoint = ClientOptions() - .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) - .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + // .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + // .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetHosts({getEnvOrDefault("CLICKHOUSE_HOST", "asasdasd"), + getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), + getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), + getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), + }) + .SetPorts({static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1212")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); From 08975773530e917b34774bf01811e1e9aa8518f9 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 24 May 2023 09:28:01 +0000 Subject: [PATCH 04/53] Rename to --- clickhouse/CMakeLists.txt | 2 +- clickhouse/base/endpoints_iterator.cpp | 45 ++++++++++++++++++ ...{hosts_iterator.h => endpoints_iterator.h} | 10 ++-- clickhouse/base/hosts_iterator.cpp | 46 ------------------- clickhouse/base/socket.cpp | 4 +- clickhouse/base/socket.h | 6 +-- clickhouse/client.cpp | 18 ++++---- tests/simple/main.cpp | 2 +- 8 files changed, 66 insertions(+), 67 deletions(-) create mode 100644 clickhouse/base/endpoints_iterator.cpp rename clickhouse/base/{hosts_iterator.h => endpoints_iterator.h} (74%) delete mode 100644 clickhouse/base/hosts_iterator.cpp diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index a9fbdb9d..4b0f018d 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -5,7 +5,7 @@ SET ( clickhouse-cpp-lib-src base/platform.cpp base/socket.cpp base/wire_format.cpp - base/hosts_iterator.cpp + base/endpoints_iterator.cpp columns/array.cpp columns/column.cpp diff --git a/clickhouse/base/endpoints_iterator.cpp b/clickhouse/base/endpoints_iterator.cpp new file mode 100644 index 00000000..dd89ad1f --- /dev/null +++ b/clickhouse/base/endpoints_iterator.cpp @@ -0,0 +1,45 @@ +#include "endpoints_iterator.h" +#include "../client.h" + +namespace clickhouse { + +RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const ClientOptions& opts) : + hosts (opts.hosts) + , ports (opts.ports) + , current_index (0) + , reseted (true) + , iteration_counter(0) +{ + +} + +const std::string RoundRobinEndpointsIterator::getHostAddr() const +{ + return hosts[current_index]; +} + +unsigned int RoundRobinEndpointsIterator::getPort() const +{ + return ports[current_index]; +} + +void RoundRobinEndpointsIterator::ResetIterations() +{ + reseted = true; + iteration_counter = 0; +} + +void RoundRobinEndpointsIterator::next() +{ + current_index = (current_index + 1) % hosts.size(); + iteration_counter++; +} + +bool RoundRobinEndpointsIterator::nextIsExist() const +{ + return iteration_counter + 1 < hosts.size(); +} + +RoundRobinEndpointsIterator::~RoundRobinEndpointsIterator() = default; + +} \ No newline at end of file diff --git a/clickhouse/base/hosts_iterator.h b/clickhouse/base/endpoints_iterator.h similarity index 74% rename from clickhouse/base/hosts_iterator.h rename to clickhouse/base/endpoints_iterator.h index 57941278..033d9b83 100644 --- a/clickhouse/base/hosts_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -7,10 +7,10 @@ namespace clickhouse { struct ClientOptions; -class HostsIteratorBase +class EndpointsIteratorBase { public: - virtual ~HostsIteratorBase() = default; + virtual ~EndpointsIteratorBase() = default; virtual void next() = 0; virtual const std::string getHostAddr() const = 0; @@ -20,17 +20,17 @@ class HostsIteratorBase }; -class RoundRobinHostsIterator : public HostsIteratorBase +class RoundRobinEndpointsIterator : public EndpointsIteratorBase { public: - RoundRobinHostsIterator(const ClientOptions& opts); + RoundRobinEndpointsIterator(const ClientOptions& opts); const std::string getHostAddr() const override; unsigned int getPort() const override; void ResetIterations() override; bool nextIsExist() const override; void next() override; - ~RoundRobinHostsIterator() override; + ~RoundRobinEndpointsIterator() override; private: diff --git a/clickhouse/base/hosts_iterator.cpp b/clickhouse/base/hosts_iterator.cpp deleted file mode 100644 index 033669f4..00000000 --- a/clickhouse/base/hosts_iterator.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "hosts_iterator.h" -#include "../client.h" - -namespace clickhouse { - -RoundRobinHostsIterator::RoundRobinHostsIterator(const ClientOptions& opts) : - hosts (opts.hosts) - , ports (opts.ports) - , current_index (0) - , reseted (true) - , iteration_counter(0) -{ - -} - -const std::string RoundRobinHostsIterator::getHostAddr() const -{ - return hosts[current_index]; -} - -unsigned int RoundRobinHostsIterator::getPort() const -{ - return ports[current_index]; -} - -void RoundRobinHostsIterator::ResetIterations() -{ - reseted = true; - iteration_counter = 0; -} - -void RoundRobinHostsIterator::next() -{ - current_index = (current_index + 1) % hosts.size(); - iteration_counter++; -} - -bool RoundRobinHostsIterator::nextIsExist() const -{ - return iteration_counter < hosts.size(); -} - -RoundRobinHostsIterator::~RoundRobinHostsIterator() = default; - - -} \ No newline at end of file diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index 36fb8e74..4bc67ba3 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -390,9 +390,9 @@ std::unique_ptr Socket::makeOutputStream() const { NonSecureSocketFactory::~NonSecureSocketFactory() {} -std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts, const std::shared_ptr hosts_iterator) { +std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts, const std::shared_ptr endpoints_iterator) { - const auto address = NetworkAddress(hosts_iterator->getHostAddr(), std::to_string(hosts_iterator->getPort())); + const auto address = NetworkAddress(endpoints_iterator->getHostAddr(), std::to_string(endpoints_iterator->getPort())); auto socket = doConnect(address, opts); setSocketOptions(*socket, opts); diff --git a/clickhouse/base/socket.h b/clickhouse/base/socket.h index 631da7d1..8cad298f 100644 --- a/clickhouse/base/socket.h +++ b/clickhouse/base/socket.h @@ -3,7 +3,7 @@ #include "platform.h" #include "input.h" #include "output.h" -#include "hosts_iterator.h" +#include "endpoints_iterator.h" #include #include @@ -89,7 +89,7 @@ class SocketFactory { // TODO: move connection-related options to ConnectionOptions structure. - virtual std::unique_ptr connect(const ClientOptions& opts, const std::shared_ptr hosts_iterator) = 0; + virtual std::unique_ptr connect(const ClientOptions& opts, const std::shared_ptr endpoints_iterator) = 0; virtual void sleepFor(const std::chrono::milliseconds& duration); }; @@ -136,7 +136,7 @@ class NonSecureSocketFactory : public SocketFactory { public: ~NonSecureSocketFactory() override; - std::unique_ptr connect(const ClientOptions& opts, const std::shared_ptr hosts_iterator) override; + std::unique_ptr connect(const ClientOptions& opts, const std::shared_ptr endpoints_iterator) override; protected: virtual std::unique_ptr doConnect(const NetworkAddress& address, const ClientOptions& opts); diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index b9ca4a5d..bf04fe4f 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -196,7 +196,7 @@ class Client::Impl { std::unique_ptr input_; std::unique_ptr output_; std::unique_ptr socket_; - std::shared_ptr hosts_iterator; + std::shared_ptr endpoints_iterator; ServerInfo server_info_; }; @@ -218,9 +218,9 @@ Client::Impl::Impl(const ClientOptions& opts, : options_(modifyClientOptions(opts)) , events_(nullptr) , socket_factory_(std::move(socket_factory)) - , hosts_iterator(new RoundRobinHostsIterator(options_)) + , endpoints_iterator(new RoundRobinEndpointsIterator(options_)) { - auto init_connection_with_host = [&](){ + auto try_make_connection_with_endpoind = [this]() { for (unsigned int i = 0; ; ) { try { ResetConnection(); @@ -234,13 +234,13 @@ Client::Impl::Impl(const ClientOptions& opts, } }; - for (; hosts_iterator->nextIsExist(); hosts_iterator->next()) + for (endpoints_iterator->ResetIterations(); ; endpoints_iterator->next()) { try { - init_connection_with_host(); + try_make_connection_with_endpoind(); } catch (const std::system_error&) { - if(!hosts_iterator->nextIsExist()) + if(!endpoints_iterator->nextIsExist()) throw; } } @@ -353,7 +353,7 @@ void Client::Impl::Ping() { } void Client::Impl::ResetConnection() { - InitializeStreams(socket_factory_->connect(options_, hosts_iterator)); + InitializeStreams(socket_factory_->connect(options_, endpoints_iterator)); if (!Handshake()) { throw ProtocolError("fail to connect to " + options_.host); @@ -885,14 +885,14 @@ bool Client::Impl::ReceiveHello() { } void Client::Impl::RetryGuard(std::function func) { - for(hosts_iterator->ResetIterations(); ; hosts_iterator->next()) + for(endpoints_iterator->ResetIterations(); ; endpoints_iterator->next()) { try { RetryToConstEndpoint(func); return; } catch (const std::system_error&) { - if (!hosts_iterator->nextIsExist()) + if (!endpoints_iterator->nextIsExist()) throw; } } diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index 4379250e..d07aa40a 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -504,7 +504,7 @@ int main() { getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), }) .SetPorts({static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1212")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), }) From f964bdbba0e8109af73bbf8dc12661da82c8af38 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 24 May 2023 10:02:35 +0000 Subject: [PATCH 05/53] Fix --- clickhouse/client.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index bf04fe4f..9bb97fe6 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -220,11 +220,11 @@ Client::Impl::Impl(const ClientOptions& opts, , socket_factory_(std::move(socket_factory)) , endpoints_iterator(new RoundRobinEndpointsIterator(options_)) { - auto try_make_connection_with_endpoind = [this]() { + auto try_make_connection_with_endpoint = [this]() { for (unsigned int i = 0; ; ) { try { ResetConnection(); - break; + return; } catch (const std::system_error&) { if (++i > options_.send_retries) { throw; @@ -233,12 +233,13 @@ Client::Impl::Impl(const ClientOptions& opts, } } }; - + for (endpoints_iterator->ResetIterations(); ; endpoints_iterator->next()) { try { - try_make_connection_with_endpoind(); + try_make_connection_with_endpoint(); + break; } catch (const std::system_error&) { if(!endpoints_iterator->nextIsExist()) throw; From 1c6aa2fbf6ca1625cf69ae1bb8e84d1b7d008153 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 24 May 2023 10:32:40 +0000 Subject: [PATCH 06/53] Equal sizes exeption --- clickhouse/client.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index 9bb97fe6..8de45b4c 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -203,9 +203,11 @@ class Client::Impl { ClientOptions modifyClientOptions(ClientOptions opts) { + if (opts.hosts.size() != opts.ports.size()) + throw ValidationError("The sizes of lists of ports and hosts must match be equal."); if (!opts.host.empty()) opts.hosts.insert(opts.hosts.begin(), opts.host); - + opts.ports.insert(opts.ports.begin(), opts.port); return opts; } From fc27ec66c11c9d32fb958510550b84651c2edcaa Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 24 May 2023 10:44:14 +0000 Subject: [PATCH 07/53] fixes --- clickhouse/client.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index 8de45b4c..05d845c3 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #if defined(WITH_OPENSSL) #include "base/sslsocket.h" #endif @@ -205,10 +204,10 @@ ClientOptions modifyClientOptions(ClientOptions opts) { if (opts.hosts.size() != opts.ports.size()) throw ValidationError("The sizes of lists of ports and hosts must match be equal."); - if (!opts.host.empty()) + if (!opts.host.empty()) { opts.hosts.insert(opts.hosts.begin(), opts.host); - - opts.ports.insert(opts.ports.begin(), opts.port); + opts.ports.insert(opts.ports.begin(), opts.port); + } return opts; } From 1f745e6174e6e9211d14261f80a90ce5b7160e42 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 24 May 2023 15:48:54 +0200 Subject: [PATCH 08/53] Updated ColumnString::LoadBody made it more thread-safe, removed piece of code that confused lots of static analyzers (false positive on dereferencing nullptr-`block` variable) --- clickhouse/columns/string.cpp | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/clickhouse/columns/string.cpp b/clickhouse/columns/string.cpp index 937e6035..f6597bb4 100644 --- a/clickhouse/columns/string.cpp +++ b/clickhouse/columns/string.cpp @@ -252,27 +252,38 @@ void ColumnString::Append(ColumnRef column) { } bool ColumnString::LoadBody(InputStream* input, size_t rows) { - items_.clear(); - blocks_.clear(); + if (rows == 0) { + items_.clear(); + blocks_.clear(); + + return true; + } - items_.reserve(rows); - Block * block = nullptr; + decltype(items_) new_items; + decltype(blocks_) new_blocks; + + new_items.reserve(rows); + + // Suboptimzal if the first row string is >DEFAULT_BLOCK_SIZE, but that must be a very rare case. + Block * block = &new_blocks.emplace_back(DEFAULT_BLOCK_SIZE); - // TODO(performance): unroll a loop to a first row (to get rid of `blocks_.size() == 0` check) and the rest. for (size_t i = 0; i < rows; ++i) { uint64_t len; if (!WireFormat::ReadUInt64(*input, &len)) return false; - if (blocks_.size() == 0 || len > block->GetAvailable()) - block = &blocks_.emplace_back(std::max(DEFAULT_BLOCK_SIZE, len)); + if (len > block->GetAvailable()) + block = &new_blocks.emplace_back(std::max(DEFAULT_BLOCK_SIZE, len)); if (!WireFormat::ReadBytes(*input, block->GetCurrentWritePos(), len)) return false; - items_.emplace_back(block->ConsumeTailAsStringViewUnsafe(len)); + new_items.emplace_back(block->ConsumeTailAsStringViewUnsafe(len)); } + items_.swap(new_items); + blocks_.swap(new_blocks); + return true; } From 615d120d8cd6fb77ffec2cac0bab6d69ade0355f Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 24 May 2023 14:53:16 +0000 Subject: [PATCH 09/53] Added unit tests --- ut/client_ut.cpp | 84 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index dfabcba9..5f453d93 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1210,3 +1210,87 @@ INSTANTIATE_TEST_SUITE_P(ClientLocalFailed, ConnectionFailedClientTest, ExpectingException{"Authentication failed: password is incorrect"} } )); + + +class ConnectionSuccessTestCase : public testing::TestWithParam {}; + +TEST_P(ConnectionSuccessTestCase, SuccessConnectionEstablished) { + const auto & client_options = GetParam(); + std::unique_ptr client; + + try { + client = std::make_unique(client_options); + SUCCEED(); + } catch (const std::exception & e) { + FAIL() << "Got an unexpected exception : " << e.what(); + } +} + + +INSTANTIATE_TEST_SUITE_P(ClientMultipleEndpoints, ConnectionSuccessTestCase, + ::testing::Values(ClientCase::ParamType{ + ClientOptions() + .SetHosts({ + getEnvOrDefault("CLICKHOUSE_HOST", "somedeadhost"), + getEnvOrDefault("CLICKHOUSE_HOST", "deadaginghost"), + getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), + getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), + }) + .SetPorts( { + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1245")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "6784")), + }) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetPingBeforeQuery(true) + .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) + .SetRetryTimeout(std::chrono::seconds(1)), + } +)); + +INSTANTIATE_TEST_SUITE_P(MultipleEndpointsFailed, ConnectionFailedClientTest, + ::testing::Values(ConnectionFailedClientTest::ParamType{ + ClientOptions() + .SetHosts({ + getEnvOrDefault("CLICKHOUSE_HOST", "somedeadhost"), + getEnvOrDefault("CLICKHOUSE_HOST", "deadaginghost"), + getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost") + }) + .SetPorts( { + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1245")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "6784")), + }) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetPingBeforeQuery(true) + .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) + .SetRetryTimeout(std::chrono::seconds(1)), + ExpectingException{"Temporary failure in name resolution"} + } +)); + +INSTANTIATE_TEST_SUITE_P(MultipleEndpointsNonValidConfig, ConnectionFailedClientTest, + ::testing::Values(ConnectionFailedClientTest::ParamType{ + ClientOptions() + .SetHosts({ + getEnvOrDefault("CLICKHOUSE_HOST", "somedeadhost"), + getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), + getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), + }) + .SetPorts( { + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + }) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetPingBeforeQuery(true) + .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) + .SetRetryTimeout(std::chrono::seconds(1)), + ExpectingException{"The sizes of lists of ports and hosts must match be equal."} + } +)); From 289343edc175ab3cb60517681e2128eaf04204bf Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 24 May 2023 15:08:58 +0000 Subject: [PATCH 10/53] minor changes --- clickhouse/base/endpoints_iterator.cpp | 4 ++-- clickhouse/base/endpoints_iterator.h | 2 +- clickhouse/base/socket.cpp | 2 +- clickhouse/client.cpp | 9 +++++---- tests/simple/main.cpp | 20 ++++++++++---------- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/clickhouse/base/endpoints_iterator.cpp b/clickhouse/base/endpoints_iterator.cpp index dd89ad1f..d8ec0d76 100644 --- a/clickhouse/base/endpoints_iterator.cpp +++ b/clickhouse/base/endpoints_iterator.cpp @@ -10,7 +10,7 @@ RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const ClientOptions& op , reseted (true) , iteration_counter(0) { - + } const std::string RoundRobinEndpointsIterator::getHostAddr() const @@ -42,4 +42,4 @@ bool RoundRobinEndpointsIterator::nextIsExist() const RoundRobinEndpointsIterator::~RoundRobinEndpointsIterator() = default; -} \ No newline at end of file +} diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h index 033d9b83..aed800d8 100644 --- a/clickhouse/base/endpoints_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -41,4 +41,4 @@ class RoundRobinEndpointsIterator : public EndpointsIteratorBase size_t iteration_counter; }; -} \ No newline at end of file +} diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index 4bc67ba3..786bb3eb 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -391,7 +391,7 @@ std::unique_ptr Socket::makeOutputStream() const { NonSecureSocketFactory::~NonSecureSocketFactory() {} std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts, const std::shared_ptr endpoints_iterator) { - + const auto address = NetworkAddress(endpoints_iterator->getHostAddr(), std::to_string(endpoints_iterator->getPort())); auto socket = doConnect(address, opts); setSocketOptions(*socket, opts); diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index 05d845c3..f0d647aa 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -14,6 +14,7 @@ #include #include #include + #if defined(WITH_OPENSSL) #include "base/sslsocket.h" #endif @@ -161,7 +162,7 @@ class Client::Impl { /// call fuc several times. void RetryGuard(std::function func); - void RetryToConstEndpoint(std::function func); + void RetryConnectToTheEndpoint(std::function& func); private: class EnsureNull { @@ -242,7 +243,7 @@ Client::Impl::Impl(const ClientOptions& opts, try_make_connection_with_endpoint(); break; } catch (const std::system_error&) { - if(!endpoints_iterator->nextIsExist()) + if(!endpoints_iterator->nextIsExist()) throw; } } @@ -891,7 +892,7 @@ void Client::Impl::RetryGuard(std::function func) { { try { - RetryToConstEndpoint(func); + RetryConnectToTheEndpoint(func); return; } catch (const std::system_error&) { if (!endpoints_iterator->nextIsExist()) @@ -900,7 +901,7 @@ void Client::Impl::RetryGuard(std::function func) { } } -void Client::Impl::RetryToConstEndpoint(std::function func) { +void Client::Impl::RetryConnectToTheEndpoint(std::function& func) { for (unsigned int i = 0; ; ++i) { try { func(); diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index d07aa40a..d0d6162e 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -496,17 +496,17 @@ static void RunTests(Client& client) { int main() { try { const auto localHostEndpoint = ClientOptions() - // .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) - // .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) - .SetHosts({getEnvOrDefault("CLICKHOUSE_HOST", "asasdasd"), - getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), - getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), - getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), + .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetHosts({ getEnvOrDefault("CLICKHOUSE_HOST", "asasdasd"), + getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), + getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), + getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), }) - .SetPorts({static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + .SetPorts({ static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + // static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1234")), + // static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "5678")), }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) From f0bf6312fb7fd664620a03d945c1ed7941957d9c Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Thu, 25 May 2023 07:58:25 +0000 Subject: [PATCH 11/53] Added knob --- clickhouse/base/endpoints_iterator.h | 14 +++++++++++++- clickhouse/client.cpp | 9 ++++++++- clickhouse/client.h | 12 ++++++++++++ tests/simple/main.cpp | 4 ++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h index aed800d8..6a97541b 100644 --- a/clickhouse/base/endpoints_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -7,19 +7,31 @@ namespace clickhouse { struct ClientOptions; +/** + * Base class for iterating through endpoints. +*/ class EndpointsIteratorBase { public: virtual ~EndpointsIteratorBase() = default; virtual void next() = 0; + // Get the address of current endpoint. virtual const std::string getHostAddr() const = 0; + + // Get the port of current endpoint. virtual unsigned int getPort() const = 0; + + // Reset iterations. virtual void ResetIterations() = 0; virtual bool nextIsExist() const = 0; }; - +/** + * Client tries to connect to those endpoints one by one, on the round-robin basis: + * first default enpoint, then each of endpoints, from begin() to end(), + * if previous are inaccessible. + */ class RoundRobinEndpointsIterator : public EndpointsIteratorBase { public: diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index f0d647aa..8109c568 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -111,6 +111,13 @@ std::unique_ptr GetSocketFactory(const ClientOptions& opts) { return std::make_unique(); } +std::unique_ptr GetEndpointsIterator(const ClientOptions& opts) { + if (opts.iteration_algo == EndpointsIterationAlgorithm::RoundRobin) { + return std::make_unique(opts); + } else + throw UnimplementedError("Unimplemented endpoints iteration alorithm."); +} + } class Client::Impl { @@ -220,7 +227,7 @@ Client::Impl::Impl(const ClientOptions& opts, : options_(modifyClientOptions(opts)) , events_(nullptr) , socket_factory_(std::move(socket_factory)) - , endpoints_iterator(new RoundRobinEndpointsIterator(options_)) + , endpoints_iterator(GetEndpointsIterator(options_)) { auto try_make_connection_with_endpoint = [this]() { for (unsigned int i = 0; ; ) { diff --git a/clickhouse/client.h b/clickhouse/client.h index f5e1fc42..accdb263 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -44,6 +44,10 @@ enum class CompressionMethod { LZ4 = 1, }; +enum class EndpointsIterationAlgorithm { + RoundRobin = 0, +}; + struct ClientOptions { // Setter goes first, so it is possible to apply 'deprecated' annotation safely. #define DECLARE_FIELD(name, type, setter, default_value) \ @@ -58,9 +62,17 @@ struct ClientOptions { /// Service port. DECLARE_FIELD(port, unsigned int, SetPort, 9000); + /* + * Client tries to connect to those endpoints one by one, on chosing alorithm basis: + * first default enpoint (set via SetHost() + SetPort()), then each of endpoints, from begin() to end(), + * the first one to establish connection is used for the rest of the session. + * If port part is not specified, default port (@see SetPort()) is used. + */ DECLARE_FIELD(hosts, std::vector, SetHosts, std::vector()); DECLARE_FIELD(ports, std::vector, SetPorts, std::vector()); + DECLARE_FIELD(iteration_algo, EndpointsIterationAlgorithm, SetEndpointsIterationAlgorithm, EndpointsIterationAlgorithm::RoundRobin); + /// Default database. DECLARE_FIELD(default_database, std::string, SetDefaultDatabase, "default"); /// User name. diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index d0d6162e..29381ff8 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -505,8 +505,8 @@ int main() { }) .SetPorts({ static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - // static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1234")), - // static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "5678")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1234")), + static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "5678")), }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) From 6833281a5e242d4cdf3486a681f42113483cfd14 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Thu, 25 May 2023 08:46:18 +0000 Subject: [PATCH 12/53] comments' --- clickhouse/base/endpoints_iterator.h | 5 ----- clickhouse/client.cpp | 12 ++++++++---- clickhouse/client.h | 16 +++++++++------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h index 6a97541b..f5011c7a 100644 --- a/clickhouse/base/endpoints_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -27,11 +27,6 @@ class EndpointsIteratorBase virtual bool nextIsExist() const = 0; }; -/** - * Client tries to connect to those endpoints one by one, on the round-robin basis: - * first default enpoint, then each of endpoints, from begin() to end(), - * if previous are inaccessible. - */ class RoundRobinEndpointsIterator : public EndpointsIteratorBase { public: diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index 8109c568..e2b69f20 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -64,8 +64,12 @@ struct ClientInfo { }; std::ostream& operator<<(std::ostream& os, const ClientOptions& opt) { - os << "Client(" << opt.user << '@' << opt.host << ":" << opt.port - << " ping_before_query:" << opt.ping_before_query + os << "Client("; + for (size_t i = 0; i < opt.hosts.size(); i++) + os << opt.user << '@' << opt.hosts[i] << ":" << opt.ports[i] + << ((i == opt.hosts[i].size() - 1) ? "" : ", "); + + os << " ping_before_query:" << opt.ping_before_query << " send_retries:" << opt.send_retries << " retry_timeout:" << opt.retry_timeout.count() << " compression_method:" @@ -111,9 +115,9 @@ std::unique_ptr GetSocketFactory(const ClientOptions& opts) { return std::make_unique(); } -std::unique_ptr GetEndpointsIterator(const ClientOptions& opts) { +std::shared_ptr GetEndpointsIterator(const ClientOptions& opts) { if (opts.iteration_algo == EndpointsIterationAlgorithm::RoundRobin) { - return std::make_unique(opts); + return std::make_shared(opts); } else throw UnimplementedError("Unimplemented endpoints iteration alorithm."); } diff --git a/clickhouse/client.h b/clickhouse/client.h index accdb263..ee7cf1e8 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -45,6 +45,11 @@ enum class CompressionMethod { }; enum class EndpointsIterationAlgorithm { +/** + * Client tries to connect to those endpoints one by one, on the round-robin basis: + * first default enpoint, then each of endpoints, from begin() to end(), + * if previous are inaccessible. + */ RoundRobin = 0, }; @@ -62,15 +67,12 @@ struct ClientOptions { /// Service port. DECLARE_FIELD(port, unsigned int, SetPort, 9000); - /* - * Client tries to connect to those endpoints one by one, on chosing alorithm basis: - * first default enpoint (set via SetHost() + SetPort()), then each of endpoints, from begin() to end(), - * the first one to establish connection is used for the rest of the session. - * If port part is not specified, default port (@see SetPort()) is used. - */ + /// Hostnames of the servers. The next host to connect is selected according to the EndpointsIterationAlgorithm. + /// Note: If SetHost and SetHosts are setted, host will be placed at the beginning of the hosts vector. DECLARE_FIELD(hosts, std::vector, SetHosts, std::vector()); + /// Ports of the servers. DECLARE_FIELD(ports, std::vector, SetPorts, std::vector()); - + /// Algorithm for selecting the next endpoint for connection. DECLARE_FIELD(iteration_algo, EndpointsIterationAlgorithm, SetEndpointsIterationAlgorithm, EndpointsIterationAlgorithm::RoundRobin); /// Default database. From 1f52a687e55cf97f500f2d0e8fbf3d330f7692b6 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Thu, 25 May 2023 11:45:01 +0000 Subject: [PATCH 13/53] Remove unused --- clickhouse/base/endpoints_iterator.cpp | 4 +--- clickhouse/base/endpoints_iterator.h | 17 ++++++++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/clickhouse/base/endpoints_iterator.cpp b/clickhouse/base/endpoints_iterator.cpp index d8ec0d76..8bee737b 100644 --- a/clickhouse/base/endpoints_iterator.cpp +++ b/clickhouse/base/endpoints_iterator.cpp @@ -7,13 +7,12 @@ RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const ClientOptions& op hosts (opts.hosts) , ports (opts.ports) , current_index (0) - , reseted (true) , iteration_counter(0) { } -const std::string RoundRobinEndpointsIterator::getHostAddr() const +std::string RoundRobinEndpointsIterator::getHostAddr() const { return hosts[current_index]; } @@ -25,7 +24,6 @@ unsigned int RoundRobinEndpointsIterator::getPort() const void RoundRobinEndpointsIterator::ResetIterations() { - reseted = true; iteration_counter = 0; } diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h index f5011c7a..0afca930 100644 --- a/clickhouse/base/endpoints_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -17,7 +17,7 @@ class EndpointsIteratorBase virtual void next() = 0; // Get the address of current endpoint. - virtual const std::string getHostAddr() const = 0; + virtual std::string getHostAddr() const = 0; // Get the port of current endpoint. virtual unsigned int getPort() const = 0; @@ -30,21 +30,20 @@ class EndpointsIteratorBase class RoundRobinEndpointsIterator : public EndpointsIteratorBase { public: - RoundRobinEndpointsIterator(const ClientOptions& opts); - const std::string getHostAddr() const override; - unsigned int getPort() const override; - void ResetIterations() override; - bool nextIsExist() const override; - void next() override; + RoundRobinEndpointsIterator(const ClientOptions& opts); + std::string getHostAddr() const override; + unsigned int getPort() const override; + void ResetIterations() override; + bool nextIsExist() const override; + void next() override; - ~RoundRobinEndpointsIterator() override; + ~RoundRobinEndpointsIterator() override; private: const std::vector& hosts; const std::vector& ports; int current_index; - bool reseted; size_t iteration_counter; }; From 8255db67f9717fd70688d239f44707073dbe8d3d Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Thu, 25 May 2023 14:08:12 +0000 Subject: [PATCH 14/53] conservations resolving --- clickhouse/base/endpoints_iterator.cpp | 8 ++++---- clickhouse/base/endpoints_iterator.h | 16 ++++++++-------- clickhouse/base/socket.cpp | 4 ++-- clickhouse/base/socket.h | 4 ++-- clickhouse/client.cpp | 15 +++++++++------ tests/simple/main.cpp | 8 ++++---- ut/client_ut.cpp | 16 ++++++++-------- ut/utils.h | 3 ++- 8 files changed, 39 insertions(+), 35 deletions(-) diff --git a/clickhouse/base/endpoints_iterator.cpp b/clickhouse/base/endpoints_iterator.cpp index 8bee737b..4fe48119 100644 --- a/clickhouse/base/endpoints_iterator.cpp +++ b/clickhouse/base/endpoints_iterator.cpp @@ -12,12 +12,12 @@ RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const ClientOptions& op } -std::string RoundRobinEndpointsIterator::getHostAddr() const +std::string RoundRobinEndpointsIterator::GetHostAddr() const { return hosts[current_index]; } -unsigned int RoundRobinEndpointsIterator::getPort() const +unsigned int RoundRobinEndpointsIterator::GetPort() const { return ports[current_index]; } @@ -27,13 +27,13 @@ void RoundRobinEndpointsIterator::ResetIterations() iteration_counter = 0; } -void RoundRobinEndpointsIterator::next() +void RoundRobinEndpointsIterator::Next() { current_index = (current_index + 1) % hosts.size(); iteration_counter++; } -bool RoundRobinEndpointsIterator::nextIsExist() const +bool RoundRobinEndpointsIterator::NextIsExist() const { return iteration_counter + 1 < hosts.size(); } diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h index 0afca930..76ce3cbb 100644 --- a/clickhouse/base/endpoints_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -15,27 +15,27 @@ class EndpointsIteratorBase public: virtual ~EndpointsIteratorBase() = default; - virtual void next() = 0; + virtual void Next() = 0; // Get the address of current endpoint. - virtual std::string getHostAddr() const = 0; + virtual std::string GetHostAddr() const = 0; // Get the port of current endpoint. - virtual unsigned int getPort() const = 0; + virtual unsigned int GetPort() const = 0; // Reset iterations. virtual void ResetIterations() = 0; - virtual bool nextIsExist() const = 0; + virtual bool NextIsExist() const = 0; }; class RoundRobinEndpointsIterator : public EndpointsIteratorBase { public: RoundRobinEndpointsIterator(const ClientOptions& opts); - std::string getHostAddr() const override; - unsigned int getPort() const override; + std::string GetHostAddr() const override; + unsigned int GetPort() const override; void ResetIterations() override; - bool nextIsExist() const override; - void next() override; + bool NextIsExist() const override; + void Next() override; ~RoundRobinEndpointsIterator() override; diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index 786bb3eb..13755a37 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -390,9 +390,9 @@ std::unique_ptr Socket::makeOutputStream() const { NonSecureSocketFactory::~NonSecureSocketFactory() {} -std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts, const std::shared_ptr endpoints_iterator) { +std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts, const std::string& host, const std::string& port) { - const auto address = NetworkAddress(endpoints_iterator->getHostAddr(), std::to_string(endpoints_iterator->getPort())); + const auto address = NetworkAddress(host, port); auto socket = doConnect(address, opts); setSocketOptions(*socket, opts); diff --git a/clickhouse/base/socket.h b/clickhouse/base/socket.h index 8cad298f..4953a4d1 100644 --- a/clickhouse/base/socket.h +++ b/clickhouse/base/socket.h @@ -89,7 +89,7 @@ class SocketFactory { // TODO: move connection-related options to ConnectionOptions structure. - virtual std::unique_ptr connect(const ClientOptions& opts, const std::shared_ptr endpoints_iterator) = 0; + virtual std::unique_ptr connect(const ClientOptions& opts, const std::string& host, const std::string& port) = 0; virtual void sleepFor(const std::chrono::milliseconds& duration); }; @@ -136,7 +136,7 @@ class NonSecureSocketFactory : public SocketFactory { public: ~NonSecureSocketFactory() override; - std::unique_ptr connect(const ClientOptions& opts, const std::shared_ptr endpoints_iterator) override; + std::unique_ptr connect(const ClientOptions& opts, const std::string& host, const std::string& port) override; protected: virtual std::unique_ptr doConnect(const NetworkAddress& address, const ClientOptions& opts); diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index e2b69f20..b967c0c9 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -216,7 +216,8 @@ ClientOptions modifyClientOptions(ClientOptions opts) { if (opts.hosts.size() != opts.ports.size()) throw ValidationError("The sizes of lists of ports and hosts must match be equal."); - if (!opts.host.empty()) { + if (!opts.host.empty() && std::find(opts.hosts.begin(), opts.hosts.end(), opts.host) == std::end(opts.hosts)) + { opts.hosts.insert(opts.hosts.begin(), opts.host); opts.ports.insert(opts.ports.begin(), opts.port); } @@ -247,14 +248,14 @@ Client::Impl::Impl(const ClientOptions& opts, } }; - for (endpoints_iterator->ResetIterations(); ; endpoints_iterator->next()) + for (endpoints_iterator->ResetIterations(); ; endpoints_iterator->Next()) { try { try_make_connection_with_endpoint(); break; } catch (const std::system_error&) { - if(!endpoints_iterator->nextIsExist()) + if(!endpoints_iterator->NextIsExist()) throw; } } @@ -367,7 +368,9 @@ void Client::Impl::Ping() { } void Client::Impl::ResetConnection() { - InitializeStreams(socket_factory_->connect(options_, endpoints_iterator)); + InitializeStreams(socket_factory_->connect(options_, endpoints_iterator->GetHostAddr(), + std::to_string(endpoints_iterator->GetPort()) + )); if (!Handshake()) { throw ProtocolError("fail to connect to " + options_.host); @@ -899,14 +902,14 @@ bool Client::Impl::ReceiveHello() { } void Client::Impl::RetryGuard(std::function func) { - for(endpoints_iterator->ResetIterations(); ; endpoints_iterator->next()) + for(endpoints_iterator->ResetIterations(); ; endpoints_iterator->Next()) { try { RetryConnectToTheEndpoint(func); return; } catch (const std::system_error&) { - if (!endpoints_iterator->nextIsExist()) + if (!endpoints_iterator->NextIsExist()) throw; } } diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index 29381ff8..f20602f4 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -503,10 +503,10 @@ int main() { getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), }) - .SetPorts({ static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1234")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "5678")), + .SetPorts({ getEnvOrDefault("CLICKHOUSE_PORT", "9000"), + getEnvOrDefault("CLICKHOUSE_PORT", "9000"), + getEnvOrDefault("CLICKHOUSE_PORT", "1234"), + getEnvOrDefault("CLICKHOUSE_PORT", "5678"), }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 5f453d93..07e53fba 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1237,10 +1237,10 @@ INSTANTIATE_TEST_SUITE_P(ClientMultipleEndpoints, ConnectionSuccessTestCase, getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), }) .SetPorts( { - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1245")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "6784")), + getEnvOrDefault("CLICKHOUSE_PORT", "9000"), + getEnvOrDefault("CLICKHOUSE_PORT", "1245"), + getEnvOrDefault("CLICKHOUSE_PORT", "9000"), + getEnvOrDefault("CLICKHOUSE_PORT", "6784"), }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) @@ -1260,9 +1260,9 @@ INSTANTIATE_TEST_SUITE_P(MultipleEndpointsFailed, ConnectionFailedClientTest, getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost") }) .SetPorts( { - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "1245")), - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "6784")), + getEnvOrDefault("CLICKHOUSE_PORT", "9000"), + getEnvOrDefault("CLICKHOUSE_PORT", "1245"), + getEnvOrDefault("CLICKHOUSE_PORT", "6784"), }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) @@ -1283,7 +1283,7 @@ INSTANTIATE_TEST_SUITE_P(MultipleEndpointsNonValidConfig, ConnectionFailedClient getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), }) .SetPorts( { - static_cast(getEnvOrDefault("CLICKHOUSE_PORT", "9000")), + getEnvOrDefault("CLICKHOUSE_PORT", "9000"), }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) diff --git a/ut/utils.h b/ut/utils.h index b9484626..41bec25c 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -51,7 +51,8 @@ auto getEnvOrDefault(const std::string& env, const char * default_val) { return std::stoll(value); } else if constexpr (std::is_unsigned_v) { if constexpr (sizeof(ResultType) <= sizeof(unsigned long)) - return std::stoul(value); + // For cases when ResultType is unsigned int. + return static_cast(std::stoul(value)); else if constexpr (sizeof(ResultType) <= sizeof(unsigned long long)) return std::stoull(value); } From a6795bbac55577871aed9c4686813b0770f316ec Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Thu, 25 May 2023 14:18:09 +0000 Subject: [PATCH 15/53] fix --- clickhouse/base/endpoints_iterator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h index 76ce3cbb..158f6246 100644 --- a/clickhouse/base/endpoints_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -30,7 +30,7 @@ class EndpointsIteratorBase class RoundRobinEndpointsIterator : public EndpointsIteratorBase { public: - RoundRobinEndpointsIterator(const ClientOptions& opts); + explicit RoundRobinEndpointsIterator(const ClientOptions& opts); std::string GetHostAddr() const override; unsigned int GetPort() const override; void ResetIterations() override; From cc40bbcca16e39e68d0ca4cf40bbc9d4c03b0363 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Tue, 6 Jun 2023 12:20:10 +0000 Subject: [PATCH 16/53] Reviews fixes --- clickhouse/base/endpoints_iterator.cpp | 15 ++-- clickhouse/base/endpoints_iterator.h | 24 +++-- clickhouse/client.cpp | 117 +++++++++++++++--------- clickhouse/client.h | 33 ++++--- tests/simple/main.cpp | 18 ++-- ut/client_ut.cpp | 119 ++++++++++++++++++------- 6 files changed, 210 insertions(+), 116 deletions(-) diff --git a/clickhouse/base/endpoints_iterator.cpp b/clickhouse/base/endpoints_iterator.cpp index 4fe48119..f687703d 100644 --- a/clickhouse/base/endpoints_iterator.cpp +++ b/clickhouse/base/endpoints_iterator.cpp @@ -1,11 +1,10 @@ #include "endpoints_iterator.h" -#include "../client.h" +#include namespace clickhouse { -RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const ClientOptions& opts) : - hosts (opts.hosts) - , ports (opts.ports) +RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const std::vector& _endpoints) : + endpoints (_endpoints) , current_index (0) , iteration_counter(0) { @@ -14,12 +13,12 @@ RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const ClientOptions& op std::string RoundRobinEndpointsIterator::GetHostAddr() const { - return hosts[current_index]; + return endpoints[current_index].host; } unsigned int RoundRobinEndpointsIterator::GetPort() const { - return ports[current_index]; + return endpoints[current_index].port; } void RoundRobinEndpointsIterator::ResetIterations() @@ -29,13 +28,13 @@ void RoundRobinEndpointsIterator::ResetIterations() void RoundRobinEndpointsIterator::Next() { - current_index = (current_index + 1) % hosts.size(); + current_index = (current_index + 1) % endpoints.size(); iteration_counter++; } bool RoundRobinEndpointsIterator::NextIsExist() const { - return iteration_counter + 1 < hosts.size(); + return iteration_counter + 1 < endpoints.size(); } RoundRobinEndpointsIterator::~RoundRobinEndpointsIterator() = default; diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h index 158f6246..dcbdd86b 100644 --- a/clickhouse/base/endpoints_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -1,6 +1,6 @@ #pragma once -#include "../client.h" +#include "clickhouse/client.h" #include namespace clickhouse { @@ -30,21 +30,19 @@ class EndpointsIteratorBase class RoundRobinEndpointsIterator : public EndpointsIteratorBase { public: - explicit RoundRobinEndpointsIterator(const ClientOptions& opts); - std::string GetHostAddr() const override; - unsigned int GetPort() const override; - void ResetIterations() override; - bool NextIsExist() const override; - void Next() override; + explicit RoundRobinEndpointsIterator(const std::vector& opts); + std::string GetHostAddr() const override; + unsigned int GetPort() const override; + void ResetIterations() override; + bool NextIsExist() const override; + void Next() override; - ~RoundRobinEndpointsIterator() override; + ~RoundRobinEndpointsIterator() override; private: - - const std::vector& hosts; - const std::vector& ports; - int current_index; - size_t iteration_counter; + const std::vector& endpoints; + int current_index; + size_t iteration_counter; }; } diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index b967c0c9..27770c16 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -65,9 +65,9 @@ struct ClientInfo { std::ostream& operator<<(std::ostream& os, const ClientOptions& opt) { os << "Client("; - for (size_t i = 0; i < opt.hosts.size(); i++) - os << opt.user << '@' << opt.hosts[i] << ":" << opt.ports[i] - << ((i == opt.hosts[i].size() - 1) ? "" : ", "); + for (size_t i = 0; i < opt.endpoints.size(); i++) + os << opt.user << '@' << opt.endpoints[i].host << ":" << opt.endpoints[i].port + << ((i == opt.endpoints.size() - 1) ? "" : ", "); os << " ping_before_query:" << opt.ping_before_query << " send_retries:" << opt.send_retries @@ -115,11 +115,8 @@ std::unique_ptr GetSocketFactory(const ClientOptions& opts) { return std::make_unique(); } -std::shared_ptr GetEndpointsIterator(const ClientOptions& opts) { - if (opts.iteration_algo == EndpointsIterationAlgorithm::RoundRobin) { - return std::make_shared(opts); - } else - throw UnimplementedError("Unimplemented endpoints iteration alorithm."); +std::unique_ptr GetEndpointsIterator(const std::vector& endpoints) { + return std::make_unique(endpoints); } } @@ -141,8 +138,12 @@ class Client::Impl { void ResetConnection(); + void ResetConnectionEndpoint(); + const ServerInfo& GetServerInfo() const; + const std::optional& GetCurrentEndpoint() const; + private: bool Handshake(); @@ -166,6 +167,8 @@ class Client::Impl { void WriteBlock(const Block& block, OutputStream& output); + void CreateConnection(); + void InitializeStreams(std::unique_ptr&& socket); private: @@ -207,19 +210,21 @@ class Client::Impl { std::unique_ptr input_; std::unique_ptr output_; std::unique_ptr socket_; - std::shared_ptr endpoints_iterator; + std::unique_ptr endpoints_iterator; + + std::optional current_endpoint_; ServerInfo server_info_; }; ClientOptions modifyClientOptions(ClientOptions opts) { - if (opts.hosts.size() != opts.ports.size()) - throw ValidationError("The sizes of lists of ports and hosts must match be equal."); - if (!opts.host.empty() && std::find(opts.hosts.begin(), opts.hosts.end(), opts.host) == std::end(opts.hosts)) - { - opts.hosts.insert(opts.hosts.begin(), opts.host); - opts.ports.insert(opts.ports.begin(), opts.port); + if (opts.host.empty()) + return opts; + + Endpoint endpoint_single({opts.host, opts.port}); + if (std::find(opts.endpoints.begin(), opts.endpoints.end(), endpoint_single) == std::end(opts.endpoints)) { + opts.endpoints.emplace(opts.endpoints.begin(),endpoint_single); } return opts; } @@ -232,33 +237,9 @@ Client::Impl::Impl(const ClientOptions& opts, : options_(modifyClientOptions(opts)) , events_(nullptr) , socket_factory_(std::move(socket_factory)) - , endpoints_iterator(GetEndpointsIterator(options_)) + , endpoints_iterator(GetEndpointsIterator(options_.endpoints)) { - auto try_make_connection_with_endpoint = [this]() { - for (unsigned int i = 0; ; ) { - try { - ResetConnection(); - return; - } catch (const std::system_error&) { - if (++i > options_.send_retries) { - throw; - } - socket_factory_->sleepFor(options_.retry_timeout); - } - } - }; - - for (endpoints_iterator->ResetIterations(); ; endpoints_iterator->Next()) - { - try - { - try_make_connection_with_endpoint(); - break; - } catch (const std::system_error&) { - if(!endpoints_iterator->NextIsExist()) - throw; - } - } + CreateConnection(); if (options_.compression_method != CompressionMethod::None) { compression_ = CompressionState::Enable; @@ -377,10 +358,51 @@ void Client::Impl::ResetConnection() { } } +void Client::Impl::ResetConnectionEndpoint() { + endpoints_iterator->ResetIterations(); + endpoints_iterator->Next(); + CreateConnection(); +} + +void Client::Impl::CreateConnection() { + current_endpoint_.reset(); + auto try_make_connection_with_endpoint = [this]() { + for (unsigned int i = 0; ; ) { + try { + ResetConnection(); + return; + } catch (const std::system_error&) { + if (++i > options_.send_retries) { + throw; + } + socket_factory_->sleepFor(options_.retry_timeout); + } + } + }; + + for (endpoints_iterator->ResetIterations(); ; endpoints_iterator->Next()) + { + try + { + try_make_connection_with_endpoint(); + current_endpoint_ = {endpoints_iterator->GetHostAddr(), endpoints_iterator->GetPort()}; + break; + } catch (const std::system_error&) { + if(!endpoints_iterator->NextIsExist()) + throw; + } + } +} + const ServerInfo& Client::Impl::GetServerInfo() const { return server_info_; } + +const std::optional& Client::Impl::GetCurrentEndpoint() const { + return current_endpoint_; +} + bool Client::Impl::Handshake() { if (!SendHello()) { return false; @@ -907,10 +929,15 @@ void Client::Impl::RetryGuard(std::function func) { try { RetryConnectToTheEndpoint(func); + if (!current_endpoint_) { + current_endpoint_ = {endpoints_iterator->GetHostAddr(), endpoints_iterator->GetPort()}; + } return; } catch (const std::system_error&) { if (!endpoints_iterator->NextIsExist()) throw; + //If the exceptions was catched here, that's mean that we should change the current_endpoint. + current_endpoint_.reset(); } } } @@ -993,6 +1020,14 @@ void Client::ResetConnection() { impl_->ResetConnection(); } +void Client::ResetConnectionEndpoint() { + impl_->ResetConnectionEndpoint(); +} + +const std::optional& Client::GetCurrentEndpoint() const { + return impl_->GetCurrentEndpoint(); +} + const ServerInfo& Client::GetServerInfo() const { return impl_->GetServerInfo(); } diff --git a/clickhouse/client.h b/clickhouse/client.h index ee7cf1e8..c40254f2 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -44,12 +44,16 @@ enum class CompressionMethod { LZ4 = 1, }; +struct Endpoint { + std::string host; + unsigned int port = 9000; + + inline bool operator==(const Endpoint& right) const { + return host == right.host && port == right.port; + } +}; + enum class EndpointsIterationAlgorithm { -/** - * Client tries to connect to those endpoints one by one, on the round-robin basis: - * first default enpoint, then each of endpoints, from begin() to end(), - * if previous are inaccessible. - */ RoundRobin = 0, }; @@ -67,13 +71,13 @@ struct ClientOptions { /// Service port. DECLARE_FIELD(port, unsigned int, SetPort, 9000); - /// Hostnames of the servers. The next host to connect is selected according to the EndpointsIterationAlgorithm. - /// Note: If SetHost and SetHosts are setted, host will be placed at the beginning of the hosts vector. - DECLARE_FIELD(hosts, std::vector, SetHosts, std::vector()); - /// Ports of the servers. - DECLARE_FIELD(ports, std::vector, SetPorts, std::vector()); - /// Algorithm for selecting the next endpoint for connection. - DECLARE_FIELD(iteration_algo, EndpointsIterationAlgorithm, SetEndpointsIterationAlgorithm, EndpointsIterationAlgorithm::RoundRobin); + /** Set endpoints (host+port), only one is used. + * Client tries to connect to those endpoints one by one, on the round-robin basis: + * first default enpoint (set via SetHost() + SetPort()), then each of endpoints, from begin() to end(), + * the first one to establish connection is used for the rest of the session. + * If port isn't specified, default(9000) value will be used. + */ + DECLARE_FIELD(endpoints, std::vector, SetEndpoints, {}); /// Default database. DECLARE_FIELD(default_database, std::string, SetDefaultDatabase, "default"); @@ -257,6 +261,11 @@ class Client { const ServerInfo& GetServerInfo() const; + /// Get current connected endpoint. + /// In case when client is not connected to any endpoint, nullopt will returned. + const std::optional& GetCurrentEndpoint() const; + + void ResetConnectionEndpoint(); private: const ClientOptions options_; diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index f20602f4..2911e559 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -8,7 +8,8 @@ #include #include #include - +#include +#include #if defined(_MSC_VER) # pragma warning(disable : 4996) #endif @@ -498,16 +499,10 @@ int main() { const auto localHostEndpoint = ClientOptions() .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) - .SetHosts({ getEnvOrDefault("CLICKHOUSE_HOST", "asasdasd"), - getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), - getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), - getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), - }) - .SetPorts({ getEnvOrDefault("CLICKHOUSE_PORT", "9000"), - getEnvOrDefault("CLICKHOUSE_PORT", "9000"), - getEnvOrDefault("CLICKHOUSE_PORT", "1234"), - getEnvOrDefault("CLICKHOUSE_PORT", "5678"), - }) + .SetEndpoints({ {"asasdasd", 9000} + ,{"localhost"} + ,{"noalocalhost", 9000} + }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); @@ -516,6 +511,7 @@ int main() { Client client(ClientOptions(localHostEndpoint) .SetPingBeforeQuery(true)); RunTests(client); + std::cout << "current endpoint : " << client.GetCurrentEndpoint().value().host << "\n"; } { diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 07e53fba..845bd801 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1217,9 +1217,12 @@ class ConnectionSuccessTestCase : public testing::TestWithParam { TEST_P(ConnectionSuccessTestCase, SuccessConnectionEstablished) { const auto & client_options = GetParam(); std::unique_ptr client; - + try { client = std::make_unique(client_options); + auto endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); SUCCEED(); } catch (const std::exception & e) { FAIL() << "Got an unexpected exception : " << e.what(); @@ -1230,17 +1233,29 @@ TEST_P(ConnectionSuccessTestCase, SuccessConnectionEstablished) { INSTANTIATE_TEST_SUITE_P(ClientMultipleEndpoints, ConnectionSuccessTestCase, ::testing::Values(ClientCase::ParamType{ ClientOptions() - .SetHosts({ - getEnvOrDefault("CLICKHOUSE_HOST", "somedeadhost"), - getEnvOrDefault("CLICKHOUSE_HOST", "deadaginghost"), - getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), - getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), + .SetEndpoints({ + {"somedeadhost", 9000} + , {"deadaginghost", 1245} + , {"localhost", 9000} + , {"noalocalhost", 6784} }) - .SetPorts( { - getEnvOrDefault("CLICKHOUSE_PORT", "9000"), - getEnvOrDefault("CLICKHOUSE_PORT", "1245"), - getEnvOrDefault("CLICKHOUSE_PORT", "9000"), - getEnvOrDefault("CLICKHOUSE_PORT", "6784"), + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetPingBeforeQuery(true) + .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) + .SetRetryTimeout(std::chrono::seconds(1)), + } +)); + +INSTANTIATE_TEST_SUITE_P(ClientMultipleEndpointsWithDefaultPort, ConnectionSuccessTestCase, + ::testing::Values(ClientCase::ParamType{ + ClientOptions() + .SetEndpoints({ + {"somedeadhost"} + , {"deadaginghost", 1245} + , {"localhost"} + , {"noalocalhost", 6784} }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) @@ -1254,15 +1269,10 @@ INSTANTIATE_TEST_SUITE_P(ClientMultipleEndpoints, ConnectionSuccessTestCase, INSTANTIATE_TEST_SUITE_P(MultipleEndpointsFailed, ConnectionFailedClientTest, ::testing::Values(ConnectionFailedClientTest::ParamType{ ClientOptions() - .SetHosts({ - getEnvOrDefault("CLICKHOUSE_HOST", "somedeadhost"), - getEnvOrDefault("CLICKHOUSE_HOST", "deadaginghost"), - getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost") - }) - .SetPorts( { - getEnvOrDefault("CLICKHOUSE_PORT", "9000"), - getEnvOrDefault("CLICKHOUSE_PORT", "1245"), - getEnvOrDefault("CLICKHOUSE_PORT", "6784"), + .SetEndpoints({ + {"deadaginghost", 9000} + ,{"somedeadhost", 1245} + ,{"noalocalhost", 6784} }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) @@ -1274,23 +1284,70 @@ INSTANTIATE_TEST_SUITE_P(MultipleEndpointsFailed, ConnectionFailedClientTest, } )); -INSTANTIATE_TEST_SUITE_P(MultipleEndpointsNonValidConfig, ConnectionFailedClientTest, - ::testing::Values(ConnectionFailedClientTest::ParamType{ +class ResetConnectionTestCase : public testing::TestWithParam {}; + +TEST_P(ResetConnectionTestCase, ResetConnectionEndpointTest) { + const auto & client_options = GetParam(); + std::unique_ptr client; + + try { + client = std::make_unique(client_options); + auto endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + client->ResetConnectionEndpoint(); + endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("127.0.0.1", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + client->ResetConnectionEndpoint(); + + endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + SUCCEED(); + } catch (const std::exception & e) { + FAIL() << "Got an unexpected exception : " << e.what(); + } +} + +TEST_P(ResetConnectionTestCase, ResetConnectionTest) { + const auto & client_options = GetParam(); + std::unique_ptr client; + + try { + client = std::make_unique(client_options); + auto endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + client->ResetConnection(); + endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + SUCCEED(); + } catch (const std::exception & e) { + FAIL() << "Got an unexpected exception : " << e.what(); + } +} + +INSTANTIATE_TEST_SUITE_P(ResetConnectionClientTest, ResetConnectionTestCase, + ::testing::Values(ResetConnectionTestCase::ParamType { ClientOptions() - .SetHosts({ - getEnvOrDefault("CLICKHOUSE_HOST", "somedeadhost"), - getEnvOrDefault("CLICKHOUSE_HOST", "localhost"), - getEnvOrDefault("CLICKHOUSE_HOST", "noalocalhost"), - }) - .SetPorts( { - getEnvOrDefault("CLICKHOUSE_PORT", "9000"), + .SetEndpoints({ + {"localhost", 9000} + ,{"somedeadhost", 1245} + ,{"noalocalhost", 6784} + ,{"127.0.0.1", 9000} }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) .SetPingBeforeQuery(true) .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) - .SetRetryTimeout(std::chrono::seconds(1)), - ExpectingException{"The sizes of lists of ports and hosts must match be equal."} + .SetRetryTimeout(std::chrono::seconds(1)) } )); From 5d535f8edb900c1a511d38b4f30fe17299ed682a Mon Sep 17 00:00:00 2001 From: wangwei <1261385937@qq.com> Date: Sat, 10 Jun 2023 14:45:19 +0800 Subject: [PATCH 17/53] add more strict syntax and fix compile error --- clickhouse/CMakeLists.txt | 11 +++++++++++ clickhouse/base/compressed.cpp | 2 +- clickhouse/base/socket.cpp | 2 +- clickhouse/client.cpp | 10 ++++------ clickhouse/columns/enum.cpp | 4 ++-- clickhouse/columns/factory.cpp | 2 +- clickhouse/columns/lowcardinality.cpp | 2 +- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 67663ec5..40813e34 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -34,6 +34,17 @@ SET ( clickhouse-cpp-lib-src query.cpp ) +if (MSVC) + add_definitions(-D_CRT_SECURE_NO_WARNINGS) + add_compile_options(/W4) + add_compile_options(/wd4996) +endif() + +if (UNIX) + set(cxx_extra_wall "-Wno-deprecated-declarations -Wempty-body -Wconversion -Wreturn-type -Wparentheses -Wno-format -Wuninitialized -Wunreachable-code -Wunused-function -Wunused-value -Wunused-variable") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall}") +endif () + IF (WITH_OPENSSL) LIST(APPEND clickhouse-cpp-lib-src base/sslsocket.cpp) ENDIF () diff --git a/clickhouse/base/compressed.cpp b/clickhouse/base/compressed.cpp index 4d5cc65d..5730e855 100644 --- a/clickhouse/base/compressed.cpp +++ b/clickhouse/base/compressed.cpp @@ -141,7 +141,7 @@ void CompressedOutput::Compress(const void * data, size_t len) { const auto compressed_size = LZ4_compress_default( (const char*)data, (char*)compressed_buffer_.data() + HEADER_SIZE, - len, + static_cast(len), static_cast(compressed_buffer_.size() - HEADER_SIZE)); if (compressed_size <= 0) throw LZ4Error("Failed to compress chunk of " + std::to_string(len) + " bytes, " diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index 48e90c73..7d7326a4 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -217,7 +217,7 @@ SOCKET SocketConnect(const NetworkAddress& addr, const SocketTimeoutParams& time fd.fd = *s; fd.events = POLLOUT; fd.revents = 0; - ssize_t rval = Poll(&fd, 1, timeout_params.connect_timeout.count()); + ssize_t rval = Poll(&fd, 1, static_cast(timeout_params.connect_timeout.count())); if (rval == -1) { throw std::system_error(getSocketErrorCode(), getErrorCategory(), "fail to connect"); diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index e4b0c7ef..2f675da0 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -411,12 +411,12 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { if (!WireFormat::ReadUInt64(*input_, &info.bytes)) { return false; } - if (REVISION >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS) { + if constexpr(REVISION >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS) { if (!WireFormat::ReadUInt64(*input_, &info.total_rows)) { return false; } } - if (REVISION >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO) + if constexpr (REVISION >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO) { if (!WireFormat::ReadUInt64(*input_, &info.written_rows)) { return false; @@ -499,13 +499,11 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { throw UnimplementedError("unimplemented " + std::to_string((int)packet_type)); break; } - - return false; } bool Client::Impl::ReadBlock(InputStream& input, Block* block) { // Additional information about block. - if (REVISION >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) { + if constexpr (REVISION >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) { uint64_t num; BlockInfo info; @@ -569,7 +567,7 @@ bool Client::Impl::ReadBlock(InputStream& input, Block* block) { bool Client::Impl::ReceiveData() { Block block; - if (REVISION >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) { + if constexpr (REVISION >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) { if (!WireFormat::SkipString(*input_)) { return false; } diff --git a/clickhouse/columns/enum.cpp b/clickhouse/columns/enum.cpp index 1361e817..cd7696ca 100644 --- a/clickhouse/columns/enum.cpp +++ b/clickhouse/columns/enum.cpp @@ -30,7 +30,7 @@ void ColumnEnum::Append(const T& value, bool checkValue) { template void ColumnEnum::Append(const std::string& name) { - data_.push_back(type_->As()->GetEnumValue(name)); + data_.push_back(static_cast(type_->As()->GetEnumValue(name))); } template @@ -63,7 +63,7 @@ void ColumnEnum::SetAt(size_t n, const T& value, bool checkValue) { template void ColumnEnum::SetNameAt(size_t n, const std::string& name) { - data_.at(n) = type_->As()->GetEnumValue(name); + data_.at(n) = static_cast(type_->As()->GetEnumValue(name)); } template diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index e003b7f5..aeacdabc 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -38,7 +38,7 @@ const auto& GetASTChildElement(const TypeAst & ast, int position) { throw ValidationError("AST child element index out of bounds: " + std::to_string(position)); if (position < 0) - position = ast.elements.size() + position; + position = static_cast(ast.elements.size() + position); return ast.elements[static_cast(position)]; } diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index d3627038..13bf17a1 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -199,7 +199,7 @@ std::uint64_t ColumnLowCardinality::getDictionaryIndex(std::uint64_t item_index) void ColumnLowCardinality::appendIndex(std::uint64_t item_index) { // TODO (nemkov): handle case when index should go from UInt8 to UInt16, etc. VisitIndexColumn([item_index](auto & arg) { - arg.Append(item_index); + arg.Append(static_cast::DataType>(item_index)); }, *index_column_); } From 82386bb5b20495e649ce36e4e4491793da03303e Mon Sep 17 00:00:00 2001 From: wangwei <1261385937@qq.com> Date: Sat, 10 Jun 2023 14:45:19 +0800 Subject: [PATCH 18/53] add more strict syntax and fix compile error --- clickhouse/CMakeLists.txt | 11 +++++++++++ clickhouse/base/compressed.cpp | 4 ++-- clickhouse/base/socket.cpp | 2 +- clickhouse/client.cpp | 10 ++++------ clickhouse/columns/enum.cpp | 4 ++-- clickhouse/columns/factory.cpp | 2 +- clickhouse/columns/lowcardinality.cpp | 2 +- 7 files changed, 22 insertions(+), 13 deletions(-) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 67663ec5..40813e34 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -34,6 +34,17 @@ SET ( clickhouse-cpp-lib-src query.cpp ) +if (MSVC) + add_definitions(-D_CRT_SECURE_NO_WARNINGS) + add_compile_options(/W4) + add_compile_options(/wd4996) +endif() + +if (UNIX) + set(cxx_extra_wall "-Wno-deprecated-declarations -Wempty-body -Wconversion -Wreturn-type -Wparentheses -Wno-format -Wuninitialized -Wunreachable-code -Wunused-function -Wunused-value -Wunused-variable") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall}") +endif () + IF (WITH_OPENSSL) LIST(APPEND clickhouse-cpp-lib-src base/sslsocket.cpp) ENDIF () diff --git a/clickhouse/base/compressed.cpp b/clickhouse/base/compressed.cpp index 4d5cc65d..135f9483 100644 --- a/clickhouse/base/compressed.cpp +++ b/clickhouse/base/compressed.cpp @@ -94,7 +94,7 @@ bool CompressedInput::Decompress() { data_ = Buffer(original); - if (LZ4_decompress_safe((const char*)tmp.data() + HEADER_SIZE, (char*)data_.data(), compressed - HEADER_SIZE, original) < 0) { + if (LZ4_decompress_safe((const char*)tmp.data() + HEADER_SIZE, (char*)data_.data(), static_cast(compressed - HEADER_SIZE), original) < 0) { throw LZ4Error("can't decompress data"); } else { mem_.Reset(data_.data(), original); @@ -141,7 +141,7 @@ void CompressedOutput::Compress(const void * data, size_t len) { const auto compressed_size = LZ4_compress_default( (const char*)data, (char*)compressed_buffer_.data() + HEADER_SIZE, - len, + static_cast(len), static_cast(compressed_buffer_.size() - HEADER_SIZE)); if (compressed_size <= 0) throw LZ4Error("Failed to compress chunk of " + std::to_string(len) + " bytes, " diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index 48e90c73..7d7326a4 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -217,7 +217,7 @@ SOCKET SocketConnect(const NetworkAddress& addr, const SocketTimeoutParams& time fd.fd = *s; fd.events = POLLOUT; fd.revents = 0; - ssize_t rval = Poll(&fd, 1, timeout_params.connect_timeout.count()); + ssize_t rval = Poll(&fd, 1, static_cast(timeout_params.connect_timeout.count())); if (rval == -1) { throw std::system_error(getSocketErrorCode(), getErrorCategory(), "fail to connect"); diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index e4b0c7ef..2f675da0 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -411,12 +411,12 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { if (!WireFormat::ReadUInt64(*input_, &info.bytes)) { return false; } - if (REVISION >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS) { + if constexpr(REVISION >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS) { if (!WireFormat::ReadUInt64(*input_, &info.total_rows)) { return false; } } - if (REVISION >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO) + if constexpr (REVISION >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO) { if (!WireFormat::ReadUInt64(*input_, &info.written_rows)) { return false; @@ -499,13 +499,11 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { throw UnimplementedError("unimplemented " + std::to_string((int)packet_type)); break; } - - return false; } bool Client::Impl::ReadBlock(InputStream& input, Block* block) { // Additional information about block. - if (REVISION >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) { + if constexpr (REVISION >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) { uint64_t num; BlockInfo info; @@ -569,7 +567,7 @@ bool Client::Impl::ReadBlock(InputStream& input, Block* block) { bool Client::Impl::ReceiveData() { Block block; - if (REVISION >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) { + if constexpr (REVISION >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) { if (!WireFormat::SkipString(*input_)) { return false; } diff --git a/clickhouse/columns/enum.cpp b/clickhouse/columns/enum.cpp index 1361e817..cd7696ca 100644 --- a/clickhouse/columns/enum.cpp +++ b/clickhouse/columns/enum.cpp @@ -30,7 +30,7 @@ void ColumnEnum::Append(const T& value, bool checkValue) { template void ColumnEnum::Append(const std::string& name) { - data_.push_back(type_->As()->GetEnumValue(name)); + data_.push_back(static_cast(type_->As()->GetEnumValue(name))); } template @@ -63,7 +63,7 @@ void ColumnEnum::SetAt(size_t n, const T& value, bool checkValue) { template void ColumnEnum::SetNameAt(size_t n, const std::string& name) { - data_.at(n) = type_->As()->GetEnumValue(name); + data_.at(n) = static_cast(type_->As()->GetEnumValue(name)); } template diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index e003b7f5..aeacdabc 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -38,7 +38,7 @@ const auto& GetASTChildElement(const TypeAst & ast, int position) { throw ValidationError("AST child element index out of bounds: " + std::to_string(position)); if (position < 0) - position = ast.elements.size() + position; + position = static_cast(ast.elements.size() + position); return ast.elements[static_cast(position)]; } diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index d3627038..13bf17a1 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -199,7 +199,7 @@ std::uint64_t ColumnLowCardinality::getDictionaryIndex(std::uint64_t item_index) void ColumnLowCardinality::appendIndex(std::uint64_t item_index) { // TODO (nemkov): handle case when index should go from UInt8 to UInt16, etc. VisitIndexColumn([item_index](auto & arg) { - arg.Append(item_index); + arg.Append(static_cast::DataType>(item_index)); }, *index_column_); } From 532aa49fdff4c3220dfa5b429cbb03896bc8079f Mon Sep 17 00:00:00 2001 From: wangwei <1261385937@qq.com> Date: Sat, 10 Jun 2023 16:53:02 +0800 Subject: [PATCH 19/53] fix compile error --- clickhouse/CMakeLists.txt | 5 ++++- clickhouse/base/sslsocket.cpp | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 40813e34..3915b26d 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -40,8 +40,9 @@ if (MSVC) add_compile_options(/wd4996) endif() -if (UNIX) +if (UNIX) set(cxx_extra_wall "-Wno-deprecated-declarations -Wempty-body -Wconversion -Wreturn-type -Wparentheses -Wno-format -Wuninitialized -Wunreachable-code -Wunused-function -Wunused-value -Wunused-variable") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall}") endif () @@ -66,6 +67,8 @@ ENDIF() IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + # a little abnormal when clang check conversion + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall} -Wno-conversion") INCLUDE (CheckCXXSourceCompiles) CHECK_CXX_SOURCE_COMPILES("#include \nint main() { return __GLIBCXX__ != 0; }" diff --git a/clickhouse/base/sslsocket.cpp b/clickhouse/base/sslsocket.cpp index 8b1971e0..76ea9d3b 100644 --- a/clickhouse/base/sslsocket.cpp +++ b/clickhouse/base/sslsocket.cpp @@ -109,7 +109,7 @@ SSL_CTX * prepareSSLContext(const clickhouse::SSLParams & context_params) { #define HANDLE_SSL_CTX_ERROR(statement) do { \ if (const auto ret_code = (statement); !ret_code) \ - throwSSLError(nullptr, ERR_peek_error(), LOCATION, #statement); \ + throwSSLError(nullptr, static_cast(ERR_peek_error()), LOCATION, #statement); \ } while(false); if (context_params.use_default_ca_locations) @@ -185,7 +185,7 @@ SSL_CTX * SSLContext::getContext() { // Allows caller to use returned value of `statement` if there was no error, throws exception otherwise. #define HANDLE_SSL_ERROR(SSL_PTR, statement) [&] { \ if (const auto ret_code = (statement); ret_code <= 0) { \ - throwSSLError(SSL_PTR, SSL_get_error(SSL_PTR, ret_code), LOCATION, #statement); \ + throwSSLError(SSL_PTR, SSL_get_error(SSL_PTR, static_cast(ret_code)), LOCATION, #statement); \ return static_cast>(0); \ } \ else \ @@ -209,7 +209,7 @@ SSLSocket::SSLSocket(const NetworkAddress& addr, const SocketTimeoutParams& time std::unique_ptr ip_addr(a2i_IPADDRESS(addr.Host().c_str()), &ASN1_OCTET_STRING_free); - HANDLE_SSL_ERROR(ssl, SSL_set_fd(ssl, handle_)); + HANDLE_SSL_ERROR(ssl, SSL_set_fd(ssl, static_cast(handle_))); if (ssl_params.use_SNI) HANDLE_SSL_ERROR(ssl, SSL_set_tlsext_host_name(ssl, addr.Host().c_str())); @@ -295,7 +295,7 @@ SSLSocketOutput::SSLSocketOutput(SSL *ssl) {} size_t SSLSocketOutput::DoWrite(const void* data, size_t len) { - return static_cast(HANDLE_SSL_ERROR(ssl_, SSL_write(ssl_, data, len))); + return static_cast(HANDLE_SSL_ERROR(ssl_, SSL_write(ssl_, data, static_cast(len)))); } #undef HANDLE_SSL_ERROR From 6f47162a55e4d67613578d8ff3f859c63e0a357c Mon Sep 17 00:00:00 2001 From: wangwei <1261385937@qq.com> Date: Sat, 10 Jun 2023 17:24:49 +0800 Subject: [PATCH 20/53] fix compile error --- clickhouse/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 3915b26d..fd16d790 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -38,13 +38,15 @@ if (MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS) add_compile_options(/W4) add_compile_options(/wd4996) -endif() - -if (UNIX) - set(cxx_extra_wall "-Wno-deprecated-declarations -Wempty-body -Wconversion -Wreturn-type -Wparentheses -Wno-format -Wuninitialized -Wunreachable-code -Wunused-function -Wunused-value -Wunused-variable") - +else() + set(cxx_extra_wall "-Wno-deprecated-declarations -Wempty-body -Wconversion -Wreturn-type -Wparentheses -Wno-format -Wuninitialized -Wunreachable-code -Wunused-function -Wunused-value -Wunused-variable") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall}") -endif () + + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + # a little abnormal when clang check conversion + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall} -Wno-conversion") + endif() +endif() IF (WITH_OPENSSL) LIST(APPEND clickhouse-cpp-lib-src base/sslsocket.cpp) @@ -67,8 +69,6 @@ ENDIF() IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - # a little abnormal when clang check conversion - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall} -Wno-conversion") INCLUDE (CheckCXXSourceCompiles) CHECK_CXX_SOURCE_COMPILES("#include \nint main() { return __GLIBCXX__ != 0; }" From 08bc16ee32f4c6d421ddbe987e3d9423d89c536b Mon Sep 17 00:00:00 2001 From: wangwei <1261385937@qq.com> Date: Sat, 10 Jun 2023 20:08:20 +0800 Subject: [PATCH 21/53] remove -Wno-* flag --- clickhouse/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index fd16d790..ec544e1b 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -37,9 +37,10 @@ SET ( clickhouse-cpp-lib-src if (MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS) add_compile_options(/W4) + # remove in 3.0 add_compile_options(/wd4996) else() - set(cxx_extra_wall "-Wno-deprecated-declarations -Wempty-body -Wconversion -Wreturn-type -Wparentheses -Wno-format -Wuninitialized -Wunreachable-code -Wunused-function -Wunused-value -Wunused-variable") + set(cxx_extra_wall "-Wempty-body -Wconversion -Wreturn-type -Wparentheses -Wuninitialized -Wunreachable-code -Wunused-function -Wunused-value -Wunused-variable") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall}") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") From 6a9f0774d7a831b2215f3c09bee263270aabcce6 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 14 Jun 2023 12:20:45 +0200 Subject: [PATCH 22/53] SSLSocketOutput::DoWrite exception in case of too much data This is a temporary solution and should be changed to invoking SSL_write() as many times are required until all data is written --- clickhouse/base/sslsocket.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clickhouse/base/sslsocket.cpp b/clickhouse/base/sslsocket.cpp index 76ea9d3b..4c5185d8 100644 --- a/clickhouse/base/sslsocket.cpp +++ b/clickhouse/base/sslsocket.cpp @@ -295,6 +295,10 @@ SSLSocketOutput::SSLSocketOutput(SSL *ssl) {} size_t SSLSocketOutput::DoWrite(const void* data, size_t len) { + if (len > std::numeric_limits::max()) + // FIXME(vnemkov): We should do multiple `SSL_write`s in this case. + throw AssertionError("Failed to write too big chunk at once " + + std::to_string(len) + " > " + std::to_string(std::numeric_limits::max())); return static_cast(HANDLE_SSL_ERROR(ssl_, SSL_write(ssl_, data, static_cast(len)))); } From 252503891bc1aed040a6766f5e96826163c0b35a Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Tue, 20 Jun 2023 14:33:13 +0000 Subject: [PATCH 23/53] Fix tests and reviews changes --- clickhouse/base/endpoints_iterator.cpp | 2 +- clickhouse/base/endpoints_iterator.h | 4 ++-- clickhouse/client.h | 5 ++--- ut/client_ut.cpp | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/clickhouse/base/endpoints_iterator.cpp b/clickhouse/base/endpoints_iterator.cpp index f687703d..a1b4ccc7 100644 --- a/clickhouse/base/endpoints_iterator.cpp +++ b/clickhouse/base/endpoints_iterator.cpp @@ -16,7 +16,7 @@ std::string RoundRobinEndpointsIterator::GetHostAddr() const return endpoints[current_index].host; } -unsigned int RoundRobinEndpointsIterator::GetPort() const +uint16_t RoundRobinEndpointsIterator::GetPort() const { return endpoints[current_index].port; } diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h index dcbdd86b..23d705d6 100644 --- a/clickhouse/base/endpoints_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -20,7 +20,7 @@ class EndpointsIteratorBase virtual std::string GetHostAddr() const = 0; // Get the port of current endpoint. - virtual unsigned int GetPort() const = 0; + virtual uint16_t GetPort() const = 0; // Reset iterations. virtual void ResetIterations() = 0; @@ -32,7 +32,7 @@ class RoundRobinEndpointsIterator : public EndpointsIteratorBase public: explicit RoundRobinEndpointsIterator(const std::vector& opts); std::string GetHostAddr() const override; - unsigned int GetPort() const override; + uint16_t GetPort() const override; void ResetIterations() override; bool NextIsExist() const override; void Next() override; diff --git a/clickhouse/client.h b/clickhouse/client.h index c40254f2..a345543f 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -46,8 +46,7 @@ enum class CompressionMethod { struct Endpoint { std::string host; - unsigned int port = 9000; - + uint16_t port = 9000; inline bool operator==(const Endpoint& right) const { return host == right.host && port == right.port; } @@ -69,7 +68,7 @@ struct ClientOptions { /// Hostname of the server. DECLARE_FIELD(host, std::string, SetHost, std::string()); /// Service port. - DECLARE_FIELD(port, unsigned int, SetPort, 9000); + DECLARE_FIELD(port, uint16_t, SetPort, 9000); /** Set endpoints (host+port), only one is used. * Client tries to connect to those endpoints one by one, on the round-robin basis: diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 845bd801..3ed1092e 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1280,7 +1280,7 @@ INSTANTIATE_TEST_SUITE_P(MultipleEndpointsFailed, ConnectionFailedClientTest, .SetPingBeforeQuery(true) .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) .SetRetryTimeout(std::chrono::seconds(1)), - ExpectingException{"Temporary failure in name resolution"} + ExpectingException{""} } )); From 87804a0bd02e58dabe030f2c19d02592771af58c Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Mon, 26 Jun 2023 07:29:06 +0000 Subject: [PATCH 24/53] Review changes --- clickhouse/base/endpoints_iterator.cpp | 34 ++----- clickhouse/base/endpoints_iterator.h | 24 +---- clickhouse/base/socket.cpp | 4 +- clickhouse/base/socket.h | 4 +- clickhouse/client.cpp | 133 +++++++++++++------------ clickhouse/client.h | 1 + 6 files changed, 88 insertions(+), 112 deletions(-) diff --git a/clickhouse/base/endpoints_iterator.cpp b/clickhouse/base/endpoints_iterator.cpp index a1b4ccc7..30d3593e 100644 --- a/clickhouse/base/endpoints_iterator.cpp +++ b/clickhouse/base/endpoints_iterator.cpp @@ -3,38 +3,16 @@ namespace clickhouse { -RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const std::vector& _endpoints) : - endpoints (_endpoints) - , current_index (0) - , iteration_counter(0) +RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const std::vector& _endpoints) + : endpoints (_endpoints) + , current_index (endpoints.size() - 1ull) { - -} - -std::string RoundRobinEndpointsIterator::GetHostAddr() const -{ - return endpoints[current_index].host; -} - -uint16_t RoundRobinEndpointsIterator::GetPort() const -{ - return endpoints[current_index].port; -} - -void RoundRobinEndpointsIterator::ResetIterations() -{ - iteration_counter = 0; -} - -void RoundRobinEndpointsIterator::Next() -{ - current_index = (current_index + 1) % endpoints.size(); - iteration_counter++; } -bool RoundRobinEndpointsIterator::NextIsExist() const +Endpoint RoundRobinEndpointsIterator::Next() { - return iteration_counter + 1 < endpoints.size(); + current_index = (current_index + 1ull) % endpoints.size(); + return endpoints[current_index]; } RoundRobinEndpointsIterator::~RoundRobinEndpointsIterator() = default; diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h index 23d705d6..ba6a850f 100644 --- a/clickhouse/base/endpoints_iterator.h +++ b/clickhouse/base/endpoints_iterator.h @@ -12,37 +12,23 @@ struct ClientOptions; */ class EndpointsIteratorBase { - public: + public: virtual ~EndpointsIteratorBase() = default; - virtual void Next() = 0; - // Get the address of current endpoint. - virtual std::string GetHostAddr() const = 0; - - // Get the port of current endpoint. - virtual uint16_t GetPort() const = 0; - - // Reset iterations. - virtual void ResetIterations() = 0; - virtual bool NextIsExist() const = 0; + virtual Endpoint Next() = 0; }; class RoundRobinEndpointsIterator : public EndpointsIteratorBase { public: explicit RoundRobinEndpointsIterator(const std::vector& opts); - std::string GetHostAddr() const override; - uint16_t GetPort() const override; - void ResetIterations() override; - bool NextIsExist() const override; - void Next() override; - + Endpoint Next() override; + ~RoundRobinEndpointsIterator() override; private: const std::vector& endpoints; - int current_index; - size_t iteration_counter; + size_t current_index; }; } diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index 13755a37..e6fee1f5 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -390,9 +390,9 @@ std::unique_ptr Socket::makeOutputStream() const { NonSecureSocketFactory::~NonSecureSocketFactory() {} -std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts, const std::string& host, const std::string& port) { +std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts, const Endpoint& endpoint) { - const auto address = NetworkAddress(host, port); + const auto address = NetworkAddress(endpoint.host, std::to_string(endpoint.port)); auto socket = doConnect(address, opts); setSocketOptions(*socket, opts); diff --git a/clickhouse/base/socket.h b/clickhouse/base/socket.h index 4953a4d1..9bd9ca34 100644 --- a/clickhouse/base/socket.h +++ b/clickhouse/base/socket.h @@ -89,7 +89,7 @@ class SocketFactory { // TODO: move connection-related options to ConnectionOptions structure. - virtual std::unique_ptr connect(const ClientOptions& opts, const std::string& host, const std::string& port) = 0; + virtual std::unique_ptr connect(const ClientOptions& opts, const Endpoint& endpoint) = 0; virtual void sleepFor(const std::chrono::milliseconds& duration); }; @@ -136,7 +136,7 @@ class NonSecureSocketFactory : public SocketFactory { public: ~NonSecureSocketFactory() override; - std::unique_ptr connect(const ClientOptions& opts, const std::string& host, const std::string& port) override; + std::unique_ptr connect(const ClientOptions& opts, const Endpoint& endpoint) override; protected: virtual std::unique_ptr doConnect(const NetworkAddress& address, const ClientOptions& opts); diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index 27770c16..41144940 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -64,7 +64,8 @@ struct ClientInfo { }; std::ostream& operator<<(std::ostream& os, const ClientOptions& opt) { - os << "Client("; + os << "Client(" << opt.user << '@' << opt.host << ":" << opt.port + << "Endpoints :"; for (size_t i = 0; i < opt.endpoints.size(); i++) os << opt.user << '@' << opt.endpoints[i].host << ":" << opt.endpoints[i].port << ((i == opt.endpoints.size() - 1) ? "" : ", "); @@ -115,8 +116,13 @@ std::unique_ptr GetSocketFactory(const ClientOptions& opts) { return std::make_unique(); } -std::unique_ptr GetEndpointsIterator(const std::vector& endpoints) { - return std::make_unique(endpoints); +std::unique_ptr GetEndpointsIterator(const ClientOptions& opts) { + if (opts.endpoints.empty()) + { + throw ValidationError("The list of endpoints is empty"); + } + + return std::make_unique(opts.endpoints); } } @@ -171,11 +177,16 @@ class Client::Impl { void InitializeStreams(std::unique_ptr&& socket); + inline size_t GetConnectionAttempts() const + { + return options_.endpoints.size() * options_.send_retries; + } + private: /// In case of network errors tries to reconnect to server and /// call fuc several times. void RetryGuard(std::function func); - + void RetryConnectToTheEndpoint(std::function& func); private: @@ -222,10 +233,8 @@ ClientOptions modifyClientOptions(ClientOptions opts) if (opts.host.empty()) return opts; - Endpoint endpoint_single({opts.host, opts.port}); - if (std::find(opts.endpoints.begin(), opts.endpoints.end(), endpoint_single) == std::end(opts.endpoints)) { - opts.endpoints.emplace(opts.endpoints.begin(),endpoint_single); - } + Endpoint default_endpoint({opts.host, opts.port}); + opts.endpoints.emplace(opts.endpoints.begin(), default_endpoint); return opts; } @@ -237,7 +246,7 @@ Client::Impl::Impl(const ClientOptions& opts, : options_(modifyClientOptions(opts)) , events_(nullptr) , socket_factory_(std::move(socket_factory)) - , endpoints_iterator(GetEndpointsIterator(options_.endpoints)) + , endpoints_iterator(GetEndpointsIterator(options_)) { CreateConnection(); @@ -349,9 +358,7 @@ void Client::Impl::Ping() { } void Client::Impl::ResetConnection() { - InitializeStreams(socket_factory_->connect(options_, endpoints_iterator->GetHostAddr(), - std::to_string(endpoints_iterator->GetPort()) - )); + InitializeStreams(socket_factory_->connect(options_, current_endpoint_.value())); if (!Handshake()) { throw ProtocolError("fail to connect to " + options_.host); @@ -359,37 +366,36 @@ void Client::Impl::ResetConnection() { } void Client::Impl::ResetConnectionEndpoint() { - endpoints_iterator->ResetIterations(); - endpoints_iterator->Next(); - CreateConnection(); -} - -void Client::Impl::CreateConnection() { current_endpoint_.reset(); - auto try_make_connection_with_endpoint = [this]() { - for (unsigned int i = 0; ; ) { - try { - ResetConnection(); - return; - } catch (const std::system_error&) { - if (++i > options_.send_retries) { - throw; - } - socket_factory_->sleepFor(options_.retry_timeout); + for (size_t i = 0; i < options_.endpoints.size();) + { + try + { + current_endpoint_ = endpoints_iterator->Next(); + ResetConnection(); + return; + } catch (const std::system_error&) { + if (++i == options_.endpoints.size()) + { + current_endpoint_.reset(); + throw; } } - }; + } +} - for (endpoints_iterator->ResetIterations(); ; endpoints_iterator->Next()) +void Client::Impl::CreateConnection() { + for (size_t i = 0; i < options_.send_retries;) { try { - try_make_connection_with_endpoint(); - current_endpoint_ = {endpoints_iterator->GetHostAddr(), endpoints_iterator->GetPort()}; - break; + ResetConnectionEndpoint(); + return; } catch (const std::system_error&) { - if(!endpoints_iterator->NextIsExist()) + if (++i == options_.send_retries) + { throw; + } } } } @@ -924,40 +930,45 @@ bool Client::Impl::ReceiveHello() { } void Client::Impl::RetryGuard(std::function func) { - for(endpoints_iterator->ResetIterations(); ; endpoints_iterator->Next()) + + if (current_endpoint_) { - try - { - RetryConnectToTheEndpoint(func); - if (!current_endpoint_) { - current_endpoint_ = {endpoints_iterator->GetHostAddr(), endpoints_iterator->GetPort()}; + for (unsigned int i = 0; ; ++i) { + try { + func(); + return; + } catch (const std::system_error&) { + bool ok = true; + + try { + socket_factory_->sleepFor(options_.retry_timeout); + ResetConnection(); + } catch (...) { + ok = false; + } + + if (!ok && i == options_.send_retries) { + break; + } } - return; - } catch (const std::system_error&) { - if (!endpoints_iterator->NextIsExist()) - throw; - //If the exceptions was catched here, that's mean that we should change the current_endpoint. - current_endpoint_.reset(); } } -} - -void Client::Impl::RetryConnectToTheEndpoint(std::function& func) { - for (unsigned int i = 0; ; ++i) { - try { + // Connectiong with current_endpoint_ are broken. + // Trying to establish with the another one from the list. + size_t connection_attempts_count = GetConnectionAttempts(); + for (size_t i = 0; i < connection_attempts_count;) + { + try + { + socket_factory_->sleepFor(options_.retry_timeout); + current_endpoint_ = endpoints_iterator->Next(); + ResetConnection(); func(); return; } catch (const std::system_error&) { - bool ok = true; - - try { - socket_factory_->sleepFor(options_.retry_timeout); - ResetConnection(); - } catch (...) { - ok = false; - } - - if (!ok && i == options_.send_retries) { + if (++i == connection_attempts_count) + { + current_endpoint_.reset(); throw; } } diff --git a/clickhouse/client.h b/clickhouse/client.h index a345543f..a66d59a6 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -264,6 +264,7 @@ class Client { /// In case when client is not connected to any endpoint, nullopt will returned. const std::optional& GetCurrentEndpoint() const; + // Try to connect to different endpoints one by one only one time. If it doesn't work, throw an exception. void ResetConnectionEndpoint(); private: const ClientOptions options_; From d488d1952e9c5050feabbe9a618ca2a6ee89dd12 Mon Sep 17 00:00:00 2001 From: alen <7587499+alenstarx@users.noreply.github.com> Date: Tue, 18 Jul 2023 10:49:12 +0800 Subject: [PATCH 25/53] Update client.cpp Fix abnormal column names in gcc7.3.1 --- clickhouse/client.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index 3b75efc0..ca57190b 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -609,9 +609,9 @@ bool Client::Impl::ReadBlock(InputStream& input, Block* block) { CreateColumnByTypeSettings create_column_settings; create_column_settings.low_cardinality_as_wrapped_column = options_.backward_compatibility_lowcardinality_as_wrapped_column; - std::string name; - std::string type; for (size_t i = 0; i < num_columns; ++i) { + std::string name; + std::string type; if (!WireFormat::ReadString(input, &name)) { return false; } From b77a1b041841194e4f8c22ffb6d7878149b8c9be Mon Sep 17 00:00:00 2001 From: alenstar Date: Sat, 22 Jul 2023 15:51:13 +0800 Subject: [PATCH 26/53] add test for Fix abnormal column names in gcc7.3.1 #317 --- ut/CMakeLists.txt | 1 + ut/abnormal_column_names_test.cpp | 47 +++++++++++++++++++++++++++++++ ut/abnormal_column_names_test.h | 18 ++++++++++++ ut/client_ut.cpp | 17 +++++++++++ 4 files changed, 83 insertions(+) create mode 100644 ut/abnormal_column_names_test.cpp create mode 100644 ut/abnormal_column_names_test.h diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index 2c9f6ee5..6b395458 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -15,6 +15,7 @@ SET ( clickhouse-cpp-ut-src performance_tests.cpp tcp_server.cpp readonly_client_test.cpp + abnormal_column_names_test.cpp connection_failed_client_test.cpp array_of_low_cardinality_tests.cpp CreateColumnByType_ut.cpp diff --git a/ut/abnormal_column_names_test.cpp b/ut/abnormal_column_names_test.cpp new file mode 100644 index 00000000..93ed2343 --- /dev/null +++ b/ut/abnormal_column_names_test.cpp @@ -0,0 +1,47 @@ +#include "abnormal_column_names_test.h" +#include "utils.h" + +#include +#include +#include +#include + +namespace { + using namespace clickhouse; +} + +void AbnormalColumnNamesTest::SetUp() { + client_ = std::make_unique(std::get<0>(GetParam())); +} + +void AbnormalColumnNamesTest::TearDown() { + client_.reset(); +} + +// Sometimes gtest fails to detect that this test is instantiated elsewhere, suppress the error explicitly. +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AbnormalColumnNamesTest); +TEST_P(AbnormalColumnNamesTest, Select) { + + const auto & queries = std::get<1>(GetParam()); + for (const auto & query : queries) { + std::unordered_set names; + size_t count = 0; + client_->Select(query, + [& query,& names, & count](const Block& block) { + if (block.GetRowCount() == 0 || block.GetColumnCount() == 0) + return; + + std::cout << "query => " << query <<"\n" << PrettyPrintBlock{block}; + for (size_t i = 0; i < block.GetColumnCount(); ++i) + { + count++; + names.insert(block.GetColumnName(i)); + } + } + ); + EXPECT_EQ(count, names.size()); + for(auto& name: names) { + std::cout << name << ", count=" << count<< std::endl; + } + } +} diff --git a/ut/abnormal_column_names_test.h b/ut/abnormal_column_names_test.h new file mode 100644 index 00000000..be5bc53c --- /dev/null +++ b/ut/abnormal_column_names_test.h @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + +#include +#include +#include + +class AbnormalColumnNamesTest : public testing::TestWithParam< + std::tuple > /*queries*/> { +protected: + void SetUp() override; + void TearDown() override; + + std::unique_ptr client_; +}; diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 3ed1092e..f023924a 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1,6 +1,7 @@ #include #include "readonly_client_test.h" +#include "abnormal_column_names_test.h" #include "connection_failed_client_test.h" #include "utils.h" @@ -1196,6 +1197,22 @@ INSTANTIATE_TEST_SUITE_P(ClientLocalReadonly, ReadonlyClientTest, } )); +INSTANTIATE_TEST_SUITE_P(ColumnNames, AbnormalColumnNamesTest, + ::testing::Values(AbnormalColumnNamesTest::ParamType{ + ClientOptions() + .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetSendRetries(1) + .SetPingBeforeQuery(true) + .SetCompressionMethod(CompressionMethod::None), + {"select 123,231,113", "select 'ABC','AAA','BBB','CCC'"} + } +)); + + INSTANTIATE_TEST_SUITE_P(ClientLocalFailed, ConnectionFailedClientTest, ::testing::Values(ConnectionFailedClientTest::ParamType{ ClientOptions() From 2a2db48556ca2886b0c986b2f65d4256df8beb5d Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 27 Jul 2023 20:31:08 +0200 Subject: [PATCH 27/53] Implemented Append/RawAt/operator[] for ColumnDate and ColumnDate32 that wourk with uint16_t/int32_t AppendRaw and tests --- clickhouse/CMakeLists.txt | 50 ++++++++++++++++++++++++++++++++++++- clickhouse/columns/date.cpp | 32 ++++++++++++++++++++++-- clickhouse/columns/date.h | 20 ++++++++++++--- ut/columns_ut.cpp | 34 +++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 7 deletions(-) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 789db754..80650388 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -17,7 +17,6 @@ SET ( clickhouse-cpp-lib-src columns/ip4.cpp columns/ip6.cpp columns/lowcardinality.cpp - columns/lowcardinalityadaptor.h columns/nullable.cpp columns/numeric.cpp columns/map.cpp @@ -33,6 +32,55 @@ SET ( clickhouse-cpp-lib-src block.cpp client.cpp query.cpp + + # Headers + base/buffer.h + base/compressed.h + base/endpoints_iterator.h + base/input.h + base/open_telemetry.h + base/output.h + base/platform.h + base/projected_iterator.h + base/singleton.h + base/socket.h + base/sslsocket.h + base/string_utils.h + base/string_view.h + base/uuid.h + base/wire_format.h + + columns/array.h + columns/column.h + columns/date.h + columns/decimal.h + columns/enum.h + columns/factory.h + columns/geo.h + columns/ip4.h + columns/ip6.h + columns/itemview.h + columns/lowcardinality.h + columns/lowcardinalityadaptor.h + columns/map.h + columns/nothing.h + columns/nullable.h + columns/numeric.h + columns/string.h + columns/tuple.h + columns/utils.h + columns/uuid.h + + types/type_parser.h + types/types.h + + block.h + client.h + error_codes.h + exceptions.h + protocol.h + query.h + server_exception.h ) if (MSVC) diff --git a/clickhouse/columns/date.cpp b/clickhouse/columns/date.cpp index 1ef67c44..d39ffda3 100644 --- a/clickhouse/columns/date.cpp +++ b/clickhouse/columns/date.cpp @@ -1,4 +1,5 @@ #include "date.h" +#include namespace clickhouse { @@ -9,7 +10,7 @@ ColumnDate::ColumnDate() } void ColumnDate::Append(const std::time_t& value) { - /// TODO: This code is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. data_->Append(static_cast(value / std::time_t(86400))); } @@ -18,9 +19,23 @@ void ColumnDate::Clear() { } std::time_t ColumnDate::At(size_t n) const { + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. return static_cast(data_->At(n)) * 86400; } + +void ColumnDate::Append(uint16_t value) { + data_->Append(value); +} + +uint16_t ColumnDate::RawAt(size_t n) const { + return data_->At(n); +} + +uint16_t ColumnDate::operator [] (size_t n) const { + return (*data_)[n]; +} + void ColumnDate::Append(ColumnRef column) { if (auto col = column->As()) { data_->Append(col->data_); @@ -70,7 +85,7 @@ ColumnDate32::ColumnDate32() } void ColumnDate32::Append(const std::time_t& value) { - /// TODO: This code is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. data_->Append(static_cast(value / std::time_t(86400))); } @@ -79,6 +94,7 @@ void ColumnDate32::Clear() { } std::time_t ColumnDate32::At(size_t n) const { + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. return static_cast(data_->At(n)) * 86400; } @@ -88,6 +104,18 @@ void ColumnDate32::Append(ColumnRef column) { } } +void ColumnDate32::Append(int32_t value) { + data_->Append(value); +} + +int32_t ColumnDate32::RawAt(size_t n) const { + return data_->At(n); +} + +int32_t ColumnDate32::operator [] (size_t n) const { + return (*data_)[n]; +} + bool ColumnDate32::LoadBody(InputStream* input, size_t rows) { return data_->LoadBody(input, rows); } diff --git a/clickhouse/columns/date.h b/clickhouse/columns/date.h index 2a240c90..2c5202ff 100644 --- a/clickhouse/columns/date.h +++ b/clickhouse/columns/date.h @@ -15,13 +15,19 @@ class ColumnDate : public Column { ColumnDate(); /// Appends one element to the end of column. - /// TODO: The implementation is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. void Append(const std::time_t& value); /// Returns element at given row number. - /// TODO: The implementation is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. std::time_t At(size_t n) const; + /// Do data as is -- number of day in Unix epoch, no conversions performed + void Append(uint16_t value); + void AppendRaw(uint16_t value) { Append(value); } + uint16_t RawAt(size_t n) const; + uint16_t operator [] (size_t n) const; + /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; @@ -56,16 +62,22 @@ class ColumnDate32 : public Column { ColumnDate32(); /// Appends one element to the end of column. - /// TODO: The implementation is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. void Append(const std::time_t& value); /// Returns element at given row number. - /// TODO: The implementation is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. std::time_t At(size_t n) const; /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; + /// Do data as is -- number of day in Unix epoch, no conversions performed + void Append(int32_t value); + void AppendRaw(int32_t value) { Append(value); } + int32_t RawAt(size_t n) const; + int32_t operator [] (size_t n) const; + /// Loads column data from input stream. bool LoadBody(InputStream* input, size_t rows) override; diff --git a/ut/columns_ut.cpp b/ut/columns_ut.cpp index eda33292..872f47e6 100644 --- a/ut/columns_ut.cpp +++ b/ut/columns_ut.cpp @@ -172,6 +172,40 @@ TEST(ColumnsCase, DateAppend) { ASSERT_EQ(col2->At(0), (now / 86400) * 86400); } +TEST(ColumnsCase, Date_UInt16_interface) { + auto col1 = std::make_shared(); + + col1->AppendRaw(1u); + col1->AppendRaw(1234u); + + ASSERT_EQ(col1->Size(), 2u); + ASSERT_EQ(col1->RawAt(0), 1u); + ASSERT_EQ(col1->RawAt(1), 1234u); + + + ASSERT_EQ((*col1)[0], 1u); + ASSERT_EQ((*col1)[1], 1234u); +} + +TEST(ColumnsCase, Date32_Int32_interface) { + auto col1 = std::make_shared(); + + col1->AppendRaw(1); + col1->AppendRaw(1234); + col1->AppendRaw(-1234); + + ASSERT_EQ(col1->Size(), 3u); + ASSERT_EQ(col1->RawAt(0), 1); + ASSERT_EQ(col1->RawAt(1), 1234); + ASSERT_EQ(col1->RawAt(1), -1234); + + + ASSERT_EQ((*col1)[0], 1); + ASSERT_EQ((*col1)[1], 1234); + ASSERT_EQ((*col1)[1], -1234); +} + + TEST(ColumnsCase, DateTime64_0) { auto column = std::make_shared(0ul); From 1701f1e27825f1806f7e489a81cf97bb559e074a Mon Sep 17 00:00:00 2001 From: alenstarx Date: Sat, 29 Jul 2023 12:28:43 +0800 Subject: [PATCH 28/53] rename AbnormalColumnNamesTest to AbnormalColumnNamesClientTest and move test suite instantiation right to the definition of the test itself --- ut/abnormal_column_names_test.cpp | 66 ++++++++++++++++++++++--------- ut/abnormal_column_names_test.h | 2 +- ut/client_ut.cpp | 16 -------- 3 files changed, 48 insertions(+), 36 deletions(-) diff --git a/ut/abnormal_column_names_test.cpp b/ut/abnormal_column_names_test.cpp index 93ed2343..cb8c81eb 100644 --- a/ut/abnormal_column_names_test.cpp +++ b/ut/abnormal_column_names_test.cpp @@ -10,38 +10,66 @@ namespace { using namespace clickhouse; } -void AbnormalColumnNamesTest::SetUp() { +void AbnormalColumnNamesClientTest::SetUp() { client_ = std::make_unique(std::get<0>(GetParam())); } -void AbnormalColumnNamesTest::TearDown() { +void AbnormalColumnNamesClientTest::TearDown() { client_.reset(); } // Sometimes gtest fails to detect that this test is instantiated elsewhere, suppress the error explicitly. -GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AbnormalColumnNamesTest); -TEST_P(AbnormalColumnNamesTest, Select) { - +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AbnormalColumnNamesClientTest); +TEST_P(AbnormalColumnNamesClientTest, Select) { + static const std::vector expect_results { + "+-------+-------+-------+\n"\ + "| 123 | 231 | 113 |\n"\ + "+-------+-------+-------+\n"\ + "| UInt8 | UInt8 | UInt8 |\n"\ + "+-------+-------+-------+\n"\ + "| 123 | 231 | 113 |\n"\ + "+-------+-------+-------+\n", + "+--------+--------+--------+--------+\n"\ + "| 'ABC' | 'AAA' | 'BBB' | 'CCC' |\n"\ + "+--------+--------+--------+--------+\n"\ + "| String | String | String | String |\n"\ + "+--------+--------+--------+--------+\n"\ + "| ABC | AAA | BBB | CCC |\n"\ + "+--------+--------+--------+--------+\n" + }; const auto & queries = std::get<1>(GetParam()); - for (const auto & query : queries) { - std::unordered_set names; - size_t count = 0; + for (size_t i = 0; i < queries.size(); ++i) { + const auto & query = queries.at(i); client_->Select(query, - [& query,& names, & count](const Block& block) { + [& queries, i](const Block& block) { if (block.GetRowCount() == 0 || block.GetColumnCount() == 0) return; + EXPECT_EQ(1UL, block.GetRowCount()); + EXPECT_EQ(i == 0 ? 3UL: 4UL, block.GetColumnCount()); - std::cout << "query => " << query <<"\n" << PrettyPrintBlock{block}; - for (size_t i = 0; i < block.GetColumnCount(); ++i) - { - count++; - names.insert(block.GetColumnName(i)); - } + std::stringstream sstr; + sstr << PrettyPrintBlock{block}; + auto result = sstr.str(); + std::cout << "query => " << queries.at(i) <<"\n" << PrettyPrintBlock{block}; + ASSERT_EQ(expect_results.at(i), result); } ); - EXPECT_EQ(count, names.size()); - for(auto& name: names) { - std::cout << name << ", count=" << count<< std::endl; - } } } + + +INSTANTIATE_TEST_SUITE_P(ClientColumnNames, AbnormalColumnNamesClientTest, + ::testing::Values(AbnormalColumnNamesClientTest::ParamType{ + ClientOptions() + .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetSendRetries(1) + .SetPingBeforeQuery(true) + .SetCompressionMethod(CompressionMethod::None), + {"select 123,231,113", "select 'ABC','AAA','BBB','CCC'"} + } +)); + diff --git a/ut/abnormal_column_names_test.h b/ut/abnormal_column_names_test.h index be5bc53c..b0f56c62 100644 --- a/ut/abnormal_column_names_test.h +++ b/ut/abnormal_column_names_test.h @@ -8,7 +8,7 @@ #include #include -class AbnormalColumnNamesTest : public testing::TestWithParam< +class AbnormalColumnNamesClientTest : public testing::TestWithParam< std::tuple > /*queries*/> { protected: void SetUp() override; diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index f023924a..d894cc36 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1,7 +1,6 @@ #include #include "readonly_client_test.h" -#include "abnormal_column_names_test.h" #include "connection_failed_client_test.h" #include "utils.h" @@ -1197,21 +1196,6 @@ INSTANTIATE_TEST_SUITE_P(ClientLocalReadonly, ReadonlyClientTest, } )); -INSTANTIATE_TEST_SUITE_P(ColumnNames, AbnormalColumnNamesTest, - ::testing::Values(AbnormalColumnNamesTest::ParamType{ - ClientOptions() - .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) - .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) - .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) - .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) - .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) - .SetSendRetries(1) - .SetPingBeforeQuery(true) - .SetCompressionMethod(CompressionMethod::None), - {"select 123,231,113", "select 'ABC','AAA','BBB','CCC'"} - } -)); - INSTANTIATE_TEST_SUITE_P(ClientLocalFailed, ConnectionFailedClientTest, ::testing::Values(ConnectionFailedClientTest::ParamType{ From f04d913ee82d742e951c223a40b247d22cdc76fb Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 31 Jul 2023 20:02:15 +0200 Subject: [PATCH 29/53] Comment on unit-tests --- ut/abnormal_column_names_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ut/abnormal_column_names_test.cpp b/ut/abnormal_column_names_test.cpp index cb8c81eb..885ed0be 100644 --- a/ut/abnormal_column_names_test.cpp +++ b/ut/abnormal_column_names_test.cpp @@ -21,6 +21,7 @@ void AbnormalColumnNamesClientTest::TearDown() { // Sometimes gtest fails to detect that this test is instantiated elsewhere, suppress the error explicitly. GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AbnormalColumnNamesClientTest); TEST_P(AbnormalColumnNamesClientTest, Select) { + // TODO(vnemkov): move expected results into the test parameters, also get rid of PrettyPrintBlock static const std::vector expect_results { "+-------+-------+-------+\n"\ "| 123 | 231 | 113 |\n"\ From dc62ca95e6c496dba42e2da9d403310ebad85a86 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 2 Aug 2023 10:40:13 +0200 Subject: [PATCH 30/53] Fixed Date32_Int32_interface test --- clickhouse/columns/date.cpp | 1 + ut/columns_ut.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/clickhouse/columns/date.cpp b/clickhouse/columns/date.cpp index d39ffda3..3bc9662b 100644 --- a/clickhouse/columns/date.cpp +++ b/clickhouse/columns/date.cpp @@ -269,6 +269,7 @@ void ColumnDateTime64::SaveBody(OutputStream* output) { void ColumnDateTime64::Clear() { data_->Clear(); } + size_t ColumnDateTime64::Size() const { return data_->Size(); } diff --git a/ut/columns_ut.cpp b/ut/columns_ut.cpp index 872f47e6..e49364ec 100644 --- a/ut/columns_ut.cpp +++ b/ut/columns_ut.cpp @@ -172,6 +172,7 @@ TEST(ColumnsCase, DateAppend) { ASSERT_EQ(col2->At(0), (now / 86400) * 86400); } + TEST(ColumnsCase, Date_UInt16_interface) { auto col1 = std::make_shared(); @@ -182,11 +183,11 @@ TEST(ColumnsCase, Date_UInt16_interface) { ASSERT_EQ(col1->RawAt(0), 1u); ASSERT_EQ(col1->RawAt(1), 1234u); - ASSERT_EQ((*col1)[0], 1u); ASSERT_EQ((*col1)[1], 1234u); } + TEST(ColumnsCase, Date32_Int32_interface) { auto col1 = std::make_shared(); @@ -197,12 +198,11 @@ TEST(ColumnsCase, Date32_Int32_interface) { ASSERT_EQ(col1->Size(), 3u); ASSERT_EQ(col1->RawAt(0), 1); ASSERT_EQ(col1->RawAt(1), 1234); - ASSERT_EQ(col1->RawAt(1), -1234); - + ASSERT_EQ(col1->RawAt(2), -1234); ASSERT_EQ((*col1)[0], 1); ASSERT_EQ((*col1)[1], 1234); - ASSERT_EQ((*col1)[1], -1234); + ASSERT_EQ((*col1)[2], -1234); } @@ -215,6 +215,7 @@ TEST(ColumnsCase, DateTime64_0) { ASSERT_EQ(0u, column->Size()); } + TEST(ColumnsCase, DateTime64_6) { auto column = std::make_shared(6ul); From e5b33f274ffd82d7ee683e3a15fb8e88f282f876 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 2 Aug 2023 11:00:19 +0200 Subject: [PATCH 31/53] Better tests --- ut/abnormal_column_names_test.cpp | 93 ++++++++++++++++--------------- ut/abnormal_column_names_test.h | 18 ------ 2 files changed, 49 insertions(+), 62 deletions(-) delete mode 100644 ut/abnormal_column_names_test.h diff --git a/ut/abnormal_column_names_test.cpp b/ut/abnormal_column_names_test.cpp index 885ed0be..11868f73 100644 --- a/ut/abnormal_column_names_test.cpp +++ b/ut/abnormal_column_names_test.cpp @@ -1,60 +1,64 @@ -#include "abnormal_column_names_test.h" -#include "utils.h" #include #include + +#include "utils.h" + +#include + #include #include namespace { - using namespace clickhouse; -} +using namespace clickhouse; -void AbnormalColumnNamesClientTest::SetUp() { - client_ = std::make_unique(std::get<0>(GetParam())); -} +std::string getColumnNames(const Block& block) { + std::string result; + for (size_t i = 0; i < block.GetColumnCount(); ++i) { + result += block.GetColumnName(i); + if (i != block.GetColumnCount() - 1) + result += ','; + } -void AbnormalColumnNamesClientTest::TearDown() { - client_.reset(); + return result; } +} + +struct AbnormalColumnNamesClientTestCase { + ClientOptions client_options; + std::vector queries; + std::vector expected_names; +}; + +class AbnormalColumnNamesClientTest : public testing::TestWithParam { +protected: + void SetUp() override { + client_ = std::make_unique(GetParam().client_options); + } + void TearDown() override { + client_.reset(); + } + + std::unique_ptr client_; +}; + -// Sometimes gtest fails to detect that this test is instantiated elsewhere, suppress the error explicitly. -GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AbnormalColumnNamesClientTest); TEST_P(AbnormalColumnNamesClientTest, Select) { - // TODO(vnemkov): move expected results into the test parameters, also get rid of PrettyPrintBlock - static const std::vector expect_results { - "+-------+-------+-------+\n"\ - "| 123 | 231 | 113 |\n"\ - "+-------+-------+-------+\n"\ - "| UInt8 | UInt8 | UInt8 |\n"\ - "+-------+-------+-------+\n"\ - "| 123 | 231 | 113 |\n"\ - "+-------+-------+-------+\n", - "+--------+--------+--------+--------+\n"\ - "| 'ABC' | 'AAA' | 'BBB' | 'CCC' |\n"\ - "+--------+--------+--------+--------+\n"\ - "| String | String | String | String |\n"\ - "+--------+--------+--------+--------+\n"\ - "| ABC | AAA | BBB | CCC |\n"\ - "+--------+--------+--------+--------+\n" - }; - const auto & queries = std::get<1>(GetParam()); + const auto & queries = GetParam().queries; for (size_t i = 0; i < queries.size(); ++i) { + const auto & query = queries.at(i); - client_->Select(query, - [& queries, i](const Block& block) { - if (block.GetRowCount() == 0 || block.GetColumnCount() == 0) - return; - EXPECT_EQ(1UL, block.GetRowCount()); - EXPECT_EQ(i == 0 ? 3UL: 4UL, block.GetColumnCount()); - - std::stringstream sstr; - sstr << PrettyPrintBlock{block}; - auto result = sstr.str(); - std::cout << "query => " << queries.at(i) <<"\n" << PrettyPrintBlock{block}; - ASSERT_EQ(expect_results.at(i), result); - } - ); + const auto & expected = GetParam().expected_names[i]; + + client_->Select(query, [query, expected](const Block& block) { + if (block.GetRowCount() == 0 || block.GetColumnCount() == 0) + return; + + EXPECT_EQ(1UL, block.GetRowCount()); + + EXPECT_EQ(expected, getColumnNames(block)) + << "For query: " << query; + }); } } @@ -70,7 +74,8 @@ INSTANTIATE_TEST_SUITE_P(ClientColumnNames, AbnormalColumnNamesClientTest, .SetSendRetries(1) .SetPingBeforeQuery(true) .SetCompressionMethod(CompressionMethod::None), - {"select 123,231,113", "select 'ABC','AAA','BBB','CCC'"} + {"select 123,231,113", "select 'ABC','AAA','BBB','CCC'"}, + {"123,231,113", "'ABC','AAA','BBB','CCC'"}, } )); diff --git a/ut/abnormal_column_names_test.h b/ut/abnormal_column_names_test.h deleted file mode 100644 index b0f56c62..00000000 --- a/ut/abnormal_column_names_test.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include - -#include - -#include -#include -#include - -class AbnormalColumnNamesClientTest : public testing::TestWithParam< - std::tuple > /*queries*/> { -protected: - void SetUp() override; - void TearDown() override; - - std::unique_ptr client_; -}; From ad07a79ff1746bb0817164e4de9415a3c2f90cea Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 3 Aug 2023 10:15:42 +0200 Subject: [PATCH 32/53] Fixed tests to generate time_t values to avoid ambiguity --- clickhouse/columns/date.cpp | 10 ++++++++-- clickhouse/columns/date.h | 4 ++-- ut/Column_ut.cpp | 10 +++++++++- ut/value_generators.cpp | 17 +++-------------- ut/value_generators.h | 20 ++++++++++++++++++-- 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/clickhouse/columns/date.cpp b/clickhouse/columns/date.cpp index 3bc9662b..4e48129f 100644 --- a/clickhouse/columns/date.cpp +++ b/clickhouse/columns/date.cpp @@ -23,11 +23,14 @@ std::time_t ColumnDate::At(size_t n) const { return static_cast(data_->At(n)) * 86400; } - void ColumnDate::Append(uint16_t value) { data_->Append(value); } +void ColumnDate::AppendRaw(uint16_t value) { + data_->Append(value); +} + uint16_t ColumnDate::RawAt(size_t n) const { return data_->At(n); } @@ -77,7 +80,6 @@ ItemView ColumnDate::GetItem(size_t index) const { } - ColumnDate32::ColumnDate32() : Column(Type::CreateDate32()) , data_(std::make_shared()) @@ -108,6 +110,10 @@ void ColumnDate32::Append(int32_t value) { data_->Append(value); } +void ColumnDate32::AppendRaw(int32_t value) { + data_->Append(value); +} + int32_t ColumnDate32::RawAt(size_t n) const { return data_->At(n); } diff --git a/clickhouse/columns/date.h b/clickhouse/columns/date.h index 2c5202ff..5d3770be 100644 --- a/clickhouse/columns/date.h +++ b/clickhouse/columns/date.h @@ -24,7 +24,7 @@ class ColumnDate : public Column { /// Do data as is -- number of day in Unix epoch, no conversions performed void Append(uint16_t value); - void AppendRaw(uint16_t value) { Append(value); } + void AppendRaw(uint16_t value); uint16_t RawAt(size_t n) const; uint16_t operator [] (size_t n) const; @@ -74,7 +74,7 @@ class ColumnDate32 : public Column { /// Do data as is -- number of day in Unix epoch, no conversions performed void Append(int32_t value); - void AppendRaw(int32_t value) { Append(value); } + void AppendRaw(int32_t value); int32_t RawAt(size_t n) const; int32_t operator [] (size_t n) const; diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index 544be975..8432009f 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -66,7 +66,7 @@ class GenericColumnTest : public testing::Test { } else if constexpr (std::is_same_v) { return GenerateVector(values_size, FromVectorGenerator{MakeFixedStrings(12)}); } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeDates()}); + return GenerateVector(values_size, FromVectorGenerator{MakeDates()}); } else if constexpr (std::is_same_v) { return GenerateVector(values_size, FromVectorGenerator{MakeDateTimes()}); } else if constexpr (std::is_same_v) { @@ -143,6 +143,14 @@ using ValueColumns = ::testing::Types< >; TYPED_TEST_SUITE(GenericColumnTest, ValueColumns); +//TestCase(TypeCreatorFunc, ValuesGeneratorFunc) + + +//class GenericColumnTestCase +//{ +// virutual +//}; + TYPED_TEST(GenericColumnTest, Construct) { auto column = this->MakeColumn(); ASSERT_EQ(0u, column->Size()); diff --git a/ut/value_generators.cpp b/ut/value_generators.cpp index 41e36a61..805f22ad 100644 --- a/ut/value_generators.cpp +++ b/ut/value_generators.cpp @@ -1,6 +1,7 @@ #include "value_generators.h" #include +#include #include namespace { @@ -53,25 +54,13 @@ std::vector MakeDateTime64s(size_t scale, size_t values_size) { }); } -std::vector MakeDates() { - // in CH Date internally a UInt16 and stores a day number - // ColumnDate expects values to be seconds, which is then - // converted to day number internally, hence the `* 86400`. - std::vector result {0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536 - 1}; - std::for_each(result.begin(), result.end(), [](auto& value) { - value *= 86400; - }); - - return result; -} - -std::vector MakeDates32() { +std::vector MakeDates32() { // in CH Date32 internally a UInt32 and stores a day number // ColumnDate expects values to be seconds, which is then // converted to day number internally, hence the `* 86400`. // 114634 * 86400 is 2282-11-10, last integer that fits into DateTime32 range // (max is 2283-11-11) - std::vector result = MakeDates(); + std::vector result = MakeDates(); // add corresponding negative values, since pre-epoch date are supported too. const auto size = result.size(); diff --git a/ut/value_generators.h b/ut/value_generators.h index 3632ca9e..28e54adc 100644 --- a/ut/value_generators.h +++ b/ut/value_generators.h @@ -33,8 +33,7 @@ std::vector MakeBools(); std::vector MakeFixedStrings(size_t string_size); std::vector MakeStrings(); std::vector MakeDateTime64s(size_t scale, size_t values_size = 200); -std::vector MakeDates(); -std::vector MakeDates32(); +std::vector MakeDates32(); std::vector MakeDateTimes(); std::vector MakeIPv4s(); std::vector MakeIPv6s(); @@ -42,6 +41,23 @@ std::vector MakeUUIDs(); std::vector MakeInt128s(); std::vector MakeDecimals(size_t precision, size_t scale); +template +std::vector MakeDates() { + std::vector result {0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536 - 1}; + + if constexpr (std::is_same_v) { + // in CH Date internally a UInt16 and stores a day number + // ColumnDate expects values to be seconds, which is then + // converted to day number internally, hence the `* 86400`. + std::for_each(result.begin(), result.end(), [](auto& value) { + value *= 86400; + }); + } + + return result; +} + + std::string FooBarGenerator(size_t i); template From dd7011b9f5434a038d29a78f6869fd8b062df23e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AE=8F=E4=BC=9F?= Date: Thu, 14 Sep 2023 08:55:22 +0800 Subject: [PATCH 33/53] update cmake and install newly endpoints_iterator.h --- clickhouse/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 789db754..29c0d61e 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -120,6 +120,7 @@ INSTALL(FILES base/string_utils.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/string_view.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/uuid.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/wire_format.h DESTINATION include/clickhouse/base/) +INSTALL(FILES base/endpoints_iterator.h DESTINATION include/clickhouse/base/) # columns INSTALL(FILES columns/array.h DESTINATION include/clickhouse/columns/) From da3048b8e1480463aa1e87bdf3bd06e96646d882 Mon Sep 17 00:00:00 2001 From: Bogdan Duminiuc Date: Thu, 14 Sep 2023 11:46:40 +0300 Subject: [PATCH 34/53] Add move semantic for ColumnEnum --- clickhouse/columns/enum.cpp | 7 +++++++ clickhouse/columns/enum.h | 1 + 2 files changed, 8 insertions(+) diff --git a/clickhouse/columns/enum.cpp b/clickhouse/columns/enum.cpp index cd7696ca..b1ac2e40 100644 --- a/clickhouse/columns/enum.cpp +++ b/clickhouse/columns/enum.cpp @@ -20,6 +20,13 @@ ColumnEnum::ColumnEnum(TypeRef type, const std::vector& data) { } +template +ColumnEnum::ColumnEnum(TypeRef type, std::vector&& data) + : Column(type) + , data_(std::move(data)) +{ +} + template void ColumnEnum::Append(const T& value, bool checkValue) { if (checkValue) { diff --git a/clickhouse/columns/enum.h b/clickhouse/columns/enum.h index c31b81ff..4c299d98 100644 --- a/clickhouse/columns/enum.h +++ b/clickhouse/columns/enum.h @@ -12,6 +12,7 @@ class ColumnEnum : public Column { ColumnEnum(TypeRef type); ColumnEnum(TypeRef type, const std::vector& data); + ColumnEnum(TypeRef type, std::vector&& data); /// Appends one element to the end of column. void Append(const T& value, bool checkValue = false); From 918a7a00adf4f76e9db6f70ad51463c38fb0899e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 14 Sep 2023 17:50:03 +0200 Subject: [PATCH 35/53] MacOS less flaky tests Disabled tests that rely on some privileges that might be unavailable on a remote server: - Client/ClientCase.Query_ID - Client/ClientCase.TracingContext --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 1a267aa8..215b2214 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -67,5 +67,5 @@ jobs: # It is impossible to start CH server in docker on macOS due to github actions limitations, # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: # - system.query_log used by 'Client/ClientCase.Query_ID' - GTEST_FILTER: "-Client/ClientCase.Query_ID*" + GTEST_FILTER: "-Client/ClientCase.Query_ID*:Client/ClientCase.TracingContext/*" run: ./clickhouse-cpp-ut ${GTEST_FILTER} From 831c2f20f38c9a27a52f8235df89a4abe2f1116e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 14 Sep 2023 17:55:11 +0200 Subject: [PATCH 36/53] Same test filter for windows mingw run --- .github/workflows/windows_mingw.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_mingw.yml b/.github/workflows/windows_mingw.yml index d99c1ef7..6c6c3e9b 100644 --- a/.github/workflows/windows_mingw.yml +++ b/.github/workflows/windows_mingw.yml @@ -85,7 +85,7 @@ jobs: # It is impossible to start CH server in docker on Windows due to github actions limitations, # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: # - system.query_log used by 'Client/ClientCase.Query_ID' - GTEST_FILTER: "-Client/ClientCase.Query_ID*" + GTEST_FILTER: "-Client/ClientCase.Query_ID*:Client/ClientCase.TracingContext/*" run: ./build/ut/clickhouse-cpp-ut.exe ${GTEST_FILTER} - name: Test (simple) From f84a4835d97bc2ab7d0a33ab936762b5f937cf77 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 14 Sep 2023 17:55:47 +0200 Subject: [PATCH 37/53] Same test filters for windows msvc run --- .github/workflows/windows_msvc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_msvc.yml b/.github/workflows/windows_msvc.yml index 149e3df4..760f5ab2 100644 --- a/.github/workflows/windows_msvc.yml +++ b/.github/workflows/windows_msvc.yml @@ -61,6 +61,6 @@ jobs: # It is impossible to start CH server in docker on Windows due to github actions limitations, # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: # - system.query_log used by 'Client/ClientCase.Query_ID' - GTEST_FILTER: "-Client/ClientCase.Query_ID*" + GTEST_FILTER: "-Client/ClientCase.Query_ID*:Client/ClientCase.TracingContext/*" working-directory: ${{github.workspace}}/build/ut run: Release\clickhouse-cpp-ut.exe "${{env.GTEST_FILTER}}" From 8fe9bda6978d9da7315162ab43f3cefc0649ef2f Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 18 Sep 2023 14:30:37 +0200 Subject: [PATCH 38/53] Changed ColumnDate and ColumnDate32 operator[] to return same as method At() --- clickhouse/columns/date.cpp | 25 ++++++++----------------- clickhouse/columns/date.h | 14 ++++++++------ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/clickhouse/columns/date.cpp b/clickhouse/columns/date.cpp index 4e48129f..ee125bff 100644 --- a/clickhouse/columns/date.cpp +++ b/clickhouse/columns/date.cpp @@ -23,10 +23,6 @@ std::time_t ColumnDate::At(size_t n) const { return static_cast(data_->At(n)) * 86400; } -void ColumnDate::Append(uint16_t value) { - data_->Append(value); -} - void ColumnDate::AppendRaw(uint16_t value) { data_->Append(value); } @@ -35,10 +31,6 @@ uint16_t ColumnDate::RawAt(size_t n) const { return data_->At(n); } -uint16_t ColumnDate::operator [] (size_t n) const { - return (*data_)[n]; -} - void ColumnDate::Append(ColumnRef column) { if (auto col = column->As()) { data_->Append(col->data_); @@ -106,10 +98,6 @@ void ColumnDate32::Append(ColumnRef column) { } } -void ColumnDate32::Append(int32_t value) { - data_->Append(value); -} - void ColumnDate32::AppendRaw(int32_t value) { data_->Append(value); } @@ -118,10 +106,6 @@ int32_t ColumnDate32::RawAt(size_t n) const { return data_->At(n); } -int32_t ColumnDate32::operator [] (size_t n) const { - return (*data_)[n]; -} - bool ColumnDate32::LoadBody(InputStream* input, size_t rows) { return data_->LoadBody(input, rows); } @@ -156,7 +140,6 @@ ItemView ColumnDate32::GetItem(size_t index) const { return ItemView{Type()->GetCode(), data_->GetItem(index)}; } - ColumnDateTime::ColumnDateTime() : Column(Type::CreateDateTime()) , data_(std::make_shared()) @@ -177,6 +160,10 @@ std::time_t ColumnDateTime::At(size_t n) const { return data_->At(n); } +std::time_t ColumnDateTime::operator[](size_t n) const { + return data_->At(n); +} + std::string ColumnDateTime::Timezone() const { return type_->As()->Timezone(); } @@ -254,6 +241,10 @@ Int64 ColumnDateTime64::At(size_t n) const { return static_cast(data_->At(n)); } +Int64 ColumnDateTime64::operator[](size_t n) const { + return this->At(n); +} + std::string ColumnDateTime64::Timezone() const { return type_->As()->Timezone(); } diff --git a/clickhouse/columns/date.h b/clickhouse/columns/date.h index 5d3770be..27b2f1b9 100644 --- a/clickhouse/columns/date.h +++ b/clickhouse/columns/date.h @@ -21,12 +21,11 @@ class ColumnDate : public Column { /// Returns element at given row number. /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. std::time_t At(size_t n) const; + inline std::time_t operator [] (size_t n) const { return this->At(n); } - /// Do data as is -- number of day in Unix epoch, no conversions performed - void Append(uint16_t value); + /// Do append data as is -- number of day in Unix epoch, no conversions performed. void AppendRaw(uint16_t value); uint16_t RawAt(size_t n) const; - uint16_t operator [] (size_t n) const; /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; @@ -72,11 +71,11 @@ class ColumnDate32 : public Column { /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; - /// Do data as is -- number of day in Unix epoch, no conversions performed - void Append(int32_t value); + inline auto operator [] (size_t n) const { return this->At(n); } + + /// Do append data as is -- number of day in Unix epoch (32bit signed), no conversions performed. void AppendRaw(int32_t value); int32_t RawAt(size_t n) const; - int32_t operator [] (size_t n) const; /// Loads column data from input stream. bool LoadBody(InputStream* input, size_t rows) override; @@ -115,6 +114,7 @@ class ColumnDateTime : public Column { /// Returns element at given row number. std::time_t At(size_t n) const; + std::time_t operator [] (size_t n) const; /// Timezone associated with a data column. std::string Timezone() const; @@ -164,6 +164,8 @@ class ColumnDateTime64 : public Column { /// Returns element at given row number. Int64 At(size_t n) const; + Int64 operator[](size_t n) const; + /// Timezone associated with a data column. std::string Timezone() const; From 5da617aa01b06eec383c60bbb895ecb20d45ff30 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 18 Sep 2023 16:22:11 +0200 Subject: [PATCH 39/53] Re-designed GenericColumnTest More control over: - how column instances are created - how values are generated --- ut/Column_ut.cpp | 163 +++++++++++++++++++++++++--------------- ut/roundtrip_tests.cpp | 5 ++ ut/utils_comparison.h | 10 ++- ut/value_generators.cpp | 2 +- ut/value_generators.h | 67 ++++++++++++++++- 5 files changed, 183 insertions(+), 64 deletions(-) diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index 8432009f..41520d09 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -43,53 +43,38 @@ std::ostream& operator<<(std::ostream& ostr, const Type::Code& type_code) { // 6. Swap: create two instances, populate one with data, swap with second, make sure has data was transferred // 7. Load/Save: create, append some data, save to buffer, load from same buffer into new column, make sure columns match. +template (*CreatorFunction)(), + typename GeneratorValueType, + typename std::vector (*GeneratorFunc)()> +struct GenericColumnTestCase +{ + using ColumnType = ColumnTypeT; + + static auto createColumn() + { + return CreatorFunction(); + } + + static auto generateValues() + { + return GeneratorFunc(); + } +}; + template class GenericColumnTest : public testing::Test { public: - using ColumnType = std::decay_t; - - static auto MakeColumn() { - if constexpr (std::is_same_v) { - return std::make_shared(12); - } else if constexpr (std::is_same_v) { - return std::make_shared(3); - } else if constexpr (std::is_same_v) { - return std::make_shared(10, 5); - } else { - return std::make_shared(); - } + using ColumnType = typename T::ColumnType; + + static auto MakeColumn() + { + return T::createColumn(); } - static auto GenerateValues(size_t values_size) { - if constexpr (std::is_same_v) { - return GenerateVector(values_size, FooBarGenerator); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeFixedStrings(12)}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeDates()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeDateTimes()}); - } else if constexpr (std::is_same_v) { - return MakeDateTime64s(3u, values_size); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeDates32()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeIPv4s()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeIPv6s()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeInt128s()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeDecimals(3, 10)}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeUUIDs()}); - } else if constexpr (std::is_integral_v) { - // ColumnUIntX and ColumnIntX - return GenerateVector(values_size, RandomGenerator()); - } else if constexpr (std::is_floating_point_v) { - // OR ColumnFloatX - return GenerateVector(values_size, RandomGenerator()); - } + static auto GenerateValues(size_t values_size) + { + return GenerateVector(values_size, FromVectorGenerator{T::generateValues()}); } template @@ -130,26 +115,79 @@ class GenericColumnTest : public testing::Test { } }; -using ValueColumns = ::testing::Types< - ColumnUInt8, ColumnUInt16, ColumnUInt32, ColumnUInt64 - , ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64 - , ColumnFloat32, ColumnFloat64 - , ColumnString, ColumnFixedString - , ColumnDate, ColumnDateTime, ColumnDateTime64, ColumnDate32 - , ColumnIPv4, ColumnIPv6 - , ColumnInt128 - , ColumnDecimal - , ColumnUUID ->; -TYPED_TEST_SUITE(GenericColumnTest, ValueColumns); +// Luckily all (non-data copying/moving) constructors have size_t params. +template +auto makeColumn() +{ + return std::make_shared(ConstructorParams...); +} + +template +struct NumberColumnTestCase : public GenericColumnTestCase, typename ColumnTypeT::ValueType, &MakeNumbers> +{ + using Base = GenericColumnTestCase, typename ColumnTypeT::ValueType, &MakeNumbers>; + + using ColumnType = typename Base::ColumnType; + using Base::createColumn; + using Base::generateValues; +}; + +template +struct DecimalColumnTestCase : public GenericColumnTestCase, clickhouse::Int128, &MakeDecimals> +{ + using Base = GenericColumnTestCase, clickhouse::Int128, &MakeDecimals>; + + using ColumnType = typename Base::ColumnType; + using Base::createColumn; + using Base::generateValues; +}; + +using TestCases = ::testing::Types< + NumberColumnTestCase, + NumberColumnTestCase, + NumberColumnTestCase, + NumberColumnTestCase, + + NumberColumnTestCase, + NumberColumnTestCase, + NumberColumnTestCase, + NumberColumnTestCase, + + NumberColumnTestCase, + NumberColumnTestCase, -//TestCase(TypeCreatorFunc, ValuesGeneratorFunc) + GenericColumnTestCase, std::string, &MakeStrings>, + GenericColumnTestCase, std::string, &MakeFixedStrings<12>>, + GenericColumnTestCase, time_t, &MakeDates>, + GenericColumnTestCase, time_t, &MakeDates>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTimes>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTime64s<0>>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTime64s<3>>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTime64s<6>>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTime64s<9>>, -//class GenericColumnTestCase -//{ -// virutual -//}; + GenericColumnTestCase, in_addr, &MakeIPv4s>, + GenericColumnTestCase, in6_addr, &MakeIPv6s>, + + GenericColumnTestCase, clickhouse::Int128, &MakeInt128s>, + GenericColumnTestCase, clickhouse::UUID, &MakeUUIDs>, + + DecimalColumnTestCase, + DecimalColumnTestCase, + DecimalColumnTestCase, + // there is an arithmetical overflow on some values in the test value generator harness + // DecimalColumnTestCase, + + DecimalColumnTestCase, + DecimalColumnTestCase, + DecimalColumnTestCase, + + DecimalColumnTestCase, + DecimalColumnTestCase + >; + +TYPED_TEST_SUITE(GenericColumnTest, TestCases); TYPED_TEST(GenericColumnTest, Construct) { auto column = this->MakeColumn(); @@ -230,7 +268,8 @@ TYPED_TEST(GenericColumnTest, GetItem) { const auto v = convertValueForGetItem(*column, values[i]); const ItemView item = column->GetItem(i); - ASSERT_TRUE(CompareRecursive(item.get(), v)); + ASSERT_TRUE(CompareRecursive(v, item.get())) + << " On item " << i << " of " << PrintContainer{values}; } } @@ -318,9 +357,11 @@ TYPED_TEST(GenericColumnTest, RoundTrip) { } TYPED_TEST(GenericColumnTest, NulableT_RoundTrip) { - using NullableType = ColumnNullableT; + using NullableType = ColumnNullableT; + auto column = std::make_shared(this->MakeColumn()); auto values = this->GenerateValues(100); + FromVectorGenerator is_null({true, false}); for (size_t i = 0; i < values.size(); ++i) { if (is_null(i)) { diff --git a/ut/roundtrip_tests.cpp b/ut/roundtrip_tests.cpp index 5f4fce9c..262ebc90 100644 --- a/ut/roundtrip_tests.cpp +++ b/ut/roundtrip_tests.cpp @@ -5,6 +5,7 @@ #include #include +#include using namespace clickhouse; @@ -213,12 +214,16 @@ TEST_P(RoundtripCase, LowCardinalityTString) { TEST_P(RoundtripCase, LowCardinalityTNullableString) { using TestColumn = ColumnLowCardinalityT>; auto col = std::make_shared(); + col->Append("abc"); col->Append("def"); col->Append("abc"); + col->Append(std::nullopt); col->Append("abc"); col->Append(std::nullopt); col->Append(std::nullopt); + col->Append("foobar"); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); EXPECT_TRUE(CompareRecursive(*col, *result_typed)); } diff --git a/ut/utils_comparison.h b/ut/utils_comparison.h index c40033b4..4c93c611 100644 --- a/ut/utils_comparison.h +++ b/ut/utils_comparison.h @@ -8,6 +8,7 @@ #include #include +#include namespace clickhouse { class Block; @@ -153,10 +154,17 @@ ::testing::AssertionResult CompareRecursive(const Left & left, const Right & rig return result << "\nExpected container: " << PrintContainer{l} << "\nActual container : " << PrintContainer{r}; } else { - if (left != right) + if (left != right) { + + if constexpr (std::is_floating_point_v && std::is_floating_point_v) { + if (std::isnan(left) && std::isnan(right)) + return ::testing::AssertionSuccess(); + } + return ::testing::AssertionFailure() << "\nExpected value: " << left << "\nActual value : " << right; + } return ::testing::AssertionSuccess(); } diff --git a/ut/value_generators.cpp b/ut/value_generators.cpp index 805f22ad..a0316a3a 100644 --- a/ut/value_generators.cpp +++ b/ut/value_generators.cpp @@ -92,7 +92,7 @@ std::vector MakeInt128s() { std::vector MakeDecimals(size_t /*precision*/, size_t scale) { const auto scale_multiplier = static_cast(std::pow(10, scale)); - const auto rhs_value = 12345678910; + const long long int rhs_value = 12345678910; const std::vector vals {0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536 - 1}; diff --git a/ut/value_generators.h b/ut/value_generators.h index 28e54adc..48483dc1 100644 --- a/ut/value_generators.h +++ b/ut/value_generators.h @@ -41,8 +41,73 @@ std::vector MakeUUIDs(); std::vector MakeInt128s(); std::vector MakeDecimals(size_t precision, size_t scale); +template ::value, bool> = true> +inline std::vector MakeNumbers() { + + std::vector result; + result.reserve(32); + + // to reach from in to max in 32 steps, it also has to be lower than 7 to work for int8 values. + const T step = static_cast(1) << (sizeof(T)*8 - 5); + + // `- step` to avoid accidential overflow + for (T i = std::numeric_limits::min(); i <= std::numeric_limits::max() - step; i += step) + { + result.push_back(i); + } + result.push_back(std::numeric_limits::max()); + + return result; +} + +template ::value, bool> = true> +inline std::vector MakeNumbers() { + + std::vector result { + std::numeric_limits::min(), + std::numeric_limits::max(), + std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + static_cast(0), + static_cast(0) + std::numeric_limits::epsilon(), + static_cast(0) - std::numeric_limits::epsilon() + }; + + const auto total_steps = 100; + const auto step = std::pow(10, (std::numeric_limits::max_exponent - std::numeric_limits::min_exponent) / total_steps); + const auto min_value = std::pow(10, std::numeric_limits::min_exponent10); + + // cover most of the precision ranges + for (T i = std::numeric_limits::max(); i >= /*std::numeric_limits::epsilon()*/ min_value * step; i /= step) + { + result.push_back(i); + result.push_back(-1 * i); + } + result.push_back(min_value); + result.push_back(-min_value); + + return result; +} + +template +inline std::vector MakeFixedStrings() { + return MakeFixedStrings(size); +} + +template +inline std::vector MakeDateTime64s() { + return MakeDateTime64s(scale); +} + +template +inline std::vector MakeDecimals() { + return MakeDecimals(precision, scale); +} + + template -std::vector MakeDates() { +inline auto MakeDates() { std::vector result {0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536 - 1}; if constexpr (std::is_same_v) { From 89ba00b94f18119dd25eb3b61771499e28b7cce7 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 18 Sep 2023 16:24:47 +0200 Subject: [PATCH 40/53] inline operator [] where possible --- clickhouse/columns/date.cpp | 8 -------- clickhouse/columns/date.h | 8 ++++---- clickhouse/columns/decimal.h | 1 + clickhouse/columns/enum.cpp | 5 ----- clickhouse/columns/enum.h | 2 +- clickhouse/columns/geo.cpp | 5 ----- clickhouse/columns/geo.h | 2 +- clickhouse/columns/map.h | 2 +- clickhouse/columns/nothing.h | 2 +- clickhouse/columns/numeric.cpp | 5 ----- clickhouse/columns/numeric.h | 2 +- clickhouse/columns/string.cpp | 9 --------- clickhouse/columns/string.h | 4 ++-- clickhouse/columns/tuple.h | 4 ++-- clickhouse/columns/uuid.cpp | 4 ---- clickhouse/columns/uuid.h | 2 +- 16 files changed, 15 insertions(+), 50 deletions(-) diff --git a/clickhouse/columns/date.cpp b/clickhouse/columns/date.cpp index ee125bff..f4d08cb4 100644 --- a/clickhouse/columns/date.cpp +++ b/clickhouse/columns/date.cpp @@ -160,10 +160,6 @@ std::time_t ColumnDateTime::At(size_t n) const { return data_->At(n); } -std::time_t ColumnDateTime::operator[](size_t n) const { - return data_->At(n); -} - std::string ColumnDateTime::Timezone() const { return type_->As()->Timezone(); } @@ -241,10 +237,6 @@ Int64 ColumnDateTime64::At(size_t n) const { return static_cast(data_->At(n)); } -Int64 ColumnDateTime64::operator[](size_t n) const { - return this->At(n); -} - std::string ColumnDateTime64::Timezone() const { return type_->As()->Timezone(); } diff --git a/clickhouse/columns/date.h b/clickhouse/columns/date.h index 27b2f1b9..be764b81 100644 --- a/clickhouse/columns/date.h +++ b/clickhouse/columns/date.h @@ -21,7 +21,7 @@ class ColumnDate : public Column { /// Returns element at given row number. /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. std::time_t At(size_t n) const; - inline std::time_t operator [] (size_t n) const { return this->At(n); } + inline std::time_t operator [] (size_t n) const { return At(n); } /// Do append data as is -- number of day in Unix epoch, no conversions performed. void AppendRaw(uint16_t value); @@ -71,7 +71,7 @@ class ColumnDate32 : public Column { /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; - inline auto operator [] (size_t n) const { return this->At(n); } + inline std::time_t operator [] (size_t n) const { return At(n); } /// Do append data as is -- number of day in Unix epoch (32bit signed), no conversions performed. void AppendRaw(int32_t value); @@ -114,7 +114,7 @@ class ColumnDateTime : public Column { /// Returns element at given row number. std::time_t At(size_t n) const; - std::time_t operator [] (size_t n) const; + inline std::time_t operator [] (size_t n) const { return At(n); } /// Timezone associated with a data column. std::string Timezone() const; @@ -164,7 +164,7 @@ class ColumnDateTime64 : public Column { /// Returns element at given row number. Int64 At(size_t n) const; - Int64 operator[](size_t n) const; + inline Int64 operator[](size_t n) const { return At(n); } /// Timezone associated with a data column. std::string Timezone() const; diff --git a/clickhouse/columns/decimal.h b/clickhouse/columns/decimal.h index d3c05ea2..4b09553a 100644 --- a/clickhouse/columns/decimal.h +++ b/clickhouse/columns/decimal.h @@ -18,6 +18,7 @@ class ColumnDecimal : public Column { void Append(const std::string& value); Int128 At(size_t i) const; + inline auto operator[](size_t i) const { return At(i); } public: void Append(ColumnRef column) override; diff --git a/clickhouse/columns/enum.cpp b/clickhouse/columns/enum.cpp index cd7696ca..3c18dc08 100644 --- a/clickhouse/columns/enum.cpp +++ b/clickhouse/columns/enum.cpp @@ -48,11 +48,6 @@ std::string_view ColumnEnum::NameAt(size_t n) const { return type_->As()->GetEnumName(data_.at(n)); } -template -const T& ColumnEnum::operator[] (size_t n) const { - return data_[n]; -} - template void ColumnEnum::SetAt(size_t n, const T& value, bool checkValue) { if (checkValue) { diff --git a/clickhouse/columns/enum.h b/clickhouse/columns/enum.h index c31b81ff..fc905d66 100644 --- a/clickhouse/columns/enum.h +++ b/clickhouse/columns/enum.h @@ -22,7 +22,7 @@ class ColumnEnum : public Column { std::string_view NameAt(size_t n) const; /// Returns element at given row number. - const T& operator[] (size_t n) const; + inline const T& operator[] (size_t n) const { return At(n); } /// Set element at given row number. void SetAt(size_t n, const T& value, bool checkValue = false); diff --git a/clickhouse/columns/geo.cpp b/clickhouse/columns/geo.cpp index ebea9895..e618fbe5 100644 --- a/clickhouse/columns/geo.cpp +++ b/clickhouse/columns/geo.cpp @@ -54,11 +54,6 @@ const typename ColumnGeo::ValueType ColumnGeoAt(n); } -template -const typename ColumnGeo::ValueType ColumnGeo::operator[](size_t n) const { - return data_->At(n); -} - template void ColumnGeo::Append(ColumnRef column) { if (auto col = column->template As()) { diff --git a/clickhouse/columns/geo.h b/clickhouse/columns/geo.h index 5f3db9b6..c3757f8a 100644 --- a/clickhouse/columns/geo.h +++ b/clickhouse/columns/geo.h @@ -26,7 +26,7 @@ class ColumnGeo : public Column { const ValueType At(size_t n) const; /// Returns element at given row number. - const ValueType operator[](size_t n) const; + inline const ValueType operator[](size_t n) const { return At(n); } public: /// Appends content of given column to the end of current one. diff --git a/clickhouse/columns/map.h b/clickhouse/columns/map.h index 24a8b4ae..ac5dc0a7 100644 --- a/clickhouse/columns/map.h +++ b/clickhouse/columns/map.h @@ -212,7 +212,7 @@ class ColumnMapT : public ColumnMap { inline auto At(size_t index) const { return MapValueView{typed_data_->At(index)}; } - inline auto operator[](size_t index) const { return MapValueView{typed_data_->At(index)}; } + inline auto operator[](size_t index) const { return At(index); } using ColumnMap::Append; diff --git a/clickhouse/columns/nothing.h b/clickhouse/columns/nothing.h index 36ddeaea..0b28d572 100644 --- a/clickhouse/columns/nothing.h +++ b/clickhouse/columns/nothing.h @@ -33,7 +33,7 @@ class ColumnNothing : public Column { std::nullptr_t At(size_t) const { return nullptr; }; /// Returns element at given row number. - std::nullptr_t operator [] (size_t) const { return nullptr; }; + inline std::nullptr_t operator [] (size_t) const { return nullptr; }; /// Makes slice of the current column. ColumnRef Slice(size_t, size_t len) const override { diff --git a/clickhouse/columns/numeric.cpp b/clickhouse/columns/numeric.cpp index 4e8d54bf..81d6c721 100644 --- a/clickhouse/columns/numeric.cpp +++ b/clickhouse/columns/numeric.cpp @@ -48,11 +48,6 @@ const T& ColumnVector::At(size_t n) const { return data_.at(n); } -template -const T& ColumnVector::operator [] (size_t n) const { - return data_[n]; -} - template void ColumnVector::Append(ColumnRef column) { if (auto col = column->As>()) { diff --git a/clickhouse/columns/numeric.h b/clickhouse/columns/numeric.h index c6da981e..dcc344bb 100644 --- a/clickhouse/columns/numeric.h +++ b/clickhouse/columns/numeric.h @@ -26,7 +26,7 @@ class ColumnVector : public Column { const T& At(size_t n) const; /// Returns element at given row number. - const T& operator [] (size_t n) const; + inline const T& operator [] (size_t n) const { return At(n); } void Erase(size_t pos, size_t count = 1); diff --git a/clickhouse/columns/string.cpp b/clickhouse/columns/string.cpp index f6597bb4..173c024f 100644 --- a/clickhouse/columns/string.cpp +++ b/clickhouse/columns/string.cpp @@ -58,11 +58,6 @@ std::string_view ColumnFixedString::At(size_t n) const { return std::string_view(&data_.at(pos), string_size_); } -std::string_view ColumnFixedString::operator [](size_t n) const { - const auto pos = n * string_size_; - return std::string_view(&data_[pos], string_size_); -} - size_t ColumnFixedString::FixedSize() const { return string_size_; } @@ -232,10 +227,6 @@ std::string_view ColumnString::At(size_t n) const { return items_.at(n); } -std::string_view ColumnString::operator [] (size_t n) const { - return items_[n]; -} - void ColumnString::Append(ColumnRef column) { if (auto col = column->As()) { const auto total_size = ComputeTotalSize(col->items_); diff --git a/clickhouse/columns/string.h b/clickhouse/columns/string.h index 9b83a088..aa78270e 100644 --- a/clickhouse/columns/string.h +++ b/clickhouse/columns/string.h @@ -34,7 +34,7 @@ class ColumnFixedString : public Column { std::string_view At(size_t n) const; /// Returns element at given row number. - std::string_view operator [] (size_t n) const; + inline std::string_view operator [] (size_t n) const { return At(n); } /// Returns the max size of the fixed string size_t FixedSize() const; @@ -101,7 +101,7 @@ class ColumnString : public Column { std::string_view At(size_t n) const; /// Returns element at given row number. - std::string_view operator [] (size_t n) const; + inline std::string_view operator [] (size_t n) const { return At(n); } public: /// Appends content of given column to the end of current one. diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index b1b5ad31..c9795565 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -17,11 +17,11 @@ class ColumnTuple : public Column { /// Returns count of columns in the tuple. size_t TupleSize() const; - ColumnRef operator [] (size_t n) const { + inline ColumnRef operator [] (size_t n) const { return columns_[n]; } - ColumnRef At(size_t n) const { + inline ColumnRef At(size_t n) const { return columns_[n]; } diff --git a/clickhouse/columns/uuid.cpp b/clickhouse/columns/uuid.cpp index 19e94761..36a7229c 100644 --- a/clickhouse/columns/uuid.cpp +++ b/clickhouse/columns/uuid.cpp @@ -34,10 +34,6 @@ const UUID ColumnUUID::At(size_t n) const { return UUID(data_->At(n * 2), data_->At(n * 2 + 1)); } -const UUID ColumnUUID::operator [] (size_t n) const { - return UUID((*data_)[n * 2], (*data_)[n * 2 + 1]); -} - void ColumnUUID::Append(ColumnRef column) { if (auto col = column->As()) { data_->Append(col->data_); diff --git a/clickhouse/columns/uuid.h b/clickhouse/columns/uuid.h index dd7d0b9d..4f6c9192 100644 --- a/clickhouse/columns/uuid.h +++ b/clickhouse/columns/uuid.h @@ -23,7 +23,7 @@ class ColumnUUID : public Column { const UUID At(size_t n) const; /// Returns element at given row number. - const UUID operator [] (size_t n) const; + inline const UUID operator [] (size_t n) const { return At(n); } public: /// Appends content of given column to the end of current one. From f07b5308d30d49f16f3ca4b0b6804a2ba67e1e58 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 18 Sep 2023 16:26:07 +0200 Subject: [PATCH 41/53] Comments and other minor fixes --- clickhouse/columns/date.h | 4 ++-- ut/Column_ut.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clickhouse/columns/date.h b/clickhouse/columns/date.h index be764b81..9b170001 100644 --- a/clickhouse/columns/date.h +++ b/clickhouse/columns/date.h @@ -101,7 +101,7 @@ class ColumnDate32 : public Column { }; -/** */ +/** DateTime64 supports date-time values (number of seconds since UNIX epoch), from 1970 up to 2130. */ class ColumnDateTime : public Column { public: using ValueType = std::time_t; @@ -147,7 +147,7 @@ class ColumnDateTime : public Column { }; -/** */ +/** DateTime64 supports date-time values of arbitrary sub-second precision, from 1900 up to 2300. */ class ColumnDateTime64 : public Column { public: using ValueType = Int64; diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index 41520d09..ace1f79d 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -237,8 +237,8 @@ inline auto convertValueForGetItem(const ColumnType& col, ValueType&& t) { using T = std::remove_cv_t>; if constexpr (std::is_same_v) { - // Since ColumnDecimal can hold 32, 64, 128-bit wide data and there is no way telling at run-time. - const ItemView item = col.GetItem(0); + // Since ColumnDecimal can hold 32, 64, 128-bit wide data and there is no way telling at run-time. + const ItemView item = col.GetItem(0); return std::string_view(reinterpret_cast(&t), item.data.size()); } else if constexpr (std::is_same_v || std::is_same_v) { From 7ddc22a8315748b89d0cfdf284727d6c1cbd55f5 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 18 Sep 2023 16:34:22 +0200 Subject: [PATCH 42/53] Fixed tests --- ut/columns_ut.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ut/columns_ut.cpp b/ut/columns_ut.cpp index e49364ec..ce477554 100644 --- a/ut/columns_ut.cpp +++ b/ut/columns_ut.cpp @@ -182,9 +182,6 @@ TEST(ColumnsCase, Date_UInt16_interface) { ASSERT_EQ(col1->Size(), 2u); ASSERT_EQ(col1->RawAt(0), 1u); ASSERT_EQ(col1->RawAt(1), 1234u); - - ASSERT_EQ((*col1)[0], 1u); - ASSERT_EQ((*col1)[1], 1234u); } @@ -199,10 +196,6 @@ TEST(ColumnsCase, Date32_Int32_interface) { ASSERT_EQ(col1->RawAt(0), 1); ASSERT_EQ(col1->RawAt(1), 1234); ASSERT_EQ(col1->RawAt(2), -1234); - - ASSERT_EQ((*col1)[0], 1); - ASSERT_EQ((*col1)[1], 1234); - ASSERT_EQ((*col1)[2], -1234); } From a5e83f215acc573ac0720b82f9bbddaf63e1ffa7 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 18 Sep 2023 18:36:28 +0200 Subject: [PATCH 43/53] Fixed tests comparing NaN wrapped in std::optional --- ut/utils_comparison.h | 17 +++++++++- ut/utils_meta.h | 7 ++++ ut/utils_ut.cpp | 78 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/ut/utils_comparison.h b/ut/utils_comparison.h index 4c93c611..0b7f805a 100644 --- a/ut/utils_comparison.h +++ b/ut/utils_comparison.h @@ -9,6 +9,7 @@ #include #include #include +#include namespace clickhouse { class Block; @@ -156,7 +157,21 @@ ::testing::AssertionResult CompareRecursive(const Left & left, const Right & rig } else { if (left != right) { - if constexpr (std::is_floating_point_v && std::is_floating_point_v) { + // Handle std::optional(nan) + // I'm too lazy to code comparison against std::nullopt, but this shpudn't be a problem in real life. + // RN comparing against std::nullopt, you'll receive an compilation error. + if constexpr (is_instantiation_of::value && is_instantiation_of::value) + { + if (left.has_value() && right.has_value()) + return CompareRecursive(*left, *right); + } + else if constexpr (is_instantiation_of::value) { + if (left) + return CompareRecursive(*left, right); + } else if constexpr (is_instantiation_of::value) { + if (right) + return CompareRecursive(left, *right); + } else if constexpr (std::is_floating_point_v && std::is_floating_point_v) { if (std::isnan(left) && std::isnan(right)) return ::testing::AssertionSuccess(); } diff --git a/ut/utils_meta.h b/ut/utils_meta.h index 707f9aca..e069dc5f 100644 --- a/ut/utils_meta.h +++ b/ut/utils_meta.h @@ -39,3 +39,10 @@ using my_result_of_t = std::result_of_t; #endif +// https://stackoverflow.com/a/11251408 +template < template class Template, typename T > +struct is_instantiation_of : std::false_type {}; + +template < template class Template, typename... Args > +struct is_instantiation_of< Template, Template > : std::true_type {}; + diff --git a/ut/utils_ut.cpp b/ut/utils_ut.cpp index 0f67c043..f69e1daa 100644 --- a/ut/utils_ut.cpp +++ b/ut/utils_ut.cpp @@ -1,9 +1,22 @@ #include #include "utils.h" +#include +#include #include -TEST(TestCompareContainer, ComparePlain) { +TEST(CompareRecursive, CompareValues) { + EXPECT_TRUE(CompareRecursive(1, 1)); + EXPECT_TRUE(CompareRecursive(1.0f, 1.0f)); + EXPECT_TRUE(CompareRecursive(1.0, 1.0)); + EXPECT_TRUE(CompareRecursive(1.0L, 1.0L)); + + EXPECT_TRUE(CompareRecursive("1.0L", "1.0L")); + EXPECT_TRUE(CompareRecursive(std::string{"1.0L"}, std::string{"1.0L"})); + EXPECT_TRUE(CompareRecursive(std::string_view{"1.0L"}, std::string_view{"1.0L"})); +} + +TEST(CompareRecursive, CompareContainers) { EXPECT_TRUE(CompareRecursive(std::vector{1, 2, 3}, std::vector{1, 2, 3})); EXPECT_TRUE(CompareRecursive(std::vector{}, std::vector{})); @@ -15,7 +28,7 @@ TEST(TestCompareContainer, ComparePlain) { } -TEST(TestCompareContainer, CompareNested) { +TEST(CompareRecursive, CompareNestedContainers) { EXPECT_TRUE(CompareRecursive( std::vector>{{{1, 2, 3}, {4, 5, 6}}}, std::vector>{{{1, 2, 3}, {4, 5, 6}}})); @@ -39,3 +52,64 @@ TEST(StringUtils, UUID) { const std::string uuid_string = "01020304-0506-0708-090a-0b0c0d0e0f10"; EXPECT_EQ(ToString(uuid), uuid_string); } + +TEST(CompareRecursive, Nan) { + /// Even though NaN == NaN is FALSE, CompareRecursive must compare those as TRUE. + + const auto NaNf = std::numeric_limits::quiet_NaN(); + const auto NaNd = std::numeric_limits::quiet_NaN(); + + EXPECT_TRUE(CompareRecursive(NaNf, NaNf)); + EXPECT_TRUE(CompareRecursive(NaNd, NaNd)); + + EXPECT_TRUE(CompareRecursive(NaNf, NaNd)); + EXPECT_TRUE(CompareRecursive(NaNd, NaNf)); + + // 1.0 is arbitrary here + EXPECT_FALSE(CompareRecursive(NaNf, 1.0)); + EXPECT_FALSE(CompareRecursive(NaNf, 1.0)); + EXPECT_FALSE(CompareRecursive(1.0, NaNd)); + EXPECT_FALSE(CompareRecursive(1.0, NaNd)); +} + +TEST(CompareRecursive, Optional) { + EXPECT_TRUE(CompareRecursive(1, std::optional{1})); + EXPECT_TRUE(CompareRecursive(std::optional{1}, 1)); + EXPECT_TRUE(CompareRecursive(std::optional{1}, std::optional{1})); + + EXPECT_FALSE(CompareRecursive(2, std::optional{1})); + EXPECT_FALSE(CompareRecursive(std::optional{1}, 2)); + EXPECT_FALSE(CompareRecursive(std::optional{2}, std::optional{1})); + EXPECT_FALSE(CompareRecursive(std::optional{1}, std::optional{2})); +} + +TEST(CompareRecursive, OptionalNan) { + // Special case for optional comparison: + // NaNs should be considered as equal (compare by unpacking value of optional) + + const auto NaNf = std::numeric_limits::quiet_NaN(); + const auto NaNd = std::numeric_limits::quiet_NaN(); + + const auto NaNfo = std::optional{NaNf}; + const auto NaNdo = std::optional{NaNd}; + + EXPECT_TRUE(CompareRecursive(NaNf, NaNf)); + EXPECT_TRUE(CompareRecursive(NaNf, NaNfo)); + EXPECT_TRUE(CompareRecursive(NaNfo, NaNf)); + EXPECT_TRUE(CompareRecursive(NaNfo, NaNfo)); + + EXPECT_TRUE(CompareRecursive(NaNd, NaNd)); + EXPECT_TRUE(CompareRecursive(NaNd, NaNdo)); + EXPECT_TRUE(CompareRecursive(NaNdo, NaNd)); + EXPECT_TRUE(CompareRecursive(NaNdo, NaNdo)); + + EXPECT_FALSE(CompareRecursive(NaNdo, std::optional{})); + EXPECT_FALSE(CompareRecursive(NaNfo, std::optional{})); + EXPECT_FALSE(CompareRecursive(std::optional{}, NaNdo)); + EXPECT_FALSE(CompareRecursive(std::optional{}, NaNfo)); + + // Too lazy to comparison code against std::nullopt, but this shouldn't be a problem in real life + // following will produce compile time error: +// EXPECT_FALSE(CompareRecursive(NaNdo, std::nullopt)); +// EXPECT_FALSE(CompareRecursive(NaNfo, std::nullopt)); +} From 484aa7ab4a117ceb205610846e64b216f58da804 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 18 Sep 2023 19:26:22 +0200 Subject: [PATCH 44/53] Minor cosmetic change --- ut/value_generators.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ut/value_generators.h b/ut/value_generators.h index 48483dc1..8ef21d6e 100644 --- a/ut/value_generators.h +++ b/ut/value_generators.h @@ -79,7 +79,7 @@ inline std::vector MakeNumbers() { const auto min_value = std::pow(10, std::numeric_limits::min_exponent10); // cover most of the precision ranges - for (T i = std::numeric_limits::max(); i >= /*std::numeric_limits::epsilon()*/ min_value * step; i /= step) + for (T i = std::numeric_limits::max(); i >= min_value * step; i /= step) { result.push_back(i); result.push_back(-1 * i); From 7c9d167c86b340bab892b5abe834ac12b2205e1b Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 18 Sep 2023 19:26:40 +0200 Subject: [PATCH 45/53] Made Performance tests to be skipped in debug builds --- ut/performance_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ut/performance_tests.cpp b/ut/performance_tests.cpp index bafa07dd..74bd4f03 100644 --- a/ut/performance_tests.cpp +++ b/ut/performance_tests.cpp @@ -84,7 +84,7 @@ class ColumnPerformanceTest : public ::testing::Test { TYPED_TEST_SUITE_P(ColumnPerformanceTest); // Turns out this is the easiest way to skip test with current version of gtest -#ifndef NDEBUG +#ifdef NDEBUG # define SKIP_IN_DEBUG_BUILDS() (void)(0) #else # define SKIP_IN_DEBUG_BUILDS() GTEST_SKIP_("Test skipped for DEBUG build...") From 7611d8354d3938ee8098cc64d94450fab9ae6af8 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 20 Sep 2023 12:35:47 +0200 Subject: [PATCH 46/53] Fixed Array::Append and LowCardinality::Append This fixes ColumnArrayT>::Append Also added some tests --- clickhouse/columns/array.cpp | 13 +-- clickhouse/columns/lowcardinality.cpp | 17 ++- ut/Column_ut.cpp | 144 ++++++++++++++++++++++---- ut/roundtrip_column.cpp | 27 ++++- ut/utils.h | 5 +- ut/utils_comparison.h | 3 +- ut/utils_meta.h | 5 + ut/utils_ut.cpp | 7 ++ ut/value_generators.cpp | 25 ++++- ut/value_generators.h | 14 +++ 10 files changed, 219 insertions(+), 41 deletions(-) diff --git a/clickhouse/columns/array.cpp b/clickhouse/columns/array.cpp index 9f66b91f..5e5e72e7 100644 --- a/clickhouse/columns/array.cpp +++ b/clickhouse/columns/array.cpp @@ -25,14 +25,9 @@ ColumnArray::ColumnArray(ColumnArray&& other) } void ColumnArray::AppendAsColumn(ColumnRef array) { - if (!data_->Type()->IsEqual(array->Type())) { - throw ValidationError( - "can't append column of type " + array->Type()->GetName() + " " - "to column type " + data_->Type()->GetName()); - } - - AddOffset(array->Size()); + // appending data may throw (i.e. due to ype check failure), so do it first to avoid partly modified state. data_->Append(array); + AddOffset(array->Size()); } ColumnRef ColumnArray::GetAsColumn(size_t n) const { @@ -59,10 +54,6 @@ ColumnRef ColumnArray::CloneEmpty() const { void ColumnArray::Append(ColumnRef column) { if (auto col = column->As()) { - if (!col->data_->Type()->IsEqual(data_->Type())) { - return; - } - for (size_t i = 0; i < col->Size(); ++i) { AppendAsColumn(col->GetAsColumn(i)); } diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index 13bf17a1..c0c12319 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -227,12 +227,21 @@ ColumnRef ColumnLowCardinality::GetDictionary() { } void ColumnLowCardinality::Append(ColumnRef col) { + // Append values from col only if it is either + // - exactly same type as `this`: LowCardinality wrapping same dictionary type + // - same type as dictionary column + auto c = col->As(); - if (!c || !dictionary_column_->Type()->IsEqual(c->dictionary_column_->Type())) - return; + // If not LowCardinality of same dictionary type + if (!c || !dictionary_column_->Type()->IsEqual(c->dictionary_column_->Type())) { + // If not column of the same type as dictionary type + if (!dictionary_column_->Type()->IsEqual(col->GetType())) { + return; + } + } - for (size_t i = 0; i < c->Size(); ++i) { - AppendUnsafe(c->GetItem(i)); + for (size_t i = 0; i < col->Size(); ++i) { + AppendUnsafe(col->GetItem(i)); } } diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index ace1f79d..ddb91335 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -16,6 +16,9 @@ #include #include +#include +#include +#include #include "utils.h" #include "roundtrip_column.h" @@ -46,10 +49,12 @@ std::ostream& operator<<(std::ostream& ostr, const Type::Code& type_code) { template (*CreatorFunction)(), typename GeneratorValueType, - typename std::vector (*GeneratorFunc)()> + typename std::vector (*GeneratorFunction)()> struct GenericColumnTestCase { using ColumnType = ColumnTypeT; + static constexpr auto Creator = CreatorFunction; + static constexpr auto Generator = GeneratorFunction; static auto createColumn() { @@ -58,7 +63,7 @@ struct GenericColumnTestCase static auto generateValues() { - return GeneratorFunc(); + return GeneratorFunction(); } }; @@ -92,7 +97,7 @@ class GenericColumnTest : public testing::Test { return std::tuple{column, values}; } - static std::optional SkipTest(clickhouse::Client& client) { + static std::optional CheckIfShouldSkipTest(clickhouse::Client& client) { if constexpr (std::is_same_v) { // Date32 first appeared in v21.9.2.17-stable const auto server_info = client.GetServerInfo(); @@ -113,6 +118,33 @@ class GenericColumnTest : public testing::Test { } return std::nullopt; } + + template + static void TestColumnRoundtrip(const std::shared_ptr & column, const ClientOptions & client_options) + { + SCOPED_TRACE(::testing::Message("Column type: ") << column->GetType().GetName()); + SCOPED_TRACE(::testing::Message("Client options: ") << client_options); + + clickhouse::Client client(client_options); + + if (auto message = CheckIfShouldSkipTest(client)) { + GTEST_SKIP() << *message; + } + + auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); + EXPECT_TRUE(CompareRecursive(*column, *result_typed)); + } + + + template + static void TestColumnRoundtrip(const ColumnType & column, const ClientOptions & client_options, CompressionMethods && compression_methods) + { + for (auto compressionMethod : compression_methods) + { + ClientOptions new_options = ClientOptions(client_options).SetCompressionMethod(compressionMethod); + TestColumnRoundtrip(column, new_options); + } + } }; // Luckily all (non-data copying/moving) constructors have size_t params. @@ -184,7 +216,17 @@ using TestCases = ::testing::Types< DecimalColumnTestCase, DecimalColumnTestCase, - DecimalColumnTestCase + DecimalColumnTestCase, + + GenericColumnTestCase, &makeColumn>, std::string, &MakeStrings> + + // Array(String) +// GenericColumnTestCase, &makeColumn>, std::vector, &MakeArrays> + +// // Array(Array(String)) +// GenericColumnTestCase>, &makeColumn>>, +// std::vector>, +// &MakeArrays, &MakeArrays>> >; TYPED_TEST_SUITE(GenericColumnTest, TestCases); @@ -262,7 +304,14 @@ TYPED_TEST(GenericColumnTest, GetItem) { auto [column, values] = this->MakeColumnWithValues(100); ASSERT_EQ(values.size(), column->Size()); - ASSERT_EQ(column->GetItem(0).type, column->GetType().GetCode()); + const auto wrapping_types = std::set{ + Type::Code::LowCardinality, Type::Code::Array, Type::Code::Nullable + }; + + // For wrapping types, type of ItemView can be different from type of column + if (wrapping_types.find(column->GetType().GetCode()) == wrapping_types.end() ) { + EXPECT_EQ(column->GetItem(0).type, column->GetType().GetCode()); + } for (size_t i = 0; i < values.size(); ++i) { const auto v = convertValueForGetItem(*column, values[i]); @@ -318,7 +367,8 @@ TYPED_TEST(GenericColumnTest, Swap) { TYPED_TEST(GenericColumnTest, LoadAndSave) { auto [column_A, values] = this->MakeColumnWithValues(100); - char buffer[4096] = {'\0'}; + // large buffer since we have pretty big values for String column + char buffer[1024*1024] = {'\0'}; { ArrayOutput output(buffer, sizeof(buffer)); // Save @@ -342,24 +392,39 @@ const auto LocalHostEndpoint = ClientOptions() .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); +const auto AllCompressionMethods = { + clickhouse::CompressionMethod::None, + clickhouse::CompressionMethod::LZ4 +}; + TYPED_TEST(GenericColumnTest, RoundTrip) { auto [column, values] = this->MakeColumnWithValues(100); EXPECT_EQ(values.size(), column->Size()); - clickhouse::Client client(LocalHostEndpoint); + this->TestColumnRoundtrip(column, LocalHostEndpoint, AllCompressionMethods); +// for (auto compressionMethod : AllCompressionMethods) +// { +// clickhouse::Client client(ClientOptions(LocalHostEndpoint) +// .SetCompressionMethod(compressionMethod)); - if (auto message = this->SkipTest(client)) { - GTEST_SKIP() << *message; - } +// if (auto message = this->CheckIfShouldSkipTest(client)) { +// GTEST_SKIP() << *message; +// } - auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); - EXPECT_TRUE(CompareRecursive(*column, *result_typed)); +// auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); +// EXPECT_TRUE(CompareRecursive(*column, *result_typed)); +// } } -TYPED_TEST(GenericColumnTest, NulableT_RoundTrip) { +TYPED_TEST(GenericColumnTest, NullableT_RoundTrip) { using NullableType = ColumnNullableT; - auto column = std::make_shared(this->MakeColumn()); + auto non_nullable_column = this->MakeColumn(); + if (non_nullable_column->GetType().GetCode() == Type::Code::LowCardinality) + // TODO (vnemkov): wrap as ColumnLowCardinalityT> instead of ColumnNullableT> + GTEST_SKIP() << "Can't wrap " << non_nullable_column->GetType().GetName() << " into Nullable"; + + auto column = std::make_shared(std::move(non_nullable_column)); auto values = this->GenerateValues(100); FromVectorGenerator is_null({true, false}); @@ -371,12 +436,53 @@ TYPED_TEST(GenericColumnTest, NulableT_RoundTrip) { } } - clickhouse::Client client(LocalHostEndpoint); + this->TestColumnRoundtrip(column, LocalHostEndpoint, AllCompressionMethods); +// for (auto compressionMethod : AllCompressionMethods) +// { +// clickhouse::Client client(ClientOptions(LocalHostEndpoint) +// .SetCompressionMethod(compressionMethod)); - if (auto message = this->SkipTest(client)) { - GTEST_SKIP() << *message; +// if (auto message = this->CheckIfShouldSkipTest(client)) { +// GTEST_SKIP() << *message; +// } + +// auto result_typed = WrapColumn(RoundtripColumnValues(client, column)); +// EXPECT_TRUE(CompareRecursive(*column, *result_typed)); +// } +} + +TYPED_TEST(GenericColumnTest, ArrayT_RoundTrip) { + using ColumnArrayType = ColumnArrayT; + + auto [nested_column, values] = this->MakeColumnWithValues(10); + + auto column = std::make_shared(nested_column->CloneEmpty()->template As()); + for (size_t i = 0; i < values.size(); ++i) + { + const std::vector> row{values.begin(), values.begin() + i}; + column->Append(values.begin(), values.begin() + i); + + EXPECT_TRUE(CompareRecursive(row, (*column)[column->Size() - 1])); } + EXPECT_EQ(values.size(), column->Size()); + + this->TestColumnRoundtrip(column, LocalHostEndpoint, AllCompressionMethods); + +// SCOPED_TRACE(::testing::Message("Column type: ") << column->GetType().GetName()); - auto result_typed = WrapColumn(RoundtripColumnValues(client, column)); - EXPECT_TRUE(CompareRecursive(*column, *result_typed)); +// for (auto compressionMethod : AllCompressionMethods) +// { +// const ClientOptions client_options = ClientOptions(LocalHostEndpoint).SetCompressionMethod(compressionMethod); +// SCOPED_TRACE(::testing::Message("Client options: ") << client_options); + +// clickhouse::Client client(client_options); + +// if (auto message = this->CheckIfShouldSkipTest(client)) { +// GTEST_SKIP() << *message; +// } + +// auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); +// EXPECT_TRUE(CompareRecursive(*column, *result_typed)); +// } } + diff --git a/ut/roundtrip_column.cpp b/ut/roundtrip_column.cpp index c4685a3b..380bfa0f 100644 --- a/ut/roundtrip_column.cpp +++ b/ut/roundtrip_column.cpp @@ -4,11 +4,30 @@ #include #include +#include +#include "clickhouse/columns/numeric.h" namespace { using namespace clickhouse; + +template +std::vector GenerateConsecutiveNumbers(size_t count, T start = 0) +{ + std::vector result; + result.reserve(count); + + T value = start; + for (size_t i = 0; i < count; ++i, ++value) + { + result.push_back(value); + } + + return result; +} + } + ColumnRef RoundtripColumnValues(Client& client, ColumnRef expected) { // Create a temporary table with a single column // insert values from `expected` @@ -16,16 +35,18 @@ ColumnRef RoundtripColumnValues(Client& client, ColumnRef expected) { auto result = expected->CloneEmpty(); const std::string type_name = result->GetType().GetName(); - client.Execute("DROP TEMPORARY TABLE IF EXISTS temporary_roundtrip_table;"); - client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS temporary_roundtrip_table (col " + type_name + ");"); + client.Execute("DROP TABLE IF EXISTS temporary_roundtrip_table;"); + // id column is to have same order of rows on SELECT + client.Execute("CREATE TABLE IF NOT EXISTS temporary_roundtrip_table (id UInt32, col " + type_name + ") Engine=MergeTree() ORDER BY id;"); { Block block; block.AppendColumn("col", expected); + block.AppendColumn("id", std::make_shared(GenerateConsecutiveNumbers(expected->Size()))); block.RefreshRowCount(); client.Insert("temporary_roundtrip_table", block); } - client.Select("SELECT col FROM temporary_roundtrip_table", [&result](const Block& b) { + client.Select("SELECT col FROM temporary_roundtrip_table ORDER BY id", [&result](const Block& b) { if (b.GetRowCount() == 0) return; diff --git a/ut/utils.h b/ut/utils.h index 41bec25c..b4b8e92d 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -167,7 +167,10 @@ std::ostream& operator<<(std::ostream & ostr, const PrintContainer& print_con for (auto i = std::begin(container); i != std::end(container); /*intentionally no ++i*/) { const auto & elem = *i; - if constexpr (is_container_v>) { + if constexpr (is_string_v) { + ostr << '"' << elem << '"'; + } + else if constexpr (is_container_v>) { ostr << PrintContainer{elem}; } else { ostr << elem; diff --git a/ut/utils_comparison.h b/ut/utils_comparison.h index 0b7f805a..e500e4f3 100644 --- a/ut/utils_comparison.h +++ b/ut/utils_comparison.h @@ -143,7 +143,8 @@ struct PrintContainer; template ::testing::AssertionResult CompareRecursive(const Left & left, const Right & right) { - if constexpr ((is_container_v || std::is_base_of_v>) + if constexpr (!is_string_v && !is_string_v + && (is_container_v || std::is_base_of_v>) && (is_container_v || std::is_base_of_v>) ) { const auto & l = maybeWrapColumnAsContainer(left); diff --git a/ut/utils_meta.h b/ut/utils_meta.h index e069dc5f..3c50da4e 100644 --- a/ut/utils_meta.h +++ b/ut/utils_meta.h @@ -2,6 +2,8 @@ #include #include // for std::begin +#include +#include // based on https://stackoverflow.com/a/31207079 template @@ -46,3 +48,6 @@ struct is_instantiation_of : std::false_type {}; template < template class Template, typename... Args > struct is_instantiation_of< Template, Template > : std::true_type {}; + +template +inline constexpr bool is_string_v = std::is_same_v> || std::is_same_v>; diff --git a/ut/utils_ut.cpp b/ut/utils_ut.cpp index f69e1daa..bd015751 100644 --- a/ut/utils_ut.cpp +++ b/ut/utils_ut.cpp @@ -1,4 +1,5 @@ #include +#include "ut/value_generators.h" #include "utils.h" #include @@ -113,3 +114,9 @@ TEST(CompareRecursive, OptionalNan) { // EXPECT_FALSE(CompareRecursive(NaNdo, std::nullopt)); // EXPECT_FALSE(CompareRecursive(NaNfo, std::nullopt)); } + + +TEST(Generators, MakeArrays) { + auto arrays = MakeArrays(); + ASSERT_LT(0u, arrays.size()); +} diff --git a/ut/value_generators.cpp b/ut/value_generators.cpp index a0316a3a..f6d7baf9 100644 --- a/ut/value_generators.cpp +++ b/ut/value_generators.cpp @@ -17,7 +17,7 @@ std::vector MakeBools() { } std::vector MakeFixedStrings(size_t string_size) { - std::vector result {"aaa", "bbb", "ccc", "ddd"}; + std::vector result = MakeStrings(); std::for_each(result.begin(), result.end(), [string_size](auto& value) { value.resize(string_size, '\0'); @@ -27,7 +27,28 @@ std::vector MakeFixedStrings(size_t string_size) { } std::vector MakeStrings() { - return {"a", "ab", "abc", "abcd"}; + return { + "a", "ab", "abc", "abcd", + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + }; } std::vector MakeUUIDs() { diff --git a/ut/value_generators.h b/ut/value_generators.h index 8ef21d6e..031f0fe9 100644 --- a/ut/value_generators.h +++ b/ut/value_generators.h @@ -122,6 +122,20 @@ inline auto MakeDates() { return result; } +template (*Generator)()> +inline auto MakeArrays() { + const auto nested_values = Generator(); + std::vector> result; + result.reserve(nested_values.size()); + + for (size_t i = 0; i < nested_values.size(); ++i) + { + result.emplace_back(nested_values.begin(), nested_values.begin() + i); + } + + return result; +} + std::string FooBarGenerator(size_t i); From 5b4c471f74b5d704ff866dc385c667742d6d2262 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 20 Sep 2023 19:34:09 +0200 Subject: [PATCH 47/53] Minor cleanup and increased number of generated values in tests --- ut/Column_ut.cpp | 61 ++++++++---------------------------------------- 1 file changed, 10 insertions(+), 51 deletions(-) diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index ddb91335..f8bacd53 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -264,7 +264,7 @@ TYPED_TEST(GenericColumnTest, EmptyColumn) { TYPED_TEST(GenericColumnTest, Append) { auto column = this->MakeColumn(); - const auto values = this->GenerateValues(100); + const auto values = this->GenerateValues(10'000); for (const auto & v : values) { EXPECT_NO_THROW(column->Append(v)); @@ -301,7 +301,7 @@ inline auto convertValueForGetItem(const ColumnType& col, ValueType&& t) { } TYPED_TEST(GenericColumnTest, GetItem) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); ASSERT_EQ(values.size(), column->Size()); const auto wrapping_types = std::set{ @@ -323,7 +323,7 @@ TYPED_TEST(GenericColumnTest, GetItem) { } TYPED_TEST(GenericColumnTest, Slice) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); auto untyped_slice = column->Slice(0, column->Size()); auto slice = untyped_slice->template AsStrict(); @@ -335,7 +335,7 @@ TYPED_TEST(GenericColumnTest, Slice) { } TYPED_TEST(GenericColumnTest, CloneEmpty) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); EXPECT_EQ(values.size(), column->Size()); auto clone_untyped = column->CloneEmpty(); @@ -347,7 +347,7 @@ TYPED_TEST(GenericColumnTest, CloneEmpty) { } TYPED_TEST(GenericColumnTest, Clear) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); EXPECT_EQ(values.size(), column->Size()); column->Clear(); @@ -355,7 +355,7 @@ TYPED_TEST(GenericColumnTest, Clear) { } TYPED_TEST(GenericColumnTest, Swap) { - auto [column_A, values] = this->MakeColumnWithValues(100); + auto [column_A, values] = this->MakeColumnWithValues(10'000); auto column_B = this->MakeColumn(); column_A->Swap(*column_B); @@ -398,22 +398,10 @@ const auto AllCompressionMethods = { }; TYPED_TEST(GenericColumnTest, RoundTrip) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); EXPECT_EQ(values.size(), column->Size()); this->TestColumnRoundtrip(column, LocalHostEndpoint, AllCompressionMethods); -// for (auto compressionMethod : AllCompressionMethods) -// { -// clickhouse::Client client(ClientOptions(LocalHostEndpoint) -// .SetCompressionMethod(compressionMethod)); - -// if (auto message = this->CheckIfShouldSkipTest(client)) { -// GTEST_SKIP() << *message; -// } - -// auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); -// EXPECT_TRUE(CompareRecursive(*column, *result_typed)); -// } } TYPED_TEST(GenericColumnTest, NullableT_RoundTrip) { @@ -422,10 +410,10 @@ TYPED_TEST(GenericColumnTest, NullableT_RoundTrip) { auto non_nullable_column = this->MakeColumn(); if (non_nullable_column->GetType().GetCode() == Type::Code::LowCardinality) // TODO (vnemkov): wrap as ColumnLowCardinalityT> instead of ColumnNullableT> - GTEST_SKIP() << "Can't wrap " << non_nullable_column->GetType().GetName() << " into Nullable"; + GTEST_SKIP() << "Can't have " << non_nullable_column->GetType().GetName() << " in Nullable"; auto column = std::make_shared(std::move(non_nullable_column)); - auto values = this->GenerateValues(100); + auto values = this->GenerateValues(10'000); FromVectorGenerator is_null({true, false}); for (size_t i = 0; i < values.size(); ++i) { @@ -437,24 +425,12 @@ TYPED_TEST(GenericColumnTest, NullableT_RoundTrip) { } this->TestColumnRoundtrip(column, LocalHostEndpoint, AllCompressionMethods); -// for (auto compressionMethod : AllCompressionMethods) -// { -// clickhouse::Client client(ClientOptions(LocalHostEndpoint) -// .SetCompressionMethod(compressionMethod)); - -// if (auto message = this->CheckIfShouldSkipTest(client)) { -// GTEST_SKIP() << *message; -// } - -// auto result_typed = WrapColumn(RoundtripColumnValues(client, column)); -// EXPECT_TRUE(CompareRecursive(*column, *result_typed)); -// } } TYPED_TEST(GenericColumnTest, ArrayT_RoundTrip) { using ColumnArrayType = ColumnArrayT; - auto [nested_column, values] = this->MakeColumnWithValues(10); + auto [nested_column, values] = this->MakeColumnWithValues(1000); auto column = std::make_shared(nested_column->CloneEmpty()->template As()); for (size_t i = 0; i < values.size(); ++i) @@ -467,22 +443,5 @@ TYPED_TEST(GenericColumnTest, ArrayT_RoundTrip) { EXPECT_EQ(values.size(), column->Size()); this->TestColumnRoundtrip(column, LocalHostEndpoint, AllCompressionMethods); - -// SCOPED_TRACE(::testing::Message("Column type: ") << column->GetType().GetName()); - -// for (auto compressionMethod : AllCompressionMethods) -// { -// const ClientOptions client_options = ClientOptions(LocalHostEndpoint).SetCompressionMethod(compressionMethod); -// SCOPED_TRACE(::testing::Message("Client options: ") << client_options); - -// clickhouse::Client client(client_options); - -// if (auto message = this->CheckIfShouldSkipTest(client)) { -// GTEST_SKIP() << *message; -// } - -// auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); -// EXPECT_TRUE(CompareRecursive(*column, *result_typed)); -// } } From f2a663e0135fc45271a247e1021a3792af0710be Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 20 Sep 2023 23:08:37 +0200 Subject: [PATCH 48/53] Fixed ColumnsCase.StringInit test --- ut/columns_ut.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ut/columns_ut.cpp b/ut/columns_ut.cpp index ce477554..152cf65f 100644 --- a/ut/columns_ut.cpp +++ b/ut/columns_ut.cpp @@ -105,9 +105,10 @@ TEST(ColumnsCase, FixedString_Append_LargeString) { } TEST(ColumnsCase, StringInit) { - auto col = std::make_shared(MakeStrings()); + auto values = MakeStrings(); + auto col = std::make_shared(values); - ASSERT_EQ(col->Size(), 4u); + ASSERT_EQ(col->Size(), values.size()); ASSERT_EQ(col->At(1), "ab"); ASSERT_EQ(col->At(3), "abcd"); } From 3d377bfd57de4f278006a37696722d9577cae7b9 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 20 Sep 2023 23:31:36 +0200 Subject: [PATCH 49/53] Using TEMPORARY table in RoundtripColumnValues --- ut/roundtrip_column.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ut/roundtrip_column.cpp b/ut/roundtrip_column.cpp index 380bfa0f..9e558500 100644 --- a/ut/roundtrip_column.cpp +++ b/ut/roundtrip_column.cpp @@ -29,15 +29,15 @@ std::vector GenerateConsecutiveNumbers(size_t count, T start = 0) ColumnRef RoundtripColumnValues(Client& client, ColumnRef expected) { - // Create a temporary table with a single column - // insert values from `expected` - // select and aggregate all values from block into `result` column + // Create a temporary table with a corresponding data column + // INSERT values from `expected` + // SELECT and collect all values from block into `result` column auto result = expected->CloneEmpty(); const std::string type_name = result->GetType().GetName(); - client.Execute("DROP TABLE IF EXISTS temporary_roundtrip_table;"); - // id column is to have same order of rows on SELECT - client.Execute("CREATE TABLE IF NOT EXISTS temporary_roundtrip_table (id UInt32, col " + type_name + ") Engine=MergeTree() ORDER BY id;"); + client.Execute("DROP TEMPORARY TABLE IF EXISTS temporary_roundtrip_table;"); + // id column is to have the same order of rows on SELECT + client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS temporary_roundtrip_table (id UInt32, col " + type_name + ") Engine=MergeTree() ORDER BY id;"); { Block block; block.AppendColumn("col", expected); From 27039a71412ba0d561f0b7f8de9eb211330bdecd Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 20 Sep 2023 23:37:36 +0200 Subject: [PATCH 50/53] Update roundtrip_column.cpp --- ut/roundtrip_column.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ut/roundtrip_column.cpp b/ut/roundtrip_column.cpp index 9e558500..5af9624a 100644 --- a/ut/roundtrip_column.cpp +++ b/ut/roundtrip_column.cpp @@ -37,7 +37,7 @@ ColumnRef RoundtripColumnValues(Client& client, ColumnRef expected) { const std::string type_name = result->GetType().GetName(); client.Execute("DROP TEMPORARY TABLE IF EXISTS temporary_roundtrip_table;"); // id column is to have the same order of rows on SELECT - client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS temporary_roundtrip_table (id UInt32, col " + type_name + ") Engine=MergeTree() ORDER BY id;"); + client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS temporary_roundtrip_table (id UInt32, col " + type_name + ");"); { Block block; block.AppendColumn("col", expected); From 7d83db4368ce200eedde11da4e9a85bf5836c087 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 21 Sep 2023 14:27:21 +0200 Subject: [PATCH 51/53] Attempt to fix test on Windows and MacOS --- ut/Column_ut.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index f8bacd53..6e5668e6 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -368,18 +368,20 @@ TYPED_TEST(GenericColumnTest, LoadAndSave) { auto [column_A, values] = this->MakeColumnWithValues(100); // large buffer since we have pretty big values for String column - char buffer[1024*1024] = {'\0'}; + auto const BufferSize = 10*1024*1024; + std::unique_ptr buffer = std::make_unique(BufferSize); + memset(buffer.get(), 0, BufferSize); { - ArrayOutput output(buffer, sizeof(buffer)); + ArrayOutput output(buffer.get(), BufferSize); // Save - EXPECT_NO_THROW(column_A->Save(&output)); + ASSERT_NO_THROW(column_A->Save(&output)); } auto column_B = this->MakeColumn(); { - ArrayInput input(buffer, sizeof(buffer)); + ArrayInput input(buffer.get(), BufferSize); // Load - EXPECT_TRUE(column_B->Load(&input, values.size())); + ASSERT_TRUE(column_B->Load(&input, values.size())); } EXPECT_TRUE(CompareRecursive(*column_A, *column_B)); @@ -430,7 +432,7 @@ TYPED_TEST(GenericColumnTest, NullableT_RoundTrip) { TYPED_TEST(GenericColumnTest, ArrayT_RoundTrip) { using ColumnArrayType = ColumnArrayT; - auto [nested_column, values] = this->MakeColumnWithValues(1000); + auto [nested_column, values] = this->MakeColumnWithValues(100); auto column = std::make_shared(nested_column->CloneEmpty()->template As()); for (size_t i = 0; i < values.size(); ++i) From f21ba79fd81926ae2a612f0b69942e51ae37127d Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 25 Sep 2023 17:25:43 +0200 Subject: [PATCH 52/53] Set backward_compatibility_lowcardinality_as_wrapped_column default value to false --- clickhouse/client.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/client.h b/clickhouse/client.h index a66d59a6..35000784 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -123,7 +123,7 @@ struct ClientOptions { * @see LowCardinalitySerializationAdaptor, CreateColumnByType */ [[deprecated("Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) ")]] - DECLARE_FIELD(backward_compatibility_lowcardinality_as_wrapped_column, bool, SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn, true); + DECLARE_FIELD(backward_compatibility_lowcardinality_as_wrapped_column, bool, SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn, false); /** Set max size data to compress if compression enabled. * From 9b2b2190508657743b5948f0bda9839679e6e47e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 26 Sep 2023 12:33:36 +0200 Subject: [PATCH 53/53] Fixed the tests --- ut/low_cardinality_nullable_tests.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ut/low_cardinality_nullable_tests.cpp b/ut/low_cardinality_nullable_tests.cpp index 8918f251..357f28a9 100644 --- a/ut/low_cardinality_nullable_tests.cpp +++ b/ut/low_cardinality_nullable_tests.cpp @@ -122,8 +122,8 @@ TEST(LowCardinalityOfNullable, InsertAndQueryEmpty) { block.AppendColumn("words", column); Client client(ClientOptions(localHostEndpoint) - .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false) - .SetPingBeforeQuery(true)); + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false) + .SetPingBeforeQuery(true)); createTable(client); @@ -141,7 +141,8 @@ TEST(LowCardinalityOfNullable, ThrowOnBackwardsCompatibleLCColumn) { block.AppendColumn("words", column); Client client(ClientOptions(localHostEndpoint) - .SetPingBeforeQuery(true)); + .SetPingBeforeQuery(true) + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(true)); createTable(client);