diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 1a267aa8..215b2214 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -67,5 +67,5 @@ jobs: # It is impossible to start CH server in docker on macOS due to github actions limitations, # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: # - system.query_log used by 'Client/ClientCase.Query_ID' - GTEST_FILTER: "-Client/ClientCase.Query_ID*" + GTEST_FILTER: "-Client/ClientCase.Query_ID*:Client/ClientCase.TracingContext/*" run: ./clickhouse-cpp-ut ${GTEST_FILTER} diff --git a/.github/workflows/windows_mingw.yml b/.github/workflows/windows_mingw.yml index d99c1ef7..6c6c3e9b 100644 --- a/.github/workflows/windows_mingw.yml +++ b/.github/workflows/windows_mingw.yml @@ -85,7 +85,7 @@ jobs: # It is impossible to start CH server in docker on Windows due to github actions limitations, # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: # - system.query_log used by 'Client/ClientCase.Query_ID' - GTEST_FILTER: "-Client/ClientCase.Query_ID*" + GTEST_FILTER: "-Client/ClientCase.Query_ID*:Client/ClientCase.TracingContext/*" run: ./build/ut/clickhouse-cpp-ut.exe ${GTEST_FILTER} - name: Test (simple) diff --git a/.github/workflows/windows_msvc.yml b/.github/workflows/windows_msvc.yml index 149e3df4..760f5ab2 100644 --- a/.github/workflows/windows_msvc.yml +++ b/.github/workflows/windows_msvc.yml @@ -61,6 +61,6 @@ jobs: # It is impossible to start CH server in docker on Windows due to github actions limitations, # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: # - system.query_log used by 'Client/ClientCase.Query_ID' - GTEST_FILTER: "-Client/ClientCase.Query_ID*" + GTEST_FILTER: "-Client/ClientCase.Query_ID*:Client/ClientCase.TracingContext/*" working-directory: ${{github.workspace}}/build/ut run: Release\clickhouse-cpp-ut.exe "${{env.GTEST_FILTER}}" diff --git a/CMakeLists.txt b/CMakeLists.txt index aed0d85c..3152b552 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,7 @@ OPTION (WITH_OPENSSL "Use OpenSSL for TLS connections" OFF) OPTION (WITH_SYSTEM_ABSEIL "Use system ABSEIL" OFF) OPTION (WITH_SYSTEM_LZ4 "Use system LZ4" OFF) OPTION (WITH_SYSTEM_CITYHASH "Use system cityhash" OFF) +OPTION (DEBUG_DEPENDENCIES "Print debug info about dependencies duting build" ON) PROJECT (CLICKHOUSE-CLIENT) @@ -99,3 +100,36 @@ PROJECT (CLICKHOUSE-CLIENT) ut ) ENDIF (BUILD_TESTS) + + if(DEBUG_DEPENDENCIES) + function(print_target_properties target) + MESSAGE("${target} properties:") + set(properties "${ARGN}") + foreach(property_name ${properties}) + get_target_property(PROPERTY ${target} ${property_name}) + MESSAGE(NOTICE "\t${property_name} : ${PROPERTY}") + endforeach() + + # Can't get path to the target file at configure time, + # so have to create a target to fetch that info at generate time. + string(REPLACE ":" "_" target_plain_name ${target}) + add_custom_target(${target_plain_name}_print_debug_info COMMAND ${CMAKE_COMMAND} -E echo "${target} : $") + add_dependencies(clickhouse-cpp-lib ${target_plain_name}_print_debug_info) + endfunction() + + function(print_target_debug_info target) + print_target_properties(${target} + INCLUDE_DIRECTORIES + BINARY_DIR + INTERFACE_INCLUDE_DIRECTORIES + INTERFACE_LINK_LIBRARIES + LINK_LIBRARIES + LINK_LIBRARIES_ONLY_TARGETS + IMPORTED_LOCATION + ) + endfunction() + + print_target_debug_info(absl::int128) + print_target_debug_info(cityhash::cityhash) + print_target_debug_info(lz4::lz4) + endif() diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 67663ec5..97f8cc56 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -5,6 +5,7 @@ SET ( clickhouse-cpp-lib-src base/platform.cpp base/socket.cpp base/wire_format.cpp + base/endpoints_iterator.cpp columns/array.cpp columns/column.cpp @@ -16,7 +17,6 @@ SET ( clickhouse-cpp-lib-src columns/ip4.cpp columns/ip6.cpp columns/lowcardinality.cpp - columns/lowcardinalityadaptor.h columns/nullable.cpp columns/numeric.cpp columns/map.cpp @@ -32,8 +32,72 @@ SET ( clickhouse-cpp-lib-src block.cpp client.cpp query.cpp + + # Headers + base/buffer.h + base/compressed.h + base/endpoints_iterator.h + base/input.h + base/open_telemetry.h + base/output.h + base/platform.h + base/projected_iterator.h + base/singleton.h + base/socket.h + base/sslsocket.h + base/string_utils.h + base/string_view.h + base/uuid.h + base/wire_format.h + + columns/array.h + columns/column.h + columns/date.h + columns/decimal.h + columns/enum.h + columns/factory.h + columns/geo.h + columns/ip4.h + columns/ip6.h + columns/itemview.h + columns/lowcardinality.h + columns/lowcardinalityadaptor.h + columns/map.h + columns/nothing.h + columns/nullable.h + columns/numeric.h + columns/string.h + columns/tuple.h + columns/utils.h + columns/uuid.h + + types/type_parser.h + types/types.h + + block.h + client.h + error_codes.h + exceptions.h + protocol.h + query.h + server_exception.h ) +if (MSVC) + add_definitions(-D_CRT_SECURE_NO_WARNINGS) + add_compile_options(/W4) + # remove in 3.0 + add_compile_options(/wd4996) +else() + set(cxx_extra_wall "-Wempty-body -Wconversion -Wreturn-type -Wparentheses -Wuninitialized -Wunreachable-code -Wunused-function -Wunused-value -Wunused-variable") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall}") + + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + # a little abnormal when clang check conversion + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_extra_wall} -Wno-conversion") + endif() +endif() + IF (WITH_OPENSSL) LIST(APPEND clickhouse-cpp-lib-src base/sslsocket.cpp) ENDIF () @@ -104,6 +168,7 @@ INSTALL(FILES base/string_utils.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/string_view.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/uuid.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/wire_format.h DESTINATION include/clickhouse/base/) +INSTALL(FILES base/endpoints_iterator.h DESTINATION include/clickhouse/base/) # columns INSTALL(FILES columns/array.h DESTINATION include/clickhouse/columns/) diff --git a/clickhouse/base/compressed.cpp b/clickhouse/base/compressed.cpp index 4d5cc65d..135f9483 100644 --- a/clickhouse/base/compressed.cpp +++ b/clickhouse/base/compressed.cpp @@ -94,7 +94,7 @@ bool CompressedInput::Decompress() { data_ = Buffer(original); - if (LZ4_decompress_safe((const char*)tmp.data() + HEADER_SIZE, (char*)data_.data(), compressed - HEADER_SIZE, original) < 0) { + if (LZ4_decompress_safe((const char*)tmp.data() + HEADER_SIZE, (char*)data_.data(), static_cast(compressed - HEADER_SIZE), original) < 0) { throw LZ4Error("can't decompress data"); } else { mem_.Reset(data_.data(), original); @@ -141,7 +141,7 @@ void CompressedOutput::Compress(const void * data, size_t len) { const auto compressed_size = LZ4_compress_default( (const char*)data, (char*)compressed_buffer_.data() + HEADER_SIZE, - len, + static_cast(len), static_cast(compressed_buffer_.size() - HEADER_SIZE)); if (compressed_size <= 0) throw LZ4Error("Failed to compress chunk of " + std::to_string(len) + " bytes, " diff --git a/clickhouse/base/endpoints_iterator.cpp b/clickhouse/base/endpoints_iterator.cpp new file mode 100644 index 00000000..30d3593e --- /dev/null +++ b/clickhouse/base/endpoints_iterator.cpp @@ -0,0 +1,20 @@ +#include "endpoints_iterator.h" +#include + +namespace clickhouse { + +RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const std::vector& _endpoints) + : endpoints (_endpoints) + , current_index (endpoints.size() - 1ull) +{ +} + +Endpoint RoundRobinEndpointsIterator::Next() +{ + current_index = (current_index + 1ull) % endpoints.size(); + return endpoints[current_index]; +} + +RoundRobinEndpointsIterator::~RoundRobinEndpointsIterator() = default; + +} diff --git a/clickhouse/base/endpoints_iterator.h b/clickhouse/base/endpoints_iterator.h new file mode 100644 index 00000000..ba6a850f --- /dev/null +++ b/clickhouse/base/endpoints_iterator.h @@ -0,0 +1,34 @@ +#pragma once + +#include "clickhouse/client.h" +#include + +namespace clickhouse { + +struct ClientOptions; + +/** + * Base class for iterating through endpoints. +*/ +class EndpointsIteratorBase +{ + public: + virtual ~EndpointsIteratorBase() = default; + + virtual Endpoint Next() = 0; +}; + +class RoundRobinEndpointsIterator : public EndpointsIteratorBase +{ + public: + explicit RoundRobinEndpointsIterator(const std::vector& opts); + Endpoint Next() override; + + ~RoundRobinEndpointsIterator() override; + + private: + const std::vector& endpoints; + size_t current_index; +}; + +} diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index 48e90c73..28be7b61 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -217,7 +217,7 @@ SOCKET SocketConnect(const NetworkAddress& addr, const SocketTimeoutParams& time fd.fd = *s; fd.events = POLLOUT; fd.revents = 0; - ssize_t rval = Poll(&fd, 1, timeout_params.connect_timeout.count()); + ssize_t rval = Poll(&fd, 1, static_cast(timeout_params.connect_timeout.count())); if (rval == -1) { throw std::system_error(getSocketErrorCode(), getErrorCategory(), "fail to connect"); @@ -390,9 +390,9 @@ std::unique_ptr Socket::makeOutputStream() const { NonSecureSocketFactory::~NonSecureSocketFactory() {} -std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts) { - const auto address = NetworkAddress(opts.host, std::to_string(opts.port)); +std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions &opts, const Endpoint& endpoint) { + const auto address = NetworkAddress(endpoint.host, std::to_string(endpoint.port)); auto socket = doConnect(address, opts); setSocketOptions(*socket, opts); diff --git a/clickhouse/base/socket.h b/clickhouse/base/socket.h index 694d0d69..9bd9ca34 100644 --- a/clickhouse/base/socket.h +++ b/clickhouse/base/socket.h @@ -3,6 +3,7 @@ #include "platform.h" #include "input.h" #include "output.h" +#include "endpoints_iterator.h" #include #include @@ -88,7 +89,7 @@ class SocketFactory { // TODO: move connection-related options to ConnectionOptions structure. - virtual std::unique_ptr connect(const ClientOptions& opts) = 0; + virtual std::unique_ptr connect(const ClientOptions& opts, const Endpoint& endpoint) = 0; virtual void sleepFor(const std::chrono::milliseconds& duration); }; @@ -135,7 +136,7 @@ class NonSecureSocketFactory : public SocketFactory { public: ~NonSecureSocketFactory() override; - std::unique_ptr connect(const ClientOptions& opts) override; + std::unique_ptr connect(const ClientOptions& opts, const Endpoint& endpoint) override; protected: virtual std::unique_ptr doConnect(const NetworkAddress& address, const ClientOptions& opts); diff --git a/clickhouse/base/sslsocket.cpp b/clickhouse/base/sslsocket.cpp index 8b1971e0..4c5185d8 100644 --- a/clickhouse/base/sslsocket.cpp +++ b/clickhouse/base/sslsocket.cpp @@ -109,7 +109,7 @@ SSL_CTX * prepareSSLContext(const clickhouse::SSLParams & context_params) { #define HANDLE_SSL_CTX_ERROR(statement) do { \ if (const auto ret_code = (statement); !ret_code) \ - throwSSLError(nullptr, ERR_peek_error(), LOCATION, #statement); \ + throwSSLError(nullptr, static_cast(ERR_peek_error()), LOCATION, #statement); \ } while(false); if (context_params.use_default_ca_locations) @@ -185,7 +185,7 @@ SSL_CTX * SSLContext::getContext() { // Allows caller to use returned value of `statement` if there was no error, throws exception otherwise. #define HANDLE_SSL_ERROR(SSL_PTR, statement) [&] { \ if (const auto ret_code = (statement); ret_code <= 0) { \ - throwSSLError(SSL_PTR, SSL_get_error(SSL_PTR, ret_code), LOCATION, #statement); \ + throwSSLError(SSL_PTR, SSL_get_error(SSL_PTR, static_cast(ret_code)), LOCATION, #statement); \ return static_cast>(0); \ } \ else \ @@ -209,7 +209,7 @@ SSLSocket::SSLSocket(const NetworkAddress& addr, const SocketTimeoutParams& time std::unique_ptr ip_addr(a2i_IPADDRESS(addr.Host().c_str()), &ASN1_OCTET_STRING_free); - HANDLE_SSL_ERROR(ssl, SSL_set_fd(ssl, handle_)); + HANDLE_SSL_ERROR(ssl, SSL_set_fd(ssl, static_cast(handle_))); if (ssl_params.use_SNI) HANDLE_SSL_ERROR(ssl, SSL_set_tlsext_host_name(ssl, addr.Host().c_str())); @@ -295,7 +295,11 @@ SSLSocketOutput::SSLSocketOutput(SSL *ssl) {} size_t SSLSocketOutput::DoWrite(const void* data, size_t len) { - return static_cast(HANDLE_SSL_ERROR(ssl_, SSL_write(ssl_, data, len))); + if (len > std::numeric_limits::max()) + // FIXME(vnemkov): We should do multiple `SSL_write`s in this case. + throw AssertionError("Failed to write too big chunk at once " + + std::to_string(len) + " > " + std::to_string(std::numeric_limits::max())); + return static_cast(HANDLE_SSL_ERROR(ssl_, SSL_write(ssl_, data, static_cast(len)))); } #undef HANDLE_SSL_ERROR diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index e4b0c7ef..ca57190b 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -65,7 +65,12 @@ struct ClientInfo { std::ostream& operator<<(std::ostream& os, const ClientOptions& opt) { os << "Client(" << opt.user << '@' << opt.host << ":" << opt.port - << " ping_before_query:" << opt.ping_before_query + << "Endpoints :"; + for (size_t i = 0; i < opt.endpoints.size(); i++) + os << opt.user << '@' << opt.endpoints[i].host << ":" << opt.endpoints[i].port + << ((i == opt.endpoints.size() - 1) ? "" : ", "); + + os << " ping_before_query:" << opt.ping_before_query << " send_retries:" << opt.send_retries << " retry_timeout:" << opt.retry_timeout.count() << " compression_method:" @@ -111,6 +116,15 @@ std::unique_ptr GetSocketFactory(const ClientOptions& opts) { return std::make_unique(); } +std::unique_ptr GetEndpointsIterator(const ClientOptions& opts) { + if (opts.endpoints.empty()) + { + throw ValidationError("The list of endpoints is empty"); + } + + return std::make_unique(opts.endpoints); +} + } class Client::Impl { @@ -130,8 +144,12 @@ class Client::Impl { void ResetConnection(); + void ResetConnectionEndpoint(); + const ServerInfo& GetServerInfo() const; + const std::optional& GetCurrentEndpoint() const; + private: bool Handshake(); @@ -155,13 +173,22 @@ class Client::Impl { void WriteBlock(const Block& block, OutputStream& output); + void CreateConnection(); + void InitializeStreams(std::unique_ptr&& socket); + inline size_t GetConnectionAttempts() const + { + return options_.endpoints.size() * options_.send_retries; + } + private: /// In case of network errors tries to reconnect to server and /// call fuc several times. void RetryGuard(std::function func); + void RetryConnectToTheEndpoint(std::function& func); + private: class EnsureNull { public: @@ -194,32 +221,34 @@ class Client::Impl { std::unique_ptr input_; std::unique_ptr output_; std::unique_ptr socket_; + std::unique_ptr endpoints_iterator; + + std::optional current_endpoint_; ServerInfo server_info_; }; +ClientOptions modifyClientOptions(ClientOptions opts) +{ + if (opts.host.empty()) + return opts; + + Endpoint default_endpoint({opts.host, opts.port}); + opts.endpoints.emplace(opts.endpoints.begin(), default_endpoint); + return opts; +} Client::Impl::Impl(const ClientOptions& opts) : Impl(opts, GetSocketFactory(opts)) {} Client::Impl::Impl(const ClientOptions& opts, std::unique_ptr socket_factory) - : options_(opts) + : options_(modifyClientOptions(opts)) , events_(nullptr) , socket_factory_(std::move(socket_factory)) + , endpoints_iterator(GetEndpointsIterator(options_)) { - for (unsigned int i = 0; ; ) { - try { - ResetConnection(); - break; - } catch (const std::system_error&) { - if (++i > options_.send_retries) { - throw; - } - - socket_factory_->sleepFor(options_.retry_timeout); - } - } + CreateConnection(); if (options_.compression_method != CompressionMethod::None) { compression_ = CompressionState::Enable; @@ -329,17 +358,57 @@ void Client::Impl::Ping() { } void Client::Impl::ResetConnection() { - InitializeStreams(socket_factory_->connect(options_)); + InitializeStreams(socket_factory_->connect(options_, current_endpoint_.value())); if (!Handshake()) { throw ProtocolError("fail to connect to " + options_.host); } } +void Client::Impl::ResetConnectionEndpoint() { + current_endpoint_.reset(); + for (size_t i = 0; i < options_.endpoints.size();) + { + try + { + current_endpoint_ = endpoints_iterator->Next(); + ResetConnection(); + return; + } catch (const std::system_error&) { + if (++i == options_.endpoints.size()) + { + current_endpoint_.reset(); + throw; + } + } + } +} + +void Client::Impl::CreateConnection() { + for (size_t i = 0; i < options_.send_retries;) + { + try + { + ResetConnectionEndpoint(); + return; + } catch (const std::system_error&) { + if (++i == options_.send_retries) + { + throw; + } + } + } +} + const ServerInfo& Client::Impl::GetServerInfo() const { return server_info_; } + +const std::optional& Client::Impl::GetCurrentEndpoint() const { + return current_endpoint_; +} + bool Client::Impl::Handshake() { if (!SendHello()) { return false; @@ -411,12 +480,12 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { if (!WireFormat::ReadUInt64(*input_, &info.bytes)) { return false; } - if (REVISION >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS) { + if constexpr(REVISION >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS) { if (!WireFormat::ReadUInt64(*input_, &info.total_rows)) { return false; } } - if (REVISION >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO) + if constexpr (REVISION >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO) { if (!WireFormat::ReadUInt64(*input_, &info.written_rows)) { return false; @@ -499,13 +568,11 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { throw UnimplementedError("unimplemented " + std::to_string((int)packet_type)); break; } - - return false; } bool Client::Impl::ReadBlock(InputStream& input, Block* block) { // Additional information about block. - if (REVISION >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) { + if constexpr (REVISION >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) { uint64_t num; BlockInfo info; @@ -542,9 +609,9 @@ bool Client::Impl::ReadBlock(InputStream& input, Block* block) { CreateColumnByTypeSettings create_column_settings; create_column_settings.low_cardinality_as_wrapped_column = options_.backward_compatibility_lowcardinality_as_wrapped_column; - std::string name; - std::string type; for (size_t i = 0; i < num_columns; ++i) { + std::string name; + std::string type; if (!WireFormat::ReadString(input, &name)) { return false; } @@ -569,7 +636,7 @@ bool Client::Impl::ReadBlock(InputStream& input, Block* block) { bool Client::Impl::ReceiveData() { Block block; - if (REVISION >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) { + if constexpr (REVISION >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) { if (!WireFormat::SkipString(*input_)) { return false; } @@ -861,21 +928,45 @@ bool Client::Impl::ReceiveHello() { } void Client::Impl::RetryGuard(std::function func) { - for (unsigned int i = 0; ; ++i) { - try { - func(); - return; - } catch (const std::system_error&) { - bool ok = true; + if (current_endpoint_) + { + for (unsigned int i = 0; ; ++i) { try { - socket_factory_->sleepFor(options_.retry_timeout); - ResetConnection(); - } catch (...) { - ok = false; + func(); + return; + } catch (const std::system_error&) { + bool ok = true; + + try { + socket_factory_->sleepFor(options_.retry_timeout); + ResetConnection(); + } catch (...) { + ok = false; + } + + if (!ok && i == options_.send_retries) { + break; + } } - - if (!ok && i == options_.send_retries) { + } + } + // Connectiong with current_endpoint_ are broken. + // Trying to establish with the another one from the list. + size_t connection_attempts_count = GetConnectionAttempts(); + for (size_t i = 0; i < connection_attempts_count;) + { + try + { + socket_factory_->sleepFor(options_.retry_timeout); + current_endpoint_ = endpoints_iterator->Next(); + ResetConnection(); + func(); + return; + } catch (const std::system_error&) { + if (++i == connection_attempts_count) + { + current_endpoint_.reset(); throw; } } @@ -938,6 +1029,14 @@ void Client::ResetConnection() { impl_->ResetConnection(); } +void Client::ResetConnectionEndpoint() { + impl_->ResetConnectionEndpoint(); +} + +const std::optional& Client::GetCurrentEndpoint() const { + return impl_->GetCurrentEndpoint(); +} + const ServerInfo& Client::GetServerInfo() const { return impl_->GetServerInfo(); } diff --git a/clickhouse/client.h b/clickhouse/client.h index 63177313..35000784 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -44,6 +44,18 @@ enum class CompressionMethod { LZ4 = 1, }; +struct Endpoint { + std::string host; + uint16_t port = 9000; + inline bool operator==(const Endpoint& right) const { + return host == right.host && port == right.port; + } +}; + +enum class EndpointsIterationAlgorithm { + RoundRobin = 0, +}; + struct ClientOptions { // Setter goes first, so it is possible to apply 'deprecated' annotation safely. #define DECLARE_FIELD(name, type, setter, default_value) \ @@ -56,7 +68,15 @@ struct ClientOptions { /// Hostname of the server. DECLARE_FIELD(host, std::string, SetHost, std::string()); /// Service port. - DECLARE_FIELD(port, unsigned int, SetPort, 9000); + DECLARE_FIELD(port, uint16_t, SetPort, 9000); + + /** Set endpoints (host+port), only one is used. + * Client tries to connect to those endpoints one by one, on the round-robin basis: + * first default enpoint (set via SetHost() + SetPort()), then each of endpoints, from begin() to end(), + * the first one to establish connection is used for the rest of the session. + * If port isn't specified, default(9000) value will be used. + */ + DECLARE_FIELD(endpoints, std::vector, SetEndpoints, {}); /// Default database. DECLARE_FIELD(default_database, std::string, SetDefaultDatabase, "default"); @@ -103,7 +123,7 @@ struct ClientOptions { * @see LowCardinalitySerializationAdaptor, CreateColumnByType */ [[deprecated("Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) ")]] - DECLARE_FIELD(backward_compatibility_lowcardinality_as_wrapped_column, bool, SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn, true); + DECLARE_FIELD(backward_compatibility_lowcardinality_as_wrapped_column, bool, SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn, false); /** Set max size data to compress if compression enabled. * @@ -240,6 +260,12 @@ class Client { const ServerInfo& GetServerInfo() const; + /// Get current connected endpoint. + /// In case when client is not connected to any endpoint, nullopt will returned. + const std::optional& GetCurrentEndpoint() const; + + // Try to connect to different endpoints one by one only one time. If it doesn't work, throw an exception. + void ResetConnectionEndpoint(); private: const ClientOptions options_; diff --git a/clickhouse/columns/array.cpp b/clickhouse/columns/array.cpp index 9f66b91f..5e5e72e7 100644 --- a/clickhouse/columns/array.cpp +++ b/clickhouse/columns/array.cpp @@ -25,14 +25,9 @@ ColumnArray::ColumnArray(ColumnArray&& other) } void ColumnArray::AppendAsColumn(ColumnRef array) { - if (!data_->Type()->IsEqual(array->Type())) { - throw ValidationError( - "can't append column of type " + array->Type()->GetName() + " " - "to column type " + data_->Type()->GetName()); - } - - AddOffset(array->Size()); + // appending data may throw (i.e. due to ype check failure), so do it first to avoid partly modified state. data_->Append(array); + AddOffset(array->Size()); } ColumnRef ColumnArray::GetAsColumn(size_t n) const { @@ -59,10 +54,6 @@ ColumnRef ColumnArray::CloneEmpty() const { void ColumnArray::Append(ColumnRef column) { if (auto col = column->As()) { - if (!col->data_->Type()->IsEqual(data_->Type())) { - return; - } - for (size_t i = 0; i < col->Size(); ++i) { AppendAsColumn(col->GetAsColumn(i)); } diff --git a/clickhouse/columns/date.cpp b/clickhouse/columns/date.cpp index 1ef67c44..f4d08cb4 100644 --- a/clickhouse/columns/date.cpp +++ b/clickhouse/columns/date.cpp @@ -1,4 +1,5 @@ #include "date.h" +#include namespace clickhouse { @@ -9,7 +10,7 @@ ColumnDate::ColumnDate() } void ColumnDate::Append(const std::time_t& value) { - /// TODO: This code is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. data_->Append(static_cast(value / std::time_t(86400))); } @@ -18,9 +19,18 @@ void ColumnDate::Clear() { } std::time_t ColumnDate::At(size_t n) const { + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. return static_cast(data_->At(n)) * 86400; } +void ColumnDate::AppendRaw(uint16_t value) { + data_->Append(value); +} + +uint16_t ColumnDate::RawAt(size_t n) const { + return data_->At(n); +} + void ColumnDate::Append(ColumnRef column) { if (auto col = column->As()) { data_->Append(col->data_); @@ -62,7 +72,6 @@ ItemView ColumnDate::GetItem(size_t index) const { } - ColumnDate32::ColumnDate32() : Column(Type::CreateDate32()) , data_(std::make_shared()) @@ -70,7 +79,7 @@ ColumnDate32::ColumnDate32() } void ColumnDate32::Append(const std::time_t& value) { - /// TODO: This code is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. data_->Append(static_cast(value / std::time_t(86400))); } @@ -79,6 +88,7 @@ void ColumnDate32::Clear() { } std::time_t ColumnDate32::At(size_t n) const { + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. return static_cast(data_->At(n)) * 86400; } @@ -88,6 +98,14 @@ void ColumnDate32::Append(ColumnRef column) { } } +void ColumnDate32::AppendRaw(int32_t value) { + data_->Append(value); +} + +int32_t ColumnDate32::RawAt(size_t n) const { + return data_->At(n); +} + bool ColumnDate32::LoadBody(InputStream* input, size_t rows) { return data_->LoadBody(input, rows); } @@ -122,7 +140,6 @@ ItemView ColumnDate32::GetItem(size_t index) const { return ItemView{Type()->GetCode(), data_->GetItem(index)}; } - ColumnDateTime::ColumnDateTime() : Column(Type::CreateDateTime()) , data_(std::make_shared()) @@ -241,6 +258,7 @@ void ColumnDateTime64::SaveBody(OutputStream* output) { void ColumnDateTime64::Clear() { data_->Clear(); } + size_t ColumnDateTime64::Size() const { return data_->Size(); } diff --git a/clickhouse/columns/date.h b/clickhouse/columns/date.h index 2a240c90..9b170001 100644 --- a/clickhouse/columns/date.h +++ b/clickhouse/columns/date.h @@ -15,12 +15,17 @@ class ColumnDate : public Column { ColumnDate(); /// Appends one element to the end of column. - /// TODO: The implementation is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. void Append(const std::time_t& value); /// Returns element at given row number. - /// TODO: The implementation is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. std::time_t At(size_t n) const; + inline std::time_t operator [] (size_t n) const { return At(n); } + + /// Do append data as is -- number of day in Unix epoch, no conversions performed. + void AppendRaw(uint16_t value); + uint16_t RawAt(size_t n) const; /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; @@ -56,16 +61,22 @@ class ColumnDate32 : public Column { ColumnDate32(); /// Appends one element to the end of column. - /// TODO: The implementation is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. void Append(const std::time_t& value); /// Returns element at given row number. - /// TODO: The implementation is fundamentally wrong. + /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving. std::time_t At(size_t n) const; /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; + inline std::time_t operator [] (size_t n) const { return At(n); } + + /// Do append data as is -- number of day in Unix epoch (32bit signed), no conversions performed. + void AppendRaw(int32_t value); + int32_t RawAt(size_t n) const; + /// Loads column data from input stream. bool LoadBody(InputStream* input, size_t rows) override; @@ -90,7 +101,7 @@ class ColumnDate32 : public Column { }; -/** */ +/** DateTime64 supports date-time values (number of seconds since UNIX epoch), from 1970 up to 2130. */ class ColumnDateTime : public Column { public: using ValueType = std::time_t; @@ -103,6 +114,7 @@ class ColumnDateTime : public Column { /// Returns element at given row number. std::time_t At(size_t n) const; + inline std::time_t operator [] (size_t n) const { return At(n); } /// Timezone associated with a data column. std::string Timezone() const; @@ -135,7 +147,7 @@ class ColumnDateTime : public Column { }; -/** */ +/** DateTime64 supports date-time values of arbitrary sub-second precision, from 1900 up to 2300. */ class ColumnDateTime64 : public Column { public: using ValueType = Int64; @@ -152,6 +164,8 @@ class ColumnDateTime64 : public Column { /// Returns element at given row number. Int64 At(size_t n) const; + inline Int64 operator[](size_t n) const { return At(n); } + /// Timezone associated with a data column. std::string Timezone() const; diff --git a/clickhouse/columns/decimal.h b/clickhouse/columns/decimal.h index d3c05ea2..4b09553a 100644 --- a/clickhouse/columns/decimal.h +++ b/clickhouse/columns/decimal.h @@ -18,6 +18,7 @@ class ColumnDecimal : public Column { void Append(const std::string& value); Int128 At(size_t i) const; + inline auto operator[](size_t i) const { return At(i); } public: void Append(ColumnRef column) override; diff --git a/clickhouse/columns/enum.cpp b/clickhouse/columns/enum.cpp index 1361e817..c84d9847 100644 --- a/clickhouse/columns/enum.cpp +++ b/clickhouse/columns/enum.cpp @@ -20,6 +20,13 @@ ColumnEnum::ColumnEnum(TypeRef type, const std::vector& data) { } +template +ColumnEnum::ColumnEnum(TypeRef type, std::vector&& data) + : Column(type) + , data_(std::move(data)) +{ +} + template void ColumnEnum::Append(const T& value, bool checkValue) { if (checkValue) { @@ -30,7 +37,7 @@ void ColumnEnum::Append(const T& value, bool checkValue) { template void ColumnEnum::Append(const std::string& name) { - data_.push_back(type_->As()->GetEnumValue(name)); + data_.push_back(static_cast(type_->As()->GetEnumValue(name))); } template @@ -48,11 +55,6 @@ std::string_view ColumnEnum::NameAt(size_t n) const { return type_->As()->GetEnumName(data_.at(n)); } -template -const T& ColumnEnum::operator[] (size_t n) const { - return data_[n]; -} - template void ColumnEnum::SetAt(size_t n, const T& value, bool checkValue) { if (checkValue) { @@ -63,7 +65,7 @@ void ColumnEnum::SetAt(size_t n, const T& value, bool checkValue) { template void ColumnEnum::SetNameAt(size_t n, const std::string& name) { - data_.at(n) = type_->As()->GetEnumValue(name); + data_.at(n) = static_cast(type_->As()->GetEnumValue(name)); } template diff --git a/clickhouse/columns/enum.h b/clickhouse/columns/enum.h index c31b81ff..1d962751 100644 --- a/clickhouse/columns/enum.h +++ b/clickhouse/columns/enum.h @@ -12,6 +12,7 @@ class ColumnEnum : public Column { ColumnEnum(TypeRef type); ColumnEnum(TypeRef type, const std::vector& data); + ColumnEnum(TypeRef type, std::vector&& data); /// Appends one element to the end of column. void Append(const T& value, bool checkValue = false); @@ -22,7 +23,7 @@ class ColumnEnum : public Column { std::string_view NameAt(size_t n) const; /// Returns element at given row number. - const T& operator[] (size_t n) const; + inline const T& operator[] (size_t n) const { return At(n); } /// Set element at given row number. void SetAt(size_t n, const T& value, bool checkValue = false); diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index e003b7f5..aeacdabc 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -38,7 +38,7 @@ const auto& GetASTChildElement(const TypeAst & ast, int position) { throw ValidationError("AST child element index out of bounds: " + std::to_string(position)); if (position < 0) - position = ast.elements.size() + position; + position = static_cast(ast.elements.size() + position); return ast.elements[static_cast(position)]; } diff --git a/clickhouse/columns/geo.cpp b/clickhouse/columns/geo.cpp index ebea9895..e618fbe5 100644 --- a/clickhouse/columns/geo.cpp +++ b/clickhouse/columns/geo.cpp @@ -54,11 +54,6 @@ const typename ColumnGeo::ValueType ColumnGeoAt(n); } -template -const typename ColumnGeo::ValueType ColumnGeo::operator[](size_t n) const { - return data_->At(n); -} - template void ColumnGeo::Append(ColumnRef column) { if (auto col = column->template As()) { diff --git a/clickhouse/columns/geo.h b/clickhouse/columns/geo.h index 5f3db9b6..c3757f8a 100644 --- a/clickhouse/columns/geo.h +++ b/clickhouse/columns/geo.h @@ -26,7 +26,7 @@ class ColumnGeo : public Column { const ValueType At(size_t n) const; /// Returns element at given row number. - const ValueType operator[](size_t n) const; + inline const ValueType operator[](size_t n) const { return At(n); } public: /// Appends content of given column to the end of current one. diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index d3627038..c0c12319 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -199,7 +199,7 @@ std::uint64_t ColumnLowCardinality::getDictionaryIndex(std::uint64_t item_index) void ColumnLowCardinality::appendIndex(std::uint64_t item_index) { // TODO (nemkov): handle case when index should go from UInt8 to UInt16, etc. VisitIndexColumn([item_index](auto & arg) { - arg.Append(item_index); + arg.Append(static_cast::DataType>(item_index)); }, *index_column_); } @@ -227,12 +227,21 @@ ColumnRef ColumnLowCardinality::GetDictionary() { } void ColumnLowCardinality::Append(ColumnRef col) { + // Append values from col only if it is either + // - exactly same type as `this`: LowCardinality wrapping same dictionary type + // - same type as dictionary column + auto c = col->As(); - if (!c || !dictionary_column_->Type()->IsEqual(c->dictionary_column_->Type())) - return; + // If not LowCardinality of same dictionary type + if (!c || !dictionary_column_->Type()->IsEqual(c->dictionary_column_->Type())) { + // If not column of the same type as dictionary type + if (!dictionary_column_->Type()->IsEqual(col->GetType())) { + return; + } + } - for (size_t i = 0; i < c->Size(); ++i) { - AppendUnsafe(c->GetItem(i)); + for (size_t i = 0; i < col->Size(); ++i) { + AppendUnsafe(col->GetItem(i)); } } diff --git a/clickhouse/columns/map.h b/clickhouse/columns/map.h index 24a8b4ae..ac5dc0a7 100644 --- a/clickhouse/columns/map.h +++ b/clickhouse/columns/map.h @@ -212,7 +212,7 @@ class ColumnMapT : public ColumnMap { inline auto At(size_t index) const { return MapValueView{typed_data_->At(index)}; } - inline auto operator[](size_t index) const { return MapValueView{typed_data_->At(index)}; } + inline auto operator[](size_t index) const { return At(index); } using ColumnMap::Append; diff --git a/clickhouse/columns/nothing.h b/clickhouse/columns/nothing.h index 36ddeaea..0b28d572 100644 --- a/clickhouse/columns/nothing.h +++ b/clickhouse/columns/nothing.h @@ -33,7 +33,7 @@ class ColumnNothing : public Column { std::nullptr_t At(size_t) const { return nullptr; }; /// Returns element at given row number. - std::nullptr_t operator [] (size_t) const { return nullptr; }; + inline std::nullptr_t operator [] (size_t) const { return nullptr; }; /// Makes slice of the current column. ColumnRef Slice(size_t, size_t len) const override { diff --git a/clickhouse/columns/numeric.cpp b/clickhouse/columns/numeric.cpp index 4e8d54bf..81d6c721 100644 --- a/clickhouse/columns/numeric.cpp +++ b/clickhouse/columns/numeric.cpp @@ -48,11 +48,6 @@ const T& ColumnVector::At(size_t n) const { return data_.at(n); } -template -const T& ColumnVector::operator [] (size_t n) const { - return data_[n]; -} - template void ColumnVector::Append(ColumnRef column) { if (auto col = column->As>()) { diff --git a/clickhouse/columns/numeric.h b/clickhouse/columns/numeric.h index c6da981e..dcc344bb 100644 --- a/clickhouse/columns/numeric.h +++ b/clickhouse/columns/numeric.h @@ -26,7 +26,7 @@ class ColumnVector : public Column { const T& At(size_t n) const; /// Returns element at given row number. - const T& operator [] (size_t n) const; + inline const T& operator [] (size_t n) const { return At(n); } void Erase(size_t pos, size_t count = 1); diff --git a/clickhouse/columns/string.cpp b/clickhouse/columns/string.cpp index 937e6035..173c024f 100644 --- a/clickhouse/columns/string.cpp +++ b/clickhouse/columns/string.cpp @@ -58,11 +58,6 @@ std::string_view ColumnFixedString::At(size_t n) const { return std::string_view(&data_.at(pos), string_size_); } -std::string_view ColumnFixedString::operator [](size_t n) const { - const auto pos = n * string_size_; - return std::string_view(&data_[pos], string_size_); -} - size_t ColumnFixedString::FixedSize() const { return string_size_; } @@ -232,10 +227,6 @@ std::string_view ColumnString::At(size_t n) const { return items_.at(n); } -std::string_view ColumnString::operator [] (size_t n) const { - return items_[n]; -} - void ColumnString::Append(ColumnRef column) { if (auto col = column->As()) { const auto total_size = ComputeTotalSize(col->items_); @@ -252,27 +243,38 @@ void ColumnString::Append(ColumnRef column) { } bool ColumnString::LoadBody(InputStream* input, size_t rows) { - items_.clear(); - blocks_.clear(); + if (rows == 0) { + items_.clear(); + blocks_.clear(); + + return true; + } - items_.reserve(rows); - Block * block = nullptr; + decltype(items_) new_items; + decltype(blocks_) new_blocks; + + new_items.reserve(rows); + + // Suboptimzal if the first row string is >DEFAULT_BLOCK_SIZE, but that must be a very rare case. + Block * block = &new_blocks.emplace_back(DEFAULT_BLOCK_SIZE); - // TODO(performance): unroll a loop to a first row (to get rid of `blocks_.size() == 0` check) and the rest. for (size_t i = 0; i < rows; ++i) { uint64_t len; if (!WireFormat::ReadUInt64(*input, &len)) return false; - if (blocks_.size() == 0 || len > block->GetAvailable()) - block = &blocks_.emplace_back(std::max(DEFAULT_BLOCK_SIZE, len)); + if (len > block->GetAvailable()) + block = &new_blocks.emplace_back(std::max(DEFAULT_BLOCK_SIZE, len)); if (!WireFormat::ReadBytes(*input, block->GetCurrentWritePos(), len)) return false; - items_.emplace_back(block->ConsumeTailAsStringViewUnsafe(len)); + new_items.emplace_back(block->ConsumeTailAsStringViewUnsafe(len)); } + items_.swap(new_items); + blocks_.swap(new_blocks); + return true; } diff --git a/clickhouse/columns/string.h b/clickhouse/columns/string.h index 9b83a088..aa78270e 100644 --- a/clickhouse/columns/string.h +++ b/clickhouse/columns/string.h @@ -34,7 +34,7 @@ class ColumnFixedString : public Column { std::string_view At(size_t n) const; /// Returns element at given row number. - std::string_view operator [] (size_t n) const; + inline std::string_view operator [] (size_t n) const { return At(n); } /// Returns the max size of the fixed string size_t FixedSize() const; @@ -101,7 +101,7 @@ class ColumnString : public Column { std::string_view At(size_t n) const; /// Returns element at given row number. - std::string_view operator [] (size_t n) const; + inline std::string_view operator [] (size_t n) const { return At(n); } public: /// Appends content of given column to the end of current one. diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index b1b5ad31..c9795565 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -17,11 +17,11 @@ class ColumnTuple : public Column { /// Returns count of columns in the tuple. size_t TupleSize() const; - ColumnRef operator [] (size_t n) const { + inline ColumnRef operator [] (size_t n) const { return columns_[n]; } - ColumnRef At(size_t n) const { + inline ColumnRef At(size_t n) const { return columns_[n]; } diff --git a/clickhouse/columns/uuid.cpp b/clickhouse/columns/uuid.cpp index 19e94761..36a7229c 100644 --- a/clickhouse/columns/uuid.cpp +++ b/clickhouse/columns/uuid.cpp @@ -34,10 +34,6 @@ const UUID ColumnUUID::At(size_t n) const { return UUID(data_->At(n * 2), data_->At(n * 2 + 1)); } -const UUID ColumnUUID::operator [] (size_t n) const { - return UUID((*data_)[n * 2], (*data_)[n * 2 + 1]); -} - void ColumnUUID::Append(ColumnRef column) { if (auto col = column->As()) { data_->Append(col->data_); diff --git a/clickhouse/columns/uuid.h b/clickhouse/columns/uuid.h index dd7d0b9d..4f6c9192 100644 --- a/clickhouse/columns/uuid.h +++ b/clickhouse/columns/uuid.h @@ -23,7 +23,7 @@ class ColumnUUID : public Column { const UUID At(size_t n) const; /// Returns element at given row number. - const UUID operator [] (size_t n) const; + inline const UUID operator [] (size_t n) const { return At(n); } public: /// Appends content of given column to the end of current one. diff --git a/cmake/Findcityhash.cmake b/cmake/Findcityhash.cmake new file mode 100644 index 00000000..5f71dcc4 --- /dev/null +++ b/cmake/Findcityhash.cmake @@ -0,0 +1,26 @@ +find_path(cityhash_INCLUDE_DIR + NAMES city.h + DOC "cityhash include directory") +mark_as_advanced(cityhash_INCLUDE_DIR) +find_library(cityhash_LIBRARY + NAMES cityhash libcityhash + DOC "cityhash library") +mark_as_advanced(cityhash_LIBRARY) + +# Unlike lz4, cityhash's version information does not seem to be available. + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(cityhash + REQUIRED_VARS cityhash_LIBRARY cityhash_INCLUDE_DIR) + +if (cityhash_FOUND) + set(cityhash_INCLUDE_DIRS "${cityhash_INCLUDE_DIR}") + set(cityhash_LIBRARIES "${cityhash_LIBRARY}") + + if (NOT TARGET cityhash::cityhash) + add_library(cityhash::cityhash UNKNOWN IMPORTED) + set_target_properties(cityhash::cityhash PROPERTIES + IMPORTED_LOCATION "${cityhash_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${cityhash_INCLUDE_DIR}") + endif () +endif () diff --git a/cmake/Findlz4.cmake b/cmake/Findlz4.cmake index ef8a366e..20917b84 100644 --- a/cmake/Findlz4.cmake +++ b/cmake/Findlz4.cmake @@ -32,6 +32,7 @@ if (lz4_FOUND) if (NOT TARGET lz4::lz4) add_library(lz4::lz4 UNKNOWN IMPORTED) set_target_properties(lz4::lz4 PROPERTIES + INCLUDE_DIRECTORIES ${lz4_INCLUDE_DIRS} IMPORTED_LOCATION "${lz4_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}") endif () diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index 51340a86..2911e559 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -8,7 +8,8 @@ #include #include #include - +#include +#include #if defined(_MSC_VER) # pragma warning(disable : 4996) #endif @@ -496,8 +497,12 @@ static void RunTests(Client& client) { int main() { try { const auto localHostEndpoint = ClientOptions() - .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetEndpoints({ {"asasdasd", 9000} + ,{"localhost"} + ,{"noalocalhost", 9000} + }) .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); @@ -506,6 +511,7 @@ int main() { Client client(ClientOptions(localHostEndpoint) .SetPingBeforeQuery(true)); RunTests(client); + std::cout << "current endpoint : " << client.GetCurrentEndpoint().value().host << "\n"; } { diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index 2c9f6ee5..6b395458 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -15,6 +15,7 @@ SET ( clickhouse-cpp-ut-src performance_tests.cpp tcp_server.cpp readonly_client_test.cpp + abnormal_column_names_test.cpp connection_failed_client_test.cpp array_of_low_cardinality_tests.cpp CreateColumnByType_ut.cpp diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index 544be975..6e5668e6 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -16,6 +16,9 @@ #include #include +#include +#include +#include #include "utils.h" #include "roundtrip_column.h" @@ -43,53 +46,40 @@ std::ostream& operator<<(std::ostream& ostr, const Type::Code& type_code) { // 6. Swap: create two instances, populate one with data, swap with second, make sure has data was transferred // 7. Load/Save: create, append some data, save to buffer, load from same buffer into new column, make sure columns match. +template (*CreatorFunction)(), + typename GeneratorValueType, + typename std::vector (*GeneratorFunction)()> +struct GenericColumnTestCase +{ + using ColumnType = ColumnTypeT; + static constexpr auto Creator = CreatorFunction; + static constexpr auto Generator = GeneratorFunction; + + static auto createColumn() + { + return CreatorFunction(); + } + + static auto generateValues() + { + return GeneratorFunction(); + } +}; + template class GenericColumnTest : public testing::Test { public: - using ColumnType = std::decay_t; - - static auto MakeColumn() { - if constexpr (std::is_same_v) { - return std::make_shared(12); - } else if constexpr (std::is_same_v) { - return std::make_shared(3); - } else if constexpr (std::is_same_v) { - return std::make_shared(10, 5); - } else { - return std::make_shared(); - } + using ColumnType = typename T::ColumnType; + + static auto MakeColumn() + { + return T::createColumn(); } - static auto GenerateValues(size_t values_size) { - if constexpr (std::is_same_v) { - return GenerateVector(values_size, FooBarGenerator); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeFixedStrings(12)}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeDates()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeDateTimes()}); - } else if constexpr (std::is_same_v) { - return MakeDateTime64s(3u, values_size); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeDates32()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeIPv4s()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeIPv6s()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeInt128s()}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeDecimals(3, 10)}); - } else if constexpr (std::is_same_v) { - return GenerateVector(values_size, FromVectorGenerator{MakeUUIDs()}); - } else if constexpr (std::is_integral_v) { - // ColumnUIntX and ColumnIntX - return GenerateVector(values_size, RandomGenerator()); - } else if constexpr (std::is_floating_point_v) { - // OR ColumnFloatX - return GenerateVector(values_size, RandomGenerator()); - } + static auto GenerateValues(size_t values_size) + { + return GenerateVector(values_size, FromVectorGenerator{T::generateValues()}); } template @@ -107,7 +97,7 @@ class GenericColumnTest : public testing::Test { return std::tuple{column, values}; } - static std::optional SkipTest(clickhouse::Client& client) { + static std::optional CheckIfShouldSkipTest(clickhouse::Client& client) { if constexpr (std::is_same_v) { // Date32 first appeared in v21.9.2.17-stable const auto server_info = client.GetServerInfo(); @@ -128,20 +118,118 @@ class GenericColumnTest : public testing::Test { } return std::nullopt; } + + template + static void TestColumnRoundtrip(const std::shared_ptr & column, const ClientOptions & client_options) + { + SCOPED_TRACE(::testing::Message("Column type: ") << column->GetType().GetName()); + SCOPED_TRACE(::testing::Message("Client options: ") << client_options); + + clickhouse::Client client(client_options); + + if (auto message = CheckIfShouldSkipTest(client)) { + GTEST_SKIP() << *message; + } + + auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); + EXPECT_TRUE(CompareRecursive(*column, *result_typed)); + } + + + template + static void TestColumnRoundtrip(const ColumnType & column, const ClientOptions & client_options, CompressionMethods && compression_methods) + { + for (auto compressionMethod : compression_methods) + { + ClientOptions new_options = ClientOptions(client_options).SetCompressionMethod(compressionMethod); + TestColumnRoundtrip(column, new_options); + } + } +}; + +// Luckily all (non-data copying/moving) constructors have size_t params. +template +auto makeColumn() +{ + return std::make_shared(ConstructorParams...); +} + +template +struct NumberColumnTestCase : public GenericColumnTestCase, typename ColumnTypeT::ValueType, &MakeNumbers> +{ + using Base = GenericColumnTestCase, typename ColumnTypeT::ValueType, &MakeNumbers>; + + using ColumnType = typename Base::ColumnType; + using Base::createColumn; + using Base::generateValues; +}; + +template +struct DecimalColumnTestCase : public GenericColumnTestCase, clickhouse::Int128, &MakeDecimals> +{ + using Base = GenericColumnTestCase, clickhouse::Int128, &MakeDecimals>; + + using ColumnType = typename Base::ColumnType; + using Base::createColumn; + using Base::generateValues; }; -using ValueColumns = ::testing::Types< - ColumnUInt8, ColumnUInt16, ColumnUInt32, ColumnUInt64 - , ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64 - , ColumnFloat32, ColumnFloat64 - , ColumnString, ColumnFixedString - , ColumnDate, ColumnDateTime, ColumnDateTime64, ColumnDate32 - , ColumnIPv4, ColumnIPv6 - , ColumnInt128 - , ColumnDecimal - , ColumnUUID ->; -TYPED_TEST_SUITE(GenericColumnTest, ValueColumns); +using TestCases = ::testing::Types< + NumberColumnTestCase, + NumberColumnTestCase, + NumberColumnTestCase, + NumberColumnTestCase, + + NumberColumnTestCase, + NumberColumnTestCase, + NumberColumnTestCase, + NumberColumnTestCase, + + NumberColumnTestCase, + NumberColumnTestCase, + + GenericColumnTestCase, std::string, &MakeStrings>, + GenericColumnTestCase, std::string, &MakeFixedStrings<12>>, + + GenericColumnTestCase, time_t, &MakeDates>, + GenericColumnTestCase, time_t, &MakeDates>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTimes>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTime64s<0>>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTime64s<3>>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTime64s<6>>, + GenericColumnTestCase, clickhouse::Int64, &MakeDateTime64s<9>>, + + GenericColumnTestCase, in_addr, &MakeIPv4s>, + GenericColumnTestCase, in6_addr, &MakeIPv6s>, + + GenericColumnTestCase, clickhouse::Int128, &MakeInt128s>, + GenericColumnTestCase, clickhouse::UUID, &MakeUUIDs>, + + DecimalColumnTestCase, + DecimalColumnTestCase, + DecimalColumnTestCase, + // there is an arithmetical overflow on some values in the test value generator harness + // DecimalColumnTestCase, + + DecimalColumnTestCase, + DecimalColumnTestCase, + DecimalColumnTestCase, + + DecimalColumnTestCase, + DecimalColumnTestCase, + + GenericColumnTestCase, &makeColumn>, std::string, &MakeStrings> + + // Array(String) +// GenericColumnTestCase, &makeColumn>, std::vector, &MakeArrays> + +// // Array(Array(String)) +// GenericColumnTestCase>, &makeColumn>>, +// std::vector>, +// &MakeArrays, &MakeArrays>> + >; + +TYPED_TEST_SUITE(GenericColumnTest, TestCases); TYPED_TEST(GenericColumnTest, Construct) { auto column = this->MakeColumn(); @@ -176,7 +264,7 @@ TYPED_TEST(GenericColumnTest, EmptyColumn) { TYPED_TEST(GenericColumnTest, Append) { auto column = this->MakeColumn(); - const auto values = this->GenerateValues(100); + const auto values = this->GenerateValues(10'000); for (const auto & v : values) { EXPECT_NO_THROW(column->Append(v)); @@ -191,8 +279,8 @@ inline auto convertValueForGetItem(const ColumnType& col, ValueType&& t) { using T = std::remove_cv_t>; if constexpr (std::is_same_v) { - // Since ColumnDecimal can hold 32, 64, 128-bit wide data and there is no way telling at run-time. - const ItemView item = col.GetItem(0); + // Since ColumnDecimal can hold 32, 64, 128-bit wide data and there is no way telling at run-time. + const ItemView item = col.GetItem(0); return std::string_view(reinterpret_cast(&t), item.data.size()); } else if constexpr (std::is_same_v || std::is_same_v) { @@ -213,21 +301,29 @@ inline auto convertValueForGetItem(const ColumnType& col, ValueType&& t) { } TYPED_TEST(GenericColumnTest, GetItem) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); ASSERT_EQ(values.size(), column->Size()); - ASSERT_EQ(column->GetItem(0).type, column->GetType().GetCode()); + const auto wrapping_types = std::set{ + Type::Code::LowCardinality, Type::Code::Array, Type::Code::Nullable + }; + + // For wrapping types, type of ItemView can be different from type of column + if (wrapping_types.find(column->GetType().GetCode()) == wrapping_types.end() ) { + EXPECT_EQ(column->GetItem(0).type, column->GetType().GetCode()); + } for (size_t i = 0; i < values.size(); ++i) { const auto v = convertValueForGetItem(*column, values[i]); const ItemView item = column->GetItem(i); - ASSERT_TRUE(CompareRecursive(item.get(), v)); + ASSERT_TRUE(CompareRecursive(v, item.get())) + << " On item " << i << " of " << PrintContainer{values}; } } TYPED_TEST(GenericColumnTest, Slice) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); auto untyped_slice = column->Slice(0, column->Size()); auto slice = untyped_slice->template AsStrict(); @@ -239,7 +335,7 @@ TYPED_TEST(GenericColumnTest, Slice) { } TYPED_TEST(GenericColumnTest, CloneEmpty) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); EXPECT_EQ(values.size(), column->Size()); auto clone_untyped = column->CloneEmpty(); @@ -251,7 +347,7 @@ TYPED_TEST(GenericColumnTest, CloneEmpty) { } TYPED_TEST(GenericColumnTest, Clear) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); EXPECT_EQ(values.size(), column->Size()); column->Clear(); @@ -259,7 +355,7 @@ TYPED_TEST(GenericColumnTest, Clear) { } TYPED_TEST(GenericColumnTest, Swap) { - auto [column_A, values] = this->MakeColumnWithValues(100); + auto [column_A, values] = this->MakeColumnWithValues(10'000); auto column_B = this->MakeColumn(); column_A->Swap(*column_B); @@ -271,18 +367,21 @@ TYPED_TEST(GenericColumnTest, Swap) { TYPED_TEST(GenericColumnTest, LoadAndSave) { auto [column_A, values] = this->MakeColumnWithValues(100); - char buffer[4096] = {'\0'}; + // large buffer since we have pretty big values for String column + auto const BufferSize = 10*1024*1024; + std::unique_ptr buffer = std::make_unique(BufferSize); + memset(buffer.get(), 0, BufferSize); { - ArrayOutput output(buffer, sizeof(buffer)); + ArrayOutput output(buffer.get(), BufferSize); // Save - EXPECT_NO_THROW(column_A->Save(&output)); + ASSERT_NO_THROW(column_A->Save(&output)); } auto column_B = this->MakeColumn(); { - ArrayInput input(buffer, sizeof(buffer)); + ArrayInput input(buffer.get(), BufferSize); // Load - EXPECT_TRUE(column_B->Load(&input, values.size())); + ASSERT_TRUE(column_B->Load(&input, values.size())); } EXPECT_TRUE(CompareRecursive(*column_A, *column_B)); @@ -295,24 +394,29 @@ const auto LocalHostEndpoint = ClientOptions() .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); +const auto AllCompressionMethods = { + clickhouse::CompressionMethod::None, + clickhouse::CompressionMethod::LZ4 +}; + TYPED_TEST(GenericColumnTest, RoundTrip) { - auto [column, values] = this->MakeColumnWithValues(100); + auto [column, values] = this->MakeColumnWithValues(10'000); EXPECT_EQ(values.size(), column->Size()); - clickhouse::Client client(LocalHostEndpoint); + this->TestColumnRoundtrip(column, LocalHostEndpoint, AllCompressionMethods); +} - if (auto message = this->SkipTest(client)) { - GTEST_SKIP() << *message; - } +TYPED_TEST(GenericColumnTest, NullableT_RoundTrip) { + using NullableType = ColumnNullableT; - auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); - EXPECT_TRUE(CompareRecursive(*column, *result_typed)); -} + auto non_nullable_column = this->MakeColumn(); + if (non_nullable_column->GetType().GetCode() == Type::Code::LowCardinality) + // TODO (vnemkov): wrap as ColumnLowCardinalityT> instead of ColumnNullableT> + GTEST_SKIP() << "Can't have " << non_nullable_column->GetType().GetName() << " in Nullable"; + + auto column = std::make_shared(std::move(non_nullable_column)); + auto values = this->GenerateValues(10'000); -TYPED_TEST(GenericColumnTest, NulableT_RoundTrip) { - using NullableType = ColumnNullableT; - auto column = std::make_shared(this->MakeColumn()); - auto values = this->GenerateValues(100); FromVectorGenerator is_null({true, false}); for (size_t i = 0; i < values.size(); ++i) { if (is_null(i)) { @@ -322,12 +426,24 @@ TYPED_TEST(GenericColumnTest, NulableT_RoundTrip) { } } - clickhouse::Client client(LocalHostEndpoint); + this->TestColumnRoundtrip(column, LocalHostEndpoint, AllCompressionMethods); +} + +TYPED_TEST(GenericColumnTest, ArrayT_RoundTrip) { + using ColumnArrayType = ColumnArrayT; - if (auto message = this->SkipTest(client)) { - GTEST_SKIP() << *message; + auto [nested_column, values] = this->MakeColumnWithValues(100); + + auto column = std::make_shared(nested_column->CloneEmpty()->template As()); + for (size_t i = 0; i < values.size(); ++i) + { + const std::vector> row{values.begin(), values.begin() + i}; + column->Append(values.begin(), values.begin() + i); + + EXPECT_TRUE(CompareRecursive(row, (*column)[column->Size() - 1])); } + EXPECT_EQ(values.size(), column->Size()); - auto result_typed = WrapColumn(RoundtripColumnValues(client, column)); - EXPECT_TRUE(CompareRecursive(*column, *result_typed)); + this->TestColumnRoundtrip(column, LocalHostEndpoint, AllCompressionMethods); } + diff --git a/ut/abnormal_column_names_test.cpp b/ut/abnormal_column_names_test.cpp new file mode 100644 index 00000000..11868f73 --- /dev/null +++ b/ut/abnormal_column_names_test.cpp @@ -0,0 +1,81 @@ + +#include +#include + +#include "utils.h" + +#include + +#include +#include + +namespace { +using namespace clickhouse; + +std::string getColumnNames(const Block& block) { + std::string result; + for (size_t i = 0; i < block.GetColumnCount(); ++i) { + result += block.GetColumnName(i); + if (i != block.GetColumnCount() - 1) + result += ','; + } + + return result; +} +} + +struct AbnormalColumnNamesClientTestCase { + ClientOptions client_options; + std::vector queries; + std::vector expected_names; +}; + +class AbnormalColumnNamesClientTest : public testing::TestWithParam { +protected: + void SetUp() override { + client_ = std::make_unique(GetParam().client_options); + } + void TearDown() override { + client_.reset(); + } + + std::unique_ptr client_; +}; + + +TEST_P(AbnormalColumnNamesClientTest, Select) { + const auto & queries = GetParam().queries; + for (size_t i = 0; i < queries.size(); ++i) { + + const auto & query = queries.at(i); + const auto & expected = GetParam().expected_names[i]; + + client_->Select(query, [query, expected](const Block& block) { + if (block.GetRowCount() == 0 || block.GetColumnCount() == 0) + return; + + EXPECT_EQ(1UL, block.GetRowCount()); + + EXPECT_EQ(expected, getColumnNames(block)) + << "For query: " << query; + }); + } +} + + +INSTANTIATE_TEST_SUITE_P(ClientColumnNames, AbnormalColumnNamesClientTest, + ::testing::Values(AbnormalColumnNamesClientTest::ParamType{ + ClientOptions() + .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetSendRetries(1) + .SetPingBeforeQuery(true) + .SetCompressionMethod(CompressionMethod::None), + {"select 123,231,113", "select 'ABC','AAA','BBB','CCC'"}, + {"123,231,113", "'ABC','AAA','BBB','CCC'"}, + } +)); + diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index dfabcba9..d894cc36 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1196,6 +1196,7 @@ INSTANTIATE_TEST_SUITE_P(ClientLocalReadonly, ReadonlyClientTest, } )); + INSTANTIATE_TEST_SUITE_P(ClientLocalFailed, ConnectionFailedClientTest, ::testing::Values(ConnectionFailedClientTest::ParamType{ ClientOptions() @@ -1210,3 +1211,144 @@ INSTANTIATE_TEST_SUITE_P(ClientLocalFailed, ConnectionFailedClientTest, ExpectingException{"Authentication failed: password is incorrect"} } )); + + +class ConnectionSuccessTestCase : public testing::TestWithParam {}; + +TEST_P(ConnectionSuccessTestCase, SuccessConnectionEstablished) { + const auto & client_options = GetParam(); + std::unique_ptr client; + + try { + client = std::make_unique(client_options); + auto endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + SUCCEED(); + } catch (const std::exception & e) { + FAIL() << "Got an unexpected exception : " << e.what(); + } +} + + +INSTANTIATE_TEST_SUITE_P(ClientMultipleEndpoints, ConnectionSuccessTestCase, + ::testing::Values(ClientCase::ParamType{ + ClientOptions() + .SetEndpoints({ + {"somedeadhost", 9000} + , {"deadaginghost", 1245} + , {"localhost", 9000} + , {"noalocalhost", 6784} + }) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetPingBeforeQuery(true) + .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) + .SetRetryTimeout(std::chrono::seconds(1)), + } +)); + +INSTANTIATE_TEST_SUITE_P(ClientMultipleEndpointsWithDefaultPort, ConnectionSuccessTestCase, + ::testing::Values(ClientCase::ParamType{ + ClientOptions() + .SetEndpoints({ + {"somedeadhost"} + , {"deadaginghost", 1245} + , {"localhost"} + , {"noalocalhost", 6784} + }) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetPingBeforeQuery(true) + .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) + .SetRetryTimeout(std::chrono::seconds(1)), + } +)); + +INSTANTIATE_TEST_SUITE_P(MultipleEndpointsFailed, ConnectionFailedClientTest, + ::testing::Values(ConnectionFailedClientTest::ParamType{ + ClientOptions() + .SetEndpoints({ + {"deadaginghost", 9000} + ,{"somedeadhost", 1245} + ,{"noalocalhost", 6784} + }) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetPingBeforeQuery(true) + .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) + .SetRetryTimeout(std::chrono::seconds(1)), + ExpectingException{""} + } +)); + +class ResetConnectionTestCase : public testing::TestWithParam {}; + +TEST_P(ResetConnectionTestCase, ResetConnectionEndpointTest) { + const auto & client_options = GetParam(); + std::unique_ptr client; + + try { + client = std::make_unique(client_options); + auto endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + client->ResetConnectionEndpoint(); + endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("127.0.0.1", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + client->ResetConnectionEndpoint(); + + endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + SUCCEED(); + } catch (const std::exception & e) { + FAIL() << "Got an unexpected exception : " << e.what(); + } +} + +TEST_P(ResetConnectionTestCase, ResetConnectionTest) { + const auto & client_options = GetParam(); + std::unique_ptr client; + + try { + client = std::make_unique(client_options); + auto endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + client->ResetConnection(); + endpoint = client->GetCurrentEndpoint().value(); + ASSERT_EQ("localhost", endpoint.host); + ASSERT_EQ(9000u, endpoint.port); + + SUCCEED(); + } catch (const std::exception & e) { + FAIL() << "Got an unexpected exception : " << e.what(); + } +} + +INSTANTIATE_TEST_SUITE_P(ResetConnectionClientTest, ResetConnectionTestCase, + ::testing::Values(ResetConnectionTestCase::ParamType { + ClientOptions() + .SetEndpoints({ + {"localhost", 9000} + ,{"somedeadhost", 1245} + ,{"noalocalhost", 6784} + ,{"127.0.0.1", 9000} + }) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")) + .SetPingBeforeQuery(true) + .SetConnectionConnectTimeout(std::chrono::milliseconds(200)) + .SetRetryTimeout(std::chrono::seconds(1)) + } +)); diff --git a/ut/columns_ut.cpp b/ut/columns_ut.cpp index eda33292..152cf65f 100644 --- a/ut/columns_ut.cpp +++ b/ut/columns_ut.cpp @@ -105,9 +105,10 @@ TEST(ColumnsCase, FixedString_Append_LargeString) { } TEST(ColumnsCase, StringInit) { - auto col = std::make_shared(MakeStrings()); + auto values = MakeStrings(); + auto col = std::make_shared(values); - ASSERT_EQ(col->Size(), 4u); + ASSERT_EQ(col->Size(), values.size()); ASSERT_EQ(col->At(1), "ab"); ASSERT_EQ(col->At(3), "abcd"); } @@ -172,6 +173,33 @@ TEST(ColumnsCase, DateAppend) { ASSERT_EQ(col2->At(0), (now / 86400) * 86400); } + +TEST(ColumnsCase, Date_UInt16_interface) { + auto col1 = std::make_shared(); + + col1->AppendRaw(1u); + col1->AppendRaw(1234u); + + ASSERT_EQ(col1->Size(), 2u); + ASSERT_EQ(col1->RawAt(0), 1u); + ASSERT_EQ(col1->RawAt(1), 1234u); +} + + +TEST(ColumnsCase, Date32_Int32_interface) { + auto col1 = std::make_shared(); + + col1->AppendRaw(1); + col1->AppendRaw(1234); + col1->AppendRaw(-1234); + + ASSERT_EQ(col1->Size(), 3u); + ASSERT_EQ(col1->RawAt(0), 1); + ASSERT_EQ(col1->RawAt(1), 1234); + ASSERT_EQ(col1->RawAt(2), -1234); +} + + TEST(ColumnsCase, DateTime64_0) { auto column = std::make_shared(0ul); @@ -181,6 +209,7 @@ TEST(ColumnsCase, DateTime64_0) { ASSERT_EQ(0u, column->Size()); } + TEST(ColumnsCase, DateTime64_6) { auto column = std::make_shared(6ul); diff --git a/ut/low_cardinality_nullable_tests.cpp b/ut/low_cardinality_nullable_tests.cpp index 8918f251..357f28a9 100644 --- a/ut/low_cardinality_nullable_tests.cpp +++ b/ut/low_cardinality_nullable_tests.cpp @@ -122,8 +122,8 @@ TEST(LowCardinalityOfNullable, InsertAndQueryEmpty) { block.AppendColumn("words", column); Client client(ClientOptions(localHostEndpoint) - .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false) - .SetPingBeforeQuery(true)); + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false) + .SetPingBeforeQuery(true)); createTable(client); @@ -141,7 +141,8 @@ TEST(LowCardinalityOfNullable, ThrowOnBackwardsCompatibleLCColumn) { block.AppendColumn("words", column); Client client(ClientOptions(localHostEndpoint) - .SetPingBeforeQuery(true)); + .SetPingBeforeQuery(true) + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(true)); createTable(client); diff --git a/ut/performance_tests.cpp b/ut/performance_tests.cpp index bafa07dd..74bd4f03 100644 --- a/ut/performance_tests.cpp +++ b/ut/performance_tests.cpp @@ -84,7 +84,7 @@ class ColumnPerformanceTest : public ::testing::Test { TYPED_TEST_SUITE_P(ColumnPerformanceTest); // Turns out this is the easiest way to skip test with current version of gtest -#ifndef NDEBUG +#ifdef NDEBUG # define SKIP_IN_DEBUG_BUILDS() (void)(0) #else # define SKIP_IN_DEBUG_BUILDS() GTEST_SKIP_("Test skipped for DEBUG build...") diff --git a/ut/roundtrip_column.cpp b/ut/roundtrip_column.cpp index c4685a3b..5af9624a 100644 --- a/ut/roundtrip_column.cpp +++ b/ut/roundtrip_column.cpp @@ -4,28 +4,49 @@ #include #include +#include +#include "clickhouse/columns/numeric.h" namespace { using namespace clickhouse; + +template +std::vector GenerateConsecutiveNumbers(size_t count, T start = 0) +{ + std::vector result; + result.reserve(count); + + T value = start; + for (size_t i = 0; i < count; ++i, ++value) + { + result.push_back(value); + } + + return result; +} + } + ColumnRef RoundtripColumnValues(Client& client, ColumnRef expected) { - // Create a temporary table with a single column - // insert values from `expected` - // select and aggregate all values from block into `result` column + // Create a temporary table with a corresponding data column + // INSERT values from `expected` + // SELECT and collect all values from block into `result` column auto result = expected->CloneEmpty(); const std::string type_name = result->GetType().GetName(); client.Execute("DROP TEMPORARY TABLE IF EXISTS temporary_roundtrip_table;"); - client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS temporary_roundtrip_table (col " + type_name + ");"); + // id column is to have the same order of rows on SELECT + client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS temporary_roundtrip_table (id UInt32, col " + type_name + ");"); { Block block; block.AppendColumn("col", expected); + block.AppendColumn("id", std::make_shared(GenerateConsecutiveNumbers(expected->Size()))); block.RefreshRowCount(); client.Insert("temporary_roundtrip_table", block); } - client.Select("SELECT col FROM temporary_roundtrip_table", [&result](const Block& b) { + client.Select("SELECT col FROM temporary_roundtrip_table ORDER BY id", [&result](const Block& b) { if (b.GetRowCount() == 0) return; diff --git a/ut/roundtrip_tests.cpp b/ut/roundtrip_tests.cpp index 5f4fce9c..262ebc90 100644 --- a/ut/roundtrip_tests.cpp +++ b/ut/roundtrip_tests.cpp @@ -5,6 +5,7 @@ #include #include +#include using namespace clickhouse; @@ -213,12 +214,16 @@ TEST_P(RoundtripCase, LowCardinalityTString) { TEST_P(RoundtripCase, LowCardinalityTNullableString) { using TestColumn = ColumnLowCardinalityT>; auto col = std::make_shared(); + col->Append("abc"); col->Append("def"); col->Append("abc"); + col->Append(std::nullopt); col->Append("abc"); col->Append(std::nullopt); col->Append(std::nullopt); + col->Append("foobar"); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); EXPECT_TRUE(CompareRecursive(*col, *result_typed)); } diff --git a/ut/utils.h b/ut/utils.h index b9484626..b4b8e92d 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -51,7 +51,8 @@ auto getEnvOrDefault(const std::string& env, const char * default_val) { return std::stoll(value); } else if constexpr (std::is_unsigned_v) { if constexpr (sizeof(ResultType) <= sizeof(unsigned long)) - return std::stoul(value); + // For cases when ResultType is unsigned int. + return static_cast(std::stoul(value)); else if constexpr (sizeof(ResultType) <= sizeof(unsigned long long)) return std::stoull(value); } @@ -166,7 +167,10 @@ std::ostream& operator<<(std::ostream & ostr, const PrintContainer& print_con for (auto i = std::begin(container); i != std::end(container); /*intentionally no ++i*/) { const auto & elem = *i; - if constexpr (is_container_v>) { + if constexpr (is_string_v) { + ostr << '"' << elem << '"'; + } + else if constexpr (is_container_v>) { ostr << PrintContainer{elem}; } else { ostr << elem; diff --git a/ut/utils_comparison.h b/ut/utils_comparison.h index c40033b4..e500e4f3 100644 --- a/ut/utils_comparison.h +++ b/ut/utils_comparison.h @@ -8,6 +8,8 @@ #include #include +#include +#include namespace clickhouse { class Block; @@ -141,7 +143,8 @@ struct PrintContainer; template ::testing::AssertionResult CompareRecursive(const Left & left, const Right & right) { - if constexpr ((is_container_v || std::is_base_of_v>) + if constexpr (!is_string_v && !is_string_v + && (is_container_v || std::is_base_of_v>) && (is_container_v || std::is_base_of_v>) ) { const auto & l = maybeWrapColumnAsContainer(left); @@ -153,10 +156,31 @@ ::testing::AssertionResult CompareRecursive(const Left & left, const Right & rig return result << "\nExpected container: " << PrintContainer{l} << "\nActual container : " << PrintContainer{r}; } else { - if (left != right) + if (left != right) { + + // Handle std::optional(nan) + // I'm too lazy to code comparison against std::nullopt, but this shpudn't be a problem in real life. + // RN comparing against std::nullopt, you'll receive an compilation error. + if constexpr (is_instantiation_of::value && is_instantiation_of::value) + { + if (left.has_value() && right.has_value()) + return CompareRecursive(*left, *right); + } + else if constexpr (is_instantiation_of::value) { + if (left) + return CompareRecursive(*left, right); + } else if constexpr (is_instantiation_of::value) { + if (right) + return CompareRecursive(left, *right); + } else if constexpr (std::is_floating_point_v && std::is_floating_point_v) { + if (std::isnan(left) && std::isnan(right)) + return ::testing::AssertionSuccess(); + } + return ::testing::AssertionFailure() << "\nExpected value: " << left << "\nActual value : " << right; + } return ::testing::AssertionSuccess(); } diff --git a/ut/utils_meta.h b/ut/utils_meta.h index 707f9aca..3c50da4e 100644 --- a/ut/utils_meta.h +++ b/ut/utils_meta.h @@ -2,6 +2,8 @@ #include #include // for std::begin +#include +#include // based on https://stackoverflow.com/a/31207079 template @@ -39,3 +41,13 @@ using my_result_of_t = std::result_of_t; #endif +// https://stackoverflow.com/a/11251408 +template < template class Template, typename T > +struct is_instantiation_of : std::false_type {}; + +template < template class Template, typename... Args > +struct is_instantiation_of< Template, Template > : std::true_type {}; + + +template +inline constexpr bool is_string_v = std::is_same_v> || std::is_same_v>; diff --git a/ut/utils_ut.cpp b/ut/utils_ut.cpp index 0f67c043..bd015751 100644 --- a/ut/utils_ut.cpp +++ b/ut/utils_ut.cpp @@ -1,9 +1,23 @@ #include +#include "ut/value_generators.h" #include "utils.h" +#include +#include #include -TEST(TestCompareContainer, ComparePlain) { +TEST(CompareRecursive, CompareValues) { + EXPECT_TRUE(CompareRecursive(1, 1)); + EXPECT_TRUE(CompareRecursive(1.0f, 1.0f)); + EXPECT_TRUE(CompareRecursive(1.0, 1.0)); + EXPECT_TRUE(CompareRecursive(1.0L, 1.0L)); + + EXPECT_TRUE(CompareRecursive("1.0L", "1.0L")); + EXPECT_TRUE(CompareRecursive(std::string{"1.0L"}, std::string{"1.0L"})); + EXPECT_TRUE(CompareRecursive(std::string_view{"1.0L"}, std::string_view{"1.0L"})); +} + +TEST(CompareRecursive, CompareContainers) { EXPECT_TRUE(CompareRecursive(std::vector{1, 2, 3}, std::vector{1, 2, 3})); EXPECT_TRUE(CompareRecursive(std::vector{}, std::vector{})); @@ -15,7 +29,7 @@ TEST(TestCompareContainer, ComparePlain) { } -TEST(TestCompareContainer, CompareNested) { +TEST(CompareRecursive, CompareNestedContainers) { EXPECT_TRUE(CompareRecursive( std::vector>{{{1, 2, 3}, {4, 5, 6}}}, std::vector>{{{1, 2, 3}, {4, 5, 6}}})); @@ -39,3 +53,70 @@ TEST(StringUtils, UUID) { const std::string uuid_string = "01020304-0506-0708-090a-0b0c0d0e0f10"; EXPECT_EQ(ToString(uuid), uuid_string); } + +TEST(CompareRecursive, Nan) { + /// Even though NaN == NaN is FALSE, CompareRecursive must compare those as TRUE. + + const auto NaNf = std::numeric_limits::quiet_NaN(); + const auto NaNd = std::numeric_limits::quiet_NaN(); + + EXPECT_TRUE(CompareRecursive(NaNf, NaNf)); + EXPECT_TRUE(CompareRecursive(NaNd, NaNd)); + + EXPECT_TRUE(CompareRecursive(NaNf, NaNd)); + EXPECT_TRUE(CompareRecursive(NaNd, NaNf)); + + // 1.0 is arbitrary here + EXPECT_FALSE(CompareRecursive(NaNf, 1.0)); + EXPECT_FALSE(CompareRecursive(NaNf, 1.0)); + EXPECT_FALSE(CompareRecursive(1.0, NaNd)); + EXPECT_FALSE(CompareRecursive(1.0, NaNd)); +} + +TEST(CompareRecursive, Optional) { + EXPECT_TRUE(CompareRecursive(1, std::optional{1})); + EXPECT_TRUE(CompareRecursive(std::optional{1}, 1)); + EXPECT_TRUE(CompareRecursive(std::optional{1}, std::optional{1})); + + EXPECT_FALSE(CompareRecursive(2, std::optional{1})); + EXPECT_FALSE(CompareRecursive(std::optional{1}, 2)); + EXPECT_FALSE(CompareRecursive(std::optional{2}, std::optional{1})); + EXPECT_FALSE(CompareRecursive(std::optional{1}, std::optional{2})); +} + +TEST(CompareRecursive, OptionalNan) { + // Special case for optional comparison: + // NaNs should be considered as equal (compare by unpacking value of optional) + + const auto NaNf = std::numeric_limits::quiet_NaN(); + const auto NaNd = std::numeric_limits::quiet_NaN(); + + const auto NaNfo = std::optional{NaNf}; + const auto NaNdo = std::optional{NaNd}; + + EXPECT_TRUE(CompareRecursive(NaNf, NaNf)); + EXPECT_TRUE(CompareRecursive(NaNf, NaNfo)); + EXPECT_TRUE(CompareRecursive(NaNfo, NaNf)); + EXPECT_TRUE(CompareRecursive(NaNfo, NaNfo)); + + EXPECT_TRUE(CompareRecursive(NaNd, NaNd)); + EXPECT_TRUE(CompareRecursive(NaNd, NaNdo)); + EXPECT_TRUE(CompareRecursive(NaNdo, NaNd)); + EXPECT_TRUE(CompareRecursive(NaNdo, NaNdo)); + + EXPECT_FALSE(CompareRecursive(NaNdo, std::optional{})); + EXPECT_FALSE(CompareRecursive(NaNfo, std::optional{})); + EXPECT_FALSE(CompareRecursive(std::optional{}, NaNdo)); + EXPECT_FALSE(CompareRecursive(std::optional{}, NaNfo)); + + // Too lazy to comparison code against std::nullopt, but this shouldn't be a problem in real life + // following will produce compile time error: +// EXPECT_FALSE(CompareRecursive(NaNdo, std::nullopt)); +// EXPECT_FALSE(CompareRecursive(NaNfo, std::nullopt)); +} + + +TEST(Generators, MakeArrays) { + auto arrays = MakeArrays(); + ASSERT_LT(0u, arrays.size()); +} diff --git a/ut/value_generators.cpp b/ut/value_generators.cpp index 41e36a61..f6d7baf9 100644 --- a/ut/value_generators.cpp +++ b/ut/value_generators.cpp @@ -1,6 +1,7 @@ #include "value_generators.h" #include +#include #include namespace { @@ -16,7 +17,7 @@ std::vector MakeBools() { } std::vector MakeFixedStrings(size_t string_size) { - std::vector result {"aaa", "bbb", "ccc", "ddd"}; + std::vector result = MakeStrings(); std::for_each(result.begin(), result.end(), [string_size](auto& value) { value.resize(string_size, '\0'); @@ -26,7 +27,28 @@ std::vector MakeFixedStrings(size_t string_size) { } std::vector MakeStrings() { - return {"a", "ab", "abc", "abcd"}; + return { + "a", "ab", "abc", "abcd", + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + "long string to test how those are handled. Here goes more text. " + }; } std::vector MakeUUIDs() { @@ -53,25 +75,13 @@ std::vector MakeDateTime64s(size_t scale, size_t values_size) { }); } -std::vector MakeDates() { - // in CH Date internally a UInt16 and stores a day number - // ColumnDate expects values to be seconds, which is then - // converted to day number internally, hence the `* 86400`. - std::vector result {0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536 - 1}; - std::for_each(result.begin(), result.end(), [](auto& value) { - value *= 86400; - }); - - return result; -} - -std::vector MakeDates32() { +std::vector MakeDates32() { // in CH Date32 internally a UInt32 and stores a day number // ColumnDate expects values to be seconds, which is then // converted to day number internally, hence the `* 86400`. // 114634 * 86400 is 2282-11-10, last integer that fits into DateTime32 range // (max is 2283-11-11) - std::vector result = MakeDates(); + std::vector result = MakeDates(); // add corresponding negative values, since pre-epoch date are supported too. const auto size = result.size(); @@ -103,7 +113,7 @@ std::vector MakeInt128s() { std::vector MakeDecimals(size_t /*precision*/, size_t scale) { const auto scale_multiplier = static_cast(std::pow(10, scale)); - const auto rhs_value = 12345678910; + const long long int rhs_value = 12345678910; const std::vector vals {0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536 - 1}; diff --git a/ut/value_generators.h b/ut/value_generators.h index 3632ca9e..031f0fe9 100644 --- a/ut/value_generators.h +++ b/ut/value_generators.h @@ -33,8 +33,7 @@ std::vector MakeBools(); std::vector MakeFixedStrings(size_t string_size); std::vector MakeStrings(); std::vector MakeDateTime64s(size_t scale, size_t values_size = 200); -std::vector MakeDates(); -std::vector MakeDates32(); +std::vector MakeDates32(); std::vector MakeDateTimes(); std::vector MakeIPv4s(); std::vector MakeIPv6s(); @@ -42,6 +41,102 @@ std::vector MakeUUIDs(); std::vector MakeInt128s(); std::vector MakeDecimals(size_t precision, size_t scale); +template ::value, bool> = true> +inline std::vector MakeNumbers() { + + std::vector result; + result.reserve(32); + + // to reach from in to max in 32 steps, it also has to be lower than 7 to work for int8 values. + const T step = static_cast(1) << (sizeof(T)*8 - 5); + + // `- step` to avoid accidential overflow + for (T i = std::numeric_limits::min(); i <= std::numeric_limits::max() - step; i += step) + { + result.push_back(i); + } + result.push_back(std::numeric_limits::max()); + + return result; +} + +template ::value, bool> = true> +inline std::vector MakeNumbers() { + + std::vector result { + std::numeric_limits::min(), + std::numeric_limits::max(), + std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + static_cast(0), + static_cast(0) + std::numeric_limits::epsilon(), + static_cast(0) - std::numeric_limits::epsilon() + }; + + const auto total_steps = 100; + const auto step = std::pow(10, (std::numeric_limits::max_exponent - std::numeric_limits::min_exponent) / total_steps); + const auto min_value = std::pow(10, std::numeric_limits::min_exponent10); + + // cover most of the precision ranges + for (T i = std::numeric_limits::max(); i >= min_value * step; i /= step) + { + result.push_back(i); + result.push_back(-1 * i); + } + result.push_back(min_value); + result.push_back(-min_value); + + return result; +} + +template +inline std::vector MakeFixedStrings() { + return MakeFixedStrings(size); +} + +template +inline std::vector MakeDateTime64s() { + return MakeDateTime64s(scale); +} + +template +inline std::vector MakeDecimals() { + return MakeDecimals(precision, scale); +} + + +template +inline auto MakeDates() { + std::vector result {0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536 - 1}; + + if constexpr (std::is_same_v) { + // in CH Date internally a UInt16 and stores a day number + // ColumnDate expects values to be seconds, which is then + // converted to day number internally, hence the `* 86400`. + std::for_each(result.begin(), result.end(), [](auto& value) { + value *= 86400; + }); + } + + return result; +} + +template (*Generator)()> +inline auto MakeArrays() { + const auto nested_values = Generator(); + std::vector> result; + result.reserve(nested_values.size()); + + for (size_t i = 0; i < nested_values.size(); ++i) + { + result.emplace_back(nested_values.begin(), nested_values.begin() + i); + } + + return result; +} + + std::string FooBarGenerator(size_t i); template