diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index caeb4497..905c0868 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,6 +1,8 @@ name: Linux on: + schedule: + - cron: '0 0 * * 1' push: branches: [ master ] pull_request: @@ -12,44 +14,67 @@ env: jobs: build: - runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - compiler: [clang-6, gcc-7, gcc-8, gcc-9] + os: [ubuntu-20.04] + compiler: [clang-6, clang-10-libc++, gcc-7, gcc-8, gcc-9] ssl: [ssl_ON, ssl_OFF] + dependencies: [dependencies_BUILT_IN] + include: - compiler: clang-6 - INSTALL: clang-6.0 + COMPILER_INSTALL: clang-6.0 libc++-dev C_COMPILER: clang-6.0 CXX_COMPILER: clang++-6.0 + - compiler: clang-10-libc++ + COMPILER_INSTALL: clang-10 libc++-dev + C_COMPILER: clang-10 + CXX_COMPILER: clang++-10 + - compiler: gcc-7 - INSTALL: gcc-7 g++-7 + COMPILER_INSTALL: gcc-7 g++-7 C_COMPILER: gcc-7 CXX_COMPILER: g++-7 - compiler: gcc-8 - INSTALL: gcc-8 g++-8 + COMPILER_INSTALL: gcc-8 g++-8 C_COMPILER: gcc-8 CXX_COMPILER: g++-8 - compiler: gcc-9 - INSTALL: gcc-9 g++-9 + COMPILER_INSTALL: gcc-9 g++-9 C_COMPILER: gcc-9 CXX_COMPILER: g++-9 - ssl: ssl_ON - INSTALL_SSL: libssl-dev - EXTRA_CMAKE_FLAGS: -DWITH_OPENSSL=ON + SSL_CMAKE_OPTION: -D WITH_OPENSSL=ON - - ssl: ssl_OFF - EXTRA_CMAKE_FLAGS: -DWITH_OPENSSL=OFF + - dependencies: dependencies_SYSTEM + compiler: compiler_SYSTEM + os: ubuntu-22.04 + COMPILER_INSTALL: gcc g++ + C_COMPILER: gcc + CXX_COMPILER: g++ + DEPENDENCIES_INSTALL: libabsl-dev liblz4-dev + DEPENDENCIES_CMAKE_OPTIONS: >- + -D WITH_SYSTEM_LZ4=ON + -D WITH_SYSTEM_ABSEIL=ON + + runs-on: ${{matrix.os}} steps: - uses: actions/checkout@v2 - name: Install dependencies - run: sudo apt-get install -y docker cmake ${{ matrix.INSTALL }} ${{ matrix.INSTALL_SSL }} + run: | + sudo apt-get update && \ + sudo apt-get install -y \ + docker \ + cmake \ + ${{matrix.COMPILER_INSTALL}} \ + ${{matrix.DEPENDENCIES_INSTALL}} - name: Install dependencies - Docker run: | @@ -60,17 +85,24 @@ jobs: sudo apt update -q sudo apt install docker-ce docker-ce-cli containerd.io - - name: Configure CMake + - name: Configure project + run: | + cmake \ + -D CMAKE_C_COMPILER=${{matrix.C_COMPILER}} \ + -D CMAKE_CXX_COMPILER=${{matrix.CXX_COMPILER}} \ + -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \ + -D BUILD_TESTS=ON \ + ${{matrix.SSL_CMAKE_OPTION}} \ + ${{matrix.DEPENDENCIES_CMAKE_OPTIONS}} \ + -S ${{github.workspace}} \ + -B ${{github.workspace}}/build + + - name: Build project run: | cmake \ - -DCMAKE_C_COMPILER=${{ matrix.C_COMPILER}} \ - -DCMAKE_CXX_COMPILER=${{ matrix.CXX_COMPILER}} \ - -B ${{github.workspace}}/build \ - -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBUILD_TESTS=ON \ - ${{ matrix.EXTRA_CMAKE_FLAGS }} - - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} --target all + --build ${{github.workspace}}/build \ + --config ${{env.BUILD_TYPE}} \ + --target all - name: Test - Start ClickHouse server in background run: | diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index a2bb171f..1a267aa8 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,6 +1,8 @@ name: macOS on: + schedule: + - cron: '0 0 * * 1' push: branches: [ master ] pull_request: @@ -21,24 +23,34 @@ jobs: build: [nossl, ssl] include: - build: nossl - extra_cmake_flags: -DWITH_OPENSSL=OFF - extra_install: - build: ssl - extra_cmake_flags: -DWITH_OPENSSL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl/ - extra_install: openssl + SSL_CMAKE_OPTION: >- + -D WITH_OPENSSL=ON + -D OPENSSL_ROOT_DIR=/usr/local/opt/openssl/ + SSL_INSTALL: openssl steps: - uses: actions/checkout@v2 - name: Install dependencies - run: brew install cmake ${{matrix.extra_install}} + run: brew install cmake ${{matrix.SSL_INSTALL}} - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBUILD_TESTS=ON ${{matrix.extra_cmake_flags}} + run: | + cmake \ + -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \ + -D BUILD_TESTS=ON \ + ${{matrix.SSL_CMAKE_OPTION}} \ + -S ${{github.workspace}} \ + -B ${{github.workspace}}/build - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} --target all + run: | + cmake \ + --build ${{github.workspace}}/build \ + --config ${{env.BUILD_TYPE}} \ + --target all - name: Start tls offoader proxy # that mimics non-secure clickhouse running on localhost @@ -53,6 +65,7 @@ jobs: working-directory: ${{github.workspace}}/build/ut env: # It is impossible to start CH server in docker on macOS due to github actions limitations, - # so limit tests to ones that do no require server interaction. - GTEST_FILTER_ONLY_LOCAL: "-Client/*" - run: ./clickhouse-cpp-ut + # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: + # - system.query_log used by 'Client/ClientCase.Query_ID' + GTEST_FILTER: "-Client/ClientCase.Query_ID*" + run: ./clickhouse-cpp-ut ${GTEST_FILTER} diff --git a/.github/workflows/windows_mingw.yml b/.github/workflows/windows_mingw.yml index 016b7a7e..d99c1ef7 100644 --- a/.github/workflows/windows_mingw.yml +++ b/.github/workflows/windows_mingw.yml @@ -1,6 +1,8 @@ name: Windows mingw on: + schedule: + - cron: '0 0 * * 1' push: branches: [ master ] pull_request: @@ -79,7 +81,12 @@ jobs: ./go-tlsoffloader.exe -l localhost:9000 -b github.demo.trial.altinity.cloud:9440 & - name: Test - run: ./build/ut/clickhouse-cpp-ut.exe + env: + # It is impossible to start CH server in docker on Windows due to github actions limitations, + # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: + # - system.query_log used by 'Client/ClientCase.Query_ID' + GTEST_FILTER: "-Client/ClientCase.Query_ID*" + run: ./build/ut/clickhouse-cpp-ut.exe ${GTEST_FILTER} - name: Test (simple) run: ./build/tests/simple/simple-test.exe diff --git a/.github/workflows/windows_msvc.yml b/.github/workflows/windows_msvc.yml index 5ac5178b..149e3df4 100644 --- a/.github/workflows/windows_msvc.yml +++ b/.github/workflows/windows_msvc.yml @@ -1,6 +1,8 @@ name: Windows on: + schedule: + - cron: '0 0 * * 1' push: branches: [ master ] pull_request: @@ -8,7 +10,6 @@ on: env: BUILD_TYPE: Release - GTEST_FILTER: --gtest_filter=-"*" CLICKHOUSE_USER: clickhouse_cpp_cicd CLICKHOUSE_PASSWORD: clickhouse_cpp_cicd # @@ -56,5 +57,10 @@ jobs: ./go-tlsoffloader.exe -l localhost:9000 -b github.demo.trial.altinity.cloud:9440 & - name: Test + env: + # It is impossible to start CH server in docker on Windows due to github actions limitations, + # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: + # - system.query_log used by 'Client/ClientCase.Query_ID' + GTEST_FILTER: "-Client/ClientCase.Query_ID*" working-directory: ${{github.workspace}}/build/ut run: Release\clickhouse-cpp-ut.exe "${{env.GTEST_FILTER}}" diff --git a/CMakeLists.txt b/CMakeLists.txt index c7635c79..aed0d85c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,12 +1,18 @@ CMAKE_MINIMUM_REQUIRED (VERSION 3.0.2) -INCLUDE (cmake/cpp17.cmake) -INCLUDE (cmake/subdirs.cmake) -INCLUDE (cmake/openssl.cmake) +LIST (APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +INCLUDE (cpp17) +INCLUDE (subdirs) +INCLUDE (openssl) OPTION (BUILD_BENCHMARK "Build benchmark" OFF) OPTION (BUILD_TESTS "Build tests" OFF) +OPTION (BUILD_SHARED_LIBS "Build shared libs" OFF) OPTION (WITH_OPENSSL "Use OpenSSL for TLS connections" OFF) +OPTION (WITH_SYSTEM_ABSEIL "Use system ABSEIL" OFF) +OPTION (WITH_SYSTEM_LZ4 "Use system LZ4" OFF) +OPTION (WITH_SYSTEM_CITYHASH "Use system cityhash" OFF) PROJECT (CLICKHOUSE-CLIENT) @@ -27,11 +33,58 @@ PROJECT (CLICKHOUSE-CLIENT) SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wno-deprecated-declarations") ENDIF () + IF (APPLE OR MSVC) + IF(BUILD_SHARED_LIBS) + MESSAGE(FATAL "Does not support shared on this platform") + ENDIF() + ENDIF() + + IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + INCLUDE (CheckCXXSourceCompiles) + + CHECK_CXX_SOURCE_COMPILES("#include \nint main() { return __GLIBCXX__ != 0; }" + CLANG_WITH_LIB_STDCXX) + ENDIF () + + IF (CLANG_WITH_LIB_STDCXX) + # there is a problem with __builtin_mul_overflow call at link time + # the error looks like: ... undefined reference to `__muloti4' ... + # caused by clang bug https://bugs.llvm.org/show_bug.cgi?id=16404 + # explicit linking to compiler-rt allows to workaround the problem + SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --rtlib=compiler-rt") + SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --rtlib=compiler-rt") + + # some workaround for linking issues on linux: + # /usr/bin/ld: CMakeFiles/simple-test.dir/main.cpp.o: undefined reference to symbol '_Unwind_Resume@@GCC_3.0' + # /usr/bin/ld: /lib/x86_64-linux-gnu/libgcc_s.so.1: error adding symbols: DSO missing from command line + # FIXME: that workaround breaks clang build on mingw + SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lgcc_s") + SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lgcc_s") + ENDIF () + + IF (WITH_SYSTEM_ABSEIL) + FIND_PACKAGE(absl REQUIRED) + ELSE () + INCLUDE_DIRECTORIES (contrib/absl) + SUBDIRS (contrib/absl/absl) + ENDIF () + + IF (WITH_SYSTEM_LZ4) + FIND_PACKAGE(lz4 REQUIRED) + ELSE () + INCLUDE_DIRECTORIES (contrib/lz4/lz4) + SUBDIRS (contrib/lz4/lz4) + ENDIF () + + IF (WITH_SYSTEM_CITYHASH) + FIND_PACKAGE(cityhash REQUIRED) + ELSE () + INCLUDE_DIRECTORIES (contrib/cityhash/cityhash) + SUBDIRS (contrib/cityhash/cityhash) + ENDIF () + SUBDIRS ( clickhouse - contrib/absl - contrib/cityhash - contrib/lz4 ) IF (BUILD_BENCHMARK) diff --git a/LICENSE b/LICENSE index 7327b333..9269404a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2018-2022 ClickHouse, Inc +Copyright 2018-2023 ClickHouse, Inc. Copyright 2017 Pavel Artemkin Apache License @@ -189,7 +189,7 @@ Copyright 2017 Pavel Artemkin same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2018-2020 Yandex LLC + Copyright 2018-2023 ClickHouse, Inc. Copyright 2017 Pavel Artemkin diff --git a/README.md b/README.md index acb4232a..04fd2485 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,19 @@ C++ client for [ClickHouse](https://clickhouse.com/). * UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 * Int128 * UUID +* Map +* Point, Ring, Polygon, MultiPolygon + +## Dependencies +In the most basic case one needs only: +- a C++-17-complaint compiler, +- `cmake` (3.12 or newer), and +- `ninja` + +Optional dependencies: +- openssl +- liblz4 +- libabsl ## Building @@ -31,7 +44,21 @@ $ cmake .. [-DBUILD_TESTS=ON] $ make ``` -## Example +Plese refer to the workflows for the reference on dependencies/build options +- https://github.com/ClickHouse/clickhouse-cpp/blob/master/.github/workflows/linux.yml +- https://github.com/ClickHouse/clickhouse-cpp/blob/master/.github/workflows/windows_msvc.yml +- https://github.com/ClickHouse/clickhouse-cpp/blob/master/.github/workflows/windows_mingw.yml +- https://github.com/ClickHouse/clickhouse-cpp/blob/master/.github/workflows/macos.yml + + +## Example application build with clickhouse-cpp + +There are various ways to integrate clickhouse-cpp with the build system of an application. Below example uses the simple approach based on +submodules presented in https://www.youtube.com/watch?v=ED-WUk440qc . + +- `mkdir clickhouse-app && cd clickhouse-app && git init` +- `git submodule add https://github.com/ClickHouse/clickhouse-cpp.git contribs/clickhouse-cpp` +- `touch app.cpp`, then copy the following C++ code into that file ```cpp #include @@ -82,12 +109,32 @@ int main() } ``` +- `touch CMakeLists.txt`, then copy the following CMake code into that file + +```cmake +cmake_minimum_required(VERSION 3.12) +project(application-example) + +set(CMAKE_CXX_STANDARD 17) + +add_subdirectory(contribs/clickhouse-cpp) + +add_executable(${PROJECT_NAME} "app.cpp") + +target_include_directories(${PROJECT_NAME} PRIVATE contribs/clickhouse-cpp/ contribs/clickhouse-cpp/contrib/absl) + +target_link_libraries(${PROJECT_NAME} PRIVATE clickhouse-cpp-lib) +``` + +- run `rm -rf build && cmake -B build -S . && cmake --build build -j32` to remove remainders of the previous builds, run CMake and build the + application. The generated binary is located in location `build/application-example`. + ## Thread-safety ⚠ Please note that `Client` instance is NOT thread-safe. I.e. you must create a separate `Client` for each thread or utilize some synchronization techniques. ⚠ ## Retries If you wish to implement some retry logic atop of `clickhouse::Client` there are few simple rules to make you life easier: -- If previous attempt threw an exception, then make sure to call `clickhouse::Client::ResetConnection()` before the next try. +- If previous attempt threw an exception, then make sure to call `clickhouse::Client::ResetConnection()` before the next try. - For `clickhouse::Client::Insert()` you can reuse a block from previous try, no need to rebuild it from scratch. See https://github.com/ClickHouse/clickhouse-cpp/issues/184 for details. diff --git a/bench/bench.cpp b/bench/bench.cpp index 4078e3f2..2b037176 100644 --- a/bench/bench.cpp +++ b/bench/bench.cpp @@ -1,6 +1,7 @@ #include #include +#include namespace clickhouse { diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index db1c8692..67663ec5 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -12,12 +12,14 @@ SET ( clickhouse-cpp-lib-src columns/decimal.cpp columns/enum.cpp columns/factory.cpp + columns/geo.cpp columns/ip4.cpp columns/ip6.cpp columns/lowcardinality.cpp columns/lowcardinalityadaptor.h columns/nullable.cpp columns/numeric.cpp + columns/map.cpp columns/string.cpp columns/tuple.cpp columns/uuid.cpp @@ -36,26 +38,21 @@ IF (WITH_OPENSSL) LIST(APPEND clickhouse-cpp-lib-src base/sslsocket.cpp) ENDIF () -ADD_LIBRARY (clickhouse-cpp-lib SHARED ${clickhouse-cpp-lib-src}) -SET_TARGET_PROPERTIES(clickhouse-cpp-lib PROPERTIES LINKER_LANGUAGE CXX) +ADD_LIBRARY (clickhouse-cpp-lib ${clickhouse-cpp-lib-src}) +SET_TARGET_PROPERTIES (clickhouse-cpp-lib PROPERTIES LINKER_LANGUAGE CXX) TARGET_LINK_LIBRARIES (clickhouse-cpp-lib - absl-lib - cityhash-lib - lz4-lib + absl::int128 + cityhash::cityhash + lz4::lz4 ) TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib PUBLIC ${PROJECT_SOURCE_DIR} ) -ADD_LIBRARY (clickhouse-cpp-lib-static STATIC ${clickhouse-cpp-lib-src}) -TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static - absl-lib - cityhash-lib - lz4-lib -) -TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib-static - PUBLIC ${PROJECT_SOURCE_DIR} -) +IF (NOT BUILD_SHARED_LIBS) + ADD_LIBRARY (clickhouse-cpp-lib-static ALIAS clickhouse-cpp-lib) +ENDIF() + IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") INCLUDE (CheckCXXSourceCompiles) @@ -75,16 +72,15 @@ IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") # /usr/bin/ld: /lib/x86_64-linux-gnu/libgcc_s.so.1: error adding symbols: DSO missing from command line # FIXME: that workaround breaks clang build on mingw TARGET_LINK_LIBRARIES (clickhouse-cpp-lib gcc_s) - TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static gcc_s) ENDIF () ENDIF () -INSTALL (TARGETS clickhouse-cpp-lib clickhouse-cpp-lib-static + +INSTALL (TARGETS clickhouse-cpp-lib ARCHIVE DESTINATION lib LIBRARY DESTINATION lib ) - # general INSTALL(FILES block.h DESTINATION include/clickhouse/) INSTALL(FILES client.h DESTINATION include/clickhouse/) @@ -101,6 +97,7 @@ INSTALL(FILES base/input.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/open_telemetry.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/output.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/platform.h DESTINATION include/clickhouse/base/) +INSTALL(FILES base/projected_iterator.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/singleton.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/socket.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/string_utils.h DESTINATION include/clickhouse/base/) @@ -115,12 +112,14 @@ INSTALL(FILES columns/date.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/decimal.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/enum.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/factory.h DESTINATION include/clickhouse/columns/) +INSTALL(FILES columns/geo.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/ip4.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/ip6.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/itemview.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/lowcardinality.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/nullable.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/numeric.h DESTINATION include/clickhouse/columns/) +INSTALL(FILES columns/map.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/string.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/tuple.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/utils.h DESTINATION include/clickhouse/columns/) @@ -132,10 +131,8 @@ INSTALL(FILES types/types.h DESTINATION include/clickhouse/types/) IF (WITH_OPENSSL) TARGET_LINK_LIBRARIES (clickhouse-cpp-lib OpenSSL::SSL) - TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static OpenSSL::SSL) ENDIF () IF (WIN32 OR MINGW) TARGET_LINK_LIBRARIES (clickhouse-cpp-lib wsock32 ws2_32) - TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static wsock32 ws2_32) ENDIF () diff --git a/clickhouse/base/compressed.cpp b/clickhouse/base/compressed.cpp index 8478ad28..4d5cc65d 100644 --- a/clickhouse/base/compressed.cpp +++ b/clickhouse/base/compressed.cpp @@ -3,8 +3,8 @@ #include "output.h" #include "../exceptions.h" -#include -#include +#include +#include #include #include diff --git a/clickhouse/base/projected_iterator.h b/clickhouse/base/projected_iterator.h new file mode 100644 index 00000000..ca5aecfe --- /dev/null +++ b/clickhouse/base/projected_iterator.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include + +namespace clickhouse { + +template ()(std::declval())), + typename Value = std::decay_t> +class ProjectedIterator { +public: + using value_type = Value; + using reference = Reference; + using pointer = Reference; + using difference_type = typename std::iterator_traits::difference_type; + using iterator_category = typename std::iterator_traits::iterator_category; + + ProjectedIterator() = default; + + inline ProjectedIterator(Iterator const& iterator, UnaryFunction functor) + : iterator_(iterator) + , functor_(std::move(functor)) { + } + + inline UnaryFunction functor() const { return functor; } + + inline Iterator const& base() const { return iterator_; } + + inline reference operator*() const { return functor_(iterator_); } + + inline ProjectedIterator& operator++() { + ++iterator_; + return *this; + } + + inline ProjectedIterator& operator--() { + --iterator_; + return *this; + } + + inline bool operator==(const ProjectedIterator& other) const { + return this->iterator_ == other.iterator_; + } + + inline bool operator!=(const ProjectedIterator& other) const { + return !(*this == other); + } + +private: + Iterator iterator_; + UnaryFunction functor_; +}; + +} // namespace clickhouse diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index e62e90df..48e90c73 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -43,6 +43,21 @@ windowsErrorCategory const& windowsErrorCategory::category() { } #endif +#if defined(_unix_) +char const* getaddrinfoErrorCategory::name() const noexcept { + return "getaddrinfoError"; +} + +std::string getaddrinfoErrorCategory::message(int c) const { + return gai_strerror(c); +} + +getaddrinfoErrorCategory const& getaddrinfoErrorCategory::category() { + static getaddrinfoErrorCategory c; + return c; +} +#endif + namespace { class LocalNames : public std::unordered_set { @@ -202,11 +217,18 @@ SOCKET SocketConnect(const NetworkAddress& addr, const SocketTimeoutParams& time fd.fd = *s; fd.events = POLLOUT; fd.revents = 0; - ssize_t rval = Poll(&fd, 1, 5000); + ssize_t rval = Poll(&fd, 1, timeout_params.connect_timeout.count()); if (rval == -1) { throw std::system_error(getSocketErrorCode(), getErrorCategory(), "fail to connect"); } + if (rval == 0) { +#if defined(_win_) + last_err = WSAETIMEDOUT; +#else + last_err = ETIMEDOUT; +#endif + } if (rval > 0) { socklen_t len = sizeof(err); getsockopt(*s, SOL_SOCKET, SO_ERROR, (char*)&err, &len); @@ -258,6 +280,12 @@ NetworkAddress::NetworkAddress(const std::string& host, const std::string& port) const int error = getaddrinfo(host.c_str(), port.c_str(), &hints, &info_); +#if defined(_unix_) + if (error && error != EAI_SYSTEM) { + throw std::system_error(error, getaddrinfoErrorCategory::category()); + } +#endif + if (error) { throw std::system_error(getSocketErrorCode(), getErrorCategory()); } @@ -372,7 +400,7 @@ std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions } std::unique_ptr NonSecureSocketFactory::doConnect(const NetworkAddress& address, const ClientOptions& opts) { - SocketTimeoutParams timeout_params { opts.connection_recv_timeout, opts.connection_send_timeout }; + SocketTimeoutParams timeout_params { opts.connection_connect_timeout, opts.connection_recv_timeout, opts.connection_send_timeout }; return std::make_unique(address, timeout_params); } diff --git a/clickhouse/base/socket.h b/clickhouse/base/socket.h index c68f250d..694d0d69 100644 --- a/clickhouse/base/socket.h +++ b/clickhouse/base/socket.h @@ -60,6 +60,18 @@ class windowsErrorCategory : public std::error_category { #endif +#if defined(_unix_) + +class getaddrinfoErrorCategory : public std::error_category { +public: + char const* name() const noexcept override final; + std::string message(int c) const override final; + + static getaddrinfoErrorCategory const& category(); +}; + +#endif + class SocketBase { public: @@ -83,6 +95,7 @@ class SocketFactory { struct SocketTimeoutParams { + std::chrono::milliseconds connect_timeout{ 5000 }; std::chrono::milliseconds recv_timeout{ 0 }; std::chrono::milliseconds send_timeout{ 0 }; }; diff --git a/clickhouse/base/sslsocket.cpp b/clickhouse/base/sslsocket.cpp index 03b064b1..8b1971e0 100644 --- a/clickhouse/base/sslsocket.cpp +++ b/clickhouse/base/sslsocket.cpp @@ -268,7 +268,7 @@ SSLSocketFactory::SSLSocketFactory(const ClientOptions& opts) SSLSocketFactory::~SSLSocketFactory() = default; std::unique_ptr SSLSocketFactory::doConnect(const NetworkAddress& address, const ClientOptions& opts) { - SocketTimeoutParams timeout_params { opts.connection_recv_timeout, opts.connection_send_timeout }; + SocketTimeoutParams timeout_params { opts.connection_connect_timeout, opts.connection_recv_timeout, opts.connection_send_timeout }; return std::make_unique(address, timeout_params, ssl_params_, *ssl_context_); } diff --git a/clickhouse/base/wire_format.h b/clickhouse/base/wire_format.h index 9bbf7959..6ff53528 100644 --- a/clickhouse/base/wire_format.h +++ b/clickhouse/base/wire_format.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace clickhouse { diff --git a/clickhouse/client.h b/clickhouse/client.h index 452226d9..63177313 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -7,11 +7,13 @@ #include "columns/date.h" #include "columns/decimal.h" #include "columns/enum.h" +#include "columns/geo.h" #include "columns/ip4.h" #include "columns/ip6.h" #include "columns/lowcardinality.h" #include "columns/nullable.h" #include "columns/numeric.h" +#include "columns/map.h" #include "columns/string.h" #include "columns/tuple.h" #include "columns/uuid.h" @@ -43,12 +45,13 @@ enum class CompressionMethod { }; struct ClientOptions { + // Setter goes first, so it is possible to apply 'deprecated' annotation safely. #define DECLARE_FIELD(name, type, setter, default_value) \ - type name = default_value; \ inline auto & setter(const type& value) { \ name = value; \ return *this; \ - } + } \ + type name = default_value /// Hostname of the server. DECLARE_FIELD(host, std::string, SetHost, std::string()); @@ -86,9 +89,12 @@ struct ClientOptions { // TCP options DECLARE_FIELD(tcp_nodelay, bool, TcpNoDelay, true); + /// Connection socket connect timeout. If the timeout is negative then the connect operation will never timeout. + DECLARE_FIELD(connection_connect_timeout, std::chrono::milliseconds, SetConnectionConnectTimeout, std::chrono::seconds(5)); + /// Connection socket timeout. If the timeout is set to zero then the operation will never timeout. - DECLARE_FIELD(connection_recv_timeout, std::chrono::seconds, SetConnectionRecvTimeout, std::chrono::seconds(0)); - DECLARE_FIELD(connection_send_timeout, std::chrono::seconds, SetConnectionSendTimeout, std::chrono::seconds(0)); + DECLARE_FIELD(connection_recv_timeout, std::chrono::milliseconds, SetConnectionRecvTimeout, std::chrono::milliseconds(0)); + DECLARE_FIELD(connection_send_timeout, std::chrono::milliseconds, SetConnectionSendTimeout, std::chrono::milliseconds(0)); /** It helps to ease migration of the old codebases, which can't afford to switch * to using ColumnLowCardinalityT or ColumnLowCardinality directly, diff --git a/clickhouse/columns/array.cpp b/clickhouse/columns/array.cpp index 9ef160b5..9f66b91f 100644 --- a/clickhouse/columns/array.cpp +++ b/clickhouse/columns/array.cpp @@ -84,7 +84,11 @@ bool ColumnArray::LoadBody(InputStream* input, size_t rows) { if (!offsets_->LoadBody(input, rows)) { return false; } - if (!data_->LoadBody(input, (*offsets_)[rows - 1])) { + const auto nested_rows = (*offsets_)[rows - 1]; + if (nested_rows == 0) { + return true; + } + if (!data_->LoadBody(input, nested_rows)) { return false; } return true; @@ -96,7 +100,9 @@ void ColumnArray::SavePrefix(OutputStream* output) { void ColumnArray::SaveBody(OutputStream* output) { offsets_->SaveBody(output); - data_->SaveBody(output); + if (data_->Size() > 0) { + data_->SaveBody(output); + } } void ColumnArray::Clear() { diff --git a/clickhouse/columns/array.h b/clickhouse/columns/array.h index 1d3eb192..0ea33d5a 100644 --- a/clickhouse/columns/array.h +++ b/clickhouse/columns/array.h @@ -2,6 +2,7 @@ #include "column.h" #include "numeric.h" +#include "utils.h" #include @@ -121,13 +122,8 @@ class ColumnArrayT : public ColumnArray { * This is a static method to make such conversion verbose. */ static auto Wrap(ColumnArray&& col) { - if constexpr (std::is_base_of_v && !std::is_same_v) { - // assuming NestedColumnType is ArrayT specialization - return std::make_shared>(NestedColumnType::Wrap(col.GetData()), col.offsets_); - } else { - auto nested_data = col.GetData()->template AsStrict(); - return std::make_shared>(nested_data, col.offsets_); - } + auto nested_data = WrapColumn(col.GetData()); + return std::make_shared>(nested_data, col.offsets_); } static auto Wrap(Column&& col) { @@ -146,7 +142,7 @@ class ColumnArrayT : public ColumnArray { const size_t size_; public: - using ValueType = typename NestedColumnType::ValueType; + using ValueType = std::decay_t().At(0))>; ArrayValueView(std::shared_ptr data, size_t offset = 0, size_t size = std::numeric_limits::max()) : typed_nested_data_(data) @@ -171,6 +167,8 @@ class ColumnArrayT : public ColumnArray { const size_t size_; size_t index_; public: + Iterator() = default; + Iterator(std::shared_ptr typed_nested_data, size_t offset, size_t size, size_t index) : typed_nested_data_(typed_nested_data) , offset_(offset) @@ -178,7 +176,7 @@ class ColumnArrayT : public ColumnArray { , index_(index) {} - using ValueType = typename NestedColumnType::ValueType; + using ValueType = typename ArrayValueView::ValueType; inline auto operator*() const { return typed_nested_data_->At(offset_ + index_); @@ -226,6 +224,22 @@ class ColumnArrayT : public ColumnArray { inline size_t Size() const { return size_; } + + inline bool operator==(const ArrayValueView& other) const { + if (size() != other.size()) { + return false; + } + for (size_t i = 0; i < size_; ++i) { + if ((*this)[i] != other[i]) { + return false; + } + } + return true; + } + + inline bool operator!=(const ArrayValueView& other) const { + return !(*this == other); + } }; inline auto At(size_t index) const { @@ -258,7 +272,7 @@ class ColumnArrayT : public ColumnArray { size_t counter = 0; while (begin != end) { - nested_data.Append(*begin); + nested_data.Append(std::move(*begin)); ++begin; ++counter; } @@ -267,6 +281,20 @@ class ColumnArrayT : public ColumnArray { AddOffset(counter); } + ColumnRef Slice(size_t begin, size_t size) const override { + return Wrap(ColumnArray::Slice(begin, size)); + } + + ColumnRef CloneEmpty() const override { + return Wrap(ColumnArray::CloneEmpty()); + } + + void Swap(Column& other) override { + auto & col = dynamic_cast &>(other); + typed_nested_data_.swap(col.typed_nested_data_); + ColumnArray::Swap(other); + } + private: /// Helper to allow wrapping a "typeless" ColumnArray ColumnArrayT(ColumnArray&& array, std::shared_ptr nested_data) diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index 38b02e1d..e003b7f5 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -4,10 +4,12 @@ #include "date.h" #include "decimal.h" #include "enum.h" +#include "geo.h" #include "ip4.h" #include "ip6.h" #include "lowcardinality.h" #include "lowcardinalityadaptor.h" +#include "map.h" #include "nothing.h" #include "nullable.h" #include "numeric.h" @@ -15,15 +17,32 @@ #include "tuple.h" #include "uuid.h" + #include "../types/type_parser.h" #include "../exceptions.h" #include +#include namespace clickhouse { namespace { +// Like Python's list's []: +// * 0 - first element +// * 1 - second element +// * -1 - last element +// * -2 - one before last, etc. +const auto& GetASTChildElement(const TypeAst & ast, int position) { + if (static_cast(abs(position)) >= ast.elements.size()) + throw ValidationError("AST child element index out of bounds: " + std::to_string(position)); + + if (position < 0) + position = ast.elements.size() + position; + + return ast.elements[static_cast(position)]; +} + static ColumnRef CreateTerminalColumn(const TypeAst& ast) { switch (ast.code) { case Type::Void: @@ -55,24 +74,24 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) { return std::make_shared(); case Type::Decimal: - return std::make_shared(ast.elements.front().value, ast.elements.back().value); + return std::make_shared(GetASTChildElement(ast, 0).value, GetASTChildElement(ast, -1).value); case Type::Decimal32: - return std::make_shared(9, ast.elements.front().value); + return std::make_shared(9, GetASTChildElement(ast, 0).value); case Type::Decimal64: - return std::make_shared(18, ast.elements.front().value); + return std::make_shared(18, GetASTChildElement(ast, 0).value); case Type::Decimal128: - return std::make_shared(38, ast.elements.front().value); + return std::make_shared(38, GetASTChildElement(ast, 0).value); case Type::String: return std::make_shared(); case Type::FixedString: - return std::make_shared(ast.elements.front().value); + return std::make_shared(GetASTChildElement(ast, 0).value); case Type::DateTime: if (ast.elements.empty()) { return std::make_shared(); } else { - return std::make_shared(ast.elements[0].value_string); + return std::make_shared(GetASTChildElement(ast, 0).value_string); } case Type::DateTime64: if (ast.elements.empty()) { @@ -86,7 +105,7 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) { case Type::Date: return std::make_shared(); case Type::Date32: - return std::make_shared(); + return std::make_shared(); case Type::IPv4: return std::make_shared(); @@ -96,6 +115,18 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) { case Type::UUID: return std::make_shared(); + case Type::Point: + return std::make_shared(); + + case Type::Ring: + return std::make_shared(); + + case Type::Polygon: + return std::make_shared(); + + case Type::MultiPolygon: + return std::make_shared(); + default: return nullptr; } @@ -105,13 +136,13 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti switch (ast.meta) { case TypeAst::Array: { return std::make_shared( - CreateColumnFromAst(ast.elements.front(), settings) + CreateColumnFromAst(GetASTChildElement(ast, 0), settings) ); } case TypeAst::Nullable: { return std::make_shared( - CreateColumnFromAst(ast.elements.front(), settings), + CreateColumnFromAst(GetASTChildElement(ast, 0), settings), std::make_shared() ); } @@ -144,9 +175,10 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti enum_items.reserve(ast.elements.size() / 2); for (size_t i = 0; i < ast.elements.size(); i += 2) { - enum_items.push_back( - Type::EnumItem{ ast.elements[i].value_string, - (int16_t)ast.elements[i + 1].value }); + enum_items.push_back(Type::EnumItem{ + ast.elements[i].value_string, + static_cast(ast.elements[i + 1].value) + }); } if (ast.code == Type::Enum8) { @@ -161,14 +193,14 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti break; } case TypeAst::LowCardinality: { - const auto nested = ast.elements.front(); + const auto nested = GetASTChildElement(ast, 0); if (settings.low_cardinality_as_wrapped_column) { switch (nested.code) { // TODO (nemkov): update this to maximize code reuse. case Type::String: return std::make_shared>(); case Type::FixedString: - return std::make_shared>(nested.elements.front().value); + return std::make_shared>(GetASTChildElement(nested, 0).value); case Type::Nullable: throw UnimplementedError("LowCardinality(" + nested.name + ") is not supported with LowCardinalityAsWrappedColumn on"); default: @@ -181,11 +213,11 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti case Type::String: return std::make_shared>(); case Type::FixedString: - return std::make_shared>(nested.elements.front().value); + return std::make_shared>(GetASTChildElement(nested, 0).value); case Type::Nullable: return std::make_shared( std::make_shared( - CreateColumnFromAst(nested.elements.front(), settings), + CreateColumnFromAst(GetASTChildElement(nested, 0), settings), std::make_shared() ) ); @@ -195,7 +227,28 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti } } case TypeAst::SimpleAggregateFunction: { - return CreateTerminalColumn(ast.elements.back()); + return CreateTerminalColumn(GetASTChildElement(ast, -1)); + } + + case TypeAst::Map: { + if (ast.elements.size() != 2) { + throw ValidationError(ast.name + " content is not correct"); + } + + std::vector columns; + + columns.reserve(ast.elements.size()); + for (const auto& elem : ast.elements) { + if (auto col = CreateColumnFromAst(elem, settings)) { + columns.push_back(col); + } else { + return nullptr; + } + } + + return std::make_shared( + std::make_shared( + std::make_shared(columns))); } case TypeAst::Assign: diff --git a/clickhouse/columns/geo.cpp b/clickhouse/columns/geo.cpp new file mode 100644 index 00000000..ebea9895 --- /dev/null +++ b/clickhouse/columns/geo.cpp @@ -0,0 +1,108 @@ +#include "geo.h" + +#include "utils.h" + +namespace { +using namespace ::clickhouse; + +template +TypeRef CreateGeoType() { + if constexpr (type_code == Type::Code::Point) { + return Type::CreatePoint(); + } else if constexpr (type_code == Type::Code::Ring) { + return Type::CreateRing(); + } else if constexpr (type_code == Type::Code::Polygon) { + return Type::CreatePolygon(); + } else if constexpr (type_code == Type::Code::MultiPolygon) { + return Type::CreateMultiPolygon(); + } +} + +template +std::shared_ptr CreateColumn() { + if constexpr (std::is_same_v>) { + return std::make_shared>( + std::make_tuple(std::make_shared(), std::make_shared())); + } else { + return std::make_shared(); + } +} + +} // namespace + +namespace clickhouse { + +template +ColumnGeo::ColumnGeo() + : Column(CreateGeoType()), + data_(CreateColumn()) { +} + +template +ColumnGeo::ColumnGeo(ColumnRef data) + : Column(CreateGeoType()) + , data_(WrapColumn(std::move(data))) { +} + +template +void ColumnGeo::Clear() { + data_->Clear(); +} + +template +const typename ColumnGeo::ValueType ColumnGeo::At(size_t n) const { + return data_->At(n); +} + +template +const typename ColumnGeo::ValueType ColumnGeo::operator[](size_t n) const { + return data_->At(n); +} + +template +void ColumnGeo::Append(ColumnRef column) { + if (auto col = column->template As()) { + data_->Append(col->data_->template As()); + } +} + +template +bool ColumnGeo::LoadBody(InputStream* input, size_t rows) { + return data_->LoadBody(input, rows); +} + +template +void ColumnGeo::SaveBody(OutputStream* output) { + data_->SaveBody(output); +} + +template +size_t ColumnGeo::Size() const { + return data_->Size(); +} + +template +ColumnRef ColumnGeo::Slice(size_t begin, size_t len) const { + return std::make_shared(data_->Slice(begin, len)); +} + +template +ColumnRef ColumnGeo::CloneEmpty() const { + return std::make_shared(); +} + +template +void ColumnGeo::Swap(Column& other) { + auto& col = dynamic_cast(other); + data_.swap(col.data_); +} + +template class ColumnGeo, Type::Code::Point>; + +template class ColumnGeo, Type::Code::Ring>; + +template class ColumnGeo, Type::Code::Polygon>; + +template class ColumnGeo, Type::Code::MultiPolygon>; + +} // namespace clickhouse diff --git a/clickhouse/columns/geo.h b/clickhouse/columns/geo.h new file mode 100644 index 00000000..5f3db9b6 --- /dev/null +++ b/clickhouse/columns/geo.h @@ -0,0 +1,76 @@ +#pragma once + +#include "array.h" +#include "column.h" +#include "numeric.h" +#include "tuple.h" + +namespace clickhouse { + +template +class ColumnGeo : public Column { +public: + using ValueType = typename NestedColumnType::ValueType; + + ColumnGeo(); + + explicit ColumnGeo(ColumnRef data); + + /// Appends one element to the end of column. + template + void Append(const T& value) { + data_->Append(value); + } + + /// Returns element at given row number. + const ValueType At(size_t n) const; + + /// Returns element at given row number. + const ValueType operator[](size_t n) const; + +public: + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override; + + /// Loads column data from input stream. + bool LoadBody(InputStream* input, size_t rows) override; + + /// Saves column data to output stream. + void SaveBody(OutputStream* output) override; + + /// Clear column data . + void Clear() override; + + /// Returns count of rows in the column. + size_t Size() const override; + + /// Makes slice of the current column. + ColumnRef Slice(size_t begin, size_t len) const override; + ColumnRef CloneEmpty() const override; + void Swap(Column& other) override; + +private: + std::shared_ptr data_; +}; + +// /** +// * Represents a Point column. +// */ +using ColumnPoint = ColumnGeo, Type::Code::Point>; + +/** + * Represents a Ring column. + */ +using ColumnRing = ColumnGeo, Type::Code::Ring>; + +/** + * Represents a Polygon column. + */ +using ColumnPolygon = ColumnGeo, Type::Code::Polygon>; + +/** + * Represents a MultiPolygon column. + */ +using ColumnMultiPolygon = ColumnGeo, Type::Code::MultiPolygon>; + +} // namespace clickhouse diff --git a/clickhouse/columns/itemview.cpp b/clickhouse/columns/itemview.cpp index a2cb69c2..12c89c31 100644 --- a/clickhouse/columns/itemview.cpp +++ b/clickhouse/columns/itemview.cpp @@ -77,6 +77,7 @@ void ItemView::ValidateData(Type::Code type, DataType data) { case Type::Code::Nullable: case Type::Code::Tuple: case Type::Code::LowCardinality: + case Type::Code::Map: throw AssertionError("Unsupported type in ItemView: " + std::string(Type::TypeName(type))); case Type::Code::IPv6: diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index 0ffef7e0..d3627038 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -4,7 +4,7 @@ #include "nullable.h" #include "../base/wire_format.h" -#include +#include #include #include @@ -281,7 +281,7 @@ auto Load(ColumnRef new_dictionary_column, InputStream& input, size_t rows) { if (auto nullable = new_dictionary_column->As()) { nullable->Append(true); - for(std::size_t i = 1; i < new_index_column->Size(); i++) { + for(std::size_t i = 1; i < dataColumn->Size(); i++) { nullable->Append(false); } } diff --git a/clickhouse/columns/lowcardinality.h b/clickhouse/columns/lowcardinality.h index 3d8581fc..afadae22 100644 --- a/clickhouse/columns/lowcardinality.h +++ b/clickhouse/columns/lowcardinality.h @@ -53,9 +53,16 @@ class ColumnLowCardinality : public Column { UniqueItems unique_items_map_; public: + ColumnLowCardinality(ColumnLowCardinality&& col) = default; // c-tor makes a deep copy of the dictionary_column. explicit ColumnLowCardinality(ColumnRef dictionary_column); explicit ColumnLowCardinality(std::shared_ptr dictionary_column); + + template + explicit ColumnLowCardinality(std::shared_ptr> dictionary_column) + : ColumnLowCardinality(dictionary_column->template As()) + {} + ~ColumnLowCardinality(); /// Appends another LowCardinality column to the end of this one, updating dictionary. @@ -117,6 +124,13 @@ class ColumnLowCardinalityT : public ColumnLowCardinality { // Type this column takes as argument of Append and returns with At() and operator[] using ValueType = typename DictionaryColumnType::ValueType; + explicit ColumnLowCardinalityT(ColumnLowCardinality&& col) + : ColumnLowCardinality(std::move(col)) + , typed_dictionary_(dynamic_cast(*GetDictionary())) + , type_(GetTypeCode(typed_dictionary_)) + { + } + template explicit ColumnLowCardinalityT(Args &&... args) : ColumnLowCardinalityT(std::make_shared(std::forward(args)...)) @@ -124,9 +138,9 @@ class ColumnLowCardinalityT : public ColumnLowCardinality { // Create LC column from existing T-column, making a deep copy of all contents. explicit ColumnLowCardinalityT(std::shared_ptr dictionary_col) - : ColumnLowCardinality(dictionary_col), - typed_dictionary_(dynamic_cast(*GetDictionary())), - type_(typed_dictionary_.Type()->GetCode()) + : ColumnLowCardinality(dictionary_col) + , typed_dictionary_(dynamic_cast(*GetDictionary())) + , type_(GetTypeCode(typed_dictionary_)) {} /// Extended interface to simplify reading/adding individual items. @@ -145,7 +159,15 @@ class ColumnLowCardinalityT : public ColumnLowCardinality { using ColumnLowCardinality::Append; inline void Append(const ValueType & value) { - AppendUnsafe(ItemView{type_, value}); + if constexpr (IsNullable) { + if (value.has_value()) { + AppendUnsafe(ItemView{type_, *value}); + } else { + AppendUnsafe(ItemView{}); + } + } else { + AppendUnsafe(ItemView{type_, value}); + } } template @@ -154,6 +176,41 @@ class ColumnLowCardinalityT : public ColumnLowCardinality { Append(item); } } + + /** Create a ColumnLowCardinalityT from a ColumnLowCardinality, without copying data and offsets, but by + * 'stealing' those from `col`. + * + * Ownership of column internals is transferred to returned object, original (argument) object + * MUST NOT BE USED IN ANY WAY, it is only safe to dispose it. + * + * Throws an exception if `col` is of wrong type, it is safe to use original col in this case. + * This is a static method to make such conversion verbose. + */ + static auto Wrap(ColumnLowCardinality&& col) { + return std::make_shared>(std::move(col)); + } + + static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast(col))); } + + // Helper to simplify integration with other APIs + static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict())); } + + ColumnRef Slice(size_t begin, size_t size) const override { + return Wrap(ColumnLowCardinality::Slice(begin, size)); + } + + ColumnRef CloneEmpty() const override { return Wrap(ColumnLowCardinality::CloneEmpty()); } + +private: + + template + static auto GetTypeCode(T& column) { + if constexpr (IsNullable) { + return GetTypeCode(*column.Nested()->template AsStrict()); + } else { + return column.Type()->GetCode(); + } + } }; } diff --git a/clickhouse/columns/lowcardinalityadaptor.h b/clickhouse/columns/lowcardinalityadaptor.h index 8b579a0d..bcde1a9b 100644 --- a/clickhouse/columns/lowcardinalityadaptor.h +++ b/clickhouse/columns/lowcardinalityadaptor.h @@ -22,7 +22,9 @@ class CodedInputStream; * @see ClientOptions, CreateColumnByType */ template -class LowCardinalitySerializationAdaptor : public AdaptedColumnType +class +[[deprecated("Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) ")]] +LowCardinalitySerializationAdaptor : public AdaptedColumnType { public: using AdaptedColumnType::AdaptedColumnType; diff --git a/clickhouse/columns/map.cpp b/clickhouse/columns/map.cpp new file mode 100644 index 00000000..3f5616df --- /dev/null +++ b/clickhouse/columns/map.cpp @@ -0,0 +1,83 @@ +#include "map.h" + +#include + +#include "../exceptions.h" +#include "utils.h" + +namespace { + +using namespace clickhouse; + +TypeRef GetMapType(const Type& data_type) { + auto array = data_type.As(); + if (!array) { + throw ValidationError("Wrong type " + data_type.GetName() + " of data for map"); + } + auto tuple = array->GetItemType()->As(); + if (!tuple) { + throw ValidationError("Wrong type " + data_type.GetName() + " of data for map"); + } + auto types = tuple->GetTupleType(); + if (types.size() != 2) { + throw ValidationError("Wrong type " + data_type.GetName() + " of data for map"); + } + return Type::CreateMap(types[0], types[1]); +} + +} // namespace + +namespace clickhouse { + +ColumnMap::ColumnMap(ColumnRef data) + : Column(GetMapType(data->GetType())), data_(data->As()) { +} + +void ColumnMap::Clear() { + data_->Clear(); +} + +void ColumnMap::Append(ColumnRef column) { + if (auto col = column->As()) { + data_->Append(col->data_); + } +} + +bool ColumnMap::LoadPrefix(InputStream* input, size_t rows) { + return data_->LoadPrefix(input, rows); +} + +bool ColumnMap::LoadBody(InputStream* input, size_t rows) { + return data_->LoadBody(input, rows); +} + +void ColumnMap::SavePrefix(OutputStream* output) { + data_->SavePrefix(output); +} + +void ColumnMap::SaveBody(OutputStream* output) { + data_->SaveBody(output); +} + +size_t ColumnMap::Size() const { + return data_->Size(); +} + +ColumnRef ColumnMap::Slice(size_t begin, size_t len) const { + return std::make_shared(data_->Slice(begin, len)); +} + +ColumnRef ColumnMap::CloneEmpty() const { + return std::make_shared(data_->CloneEmpty()); +} + +void ColumnMap::Swap(Column& other) { + auto& col = dynamic_cast(other); + data_.swap(col.data_); +} + +ColumnRef ColumnMap::GetAsColumn(size_t n) const { + return data_->GetAsColumn(n); +} + +} // namespace clickhouse diff --git a/clickhouse/columns/map.h b/clickhouse/columns/map.h new file mode 100644 index 00000000..24a8b4ae --- /dev/null +++ b/clickhouse/columns/map.h @@ -0,0 +1,254 @@ +#pragma once + +#include "../base/projected_iterator.h" +#include "array.h" +#include "column.h" +#include "tuple.h" + +#include +#include + +namespace clickhouse { + +template +class ColumnMapT; + +/** + * Represents column of Map(K, V). + */ +class ColumnMap : public Column { +public: + /** Create a map of given type, with actual values and offsets. + * + * Both `data` and `offsets` are used (and modified) internally bye ColumnArray. + * Users are strongly advised against modifying contents of `data` or `offsets` afterwards. + */ + explicit ColumnMap(ColumnRef data); + + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override; + + /// Loads column prefix from input stream. + bool LoadPrefix(InputStream* input, size_t rows) override; + + /// Loads column data from input stream. + bool LoadBody(InputStream* input, size_t rows) override; + + /// Saves column prefix to output stream. + void SavePrefix(OutputStream* output) override; + + /// Saves column data to output stream. + void SaveBody(OutputStream* output) override; + + /// Clear column data . + void Clear() override; + + /// Returns count of rows in the column. + size_t Size() const override; + + /// Makes slice of the current column. + ColumnRef Slice(size_t, size_t) const override; + ColumnRef CloneEmpty() const override; + void Swap(Column&) override; + + /// Converts map at pos n to column. + /// Type of row is tuple {key, value}. + ColumnRef GetAsColumn(size_t n) const; + +protected: + template + friend class ColumnMapT; + + ColumnMap(ColumnMap&& map); + +private: + std::shared_ptr data_; +}; + +template +class ColumnMapT : public ColumnMap { +public: + using KeyColumnType = K; + using ValueColumnType = V; + using Key = std::decay_t().At(0))>; + using Value = std::decay_t().At(0))>; + using TupleColumnType = ColumnTupleT; + using ArrayColumnType = ColumnArrayT; + + ColumnMapT(ColumnRef data) + : ColumnMap(data), typed_data_(data->AsStrict>()) {} + + ColumnMapT(std::shared_ptr keys, std::shared_ptr values) + : ColumnMap(std::make_shared(std::make_shared( + std::make_tuple(std::move(keys), std::move(values))))), + typed_data_(data_->template As()) {} + + ColumnRef Slice(size_t begin, size_t len) const override { + return std::make_shared>(typed_data_->Slice(begin, len)); + } + + ColumnRef CloneEmpty() const override { + return std::make_shared>(typed_data_->CloneEmpty()); + } + + void Swap(Column& other) override { + auto& col = dynamic_cast&>(other); + col.typed_data_.swap(typed_data_); + ColumnMap::Swap(other); + } + + /// A single (row) value of the Map-column i.e. read-only map. + /// It has a linear time complexity to access items + /// Because data base type has same structure + /// "This lookup works now with a linear complexity." + /// https://clickhouse.com/docs/en/sql-reference/data-types/map + /// Convert it to a suitable container required to access more than one element + + class MapValueView { + const typename ArrayColumnType::ArrayValueView data_; + + public: + using ValueType = std::pair; + + MapValueView(typename ArrayColumnType::ArrayValueView data) : data_(std::move(data)) {} + + inline auto operator[](const Key& key) const { return (*Find(key)).second; } + + inline auto At(const Key& key) const { + auto it = Find(key); + if (it == end()) throw ValidationError("ColumnMap value key not found"); + return (*it).second; + } + + class Iterator { + typename ArrayColumnType::ArrayValueView::Iterator data_iterator_; + + public: + Iterator() = default; + + Iterator(typename ArrayColumnType::ArrayValueView::Iterator data_iterator) + : data_iterator_(data_iterator) {} + + using ValueType = std::pair; + using difference_type = size_t; + using value_type = ValueType; + using pointer = void; + using reference = ValueType&; + using iterator_category = std::forward_iterator_tag; + + inline auto operator*() const { + auto tuple = *data_iterator_; + return ValueType{std::get<0>(tuple), std::get<1>(tuple)}; + } + + inline Iterator& operator++() { + ++data_iterator_; + return *this; + } + + inline bool operator==(const Iterator& other) const { + return this->data_iterator_ == other.data_iterator_; + } + + inline bool operator!=(const Iterator& other) const { return !(*this == other); } + }; + + // minimalistic stl-like container interface, hence the lowercase + inline Iterator begin() const { return Iterator{data_.begin()}; } + + inline Iterator cbegin() const { return Iterator{data_.cbegin()}; } + + inline Iterator end() const { return Iterator{data_.end()}; } + + inline Iterator cend() const { return Iterator{data_.cend()}; } + + inline size_t size() const { return data_.size(); } + + // It is ugly to have both size() and Size(), but it is for compatitability with both STL + // and rest of the clickhouse-cpp. + inline size_t Size() const { return data_.Size(); } + + inline size_t Count(const Key& key) const { + size_t result = 0; + for (auto item : data_) { + if (std::get<0>(item) == key) { + ++result; + } + } + return result; + } + + inline Iterator Find(const Key& key) const { + for (auto it = data_.begin(); it != data_.end(); ++it) { + if (std::get<0>(*it) == key) { + return Iterator{it}; + } + } + return end(); + } + + inline bool operator==(const MapValueView& other) const { + if (size() != other.size()) { + return false; + } + const auto make_index = [](const auto& data) { + std::vector result{data.Size()}; + std::generate(result.begin(), result.end(), [i = 0] () mutable {return i++;}); + std::sort(result.begin(), result.end(), [&data](size_t l, size_t r) {return data[l] < data[r];}); + return result; + }; + const auto index = make_index(data_); + for (const auto& val : other.data_) { + if (!std::binary_search(index.begin(), index.end(), val, + [&data = data_](const auto& l, size_t r) {return l < data[r];})) { + return false; + } + } + return true; + } + + inline bool operator!=(const MapValueView& other) const { return !(*this == other); } + }; + + inline auto At(size_t index) const { return MapValueView{typed_data_->At(index)}; } + + inline auto operator[](size_t index) const { return MapValueView{typed_data_->At(index)}; } + + using ColumnMap::Append; + + inline void Append(const MapValueView& value) { typed_data_->Append(value.data_); } + + inline void Append(const std::vector>& tuples) { + typed_data_->Append(tuples.begin(), tuples.end()); + } + + template + inline void Append(const T& value) { + using BaseIter = decltype(value.begin()); + using KeyOfT = decltype(std::declval()->first); + using ValOfT = decltype(std::declval()->second); + using Functor = std::function(const BaseIter&)>; + using Iterator = ProjectedIterator; + + Functor functor = [](const BaseIter& i) { + return std::make_tuple(std::cref(i->first), std::cref(i->second)); + }; + + typed_data_->Append(Iterator{value.begin(), functor}, Iterator{value.end(), functor}); + } + + static auto Wrap(ColumnMap&& col) { + auto data = ArrayColumnType::Wrap(std::move(col.data_)); + return std::make_shared>(std::move(data)); + } + + static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast(col))); } + + // Helper to simplify integration with other APIs + static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict())); } + +private: + std::shared_ptr typed_data_; +}; + +} // namespace clickhouse diff --git a/clickhouse/columns/nullable.h b/clickhouse/columns/nullable.h index 41806624..c1924af0 100644 --- a/clickhouse/columns/nullable.h +++ b/clickhouse/columns/nullable.h @@ -3,6 +3,8 @@ #include "column.h" #include "numeric.h" +#include + namespace clickhouse { /** @@ -42,7 +44,7 @@ class ColumnNullable : public Column { /// Clear column data . void Clear() override; - + /// Returns count of rows in the column. size_t Size() const override; @@ -58,4 +60,91 @@ class ColumnNullable : public Column { std::shared_ptr nulls_; }; +template +class ColumnNullableT : public ColumnNullable { +public: + using NestedColumnType = ColumnType; + using ValueType = std::optional().At(0))>>; + + ColumnNullableT(std::shared_ptr data, std::shared_ptr nulls) + : ColumnNullable(data, nulls) + , typed_nested_data_(data) + {} + + explicit ColumnNullableT(std::shared_ptr data) + : ColumnNullableT(data, FillNulls(data->Size())) + {} + + template + explicit ColumnNullableT(Args &&... args) + : ColumnNullableT(std::make_shared(std::forward(args)...)) + {} + + inline ValueType At(size_t index) const { + return IsNull(index) ? ValueType{} : ValueType{typed_nested_data_->At(index)}; + } + + inline ValueType operator[](size_t index) const { return At(index); } + + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override { + ColumnNullable::Append(std::move(column)); + } + + inline void Append(ValueType value) { + ColumnNullable::Append(!value.has_value()); + if (value.has_value()) { + typed_nested_data_->Append(std::move(*value)); + } else { + typed_nested_data_->Append(typename ValueType::value_type{}); + } + } + + /** Create a ColumnNullableT from a ColumnNullable, without copying data and offsets, but by + * 'stealing' those from `col`. + * + * Ownership of column internals is transferred to returned object, original (argument) object + * MUST NOT BE USED IN ANY WAY, it is only safe to dispose it. + * + * Throws an exception if `col` is of wrong type, it is safe to use original col in this case. + * This is a static method to make such conversion verbose. + */ + static auto Wrap(ColumnNullable&& col) { + return std::make_shared>( + col.Nested()->AsStrict(), + col.Nulls()->AsStrict()) ; + } + + static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast(col))); } + + // Helper to simplify integration with other APIs + static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict())); } + + ColumnRef Slice(size_t begin, size_t size) const override { + return Wrap(ColumnNullable::Slice(begin, size)); + } + + ColumnRef CloneEmpty() const override { return Wrap(ColumnNullable::CloneEmpty()); } + + void Swap(Column& other) override { + auto& col = dynamic_cast&>(other); + typed_nested_data_.swap(col.typed_nested_data_); + ColumnNullable::Swap(other); + } + +private: + static inline auto FillNulls(size_t n){ + auto result = std::make_shared(); + for (size_t i = 0; i < n; ++i) { + result->Append(0); + } + return result; + } + + std::shared_ptr typed_nested_data_; +}; + +template +constexpr bool IsNullable = std::is_base_of_v; + } diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index 63cfc689..b1b5ad31 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -1,6 +1,7 @@ #pragma once #include "column.h" +#include "utils.h" #include @@ -20,6 +21,10 @@ class ColumnTuple : public Column { return columns_[n]; } + ColumnRef At(size_t n) const { + return columns_[n]; + } + public: /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; @@ -51,4 +56,120 @@ class ColumnTuple : public Column { std::vector columns_; }; -} +template +class ColumnTupleT : public ColumnTuple { +public: + using TupleOfColumns = std::tuple...>; + + using ValueType = std::tuple().At(0))>...>; + + ColumnTupleT(std::tuple...> columns) + : ColumnTuple(TupleToVector(columns)), typed_columns_(std::move(columns)) {} + + ColumnTupleT(std::vector columns) + : ColumnTuple(columns), typed_columns_(VectorToTuple(std::move(columns))) {} + + ColumnTupleT(const std::initializer_list columns) + : ColumnTuple(columns), typed_columns_(VectorToTuple(std::move(columns))) {} + + inline ValueType At(size_t index) const { return GetTupleOfValues(index); } + + inline ValueType operator[](size_t index) const { return GetTupleOfValues(index); } + + using ColumnTuple::Append; + + template + inline void Append(std::tuple value) { + AppendTuple(std::move(value)); + } + + /** Create a ColumnTupleT from a ColumnTuple, without copying data and offsets, but by + * 'stealing' those from `col`. + * + * Ownership of column internals is transferred to returned object, original (argument) object + * MUST NOT BE USED IN ANY WAY, it is only safe to dispose it. + * + * Throws an exception if `col` is of wrong type, it is safe to use original col in this case. + * This is a static method to make such conversion verbose. + */ + static auto Wrap(ColumnTuple&& col) { + if (col.TupleSize() != std::tuple_size_v) { + throw ValidationError("Can't wrap from " + col.GetType().GetName()); + } + return std::make_shared>(VectorToTuple(std::move(col))); + } + + static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast(col))); } + + // Helper to simplify integration with other APIs + static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict())); } + + ColumnRef Slice(size_t begin, size_t size) const override { + return Wrap(ColumnTuple::Slice(begin, size)); + } + + ColumnRef CloneEmpty() const override { return Wrap(ColumnTuple::CloneEmpty()); } + + void Swap(Column& other) override { + auto& col = dynamic_cast&>(other); + typed_columns_.swap(col.typed_columns_); + ColumnTuple::Swap(other); + } + +private: + template > + inline void AppendTuple([[maybe_unused]] T value) { + static_assert(index <= std::tuple_size_v); + static_assert(std::tuple_size_v == std::tuple_size_v); + if constexpr (index == 0) { + return; + } else { + std::get(typed_columns_)->Append(std::move(std::get(value))); + AppendTuple(std::move(value)); + } + } + + template > + inline static std::vector TupleToVector([[maybe_unused]] const T& value) { + static_assert(index <= std::tuple_size_v); + if constexpr (index == 0) { + std::vector result; + result.reserve(std::tuple_size_v); + return result; + } else { + auto result = TupleToVector(value); + result.push_back(std::get(value)); + return result; + } + } + + template > + inline static auto VectorToTuple([[maybe_unused]] T columns) { + static_assert(column_index <= std::tuple_size_v); + if constexpr (column_index == 0) { + return std::make_tuple(); + } else { + using ColumnType = + typename std::tuple_element::type::element_type; + auto column = WrapColumn(columns[column_index - 1]); + return std::tuple_cat(std::move(VectorToTuple(std::move(columns))), + std::make_tuple(std::move(column))); + } + } + + template > + inline auto GetTupleOfValues([[maybe_unused]]size_t index) const { + static_assert(column_index <= std::tuple_size_v); + if constexpr (column_index == 0) { + return std::make_tuple(); + } else { + return std::tuple_cat( + std::move(GetTupleOfValues(index)), + std::move(std::make_tuple(std::get(typed_columns_)->At(index)))); + } + } + + TupleOfColumns typed_columns_; +}; + +} // namespace clickhouse diff --git a/clickhouse/columns/utils.h b/clickhouse/columns/utils.h index bcc5b988..9fbafa16 100644 --- a/clickhouse/columns/utils.h +++ b/clickhouse/columns/utils.h @@ -17,4 +17,24 @@ std::vector SliceVector(const std::vector& vec, size_t begin, size_t len) return result; } +template +struct HasWrapMethod { +private: + static int detect(...); + template + static decltype(U::Wrap(std::move(std::declval()))) detect(const U&); + +public: + static constexpr bool value = !std::is_same()))>::value; +}; + +template +inline std::shared_ptr WrapColumn(ColumnRef&& column) { + if constexpr (HasWrapMethod::value) { + return T::Wrap(std::move(column)); + } else { + return column->template AsStrict(); + } +} + } diff --git a/clickhouse/query.h b/clickhouse/query.h index 21d7231f..b6551803 100644 --- a/clickhouse/query.h +++ b/clickhouse/query.h @@ -19,6 +19,7 @@ struct QuerySettingsField { { IMPORTANT = 0x01, CUSTOM = 0x02, + OBSOLETE = 0x04, }; std::string value; uint64_t flags{0}; @@ -79,6 +80,7 @@ using SelectCallback = std::function; using SelectCancelableCallback = std::function; using SelectServerLogCallback = std::function; using ProfileEventsCallback = std::function; +using ProfileCallbak = std::function; class Query : public QueryEvents { @@ -158,6 +160,11 @@ class Query : public QueryEvents { return *this; } + inline Query& OnProfile(ProfileCallbak cb) { + profile_callback_cb_ = std::move(cb); + return *this; + } + static const std::string default_query_id; private: @@ -182,7 +189,8 @@ class Query : public QueryEvents { } void OnProfile(const Profile& profile) override { - (void)profile; + if (profile_callback_cb_) + profile_callback_cb_(profile); } void OnProgress(const Progress& progress) override { @@ -217,6 +225,7 @@ class Query : public QueryEvents { SelectCancelableCallback select_cancelable_cb_; SelectServerLogCallback select_server_log_cb_; ProfileEventsCallback profile_events_callback_cb_; + ProfileCallbak profile_callback_cb_; }; } diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index 37a049a0..6142dec2 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -1,10 +1,21 @@ #include "type_parser.h" +#include "clickhouse/exceptions.h" +#include "clickhouse/base/platform.h" // for _win_ + #include +#include #include #include #include +#if defined _win_ +#include +#else +#include +#endif + + namespace clickhouse { bool TypeAst::operator==(const TypeAst & other) const { @@ -16,10 +27,12 @@ bool TypeAst::operator==(const TypeAst & other) const { } static const std::unordered_map kTypeCode = { + { "Void", Type::Void }, { "Int8", Type::Int8 }, { "Int16", Type::Int16 }, { "Int32", Type::Int32 }, { "Int64", Type::Int64 }, + { "Bool", Type::UInt8 }, { "UInt8", Type::UInt8 }, { "UInt16", Type::UInt16 }, { "UInt32", Type::UInt32 }, @@ -41,18 +54,38 @@ static const std::unordered_map kTypeCode = { { "IPv4", Type::IPv4 }, { "IPv6", Type::IPv6 }, { "Int128", Type::Int128 }, +// { "UInt128", Type::UInt128 }, { "Decimal", Type::Decimal }, { "Decimal32", Type::Decimal32 }, { "Decimal64", Type::Decimal64 }, { "Decimal128", Type::Decimal128 }, { "LowCardinality", Type::LowCardinality }, + { "Map", Type::Map }, + { "Point", Type::Point }, + { "Ring", Type::Ring }, + { "Polygon", Type::Polygon }, + { "MultiPolygon", Type::MultiPolygon }, }; +template +inline int CompateStringsCaseInsensitive(const L& left, const R& right) { + int64_t size_diff = left.size() - right.size(); + if (size_diff != 0) + return size_diff > 0 ? 1 : -1; + +#if defined _win_ + return _strnicmp(left.data(), right.data(), left.size()); +#else + return strncasecmp(left.data(), right.data(), left.size()); +#endif +} + static Type::Code GetTypeCode(const std::string& name) { auto it = kTypeCode.find(name); if (it != kTypeCode.end()) { return it->second; } + return Type::Void; } @@ -85,9 +118,24 @@ static TypeAst::Meta GetTypeMeta(const StringView& name) { return TypeAst::SimpleAggregateFunction; } + if (name == "Map") { + return TypeAst::Map; + } + return TypeAst::Terminal; } +bool ValidateAST(const TypeAst& ast) { + // Void terminal that is not actually "void" produced when unknown type is encountered. + if (ast.meta == TypeAst::Terminal + && ast.code == Type::Void + && CompateStringsCaseInsensitive(ast.name, std::string_view("void")) != 0) + //throw UnimplementedError("Unsupported type: " + ast.name); + return false; + + return true; +} + TypeParser::TypeParser(const StringView& name) : cur_(name.data()) @@ -102,6 +150,7 @@ bool TypeParser::Parse(TypeAst* type) { type_ = type; open_elements_.push(type_); + size_t processed_tokens = 0; do { const Token & token = NextToken(); switch (token.type) { @@ -150,11 +199,17 @@ bool TypeParser::Parse(TypeAst* type) { // Ubalanced braces, brackets, etc is an error. if (open_elements_.size() != 1) return false; - return true; + + // Empty input string, no tokens produced + if (processed_tokens == 0) + return false; + + return ValidateAST(*type); } case Token::Invalid: return false; } + ++processed_tokens; } while (true); } diff --git a/clickhouse/types/type_parser.h b/clickhouse/types/type_parser.h index 2c81d2e5..2f8f2f6f 100644 --- a/clickhouse/types/type_parser.h +++ b/clickhouse/types/type_parser.h @@ -21,7 +21,8 @@ struct TypeAst { Tuple, Enum, LowCardinality, - SimpleAggregateFunction + SimpleAggregateFunction, + Map }; /// Type's category. diff --git a/clickhouse/types/types.cpp b/clickhouse/types/types.cpp index c34c0b4e..79e8be48 100644 --- a/clickhouse/types/types.cpp +++ b/clickhouse/types/types.cpp @@ -2,7 +2,7 @@ #include "../exceptions.h" -#include +#include #include @@ -46,6 +46,11 @@ const char* Type::TypeName(Type::Code code) { case Type::Code::LowCardinality: return "LowCardinality"; case Type::Code::DateTime64: return "DateTime64"; case Type::Code::Date32: return "Date32"; + case Type::Code::Map: return "Map"; + case Type::Code::Point: return "Point"; + case Type::Code::Ring: return "Ring"; + case Type::Code::Polygon: return "Polygon"; + case Type::Code::MultiPolygon: return "MultiPolygon"; } return "Unknown type"; @@ -71,6 +76,10 @@ std::string Type::GetName() const { case IPv6: case Date: case Date32: + case Point: + case Ring: + case Polygon: + case MultiPolygon: return TypeName(code_); case FixedString: return As()->GetName(); @@ -94,6 +103,8 @@ std::string Type::GetName() const { return As()->GetName(); case LowCardinality: return As()->GetName(); + case Map: + return As()->GetName(); } // XXX: NOT REACHED! @@ -123,6 +134,10 @@ uint64_t Type::GetTypeUniqueId() const { case IPv6: case Date: case Date32: + case Point: + case Ring: + case Polygon: + case MultiPolygon: // For simple types, unique ID is the same as Type::Code return code_; @@ -138,7 +153,8 @@ uint64_t Type::GetTypeUniqueId() const { case Decimal32: case Decimal64: case Decimal128: - case LowCardinality: { + case LowCardinality: + case Map: { // For complex types, exact unique ID depends on nested types and/or parameters, // the easiest way is to lazy-compute unique ID from name once. // Here we do not care if multiple threads are computing value simultaneosly since it is both: @@ -225,6 +241,26 @@ TypeRef Type::CreateLowCardinality(TypeRef item_type) { return std::make_shared(item_type); } +TypeRef Type::CreateMap(TypeRef key_type, TypeRef value_type) { + return std::make_shared(key_type, value_type); +} + +TypeRef Type::CreatePoint() { + return TypeRef(new Type(Type::Point)); +} + +TypeRef Type::CreateRing() { + return TypeRef(new Type(Type::Ring)); +} + +TypeRef Type::CreatePolygon() { + return TypeRef(new Type(Type::Polygon)); +} + +TypeRef Type::CreateMultiPolygon() { + return TypeRef(new Type(Type::MultiPolygon)); +} + /// class ArrayType ArrayType::ArrayType(TypeRef item_type) : Type(Array), item_type_(item_type) { @@ -404,4 +440,15 @@ std::string TupleType::GetName() const { return result; } +/// class MapType +MapType::MapType(TypeRef key_type, TypeRef value_type) + : Type(Map) + , key_type_(key_type) + , value_type_(value_type) { +} + +std::string MapType::GetName() const { + return std::string("Map(") + key_type_->GetName() + ", " +value_type_->GetName() + ")"; +} + } // namespace clickhouse diff --git a/clickhouse/types/types.h b/clickhouse/types/types.h index 8a27257a..423a6f70 100644 --- a/clickhouse/types/types.h +++ b/clickhouse/types/types.h @@ -50,6 +50,11 @@ class Type { LowCardinality, DateTime64, Date32, + Map, + Point, + Ring, + Polygon, + MultiPolygon }; using EnumItem = std::pair; @@ -92,7 +97,7 @@ class Type { static TypeRef CreateDate(); - static TypeRef CreateDate32(); + static TypeRef CreateDate32(); static TypeRef CreateDateTime(std::string timezone = std::string()); @@ -125,6 +130,16 @@ class Type { static TypeRef CreateLowCardinality(TypeRef item_type); + static TypeRef CreateMap(TypeRef key_type, TypeRef value_type); + + static TypeRef CreatePoint(); + + static TypeRef CreateRing(); + + static TypeRef CreatePolygon(); + + static TypeRef CreateMultiPolygon(); + private: uint64_t GetTypeUniqueId() const; @@ -280,6 +295,23 @@ class LowCardinalityType : public Type { TypeRef nested_type_; }; +class MapType : public Type { +public: + explicit MapType(TypeRef key_type, TypeRef value_type); + + std::string GetName() const; + + /// Type of keys. + TypeRef GetKeyType() const { return key_type_; } + + /// Type of values. + TypeRef GetValueType() const { return value_type_; } + +private: + TypeRef key_type_; + TypeRef value_type_; +}; + template <> inline TypeRef Type::CreateSimple() { return TypeRef(new Type(Int8)); diff --git a/cmake/Findlz4.cmake b/cmake/Findlz4.cmake new file mode 100644 index 00000000..ef8a366e --- /dev/null +++ b/cmake/Findlz4.cmake @@ -0,0 +1,38 @@ +find_path(lz4_INCLUDE_DIR + NAMES lz4.h + DOC "lz4 include directory") +mark_as_advanced(lz4_INCLUDE_DIR) +find_library(lz4_LIBRARY + NAMES lz4 liblz4 + DOC "lz4 library") +mark_as_advanced(lz4_LIBRARY) + +if (lz4_INCLUDE_DIR) + file(STRINGS "${lz4_INCLUDE_DIR}/lz4.h" _lz4_version_lines + REGEX "#define[ \t]+LZ4_VERSION_(MAJOR|MINOR|RELEASE)") + string(REGEX REPLACE ".*LZ4_VERSION_MAJOR *\([0-9]*\).*" "\\1" _lz4_version_major "${_lz4_version_lines}") + string(REGEX REPLACE ".*LZ4_VERSION_MINOR *\([0-9]*\).*" "\\1" _lz4_version_minor "${_lz4_version_lines}") + string(REGEX REPLACE ".*LZ4_VERSION_RELEASE *\([0-9]*\).*" "\\1" _lz4_version_release "${_lz4_version_lines}") + set(lz4_VERSION "${_lz4_version_major}.${_lz4_version_minor}.${_lz4_version_release}") + unset(_lz4_version_major) + unset(_lz4_version_minor) + unset(_lz4_version_release) + unset(_lz4_version_lines) +endif () + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(lz4 + REQUIRED_VARS lz4_LIBRARY lz4_INCLUDE_DIR + VERSION_VAR lz4_VERSION) + +if (lz4_FOUND) + set(lz4_INCLUDE_DIRS "${lz4_INCLUDE_DIR}") + set(lz4_LIBRARIES "${lz4_LIBRARY}") + + if (NOT TARGET lz4::lz4) + add_library(lz4::lz4 UNKNOWN IMPORTED) + set_target_properties(lz4::lz4 PROPERTIES + IMPORTED_LOCATION "${lz4_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}") + endif () +endif () diff --git a/contrib/absl/CMakeLists.txt b/contrib/absl/CMakeLists.txt deleted file mode 100644 index 2cd0f2be..00000000 --- a/contrib/absl/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -ADD_LIBRARY (absl-lib STATIC - numeric/int128.cc -) -TARGET_INCLUDE_DIRECTORIES (absl-lib - PUBLIC ${PROJECT_SOURCE_DIR}/contrib -) diff --git a/contrib/absl/absl/CMakeLists.txt b/contrib/absl/absl/CMakeLists.txt new file mode 100644 index 00000000..69f088f4 --- /dev/null +++ b/contrib/absl/absl/CMakeLists.txt @@ -0,0 +1,9 @@ +ADD_LIBRARY (absl_int128 STATIC + numeric/int128.cc +) + +TARGET_INCLUDE_DIRECTORIES (absl_int128 + PUBLIC ${PROJECT_SOURCE_DIR}/contrib/absl +) + +ADD_LIBRARY (absl::int128 ALIAS absl_int128) diff --git a/contrib/absl/base/attributes.h b/contrib/absl/absl/base/attributes.h similarity index 100% rename from contrib/absl/base/attributes.h rename to contrib/absl/absl/base/attributes.h diff --git a/contrib/absl/base/config.h b/contrib/absl/absl/base/config.h similarity index 100% rename from contrib/absl/base/config.h rename to contrib/absl/absl/base/config.h diff --git a/contrib/absl/base/internal/bits.h b/contrib/absl/absl/base/internal/bits.h similarity index 100% rename from contrib/absl/base/internal/bits.h rename to contrib/absl/absl/base/internal/bits.h diff --git a/contrib/absl/base/macros.h b/contrib/absl/absl/base/macros.h similarity index 100% rename from contrib/absl/base/macros.h rename to contrib/absl/absl/base/macros.h diff --git a/contrib/absl/base/optimization.h b/contrib/absl/absl/base/optimization.h similarity index 100% rename from contrib/absl/base/optimization.h rename to contrib/absl/absl/base/optimization.h diff --git a/contrib/absl/base/options.h b/contrib/absl/absl/base/options.h similarity index 100% rename from contrib/absl/base/options.h rename to contrib/absl/absl/base/options.h diff --git a/contrib/absl/base/policy_checks.h b/contrib/absl/absl/base/policy_checks.h similarity index 100% rename from contrib/absl/base/policy_checks.h rename to contrib/absl/absl/base/policy_checks.h diff --git a/contrib/absl/base/port.h b/contrib/absl/absl/base/port.h similarity index 100% rename from contrib/absl/base/port.h rename to contrib/absl/absl/base/port.h diff --git a/contrib/absl/numeric/int128.cc b/contrib/absl/absl/numeric/int128.cc similarity index 100% rename from contrib/absl/numeric/int128.cc rename to contrib/absl/absl/numeric/int128.cc diff --git a/contrib/absl/numeric/int128.h b/contrib/absl/absl/numeric/int128.h similarity index 100% rename from contrib/absl/numeric/int128.h rename to contrib/absl/absl/numeric/int128.h diff --git a/contrib/absl/numeric/int128_have_intrinsic.inc b/contrib/absl/absl/numeric/int128_have_intrinsic.inc similarity index 100% rename from contrib/absl/numeric/int128_have_intrinsic.inc rename to contrib/absl/absl/numeric/int128_have_intrinsic.inc diff --git a/contrib/absl/numeric/int128_no_intrinsic.inc b/contrib/absl/absl/numeric/int128_no_intrinsic.inc similarity index 100% rename from contrib/absl/numeric/int128_no_intrinsic.inc rename to contrib/absl/absl/numeric/int128_no_intrinsic.inc diff --git a/contrib/cityhash/CMakeLists.txt b/contrib/cityhash/CMakeLists.txt deleted file mode 100644 index e31bc245..00000000 --- a/contrib/cityhash/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -ADD_LIBRARY (cityhash-lib STATIC - city.cc -) - -set_property(TARGET cityhash-lib PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/contrib/cityhash/BUCK b/contrib/cityhash/cityhash/BUCK similarity index 100% rename from contrib/cityhash/BUCK rename to contrib/cityhash/cityhash/BUCK diff --git a/contrib/cityhash/cityhash/CMakeLists.txt b/contrib/cityhash/cityhash/CMakeLists.txt new file mode 100644 index 00000000..d1311562 --- /dev/null +++ b/contrib/cityhash/cityhash/CMakeLists.txt @@ -0,0 +1,7 @@ +ADD_LIBRARY (cityhash STATIC + city.cc +) + +set_property(TARGET cityhash PROPERTY POSITION_INDEPENDENT_CODE ON) + +ADD_LIBRARY (cityhash::cityhash ALIAS cityhash) diff --git a/contrib/cityhash/COPYING b/contrib/cityhash/cityhash/COPYING similarity index 100% rename from contrib/cityhash/COPYING rename to contrib/cityhash/cityhash/COPYING diff --git a/contrib/cityhash/city.cc b/contrib/cityhash/cityhash/city.cc similarity index 100% rename from contrib/cityhash/city.cc rename to contrib/cityhash/cityhash/city.cc diff --git a/contrib/cityhash/city.h b/contrib/cityhash/cityhash/city.h similarity index 100% rename from contrib/cityhash/city.h rename to contrib/cityhash/cityhash/city.h diff --git a/contrib/cityhash/citycrc.h b/contrib/cityhash/cityhash/citycrc.h similarity index 100% rename from contrib/cityhash/citycrc.h rename to contrib/cityhash/cityhash/citycrc.h diff --git a/contrib/cityhash/config.h b/contrib/cityhash/cityhash/config.h similarity index 100% rename from contrib/cityhash/config.h rename to contrib/cityhash/cityhash/config.h diff --git a/contrib/lz4/CMakeLists.txt b/contrib/lz4/CMakeLists.txt deleted file mode 100644 index 7b471da3..00000000 --- a/contrib/lz4/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -ADD_LIBRARY (lz4-lib STATIC - lz4.c - lz4hc.c -) - -set_property(TARGET lz4-lib PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/contrib/lz4/BUCK b/contrib/lz4/lz4/BUCK similarity index 100% rename from contrib/lz4/BUCK rename to contrib/lz4/lz4/BUCK diff --git a/contrib/lz4/lz4/CMakeLists.txt b/contrib/lz4/lz4/CMakeLists.txt new file mode 100644 index 00000000..1ec7fb07 --- /dev/null +++ b/contrib/lz4/lz4/CMakeLists.txt @@ -0,0 +1,8 @@ +ADD_LIBRARY (lz4 STATIC + lz4.c + lz4hc.c +) + +set_property(TARGET lz4 PROPERTY POSITION_INDEPENDENT_CODE ON) + +ADD_LIBRARY(lz4::lz4 ALIAS lz4) diff --git a/contrib/lz4/LICENSE b/contrib/lz4/lz4/LICENSE similarity index 100% rename from contrib/lz4/LICENSE rename to contrib/lz4/lz4/LICENSE diff --git a/contrib/lz4/lz4.c b/contrib/lz4/lz4/lz4.c similarity index 100% rename from contrib/lz4/lz4.c rename to contrib/lz4/lz4/lz4.c diff --git a/contrib/lz4/lz4.h b/contrib/lz4/lz4/lz4.h similarity index 100% rename from contrib/lz4/lz4.h rename to contrib/lz4/lz4/lz4.h diff --git a/contrib/lz4/lz4hc.c b/contrib/lz4/lz4/lz4hc.c similarity index 100% rename from contrib/lz4/lz4hc.c rename to contrib/lz4/lz4/lz4hc.c diff --git a/contrib/lz4/lz4hc.h b/contrib/lz4/lz4/lz4hc.h similarity index 100% rename from contrib/lz4/lz4hc.h rename to contrib/lz4/lz4/lz4hc.h diff --git a/tests/simple/CMakeLists.txt b/tests/simple/CMakeLists.txt index cd102ec3..3804fcec 100644 --- a/tests/simple/CMakeLists.txt +++ b/tests/simple/CMakeLists.txt @@ -4,13 +4,5 @@ ADD_EXECUTABLE (simple-test ) TARGET_LINK_LIBRARIES (simple-test - clickhouse-cpp-lib-static + clickhouse-cpp-lib ) - -IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - # there is a problem with __builtin_mul_overflow call at link time - # the error looks like: ... undefined reference to `__muloti4' ... - # caused by clang bug https://bugs.llvm.org/show_bug.cgi?id=16404 - # explicit linking to compiler-rt allows to workaround the problem - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --rtlib=compiler-rt") -ENDIF () diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index 86066965..2c9f6ee5 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -20,6 +20,7 @@ SET ( clickhouse-cpp-ut-src CreateColumnByType_ut.cpp Column_ut.cpp roundtrip_column.cpp + roundtrip_tests.cpp utils.cpp value_generators.cpp @@ -35,9 +36,10 @@ ADD_EXECUTABLE (clickhouse-cpp-ut ) TARGET_LINK_LIBRARIES (clickhouse-cpp-ut - clickhouse-cpp-lib-static + clickhouse-cpp-lib gtest-lib ) -IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --rtlib=compiler-rt") -ENDIF () + +IF (MSVC) + TARGET_COMPILE_OPTIONS(clickhouse-cpp-ut PRIVATE /bigobj) +ENDIF() diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index 51cd4980..544be975 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -106,6 +106,28 @@ class GenericColumnTest : public testing::Test { return std::tuple{column, values}; } + + static std::optional SkipTest(clickhouse::Client& client) { + if constexpr (std::is_same_v) { + // Date32 first appeared in v21.9.2.17-stable + const auto server_info = client.GetServerInfo(); + if (versionNumber(server_info) < versionNumber(21, 9)) { + std::stringstream buffer; + buffer << "Date32 is available since v21.9.2.17-stable and can't be tested against server: " << server_info; + return buffer.str(); + } + } + + if constexpr (std::is_same_v) { + const auto server_info = client.GetServerInfo(); + if (versionNumber(server_info) < versionNumber(21, 7)) { + std::stringstream buffer; + buffer << "ColumnInt128 is available since v21.7.2.7-stable and can't be tested against server: " << server_info; + return buffer.str(); + } + } + return std::nullopt; + } }; using ValueColumns = ::testing::Types< @@ -279,21 +301,33 @@ TYPED_TEST(GenericColumnTest, RoundTrip) { clickhouse::Client client(LocalHostEndpoint); - if constexpr (std::is_same_v) { - // Date32 first appeared in v21.9.2.17-stable - const auto server_info = client.GetServerInfo(); - if (versionNumber(server_info) < versionNumber(21, 9)) { - GTEST_SKIP() << "Date32 is available since v21.9.2.17-stable and can't be tested against server: " << server_info; - } + if (auto message = this->SkipTest(client)) { + GTEST_SKIP() << *message; } - if constexpr (std::is_same_v) { - const auto server_info = client.GetServerInfo(); - if (versionNumber(server_info) < versionNumber(21, 7)) { - GTEST_SKIP() << "ColumnInt128 is available since v21.7.2.7-stable and can't be tested against server: " << server_info; + auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); + EXPECT_TRUE(CompareRecursive(*column, *result_typed)); +} + +TYPED_TEST(GenericColumnTest, NulableT_RoundTrip) { + using NullableType = ColumnNullableT; + auto column = std::make_shared(this->MakeColumn()); + auto values = this->GenerateValues(100); + FromVectorGenerator is_null({true, false}); + for (size_t i = 0; i < values.size(); ++i) { + if (is_null(i)) { + column->Append(std::nullopt); + } else { + column->Append(values[i]); } } - auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); + clickhouse::Client client(LocalHostEndpoint); + + if (auto message = this->SkipTest(client)) { + GTEST_SKIP() << *message; + } + + auto result_typed = WrapColumn(RoundtripColumnValues(client, column)); EXPECT_TRUE(CompareRecursive(*column, *result_typed)); } diff --git a/ut/CreateColumnByType_ut.cpp b/ut/CreateColumnByType_ut.cpp index fb7ffd85..b6794275 100644 --- a/ut/CreateColumnByType_ut.cpp +++ b/ut/CreateColumnByType_ut.cpp @@ -50,9 +50,21 @@ TEST(CreateColumnByType, DateTime) { ASSERT_EQ(CreateColumnByType("DateTime64(3, 'UTC')")->As()->Timezone(), "UTC"); } +TEST(CreateColumnByType, AggregateFunction) { + EXPECT_EQ(nullptr, CreateColumnByType("AggregateFunction(argMax, Int32, DateTime64(3))")); + EXPECT_EQ(nullptr, CreateColumnByType("AggregateFunction(argMax, FIxedString(10), DateTime64(3, 'UTC'))")); +} + + class CreateColumnByTypeWithName : public ::testing::TestWithParam {}; +TEST(CreateColumnByType, Bool) { + const auto col = CreateColumnByType("Bool"); + ASSERT_NE(nullptr, col); + EXPECT_EQ(col->GetType().GetName(), "UInt8"); +} + TEST_P(CreateColumnByTypeWithName, CreateColumnByType) { const auto col = CreateColumnByType(GetParam()); diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 6a0af56b..dfabcba9 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -3,10 +3,10 @@ #include "readonly_client_test.h" #include "connection_failed_client_test.h" #include "utils.h" -#include "roundtrip_column.h" #include +#include #include #include @@ -273,16 +273,17 @@ TEST_P(ClientCase, LowCardinalityString_AsString) { TEST_P(ClientCase, Generic) { client_->Execute( - "CREATE TEMPORARY TABLE IF NOT EXISTS test_clickhouse_cpp_client (id UInt64, name String) "); + "CREATE TEMPORARY TABLE IF NOT EXISTS test_clickhouse_cpp_client (id UInt64, name String, f Bool) "); const struct { uint64_t id; std::string name; + bool f; } TEST_DATA[] = { - { 1, "id" }, - { 3, "foo" }, - { 5, "bar" }, - { 7, "name" }, + { 1, "id", true }, + { 3, "foo", false }, + { 5, "bar", true }, + { 7, "name", false }, }; /// Insert some values. @@ -291,20 +292,23 @@ TEST_P(ClientCase, Generic) { auto id = std::make_shared(); auto name = std::make_shared(); + auto f = std::make_shared (); for (auto const& td : TEST_DATA) { id->Append(td.id); name->Append(td.name); + f->Append(td.f); } block.AppendColumn("id" , id); block.AppendColumn("name", name); + block.AppendColumn("f", f); client_->Insert("test_clickhouse_cpp_client", block); } /// Select values inserted in the previous step. size_t row = 0; - client_->Select("SELECT id, name FROM test_clickhouse_cpp_client", [TEST_DATA, &row](const Block& block) + client_->Select("SELECT id, name, f FROM test_clickhouse_cpp_client", [TEST_DATA, &row](const Block& block) { if (block.GetRowCount() == 0) { return; @@ -314,6 +318,7 @@ TEST_P(ClientCase, Generic) { for (size_t c = 0; c < block.GetRowCount(); ++c, ++row) { EXPECT_EQ(TEST_DATA[row].id, (*block[0]->As())[c]); EXPECT_EQ(TEST_DATA[row].name, (*block[1]->As())[c]); + EXPECT_EQ(TEST_DATA[row].f, (*block[2]->As())[c]); } } ); @@ -963,65 +968,6 @@ TEST_P(ClientCase, DISABLED_ArrayArrayUInt64) { } } -TEST_P(ClientCase, RoundtripArrayTUint64) { - auto array = std::make_shared>(); - array->Append({0, 1, 2}); - - auto result = RoundtripColumnValues(*client_, array)->AsStrict(); - auto row = result->GetAsColumn(0)->As(); - - EXPECT_EQ(0u, row->At(0)); - EXPECT_EQ(1u, (*row)[1]); - EXPECT_EQ(2u, (*row)[2]); -} - -TEST_P(ClientCase, RoundtripArrayTArrayTUint64) { - const std::vector> row_values = { - {1, 2, 3}, - {4, 5, 6}, - {7, 8, 9, 10} - }; - - auto array = std::make_shared>>(); - array->Append(row_values); - - auto result_typed = ColumnArrayT>::Wrap(RoundtripColumnValues(*client_, array)); - EXPECT_TRUE(CompareRecursive(*array, *result_typed)); -} - -TEST_P(ClientCase, RoundtripArrayTArrayTArrayTUint64) { - using ColumnType = ColumnArrayT>>; - const std::vector>> row_values = { - {{1, 2, 3}, {3, 2, 1}}, - {{4, 5, 6}, {6, 5, 4}}, - {{7, 8, 9, 10}, {}}, - {{}, {10, 9, 8, 7}} - }; - - auto array = std::make_shared(); - array->Append(row_values); - - auto result_typed = ColumnType::Wrap(RoundtripColumnValues(*client_, array)); - EXPECT_TRUE(CompareRecursive(*array, *result_typed)); -} - - -TEST_P(ClientCase, RoundtripArrayTFixedString) { - auto array = std::make_shared>(6); - array->Append({"hello", "world"}); - - auto result_typed = ColumnArrayT::Wrap(RoundtripColumnValues(*client_, array)); - EXPECT_TRUE(CompareRecursive(*array, *result_typed)); -} - -TEST_P(ClientCase, RoundtripArrayTString) { - auto array = std::make_shared>(); - array->Append({"hello", "world"}); - - auto result_typed = ColumnArrayT::Wrap(RoundtripColumnValues(*client_, array)); - EXPECT_TRUE(CompareRecursive(*array, *result_typed)); -} - TEST_P(ClientCase, OnProgress) { Block block; createTableWithOneColumn(block); @@ -1171,6 +1117,46 @@ TEST_P(ClientCase, OnProfileEvents) { } } +TEST_P(ClientCase, OnProfile) { + Query query("SELECT * FROM system.numbers LIMIT 10;"); + + std::optional profile; + query.OnProfile([&profile](const Profile & new_profile) { + profile = new_profile; + }); + + client_->Execute(query); + + // Make sure that profile event came through + ASSERT_NE(profile, std::nullopt); + + EXPECT_GE(profile->rows, 10u); + EXPECT_GE(profile->blocks, 1u); + EXPECT_GT(profile->bytes, 1u); + EXPECT_GE(profile->rows_before_limit, 10u); + EXPECT_EQ(profile->applied_limit, true); + EXPECT_EQ(profile->calculated_rows_before_limit, true); +} + +TEST_P(ClientCase, SelectAggregateFunction) { + // Verifies that perofing SELECT value of type AggregateFunction(...) doesn't crash the client. + // For details: https://github.com/ClickHouse/clickhouse-cpp/issues/266 + client_->Execute("CREATE TEMPORARY TABLE IF NOT EXISTS tableplus_crash_example (col AggregateFunction(argMax, Int32, DateTime(3))) engine = Memory"); + client_->Execute("insert into tableplus_crash_example values (unhex('010000000001089170A883010000'))"); + + client_->Select("select version()", + [&](const Block& block) { + std::cerr << PrettyPrintBlock{block} << std::endl; + }); + + // Column type `AggregateFunction` is not supported. + EXPECT_THROW(client_->Select("select toTypeName(col), col from tableplus_crash_example", + [&](const Block& block) { + std::cerr << PrettyPrintBlock{block} << std::endl; + }), clickhouse::UnimplementedError); +} + + const auto LocalHostEndpoint = ClientOptions() .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) @@ -1196,7 +1182,6 @@ const auto QUERIES = std::vector{ "SELECT fqdn()", "SELECT buildId()", "SELECT uptime()", - "SELECT filesystemFree()", "SELECT now()" }; } diff --git a/ut/columns_ut.cpp b/ut/columns_ut.cpp index 604c94a4..eda33292 100644 --- a/ut/columns_ut.cpp +++ b/ut/columns_ut.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -787,3 +788,103 @@ TEST(ColumnsCase, ColumnLowCardinalityString_WithEmptyString_3) { EXPECT_EQ(values[i], col.At(i)) << " at pos: " << i; } } + + +TEST(ColumnsCase, ColumnTupleT) { + using TestTuple = ColumnTupleT; + + TestTuple col( + std::make_tuple( + std::make_shared(), + std::make_shared(), + std::make_shared(3) + ) + ); + const auto val = std::make_tuple(1, "a", "bcd"); + col.Append(val); + static_assert(std::is_same_v::type>); + static_assert(std::is_same_v::type>); + static_assert(std::is_same_v::type>); + EXPECT_EQ(val, col.At(0)); +} + +TEST(ColumnsCase, ColumnTupleT_Wrap) { + ColumnTuple col ({ + std::make_shared(), + std::make_shared(), + std::make_shared(3) + } + ); + + const auto val = std::make_tuple(1, "a", "bcd"); + + col[0]->AsStrict()->Append(std::get<0>(val)); + col[1]->AsStrict()->Append(std::get<1>(val)); + col[2]->AsStrict()->Append(std::get<2>(val)); + + using TestTuple = ColumnTupleT; + auto wrapped_col = TestTuple::Wrap(std::move(col)); + + EXPECT_EQ(wrapped_col->Size(), 1u); + EXPECT_EQ(val, wrapped_col->At(0)); +} + +TEST(ColumnsCase, ColumnTupleT_Empty) { + using TestTuple = ColumnTupleT<>; + + TestTuple col(std::make_tuple()); + const auto val = std::make_tuple(); + col.Append(val); + EXPECT_EQ(col.Size(), 0u); +} + +TEST(ColumnsCase, ColumnMapT) { + ColumnMapT col( + std::make_shared(), + std::make_shared()); + + std::map val; + val[1] = "123"; + val[2] = "abc"; + col.Append(val); + + auto map_view = col.At(0); + + EXPECT_THROW(map_view.At(0), ValidationError); + EXPECT_EQ(val[1], map_view.At(1)); + EXPECT_EQ(val[2], map_view.At(2)); + + std::map map{map_view.begin(), map_view.end()}; + + EXPECT_EQ(val[1], map.at(1)); + EXPECT_EQ(val[2], map.at(2)); +} + +TEST(ColumnsCase, ColumnMapT_Wrap) { + auto tupls = std::make_shared(std::vector{ + std::make_shared(), + std::make_shared()}); + + auto data = std::make_shared(tupls); + + auto val = tupls->CloneEmpty()->As(); + + (*val)[0]->AsStrict()->Append(1); + (*val)[1]->AsStrict()->Append("123"); + + (*val)[0]->AsStrict()->Append(2); + (*val)[1]->AsStrict()->Append("abc"); + + data->AppendAsColumn(val); + + ColumnMap col{data}; + + using TestMap = ColumnMapT; + auto wrapped_col = TestMap::Wrap(std::move(col)); + + auto map_view = wrapped_col->At(0); + + EXPECT_THROW(map_view.At(0), ValidationError); + EXPECT_EQ("123", map_view.At(1)); + EXPECT_EQ("abc", map_view.At(2)); +} diff --git a/ut/itemview_ut.cpp b/ut/itemview_ut.cpp index 40da5027..6413e190 100644 --- a/ut/itemview_ut.cpp +++ b/ut/itemview_ut.cpp @@ -80,6 +80,7 @@ TEST(ItemView, ErrorTypes) { EXPECT_ITEMVIEW_ERROR(Type::Code::Nullable, int); EXPECT_ITEMVIEW_ERROR(Type::Code::Tuple, int); EXPECT_ITEMVIEW_ERROR(Type::Code::LowCardinality, int); + EXPECT_ITEMVIEW_ERROR(Type::Code::Map, int); } TEST(ItemView, TypeSizeMismatch) { diff --git a/ut/low_cardinality_nullable_tests.cpp b/ut/low_cardinality_nullable_tests.cpp index 41c0d3c5..8918f251 100644 --- a/ut/low_cardinality_nullable_tests.cpp +++ b/ut/low_cardinality_nullable_tests.cpp @@ -78,6 +78,43 @@ TEST(LowCardinalityOfNullable, InsertAndQuery) { }); } +TEST(LowCardinalityOfNullable, InsertAndQueryOneRow) { + const auto rowsData = std::vector { + "eminem" + }; + + const auto nulls = std::vector { + false + }; + + auto column = buildTestColumn(rowsData, nulls); + + Block block; + block.AppendColumn("words", column); + + Client client(ClientOptions(localHostEndpoint) + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false) + .SetPingBeforeQuery(true)); + + createTable(client); + + client.Insert("lc_of_nullable", block); + + client.Select("SELECT * FROM lc_of_nullable", [&](const Block& bl) { + for (size_t row = 0; row < bl.GetRowCount(); row++) { + auto lc_col = bl[0]->As(); + auto item = lc_col->GetItem(row); + + if (nulls[row]) { + ASSERT_EQ(Type::Code::Void, item.type); + } else { + ASSERT_EQ(rowsData[row], item.get()); + } + } + }); +} + + TEST(LowCardinalityOfNullable, InsertAndQueryEmpty) { auto column = buildTestColumn({}, {}); @@ -113,4 +150,4 @@ TEST(LowCardinalityOfNullable, ThrowOnBackwardsCompatibleLCColumn) { client.Select("SELECT * FROM lc_of_nullable", [&](const Block& bl) { ASSERT_EQ(bl.GetRowCount(), 0u); }); -} \ No newline at end of file +} diff --git a/ut/roundtrip_tests.cpp b/ut/roundtrip_tests.cpp new file mode 100644 index 00000000..5f4fce9c --- /dev/null +++ b/ut/roundtrip_tests.cpp @@ -0,0 +1,313 @@ +#include + +#include "utils.h" +#include "roundtrip_column.h" + +#include +#include + +using namespace clickhouse; + +// Use value-parameterized tests to run same tests with different client +// options. +class RoundtripCase : public testing::TestWithParam { +protected: + void SetUp() override { + client_ = std::make_unique(GetParam()); + } + + void TearDown() override { + client_.reset(); + } + + std::string GetSettingValue(const std::string& name) { + std::string result; + client_->Select("SELECT value FROM system.settings WHERE name = \'" + name + "\'", + [&result](const Block& block) + { + if (block.GetRowCount() == 0) { + return; + } + result = block[0]->AsStrict()->At(0); + } + ); + return result; + } + + + std::unique_ptr client_; +}; + +TEST_P(RoundtripCase, ArrayTUint64) { + auto array = std::make_shared>(); + array->Append({0, 1, 2}); + + auto result = RoundtripColumnValues(*client_, array)->AsStrict(); + auto row = result->GetAsColumn(0)->As(); + + EXPECT_EQ(0u, row->At(0)); + EXPECT_EQ(1u, (*row)[1]); + EXPECT_EQ(2u, (*row)[2]); +} + +TEST_P(RoundtripCase, ArrayTArrayTUint64) { + const std::vector> row_values = { + {1, 2, 3}, + {4, 5, 6}, + {7, 8, 9, 10} + }; + + auto array = std::make_shared>>(); + array->Append(row_values); + + auto result_typed = ColumnArrayT>::Wrap(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + +TEST_P(RoundtripCase, ArrayTArrayTArrayTUint64) { + using ColumnType = ColumnArrayT>>; + const std::vector>> row_values = { + {{1, 2, 3}, {3, 2, 1}}, + {{4, 5, 6}, {6, 5, 4}}, + {{7, 8, 9, 10}, {}}, + {{}, {10, 9, 8, 7}} + }; + + auto array = std::make_shared(); + array->Append(row_values); + + auto result_typed = ColumnType::Wrap(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + + +TEST_P(RoundtripCase, ArrayTFixedString) { + auto array = std::make_shared>(6); + array->Append({"hello", "world"}); + + auto result_typed = ColumnArrayT::Wrap(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + +TEST_P(RoundtripCase, ArrayTString) { + auto array = std::make_shared>(); + array->Append({"hello", "world"}); + + auto result_typed = ColumnArrayT::Wrap(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + +TEST_P(RoundtripCase, MapTUint64String) { + using Map = ColumnMapT; + auto map = std::make_shared(std::make_shared(), std::make_shared()); + + std::map row; + row[1] = "hello"; + row[2] = "world"; + map->Append(row); + + auto result_typed = Map::Wrap(RoundtripColumnValues(*client_, map)); + EXPECT_TRUE(CompareRecursive(*map, *result_typed)); +} + +TEST_P(RoundtripCase, MapUUID_Tuple_String_Array_Uint64) { + using Tuple = ColumnTupleT>; + using Map = ColumnMapT; + auto map = std::make_shared(std::make_shared(), std::make_shared( + std::make_tuple(std::make_shared(), std::make_shared>()))); + + + std::map>> row; + row[UUID{1, 1}] = std::make_tuple("hello", std::vector{1, 2, 3}) ; + row[UUID{2, 2}] = std::make_tuple("world", std::vector{4, 5, 6}) ; + map->Append(row); + + auto result_typed = Map::Wrap(RoundtripColumnValues(*client_, map)); + EXPECT_TRUE(CompareRecursive(*map, *result_typed)); +} + +TEST_P(RoundtripCase, Point) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + col->Append({1.0, 2.0}); + col->Append({0.1, 0.2}); + + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, Ring) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector ring{{1.0, 2.0}, {3.0, 4.0}}; + col->Append(ring); + } + { + std::vector ring{{0.1, 0.2}, {0.3, 0.4}}; + col->Append(ring); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, Polygon) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector> polygon + {{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}; + col->Append(polygon); + } + { + std::vector> polygon + {{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}; + col->Append(polygon); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, MultiPolygon) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector>> multi_polygon + {{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}, + {{{1.1, 2.2}, {3.3, 4.4}}, {{5.5, 6.6}, {7.7, 8.8}}}}; + col->Append(multi_polygon); + } + { + std::vector>> multi_polygon + {{{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}, + {{{1.1, 1.2}, {1.3, 1.4}}, {{1.5, 1.6}, {1.7, 1.8}}}}; + col->Append(multi_polygon); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, LowCardinalityTString) { + using TestColumn = ColumnLowCardinalityT; + auto col = std::make_shared(); + col->Append("abc"); + col->Append("def"); + col->Append("abc"); + col->Append("abc"); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, LowCardinalityTNullableString) { + using TestColumn = ColumnLowCardinalityT>; + auto col = std::make_shared(); + col->Append("abc"); + col->Append("def"); + col->Append("abc"); + col->Append("abc"); + col->Append(std::nullopt); + col->Append(std::nullopt); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, ArrayTNullableString) { + using TestColumn = ColumnArrayT>; + auto col = std::make_shared(); + col->Append({std::nullopt, std::nullopt, std::nullopt}); + col->Append(std::vector>{"abc", std::nullopt}); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, TupleTNullableString) { + using TestColumn = ColumnTupleT>; + auto col = std::make_shared(std::make_tuple(std::make_shared>())); + col->Append(std::make_tuple(std::nullopt)); + col->Append(std::make_tuple("abc")); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, Map_TString_TNullableString) { + using Key = ColumnString; + using Value = ColumnNullableT; + using TestColumn = ColumnMapT; + auto col = std::make_shared(std::make_shared(), std::make_shared()); + { + std::map> value; + value["1"]= "one"; + value["2"]= std::nullopt; + col->Append(value); + } + { + std::map> value; + value["4"]= "one"; + value["2"]= std::nullopt; + col->Append(value); + } + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, Map_LowCardinalityTString_LowCardinalityTNullableString) { + using Key = ColumnLowCardinalityT; + using Value = ColumnLowCardinalityT>; + using TestColumn = ColumnMapT; + auto col = std::make_shared(std::make_shared(), std::make_shared()); + { + std::map> value; + value["1"]= "one"; + value["2"]= std::nullopt; + col->Append(value); + } + { + std::map> value; + value["4"]= "one"; + value["2"]= std::nullopt; + col->Append(value); + } + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, RoundtripArrayLowCardinalityTString) { + using TestColumn = ColumnArrayT>; + auto array = std::make_shared(); + array->Append(std::vector{}); + array->Append(std::vector{}); + + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + +const auto LocalHostEndpoint = ClientOptions() + .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); + +INSTANTIATE_TEST_SUITE_P( + Roundtrip, RoundtripCase, + ::testing::Values( + ClientOptions(LocalHostEndpoint) + .SetPingBeforeQuery(true) + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false), + ClientOptions(LocalHostEndpoint) + .SetPingBeforeQuery(false) + .SetCompressionMethod(CompressionMethod::LZ4) + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false) + )); diff --git a/ut/socket_ut.cpp b/ut/socket_ut.cpp index 5a263435..ee531544 100644 --- a/ut/socket_ut.cpp +++ b/ut/socket_ut.cpp @@ -8,6 +8,13 @@ #include #include +// for EAI_* error codes +#if defined(_win_) +# include +#else +# include +#endif + using namespace clickhouse; TEST(Socketcase, connecterror) { @@ -43,7 +50,7 @@ TEST(Socketcase, timeoutrecv) { std::this_thread::sleep_for(std::chrono::seconds(1)); try { - Socket socket(addr, SocketTimeoutParams { Seconds(5), Seconds(5) }); + Socket socket(addr, SocketTimeoutParams { Seconds(5), Seconds(5), Seconds(5) }); std::unique_ptr ptr_input_stream = socket.makeInputStream(); char buf[1024]; @@ -63,6 +70,49 @@ TEST(Socketcase, timeoutrecv) { server.stop(); } +TEST(Socketcase, gaierror) { + try { + NetworkAddress addr("host.invalid", "80"); // never resolves + FAIL(); + } catch (const std::system_error& e) { + ASSERT_PRED1([](int error) { return error == EAI_NONAME || error == EAI_AGAIN || error == EAI_FAIL; }, e.code().value()); + } +} + +TEST(Socketcase, connecttimeout) { + using Clock = std::chrono::steady_clock; + + try { + NetworkAddress("::1", "19980"); + } catch (const std::system_error& e) { + GTEST_SKIP() << "missing IPv6 support"; + } + + NetworkAddress addr("100::1", "19980"); // "discard" IPv6 address + + const auto connect_start = Clock::now(); + try { + Socket socket(addr, SocketTimeoutParams{std::chrono::milliseconds(100)}); + FAIL(); + } catch (const std::system_error& e) { + const int error = e.code().value(); + if (error == ENETUNREACH || error == EHOSTUNREACH +#if defined(_win_) + || error == WSAENETUNREACH +#endif + ) { + GTEST_SKIP() << "missing IPv6 support"; + } +#if defined(_win_) + const auto expected = WSAETIMEDOUT; +#else + const auto expected = ETIMEDOUT; +#endif + EXPECT_EQ(expected, error); + EXPECT_LT(Clock::now() - connect_start, std::chrono::seconds(5)); + } +} + // Test to verify that reading from empty socket doesn't hangs. //TEST(Socketcase, ReadFromEmptySocket) { // const int port = 12345; diff --git a/ut/ssl_ut.cpp b/ut/ssl_ut.cpp index f68db08c..cd567422 100644 --- a/ut/ssl_ut.cpp +++ b/ut/ssl_ut.cpp @@ -16,7 +16,6 @@ namespace { "SELECT fqdn()", "SELECT buildId()", "SELECT uptime()", - "SELECT filesystemFree()", "SELECT now()" }; } diff --git a/ut/type_parser_ut.cpp b/ut/type_parser_ut.cpp index e7fe6bf0..b0193ded 100644 --- a/ut/type_parser_ut.cpp +++ b/ut/type_parser_ut.cpp @@ -226,3 +226,47 @@ TEST(TypeParserCase, ParseDateTime64) { ASSERT_EQ(ast.elements[1].value_string, "UTC"); ASSERT_EQ(ast.elements[1].value, 0); } + +TEST(TypeParserCase, ParseMap) { + TypeAst ast; + TypeParser("Map(Int32, String)").Parse(&ast); + ASSERT_EQ(ast.meta, TypeAst::Map); + ASSERT_EQ(ast.name, "Map"); + ASSERT_EQ(ast.code, Type::Map); + ASSERT_EQ(ast.elements.size(), 2u); + ASSERT_EQ(ast.elements[0].meta, TypeAst::Terminal); + ASSERT_EQ(ast.elements[0].name, "Int32"); + ASSERT_EQ(ast.elements[1].meta, TypeAst::Terminal); + ASSERT_EQ(ast.elements[1].name, "String"); +} + +TEST(TypeParser, EmptyName) { + { + TypeAst ast; + EXPECT_EQ(false, TypeParser("").Parse(&ast)); + } + + { + TypeAst ast; + EXPECT_EQ(false, TypeParser(" ").Parse(&ast)); + } +} + +TEST(ParseTypeName, EmptyName) { + // Empty and invalid names shouldn't produce any AST and shoudn't crash + EXPECT_EQ(nullptr, ParseTypeName("")); + EXPECT_EQ(nullptr, ParseTypeName(" ")); + EXPECT_EQ(nullptr, ParseTypeName(std::string(5, '\0'))); +} + +TEST(TypeParser, AggregateFunction) { + { + TypeAst ast; + EXPECT_FALSE(TypeParser("AggregateFunction(argMax, Int32, DateTime(3))").Parse(&ast)); + } + + { + TypeAst ast; + EXPECT_FALSE(TypeParser("AggregateFunction(argMax, LowCardinality(Nullable(FixedString(4))), DateTime(3, 'UTC'))").Parse(&ast)); + } +} diff --git a/ut/types_ut.cpp b/ut/types_ut.cpp index c5922d0e..7af343b5 100644 --- a/ut/types_ut.cpp +++ b/ut/types_ut.cpp @@ -32,6 +32,8 @@ TEST(TypesCase, TypeName) { Type::CreateEnum8({})->GetName(), "Enum8()" ); + + ASSERT_EQ(Type::CreateMap(Type::CreateSimple(), Type::CreateString())->GetName(), "Map(Int32, String)"); } TEST(TypesCase, NullableType) { @@ -78,7 +80,7 @@ TEST(TypesCase, IsEqual) { const std::string type_names[] = { "UInt8", "Int8", - "UInt128", +// "UInt128", "String", "FixedString(0)", "FixedString(10000)", @@ -103,6 +105,15 @@ TEST(TypesCase, IsEqual) { "Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime)))))", "Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC')))))))" "Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC'), Tuple(LowCardinality(String), Enum8('READ'=1, 'WRITE'=0))))))))", + "Map(String, Int8)", + "Map(String, Tuple(String, Int8, Date, DateTime))", + "Map(UUID, Array(Tuple(String, Int8, Date, DateTime)))", + "Map(String, Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime))))))", + "Map(LowCardinality(FixedString(10)), Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC'))))))))", + "Point", + "Ring", + "Polygon", + "MultiPolygon" }; // Check that Type::IsEqual returns true only if: @@ -117,7 +128,11 @@ TEST(TypesCase, IsEqual) { EXPECT_TRUE(type->IsEqual(*type)); for (const auto & other_type_name : type_names) { - const auto other_type = clickhouse::CreateColumnByType(other_type_name)->Type(); + SCOPED_TRACE(other_type_name); + const auto other_column = clickhouse::CreateColumnByType(other_type_name); + ASSERT_NE(nullptr, other_column); + + const auto other_type = other_column->Type(); const auto should_be_equal = type_name == other_type_name; EXPECT_EQ(should_be_equal, type->IsEqual(other_type)) @@ -133,9 +148,9 @@ TEST(TypesCase, ErrorEnumContent) { "Enum8('ONE'=1,'TWO')", "Enum16('TWO'=,'TWO')", }; - + for (const auto& type_name : type_names) { SCOPED_TRACE(type_name); EXPECT_THROW(clickhouse::CreateColumnByType(type_name)->Type(), ValidationError); } -} \ No newline at end of file +} diff --git a/ut/utils.cpp b/ut/utils.cpp index e624f45c..bfa4872e 100644 --- a/ut/utils.cpp +++ b/ut/utils.cpp @@ -7,9 +7,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -19,6 +21,7 @@ #include #include #include +#include namespace { @@ -46,10 +49,16 @@ std::ostream& operator<<(std::ostream & ostr, const DateTimeValue & time) { return ostr << buffer; } -template +template ().At(0)) > bool doPrintValue(const ColumnRef & c, const size_t row, std::ostream & ostr) { if (const auto & casted_c = c->As()) { - ostr << static_cast(casted_c->At(row)); + if constexpr (is_container_v> + && !std::is_same_v + && !std::is_same_v) { + ostr << PrintContainer{static_cast(casted_c->At(row))}; + } else { + ostr << static_cast(casted_c->At(row)); + } return true; } return false; @@ -125,6 +134,27 @@ bool doPrintValue(const ColumnRef & c, const size_t row, std:: return false; } +template <> +bool doPrintValue(const ColumnRef & c, const size_t row, std::ostream & ostr) { + // via temporary stream to preserve fill and alignment of the ostr + std::stringstream sstr; + if (const auto & map_col = c->As()) { + sstr << "{"; + const auto tuples = map_col->GetAsColumn(row); + for (size_t i = 0; i < tuples->Size(); ++i) { + printColumnValue(tuples, i, sstr); + + if (i < tuples->Size() - 1) + sstr << ", "; + } + + sstr << "}"; + ostr << sstr.str(); + return true; + } + return false; +} + std::ostream & printColumnValue(const ColumnRef& c, const size_t row, std::ostream & ostr) { const auto r = false @@ -150,7 +180,12 @@ std::ostream & printColumnValue(const ColumnRef& c, const size_t row, std::ostre || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) - || doPrintValue(c, row, ostr); + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr); if (!r) ostr << "Unable to print value of type " << c->GetType().GetName(); @@ -278,6 +313,25 @@ std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info) { << " (" << server_info.revision << ")"; } +std::ostream & operator<<(std::ostream & ostr, const Profile & profile) { + return ostr + << "rows : " << profile.rows + << " blocks : " << profile.blocks + << " bytes : " << profile.bytes + << " rows_before_limit : " << profile.rows_before_limit + << " applied_limit : " << profile.applied_limit + << " calculated_rows_before_limit : " << profile.calculated_rows_before_limit; +} + +std::ostream & operator<<(std::ostream & ostr, const Progress & progress) { + return ostr + << "rows : " << progress.rows + << " bytes : " << progress.bytes + << " total_rows : " << progress.total_rows + << " written_rows : " << progress.written_rows + << " written_bytes : " << progress.written_bytes; +} + } uint64_t versionNumber(const ServerInfo & server_info) { diff --git a/ut/utils.h b/ut/utils.h index 621119fb..b9484626 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -3,9 +3,11 @@ #include #include +#include "clickhouse/query.h" #include "utils_meta.h" #include "utils_comparison.h" +#include #include #include #include @@ -23,6 +25,9 @@ namespace clickhouse { class Block; class Type; struct ServerInfo; + struct Profile; + struct QuerySettingsField; + struct Progress; } template @@ -76,6 +81,27 @@ inline const char * getPrefix() { return prefix; } +template > +inline std::ostream & printTuple(std::ostream & ostr, [[maybe_unused]] const T & t) { + static_assert(index <= std::tuple_size_v); + if constexpr (index == 0) { + return ostr << "( "; + } else { + printTuple(ostr, t); + using ElementType = std::tuple_element_t; + if constexpr (is_container_v) { + ostr << PrintContainer{std::get(t)}; + } else { + ostr << std::get<0>(t); + } + if constexpr (index == std::tuple_size_v) { + return ostr << " )"; + } else { + return ostr << ", "; + } + } +} + namespace std { template inline ostream & operator<<(ostream & ostr, const chrono::duration & d) { @@ -86,6 +112,20 @@ template inline ostream & operator<<(ostream & ostr, const pair & t) { return ostr << "{ " << t.first << ", " << t.second << " }"; } + +template +inline ostream & operator<<(ostream & ostr, const tuple & t) { + return printTuple(ostr, t); +} + +template +inline ostream & operator<<(ostream & ostr, const optional & t) { + if (t.has_value()) { + return ostr << *t; + } else { + return ostr << "NULL"; + } +} } @@ -100,6 +140,8 @@ namespace clickhouse { std::ostream& operator<<(std::ostream & ostr, const Block & block); std::ostream& operator<<(std::ostream & ostr, const Type & type); std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info); +std::ostream & operator<<(std::ostream & ostr, const Profile & profile); +std::ostream & operator<<(std::ostream & ostr, const Progress & progress); } std::ostream& operator<<(std::ostream & ostr, const PrettyPrintBlock & block);