From b5b85e39bd16f7132107ee8ae577b1b48ad27a06 Mon Sep 17 00:00:00 2001 From: Chengx Date: Mon, 31 Oct 2022 14:14:50 +0800 Subject: [PATCH 01/44] Update bench.cpp fix bench compile error --- bench/bench.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/bench.cpp b/bench/bench.cpp index 4078e3f2..2b037176 100644 --- a/bench/bench.cpp +++ b/bench/bench.cpp @@ -1,6 +1,7 @@ #include #include +#include namespace clickhouse { From f065736423ca4c1075a3031aac1aac481dea6048 Mon Sep 17 00:00:00 2001 From: wangwei <1261385937@qq.com> Date: Sun, 27 Nov 2022 20:56:14 +0800 Subject: [PATCH 02/44] timeout from seconds to milliseconds --- clickhouse/client.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse/client.h b/clickhouse/client.h index 452226d9..e35fcb65 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -87,8 +87,8 @@ struct ClientOptions { DECLARE_FIELD(tcp_nodelay, bool, TcpNoDelay, true); /// Connection socket timeout. If the timeout is set to zero then the operation will never timeout. - DECLARE_FIELD(connection_recv_timeout, std::chrono::seconds, SetConnectionRecvTimeout, std::chrono::seconds(0)); - DECLARE_FIELD(connection_send_timeout, std::chrono::seconds, SetConnectionSendTimeout, std::chrono::seconds(0)); + DECLARE_FIELD(connection_recv_timeout, std::chrono::milliseconds, SetConnectionRecvTimeout, std::chrono::milliseconds(0)); + DECLARE_FIELD(connection_send_timeout, std::chrono::milliseconds, SetConnectionSendTimeout, std::chrono::milliseconds(0)); /** It helps to ease migration of the old codebases, which can't afford to switch * to using ColumnLowCardinalityT or ColumnLowCardinality directly, From 5ad9b9cd086bcda5867b40236753ced13de4f1bc Mon Sep 17 00:00:00 2001 From: den818 Date: Sun, 27 Nov 2022 17:19:06 +0400 Subject: [PATCH 03/44] Support column type: map --- README.md | 3 +- clickhouse/CMakeLists.txt | 2 + clickhouse/client.h | 1 + clickhouse/columns/array.h | 44 ++++-- clickhouse/columns/factory.cpp | 24 +++- clickhouse/columns/itemview.cpp | 1 + clickhouse/columns/map.cpp | 83 +++++++++++ clickhouse/columns/map.h | 235 +++++++++++++++++++++++++++++++ clickhouse/columns/tuple.h | 112 ++++++++++++++- clickhouse/columns/utils.h | 20 +++ clickhouse/types/type_parser.cpp | 5 + clickhouse/types/type_parser.h | 3 +- clickhouse/types/types.cpp | 21 ++- clickhouse/types/types.h | 22 ++- ut/client_ut.cpp | 29 ++++ ut/columns_ut.cpp | 101 +++++++++++++ ut/itemview_ut.cpp | 1 + ut/type_parser_ut.cpp | 13 ++ ut/types_ut.cpp | 11 +- ut/utils.cpp | 25 +++- ut/utils.h | 26 ++++ 21 files changed, 764 insertions(+), 18 deletions(-) create mode 100644 clickhouse/columns/map.cpp create mode 100644 clickhouse/columns/map.h diff --git a/README.md b/README.md index acb4232a..b35d5c55 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ C++ client for [ClickHouse](https://clickhouse.com/). * UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 * Int128 * UUID +* Map ## Building @@ -87,7 +88,7 @@ int main() ## Retries If you wish to implement some retry logic atop of `clickhouse::Client` there are few simple rules to make you life easier: -- If previous attempt threw an exception, then make sure to call `clickhouse::Client::ResetConnection()` before the next try. +- If previous attempt threw an exception, then make sure to call `clickhouse::Client::ResetConnection()` before the next try. - For `clickhouse::Client::Insert()` you can reuse a block from previous try, no need to rebuild it from scratch. See https://github.com/ClickHouse/clickhouse-cpp/issues/184 for details. diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index db1c8692..2caeebba 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -18,6 +18,7 @@ SET ( clickhouse-cpp-lib-src columns/lowcardinalityadaptor.h columns/nullable.cpp columns/numeric.cpp + columns/map.cpp columns/string.cpp columns/tuple.cpp columns/uuid.cpp @@ -121,6 +122,7 @@ INSTALL(FILES columns/itemview.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/lowcardinality.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/nullable.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/numeric.h DESTINATION include/clickhouse/columns/) +INSTALL(FILES columns/map.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/string.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/tuple.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/utils.h DESTINATION include/clickhouse/columns/) diff --git a/clickhouse/client.h b/clickhouse/client.h index 452226d9..7dfc75bd 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -12,6 +12,7 @@ #include "columns/lowcardinality.h" #include "columns/nullable.h" #include "columns/numeric.h" +#include "columns/map.h" #include "columns/string.h" #include "columns/tuple.h" #include "columns/uuid.h" diff --git a/clickhouse/columns/array.h b/clickhouse/columns/array.h index 1d3eb192..f3bb23d6 100644 --- a/clickhouse/columns/array.h +++ b/clickhouse/columns/array.h @@ -2,6 +2,7 @@ #include "column.h" #include "numeric.h" +#include "utils.h" #include @@ -121,13 +122,8 @@ class ColumnArrayT : public ColumnArray { * This is a static method to make such conversion verbose. */ static auto Wrap(ColumnArray&& col) { - if constexpr (std::is_base_of_v && !std::is_same_v) { - // assuming NestedColumnType is ArrayT specialization - return std::make_shared>(NestedColumnType::Wrap(col.GetData()), col.offsets_); - } else { - auto nested_data = col.GetData()->template AsStrict(); - return std::make_shared>(nested_data, col.offsets_); - } + auto nested_data = WrapColumn(col.GetData()); + return std::make_shared>(nested_data, col.offsets_); } static auto Wrap(Column&& col) { @@ -146,7 +142,7 @@ class ColumnArrayT : public ColumnArray { const size_t size_; public: - using ValueType = typename NestedColumnType::ValueType; + using ValueType = std::decay_t().At(0))>; ArrayValueView(std::shared_ptr data, size_t offset = 0, size_t size = std::numeric_limits::max()) : typed_nested_data_(data) @@ -178,7 +174,7 @@ class ColumnArrayT : public ColumnArray { , index_(index) {} - using ValueType = typename NestedColumnType::ValueType; + using ValueType = typename ArrayValueView::ValueType; inline auto operator*() const { return typed_nested_data_->At(offset_ + index_); @@ -226,6 +222,22 @@ class ColumnArrayT : public ColumnArray { inline size_t Size() const { return size_; } + + inline bool operator==(const ArrayValueView& other) const { + if (size() != other.size()) { + return false; + } + for (size_t i = 0; i < size_; ++i) { + if ((*this)[i] != other[i]) { + return false; + } + } + return true; + } + + inline bool operator!=(const ArrayValueView& other) const { + return !(*this == other); + } }; inline auto At(size_t index) const { @@ -267,6 +279,20 @@ class ColumnArrayT : public ColumnArray { AddOffset(counter); } + ColumnRef Slice(size_t begin, size_t size) const override { + return Wrap(ColumnArray::Slice(begin, size)); + } + + ColumnRef CloneEmpty() const override { + return Wrap(ColumnArray::CloneEmpty()); + } + + void Swap(Column& other) override { + auto & col = dynamic_cast &>(other); + typed_nested_data_.swap(col.typed_nested_data_); + ColumnArray::Swap(other); + } + private: /// Helper to allow wrapping a "typeless" ColumnArray ColumnArrayT(ColumnArray&& array, std::shared_ptr nested_data) diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index 38b02e1d..2ea7b6f5 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -14,6 +14,7 @@ #include "string.h" #include "tuple.h" #include "uuid.h" +#include "map.h" #include "../types/type_parser.h" @@ -86,7 +87,7 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) { case Type::Date: return std::make_shared(); case Type::Date32: - return std::make_shared(); + return std::make_shared(); case Type::IPv4: return std::make_shared(); @@ -198,6 +199,27 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti return CreateTerminalColumn(ast.elements.back()); } + case TypeAst::Map: { + if (ast.elements.size() != 2) { + throw ValidationError(ast.name + " content is not correct"); + } + + std::vector columns; + + columns.reserve(ast.elements.size()); + for (const auto& elem : ast.elements) { + if (auto col = CreateColumnFromAst(elem, settings)) { + columns.push_back(col); + } else { + return nullptr; + } + } + + return std::make_shared( + std::make_shared( + std::make_shared(columns))); + } + case TypeAst::Assign: case TypeAst::Null: case TypeAst::Number: diff --git a/clickhouse/columns/itemview.cpp b/clickhouse/columns/itemview.cpp index a2cb69c2..12c89c31 100644 --- a/clickhouse/columns/itemview.cpp +++ b/clickhouse/columns/itemview.cpp @@ -77,6 +77,7 @@ void ItemView::ValidateData(Type::Code type, DataType data) { case Type::Code::Nullable: case Type::Code::Tuple: case Type::Code::LowCardinality: + case Type::Code::Map: throw AssertionError("Unsupported type in ItemView: " + std::string(Type::TypeName(type))); case Type::Code::IPv6: diff --git a/clickhouse/columns/map.cpp b/clickhouse/columns/map.cpp new file mode 100644 index 00000000..3f5616df --- /dev/null +++ b/clickhouse/columns/map.cpp @@ -0,0 +1,83 @@ +#include "map.h" + +#include + +#include "../exceptions.h" +#include "utils.h" + +namespace { + +using namespace clickhouse; + +TypeRef GetMapType(const Type& data_type) { + auto array = data_type.As(); + if (!array) { + throw ValidationError("Wrong type " + data_type.GetName() + " of data for map"); + } + auto tuple = array->GetItemType()->As(); + if (!tuple) { + throw ValidationError("Wrong type " + data_type.GetName() + " of data for map"); + } + auto types = tuple->GetTupleType(); + if (types.size() != 2) { + throw ValidationError("Wrong type " + data_type.GetName() + " of data for map"); + } + return Type::CreateMap(types[0], types[1]); +} + +} // namespace + +namespace clickhouse { + +ColumnMap::ColumnMap(ColumnRef data) + : Column(GetMapType(data->GetType())), data_(data->As()) { +} + +void ColumnMap::Clear() { + data_->Clear(); +} + +void ColumnMap::Append(ColumnRef column) { + if (auto col = column->As()) { + data_->Append(col->data_); + } +} + +bool ColumnMap::LoadPrefix(InputStream* input, size_t rows) { + return data_->LoadPrefix(input, rows); +} + +bool ColumnMap::LoadBody(InputStream* input, size_t rows) { + return data_->LoadBody(input, rows); +} + +void ColumnMap::SavePrefix(OutputStream* output) { + data_->SavePrefix(output); +} + +void ColumnMap::SaveBody(OutputStream* output) { + data_->SaveBody(output); +} + +size_t ColumnMap::Size() const { + return data_->Size(); +} + +ColumnRef ColumnMap::Slice(size_t begin, size_t len) const { + return std::make_shared(data_->Slice(begin, len)); +} + +ColumnRef ColumnMap::CloneEmpty() const { + return std::make_shared(data_->CloneEmpty()); +} + +void ColumnMap::Swap(Column& other) { + auto& col = dynamic_cast(other); + data_.swap(col.data_); +} + +ColumnRef ColumnMap::GetAsColumn(size_t n) const { + return data_->GetAsColumn(n); +} + +} // namespace clickhouse diff --git a/clickhouse/columns/map.h b/clickhouse/columns/map.h new file mode 100644 index 00000000..b770f6b8 --- /dev/null +++ b/clickhouse/columns/map.h @@ -0,0 +1,235 @@ +#pragma once + +#include "array.h" +#include "column.h" +#include "tuple.h" + +#include + +namespace clickhouse { + +template +class ColumnMapT; + +/** + * Represents column of Map(K, V). + */ +class ColumnMap : public Column { +public: + /** Create a map of given type, with actual values and offsets. + * + * Both `data` and `offsets` are used (and modified) internally bye ColumnArray. + * Users are strongly advised against modifying contents of `data` or `offsets` afterwards. + */ + explicit ColumnMap(ColumnRef data); + + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override; + + /// Loads column prefix from input stream. + bool LoadPrefix(InputStream* input, size_t rows) override; + + /// Loads column data from input stream. + bool LoadBody(InputStream* input, size_t rows) override; + + /// Saves column prefix to output stream. + void SavePrefix(OutputStream* output) override; + + /// Saves column data to output stream. + void SaveBody(OutputStream* output) override; + + /// Clear column data . + void Clear() override; + + /// Returns count of rows in the column. + size_t Size() const override; + + /// Makes slice of the current column. + ColumnRef Slice(size_t, size_t) const override; + ColumnRef CloneEmpty() const override; + void Swap(Column&) override; + + /// Converts map at pos n to column. + /// Type of row is tuple {key, value}. + ColumnRef GetAsColumn(size_t n) const; + +protected: + template + friend class ColumnMapT; + + ColumnMap(ColumnMap&& map); + +private: + std::shared_ptr data_; +}; + +template +class ColumnMapT : public ColumnMap { +public: + using KeyColumnType = K; + using ValueColumnType = V; + using Key = std::decay_t().At(0))>; + using Value = std::decay_t().At(0))>; + using TupleColumnType = ColumnTupleT; + using ArrayColumnType = ColumnArrayT; + + ColumnMapT(ColumnRef data) + : ColumnMap(data), typed_data_(data->AsStrict>()) {} + + ColumnMapT(std::shared_ptr keys, std::shared_ptr values) + : ColumnMap(std::make_shared(std::make_shared( + std::make_tuple(std::move(keys), std::move(values))))), + typed_data_(data_->template As()) {} + + ColumnRef Slice(size_t begin, size_t len) const override { + return std::make_shared>(typed_data_->Slice(begin, len)); + } + + ColumnRef CloneEmpty() const override { + return std::make_shared>(typed_data_->CloneEmpty()); + } + + void Swap(Column& other) override { + auto& col = dynamic_cast&>(other); + col.typed_data_.swap(typed_data_); + ColumnMap::Swap(other); + } + + /// A single (row) value of the Map-column i.e. read-only map. + /// It has a linear time complexity to access items + /// Because data base type has same structure + /// "This lookup works now with a linear complexity." + /// https://clickhouse.com/docs/en/sql-reference/data-types/map + /// Convert it to a suitable container required to access more than one element + + class MapValueView { + const typename ArrayColumnType::ArrayValueView data_; + + public: + using ValueType = std::pair; + + MapValueView(typename ArrayColumnType::ArrayValueView data) : data_(std::move(data)) {} + + inline auto operator[](const Key& key) const { return (*Find(key)).second; } + + inline auto At(const Key& key) const { + auto it = Find(key); + if (it == end()) throw ValidationError("ColumnMap value key not found"); + return (*it).second; + } + + class Iterator { + typename ArrayColumnType::ArrayValueView::Iterator data_iterator_; + + public: + Iterator(typename ArrayColumnType::ArrayValueView::Iterator data_iterator) + : data_iterator_(data_iterator) {} + + using ValueType = std::pair; + + inline auto operator*() const { + auto tuple = *data_iterator_; + return ValueType{std::get<0>(tuple), std::get<1>(tuple)}; + } + + inline Iterator& operator++() { + ++data_iterator_; + return *this; + } + + inline bool operator==(const Iterator& other) const { + return this->data_iterator_ == other.data_iterator_; + } + + inline bool operator!=(const Iterator& other) const { return !(*this == other); } + }; + + // minimalistic stl-like container interface, hence the lowercase + inline Iterator begin() const { return Iterator{data_.begin()}; } + + inline Iterator cbegin() const { return Iterator{data_.cbegin()}; } + + inline Iterator end() const { return Iterator{data_.end()}; } + + inline Iterator cend() const { return Iterator{data_.cend()}; } + + inline size_t size() const { return data_.size(); } + + // It is ugly to have both size() and Size(), but it is for compatitability with both STL + // and rest of the clickhouse-cpp. + inline size_t Size() const { return data_.Size(); } + + inline size_t Count(const Key& key) const { + size_t result = 0; + for (auto item : data_) { + if (std::get<0>(item) == key) { + ++result; + } + } + return result; + } + + inline Iterator Find(const Key& key) const { + for (auto it = data_.begin(); it != data_.end(); ++it) { + if (std::get<0>(*it) == key) { + return Iterator{it}; + } + } + return end(); + } + + inline bool operator==(const MapValueView& other) const { + if (size() != other.size()) { + return false; + } + using Vector = std::vector>; + Vector l(begin(), end()); + Vector r(other.begin(), other.end()); + auto comp = [](const auto& l, const auto& r) { return l.frist < r.first; }; + std::sort(l.begin(), l.end(), comp); + std::sort(r.begin(), r.end(), comp); + return std::equal(l.begin(), l.end(), r.begin(), r.end()); + return true; + } + + inline bool operator!=(const MapValueView& other) const { return !(*this == other); } + }; + + inline auto At(size_t index) const { return MapValueView{typed_data_->At(index)}; } + + inline auto operator[](size_t index) const { return MapValueView{typed_data_->At(index)}; } + + using ColumnMap::Append; + + inline void Append(const MapValueView& value) { typed_data_->Append(value.data_); } + + inline void Append(const std::vector>& tuples) { + typed_data_->Append(tuples.begin(), tuples.end()); + } + + template + inline void Append(const T& value) { + // TODO Refuse to copy. + std::vector> container; + container.reserve(value.size()); + for (const auto& i : value) { + container.emplace_back(i.first, i.second); + } + typed_data_->Append(container.begin(), container.end()); + } + + static auto Wrap(ColumnMap&& col) { + auto data = ArrayColumnType::Wrap(std::move(col.data_)); + return std::make_shared>(std::move(data)); + } + + static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast(col))); } + + // Helper to simplify integration with other APIs + static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict())); } + +private: + std::shared_ptr typed_data_; +}; + +} // namespace clickhouse diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index 63cfc689..87303c2f 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -1,6 +1,7 @@ #pragma once #include "column.h" +#include "utils.h" #include @@ -51,4 +52,113 @@ class ColumnTuple : public Column { std::vector columns_; }; -} +template +class ColumnTupleT : public ColumnTuple { +public: + using TupleOfColumns = std::tuple...>; + + using ValueType = std::tuple().At(0))>...>; + + ColumnTupleT(std::tuple...> columns) + : ColumnTuple(TupleToVector(columns)), typed_columns_(std::move(columns)) {} + + ColumnTupleT(std::vector columns) + : ColumnTuple(columns), typed_columns_(VectorToTuple(std::move(columns))) {} + + ColumnTupleT(const std::initializer_list columns) + : ColumnTuple(columns), typed_columns_(VectorToTuple(std::move(columns))) {} + + inline ValueType At(size_t index) const { return GetTupleOfValues(index); } + + inline ValueType operator[](size_t index) const { return GetTupleOfValues(index); } + + template > + inline void Append(T value) { + static_assert(index <= std::tuple_size_v); + static_assert(std::tuple_size_v == std::tuple_size_v); + if constexpr (index == 0) { + return; + } else { + std::get(typed_columns_)->Append(std::move(std::get(value))); + Append(std::move(value)); + } + } + + /** Create a ColumnTupleT from a ColumnTuple, without copying data and offsets, but by + * 'stealing' those from `col`. + * + * Ownership of column internals is transferred to returned object, original (argument) object + * MUST NOT BE USED IN ANY WAY, it is only safe to dispose it. + * + * Throws an exception if `col` is of wrong type, it is safe to use original col in this case. + * This is a static method to make such conversion verbose. + */ + static auto Wrap(ColumnTuple&& col) { + if (col.TupleSize() != std::tuple_size_v) { + throw ValidationError("Can't wrap from " + col.GetType().GetName()); + } + return std::make_shared>(VectorToTuple(std::move(col))); + } + + static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast(col))); } + + // Helper to simplify integration with other APIs + static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict())); } + + ColumnRef Slice(size_t begin, size_t size) const override { + return Wrap(ColumnTuple::Slice(begin, size)); + } + + ColumnRef CloneEmpty() const override { return Wrap(ColumnTuple::CloneEmpty()); } + + void Swap(Column& other) override { + auto& col = dynamic_cast&>(other); + typed_columns_.swap(col.typed_columns_); + ColumnTuple::Swap(other); + } + +private: + template > + inline static std::vector TupleToVector(const T& value) { + static_assert(index <= std::tuple_size_v); + if constexpr (index == 0) { + std::vector result; + result.reserve(std::tuple_size_v); + return result; + } else { + auto result = std::move(TupleToVector(value)); + result.push_back(std::get(value)); + return result; + } + } + + template > + inline static auto VectorToTuple(T columns) { + static_assert(column_index <= std::tuple_size_v); + if constexpr (column_index == 0) { + return std::make_tuple(); + } else { + using ColumnType = + typename std::tuple_element::type::element_type; + auto column = WrapColumn(columns[column_index - 1]); + return std::tuple_cat(std::move(VectorToTuple(std::move(columns))), + std::make_tuple(std::move(column))); + } + } + + template > + inline auto GetTupleOfValues(size_t index) const { + static_assert(column_index <= std::tuple_size_v); + if constexpr (column_index == 0) { + return std::make_tuple(); + } else { + return std::tuple_cat( + std::move(GetTupleOfValues(index)), + std::move(std::make_tuple(std::get(typed_columns_)->At(index)))); + } + } + + TupleOfColumns typed_columns_; +}; + +} // namespace clickhouse diff --git a/clickhouse/columns/utils.h b/clickhouse/columns/utils.h index bcc5b988..9fbafa16 100644 --- a/clickhouse/columns/utils.h +++ b/clickhouse/columns/utils.h @@ -17,4 +17,24 @@ std::vector SliceVector(const std::vector& vec, size_t begin, size_t len) return result; } +template +struct HasWrapMethod { +private: + static int detect(...); + template + static decltype(U::Wrap(std::move(std::declval()))) detect(const U&); + +public: + static constexpr bool value = !std::is_same()))>::value; +}; + +template +inline std::shared_ptr WrapColumn(ColumnRef&& column) { + if constexpr (HasWrapMethod::value) { + return T::Wrap(std::move(column)); + } else { + return column->template AsStrict(); + } +} + } diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index 37a049a0..ee05a244 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -46,6 +46,7 @@ static const std::unordered_map kTypeCode = { { "Decimal64", Type::Decimal64 }, { "Decimal128", Type::Decimal128 }, { "LowCardinality", Type::LowCardinality }, + { "Map", Type::Map}, }; static Type::Code GetTypeCode(const std::string& name) { @@ -85,6 +86,10 @@ static TypeAst::Meta GetTypeMeta(const StringView& name) { return TypeAst::SimpleAggregateFunction; } + if (name == "Map") { + return TypeAst::Map; + } + return TypeAst::Terminal; } diff --git a/clickhouse/types/type_parser.h b/clickhouse/types/type_parser.h index 2c81d2e5..2f8f2f6f 100644 --- a/clickhouse/types/type_parser.h +++ b/clickhouse/types/type_parser.h @@ -21,7 +21,8 @@ struct TypeAst { Tuple, Enum, LowCardinality, - SimpleAggregateFunction + SimpleAggregateFunction, + Map }; /// Type's category. diff --git a/clickhouse/types/types.cpp b/clickhouse/types/types.cpp index c34c0b4e..370bc47b 100644 --- a/clickhouse/types/types.cpp +++ b/clickhouse/types/types.cpp @@ -46,6 +46,7 @@ const char* Type::TypeName(Type::Code code) { case Type::Code::LowCardinality: return "LowCardinality"; case Type::Code::DateTime64: return "DateTime64"; case Type::Code::Date32: return "Date32"; + case Type::Code::Map: return "Map"; } return "Unknown type"; @@ -94,6 +95,8 @@ std::string Type::GetName() const { return As()->GetName(); case LowCardinality: return As()->GetName(); + case Map: + return As()->GetName(); } // XXX: NOT REACHED! @@ -138,7 +141,8 @@ uint64_t Type::GetTypeUniqueId() const { case Decimal32: case Decimal64: case Decimal128: - case LowCardinality: { + case LowCardinality: + case Map: { // For complex types, exact unique ID depends on nested types and/or parameters, // the easiest way is to lazy-compute unique ID from name once. // Here we do not care if multiple threads are computing value simultaneosly since it is both: @@ -225,6 +229,10 @@ TypeRef Type::CreateLowCardinality(TypeRef item_type) { return std::make_shared(item_type); } +TypeRef Type::CreateMap(TypeRef key_type, TypeRef value_type) { + return std::make_shared(key_type, value_type); +} + /// class ArrayType ArrayType::ArrayType(TypeRef item_type) : Type(Array), item_type_(item_type) { @@ -404,4 +412,15 @@ std::string TupleType::GetName() const { return result; } +/// class MapType +MapType::MapType(TypeRef key_type, TypeRef value_type) + : Type(Map) + , key_type_(key_type) + , value_type_(value_type) { +} + +std::string MapType::GetName() const { + return std::string("Map(") + key_type_->GetName() + ", " +value_type_->GetName() + ")"; +} + } // namespace clickhouse diff --git a/clickhouse/types/types.h b/clickhouse/types/types.h index 8a27257a..16570f34 100644 --- a/clickhouse/types/types.h +++ b/clickhouse/types/types.h @@ -50,6 +50,7 @@ class Type { LowCardinality, DateTime64, Date32, + Map }; using EnumItem = std::pair; @@ -92,7 +93,7 @@ class Type { static TypeRef CreateDate(); - static TypeRef CreateDate32(); + static TypeRef CreateDate32(); static TypeRef CreateDateTime(std::string timezone = std::string()); @@ -125,6 +126,8 @@ class Type { static TypeRef CreateLowCardinality(TypeRef item_type); + static TypeRef CreateMap(TypeRef key_type, TypeRef value_type); + private: uint64_t GetTypeUniqueId() const; @@ -280,6 +283,23 @@ class LowCardinalityType : public Type { TypeRef nested_type_; }; +class MapType : public Type { +public: + explicit MapType(TypeRef key_type, TypeRef value_type); + + std::string GetName() const; + + /// Type of keys. + TypeRef GetKeyType() const { return key_type_; } + + /// Type of values. + TypeRef GetValueType() const { return value_type_; } + +private: + TypeRef key_type_; + TypeRef value_type_; +}; + template <> inline TypeRef Type::CreateSimple() { return TypeRef(new Type(Int8)); diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 6a0af56b..58d6304b 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1022,6 +1022,35 @@ TEST_P(ClientCase, RoundtripArrayTString) { EXPECT_TRUE(CompareRecursive(*array, *result_typed)); } +TEST_P(ClientCase, RoundtripMapTUint64String) { + using Map = ColumnMapT; + auto map = std::make_shared(std::make_shared(), std::make_shared()); + + std::map row; + row[1] = "hello"; + row[2] = "world"; + map->Append(row); + + auto result_typed = Map::Wrap(RoundtripColumnValues(*client_, map)); + EXPECT_TRUE(CompareRecursive(*map, *result_typed)); +} + +TEST_P(ClientCase, RoundtripMapUUID_Tuple_String_Array_Uint64) { + using Tuple = ColumnTupleT>; + using Map = ColumnMapT; + auto map = std::make_shared(std::make_shared(), std::make_shared( + std::make_tuple(std::make_shared(), std::make_shared>()))); + + + std::map>> row; + row[UUID{1, 1}] = std::make_tuple("hello", std::vector{1, 2, 3}) ; + row[UUID{2, 2}] = std::make_tuple("world", std::vector{4, 5, 6}) ; + map->Append(row); + + auto result_typed = Map::Wrap(RoundtripColumnValues(*client_, map)); + EXPECT_TRUE(CompareRecursive(*map, *result_typed)); +} + TEST_P(ClientCase, OnProgress) { Block block; createTableWithOneColumn(block); diff --git a/ut/columns_ut.cpp b/ut/columns_ut.cpp index 604c94a4..eda33292 100644 --- a/ut/columns_ut.cpp +++ b/ut/columns_ut.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -787,3 +788,103 @@ TEST(ColumnsCase, ColumnLowCardinalityString_WithEmptyString_3) { EXPECT_EQ(values[i], col.At(i)) << " at pos: " << i; } } + + +TEST(ColumnsCase, ColumnTupleT) { + using TestTuple = ColumnTupleT; + + TestTuple col( + std::make_tuple( + std::make_shared(), + std::make_shared(), + std::make_shared(3) + ) + ); + const auto val = std::make_tuple(1, "a", "bcd"); + col.Append(val); + static_assert(std::is_same_v::type>); + static_assert(std::is_same_v::type>); + static_assert(std::is_same_v::type>); + EXPECT_EQ(val, col.At(0)); +} + +TEST(ColumnsCase, ColumnTupleT_Wrap) { + ColumnTuple col ({ + std::make_shared(), + std::make_shared(), + std::make_shared(3) + } + ); + + const auto val = std::make_tuple(1, "a", "bcd"); + + col[0]->AsStrict()->Append(std::get<0>(val)); + col[1]->AsStrict()->Append(std::get<1>(val)); + col[2]->AsStrict()->Append(std::get<2>(val)); + + using TestTuple = ColumnTupleT; + auto wrapped_col = TestTuple::Wrap(std::move(col)); + + EXPECT_EQ(wrapped_col->Size(), 1u); + EXPECT_EQ(val, wrapped_col->At(0)); +} + +TEST(ColumnsCase, ColumnTupleT_Empty) { + using TestTuple = ColumnTupleT<>; + + TestTuple col(std::make_tuple()); + const auto val = std::make_tuple(); + col.Append(val); + EXPECT_EQ(col.Size(), 0u); +} + +TEST(ColumnsCase, ColumnMapT) { + ColumnMapT col( + std::make_shared(), + std::make_shared()); + + std::map val; + val[1] = "123"; + val[2] = "abc"; + col.Append(val); + + auto map_view = col.At(0); + + EXPECT_THROW(map_view.At(0), ValidationError); + EXPECT_EQ(val[1], map_view.At(1)); + EXPECT_EQ(val[2], map_view.At(2)); + + std::map map{map_view.begin(), map_view.end()}; + + EXPECT_EQ(val[1], map.at(1)); + EXPECT_EQ(val[2], map.at(2)); +} + +TEST(ColumnsCase, ColumnMapT_Wrap) { + auto tupls = std::make_shared(std::vector{ + std::make_shared(), + std::make_shared()}); + + auto data = std::make_shared(tupls); + + auto val = tupls->CloneEmpty()->As(); + + (*val)[0]->AsStrict()->Append(1); + (*val)[1]->AsStrict()->Append("123"); + + (*val)[0]->AsStrict()->Append(2); + (*val)[1]->AsStrict()->Append("abc"); + + data->AppendAsColumn(val); + + ColumnMap col{data}; + + using TestMap = ColumnMapT; + auto wrapped_col = TestMap::Wrap(std::move(col)); + + auto map_view = wrapped_col->At(0); + + EXPECT_THROW(map_view.At(0), ValidationError); + EXPECT_EQ("123", map_view.At(1)); + EXPECT_EQ("abc", map_view.At(2)); +} diff --git a/ut/itemview_ut.cpp b/ut/itemview_ut.cpp index 40da5027..6413e190 100644 --- a/ut/itemview_ut.cpp +++ b/ut/itemview_ut.cpp @@ -80,6 +80,7 @@ TEST(ItemView, ErrorTypes) { EXPECT_ITEMVIEW_ERROR(Type::Code::Nullable, int); EXPECT_ITEMVIEW_ERROR(Type::Code::Tuple, int); EXPECT_ITEMVIEW_ERROR(Type::Code::LowCardinality, int); + EXPECT_ITEMVIEW_ERROR(Type::Code::Map, int); } TEST(ItemView, TypeSizeMismatch) { diff --git a/ut/type_parser_ut.cpp b/ut/type_parser_ut.cpp index e7fe6bf0..ee1258b7 100644 --- a/ut/type_parser_ut.cpp +++ b/ut/type_parser_ut.cpp @@ -226,3 +226,16 @@ TEST(TypeParserCase, ParseDateTime64) { ASSERT_EQ(ast.elements[1].value_string, "UTC"); ASSERT_EQ(ast.elements[1].value, 0); } + +TEST(TypeParserCase, ParseMap) { + TypeAst ast; + TypeParser("Map(Int32, String)").Parse(&ast); + ASSERT_EQ(ast.meta, TypeAst::Map); + ASSERT_EQ(ast.name, "Map"); + ASSERT_EQ(ast.code, Type::Map); + ASSERT_EQ(ast.elements.size(), 2u); + ASSERT_EQ(ast.elements[0].meta, TypeAst::Terminal); + ASSERT_EQ(ast.elements[0].name, "Int32"); + ASSERT_EQ(ast.elements[1].meta, TypeAst::Terminal); + ASSERT_EQ(ast.elements[1].name, "String"); +} diff --git a/ut/types_ut.cpp b/ut/types_ut.cpp index c5922d0e..8e2f943c 100644 --- a/ut/types_ut.cpp +++ b/ut/types_ut.cpp @@ -32,6 +32,8 @@ TEST(TypesCase, TypeName) { Type::CreateEnum8({})->GetName(), "Enum8()" ); + + ASSERT_EQ(Type::CreateMap(Type::CreateSimple(), Type::CreateString())->GetName(), "Map(Int32, String)"); } TEST(TypesCase, NullableType) { @@ -103,6 +105,11 @@ TEST(TypesCase, IsEqual) { "Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime)))))", "Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC')))))))" "Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC'), Tuple(LowCardinality(String), Enum8('READ'=1, 'WRITE'=0))))))))", + "Map(String, Int8)", + "Map(String, Tuple(String, Int8, Date, DateTime))", + "Map(UUID, Array(Tuple(String, Int8, Date, DateTime)))", + "Map(String, Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime))))))", + "Map(LowCardinality(FixedString(10)), Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC'))))))))" }; // Check that Type::IsEqual returns true only if: @@ -133,9 +140,9 @@ TEST(TypesCase, ErrorEnumContent) { "Enum8('ONE'=1,'TWO')", "Enum16('TWO'=,'TWO')", }; - + for (const auto& type_name : type_names) { SCOPED_TRACE(type_name); EXPECT_THROW(clickhouse::CreateColumnByType(type_name)->Type(), ValidationError); } -} \ No newline at end of file +} diff --git a/ut/utils.cpp b/ut/utils.cpp index e624f45c..9f8d9cab 100644 --- a/ut/utils.cpp +++ b/ut/utils.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -125,6 +126,27 @@ bool doPrintValue(const ColumnRef & c, const size_t row, std:: return false; } +template <> +bool doPrintValue(const ColumnRef & c, const size_t row, std::ostream & ostr) { + // via temporary stream to preserve fill and alignment of the ostr + std::stringstream sstr; + if (const auto & map_col = c->As()) { + sstr << "{"; + const auto tuples = map_col->GetAsColumn(row); + for (size_t i = 0; i < tuples->Size(); ++i) { + printColumnValue(tuples, i, sstr); + + if (i < tuples->Size() - 1) + sstr << ", "; + } + + sstr << "}"; + ostr << sstr.str(); + return true; + } + return false; +} + std::ostream & printColumnValue(const ColumnRef& c, const size_t row, std::ostream & ostr) { const auto r = false @@ -150,7 +172,8 @@ std::ostream & printColumnValue(const ColumnRef& c, const size_t row, std::ostre || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) - || doPrintValue(c, row, ostr); + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr); if (!r) ostr << "Unable to print value of type " << c->GetType().GetName(); diff --git a/ut/utils.h b/ut/utils.h index 621119fb..d11a1fbd 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -76,6 +76,27 @@ inline const char * getPrefix() { return prefix; } +template > +inline std::ostream & printTuple(std::ostream & ostr, const T & t) { + static_assert(index <= std::tuple_size_v); + if constexpr (index == 0) { + return ostr << "( "; + } else { + printTuple(ostr, t); + using ElementType = std::tuple_element_t; + if constexpr (is_container_v) { + ostr << PrintContainer{std::get(t)}; + } else { + ostr << std::get<0>(t); + } + if constexpr (index == std::tuple_size_v) { + return ostr << " )"; + } else { + return ostr << ", "; + } + } +} + namespace std { template inline ostream & operator<<(ostream & ostr, const chrono::duration & d) { @@ -86,6 +107,11 @@ template inline ostream & operator<<(ostream & ostr, const pair & t) { return ostr << "{ " << t.first << ", " << t.second << " }"; } + +template +inline ostream & operator<<(ostream & ostr, const tuple & t) { + return printTuple(ostr, t); +} } From 9cdb643eabc824eaeea2bb3795237fc0f0833230 Mon Sep 17 00:00:00 2001 From: den818 Date: Sun, 27 Nov 2022 17:49:01 +0400 Subject: [PATCH 04/44] fix --- clickhouse/columns/map.h | 1 + clickhouse/columns/tuple.h | 6 +++--- ut/utils.h | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/clickhouse/columns/map.h b/clickhouse/columns/map.h index b770f6b8..f57cd6c4 100644 --- a/clickhouse/columns/map.h +++ b/clickhouse/columns/map.h @@ -126,6 +126,7 @@ class ColumnMapT : public ColumnMap { : data_iterator_(data_iterator) {} using ValueType = std::pair; + using value_type = ValueType; inline auto operator*() const { auto tuple = *data_iterator_; diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index 87303c2f..178074fd 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -119,7 +119,7 @@ class ColumnTupleT : public ColumnTuple { private: template > - inline static std::vector TupleToVector(const T& value) { + inline static std::vector TupleToVector([[maybe_unused]] const T& value) { static_assert(index <= std::tuple_size_v); if constexpr (index == 0) { std::vector result; @@ -133,7 +133,7 @@ class ColumnTupleT : public ColumnTuple { } template > - inline static auto VectorToTuple(T columns) { + inline static auto VectorToTuple([[maybe_unused]] T columns) { static_assert(column_index <= std::tuple_size_v); if constexpr (column_index == 0) { return std::make_tuple(); @@ -147,7 +147,7 @@ class ColumnTupleT : public ColumnTuple { } template > - inline auto GetTupleOfValues(size_t index) const { + inline auto GetTupleOfValues([[maybe_unused]]size_t index) const { static_assert(column_index <= std::tuple_size_v); if constexpr (column_index == 0) { return std::make_tuple(); diff --git a/ut/utils.h b/ut/utils.h index d11a1fbd..a0d136fd 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -77,7 +77,7 @@ inline const char * getPrefix() { } template > -inline std::ostream & printTuple(std::ostream & ostr, const T & t) { +inline std::ostream & printTuple(std::ostream & ostr, [[maybe_unused]] const T & t) { static_assert(index <= std::tuple_size_v); if constexpr (index == 0) { return ostr << "( "; From 8c8c7a6628056f4529c9b30c8100456fd11359a6 Mon Sep 17 00:00:00 2001 From: den818 Date: Sun, 27 Nov 2022 18:08:38 +0400 Subject: [PATCH 05/44] fix --- clickhouse/columns/map.h | 4 ++++ clickhouse/columns/tuple.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/clickhouse/columns/map.h b/clickhouse/columns/map.h index f57cd6c4..298d2c80 100644 --- a/clickhouse/columns/map.h +++ b/clickhouse/columns/map.h @@ -126,7 +126,11 @@ class ColumnMapT : public ColumnMap { : data_iterator_(data_iterator) {} using ValueType = std::pair; + using difference_type = size_t; using value_type = ValueType; + using pointer = void; + using reference = ValueType&; + using iterator_category = std::forward_iterator_tag; inline auto operator*() const { auto tuple = *data_iterator_; diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index 178074fd..a5f3a25f 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -73,7 +73,7 @@ class ColumnTupleT : public ColumnTuple { inline ValueType operator[](size_t index) const { return GetTupleOfValues(index); } template > - inline void Append(T value) { + inline void Append([[maybe_unused]] T value) { static_assert(index <= std::tuple_size_v); static_assert(std::tuple_size_v == std::tuple_size_v); if constexpr (index == 0) { From d95f85f22b0770228b23d7c073473fd9259efd94 Mon Sep 17 00:00:00 2001 From: den818 Date: Wed, 30 Nov 2022 21:52:22 +0400 Subject: [PATCH 06/44] improvement ColumnMap --- clickhouse/CMakeLists.txt | 1 + clickhouse/base/projected_iterator.h | 55 ++++++++++++++++++++++++++++ clickhouse/columns/array.h | 2 + clickhouse/columns/map.h | 38 ++++++++++++------- 4 files changed, 82 insertions(+), 14 deletions(-) create mode 100644 clickhouse/base/projected_iterator.h diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 2caeebba..dd76059c 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -102,6 +102,7 @@ INSTALL(FILES base/input.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/open_telemetry.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/output.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/platform.h DESTINATION include/clickhouse/base/) +INSTALL(FILES base/projected_iterator.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/singleton.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/socket.h DESTINATION include/clickhouse/base/) INSTALL(FILES base/string_utils.h DESTINATION include/clickhouse/base/) diff --git a/clickhouse/base/projected_iterator.h b/clickhouse/base/projected_iterator.h new file mode 100644 index 00000000..ca5aecfe --- /dev/null +++ b/clickhouse/base/projected_iterator.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include + +namespace clickhouse { + +template ()(std::declval())), + typename Value = std::decay_t> +class ProjectedIterator { +public: + using value_type = Value; + using reference = Reference; + using pointer = Reference; + using difference_type = typename std::iterator_traits::difference_type; + using iterator_category = typename std::iterator_traits::iterator_category; + + ProjectedIterator() = default; + + inline ProjectedIterator(Iterator const& iterator, UnaryFunction functor) + : iterator_(iterator) + , functor_(std::move(functor)) { + } + + inline UnaryFunction functor() const { return functor; } + + inline Iterator const& base() const { return iterator_; } + + inline reference operator*() const { return functor_(iterator_); } + + inline ProjectedIterator& operator++() { + ++iterator_; + return *this; + } + + inline ProjectedIterator& operator--() { + --iterator_; + return *this; + } + + inline bool operator==(const ProjectedIterator& other) const { + return this->iterator_ == other.iterator_; + } + + inline bool operator!=(const ProjectedIterator& other) const { + return !(*this == other); + } + +private: + Iterator iterator_; + UnaryFunction functor_; +}; + +} // namespace clickhouse diff --git a/clickhouse/columns/array.h b/clickhouse/columns/array.h index f3bb23d6..06408fe7 100644 --- a/clickhouse/columns/array.h +++ b/clickhouse/columns/array.h @@ -167,6 +167,8 @@ class ColumnArrayT : public ColumnArray { const size_t size_; size_t index_; public: + Iterator() = default; + Iterator(std::shared_ptr typed_nested_data, size_t offset, size_t size, size_t index) : typed_nested_data_(typed_nested_data) , offset_(offset) diff --git a/clickhouse/columns/map.h b/clickhouse/columns/map.h index 298d2c80..3897be5e 100644 --- a/clickhouse/columns/map.h +++ b/clickhouse/columns/map.h @@ -1,5 +1,6 @@ #pragma once +#include "../base/projected_iterator.h" #include "array.h" #include "column.h" #include "tuple.h" @@ -122,6 +123,8 @@ class ColumnMapT : public ColumnMap { typename ArrayColumnType::ArrayValueView::Iterator data_iterator_; public: + Iterator() = default; + Iterator(typename ArrayColumnType::ArrayValueView::Iterator data_iterator) : data_iterator_(data_iterator) {} @@ -187,13 +190,16 @@ class ColumnMapT : public ColumnMap { if (size() != other.size()) { return false; } - using Vector = std::vector>; - Vector l(begin(), end()); - Vector r(other.begin(), other.end()); - auto comp = [](const auto& l, const auto& r) { return l.frist < r.first; }; - std::sort(l.begin(), l.end(), comp); - std::sort(r.begin(), r.end(), comp); - return std::equal(l.begin(), l.end(), r.begin(), r.end()); + const auto make_index = [](const auto& data) { + std::vector result{data.Size()}; + std::generate(result.begin(), result.end(), [i = 0] () mutable { return i++; }); + std::sort(result.begin(), result.end(), [&data](size_t l, size_t r) {return data[l] < data[r];}); + return result; + }; + const auto l_index = make_index(data_); + const auto r_index = make_index(other.data_); + return std::equal(l_index.begin(), l_index.end(), r_index.begin(), r_index.end(), + [&l_data = data_, &r_data = other.data_](size_t l, size_t r) { return l_data[l] == r_data[r];}); return true; } @@ -214,13 +220,17 @@ class ColumnMapT : public ColumnMap { template inline void Append(const T& value) { - // TODO Refuse to copy. - std::vector> container; - container.reserve(value.size()); - for (const auto& i : value) { - container.emplace_back(i.first, i.second); - } - typed_data_->Append(container.begin(), container.end()); + using BaseIter = typename T::const_iterator; + using KeyOfT = decltype(std::declval()->first); + using ValOfT = decltype(std::declval()->second); + using Functor = std::function(const BaseIter&)>; + using Iterator = ProjectedIterator; + + Functor functor = [](const BaseIter& i) { + return std::make_tuple(i->first, i->second); + }; + + typed_data_->Append(Iterator{value.begin(), functor}, Iterator{value.end(), functor}); } static auto Wrap(ColumnMap&& col) { From 739b2fda51d30ad6e29a9958ac1f27d80415ad44 Mon Sep 17 00:00:00 2001 From: den818 Date: Wed, 30 Nov 2022 22:04:27 +0400 Subject: [PATCH 07/44] fix --- clickhouse/columns/map.h | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse/columns/map.h b/clickhouse/columns/map.h index 3897be5e..0249e97b 100644 --- a/clickhouse/columns/map.h +++ b/clickhouse/columns/map.h @@ -5,6 +5,7 @@ #include "column.h" #include "tuple.h" +#include #include namespace clickhouse { From bac29677ed2e392b47825d3e4a0445b3d446c03c Mon Sep 17 00:00:00 2001 From: den818 Date: Thu, 1 Dec 2022 01:02:36 +0400 Subject: [PATCH 08/44] Support geo column types --- README.md | 1 + clickhouse/CMakeLists.txt | 2 + clickhouse/client.h | 1 + clickhouse/columns/factory.cpp | 16 ++++- clickhouse/columns/geo.cpp | 108 +++++++++++++++++++++++++++++++ clickhouse/columns/geo.h | 76 ++++++++++++++++++++++ clickhouse/columns/tuple.h | 27 +++++--- clickhouse/types/type_parser.cpp | 6 +- clickhouse/types/types.cpp | 28 ++++++++ clickhouse/types/types.h | 14 +++- ut/client_ut.cpp | 88 +++++++++++++++++++++++++ ut/types_ut.cpp | 6 +- ut/utils.cpp | 15 ++++- 13 files changed, 371 insertions(+), 17 deletions(-) create mode 100644 clickhouse/columns/geo.cpp create mode 100644 clickhouse/columns/geo.h diff --git a/README.md b/README.md index b35d5c55..ce1f68fe 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ C++ client for [ClickHouse](https://clickhouse.com/). * Int128 * UUID * Map +* Point, Ring, Polygon, MultiPolygon ## Building diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 2caeebba..3c8f03a9 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -12,6 +12,7 @@ SET ( clickhouse-cpp-lib-src columns/decimal.cpp columns/enum.cpp columns/factory.cpp + columns/geo.cpp columns/ip4.cpp columns/ip6.cpp columns/lowcardinality.cpp @@ -116,6 +117,7 @@ INSTALL(FILES columns/date.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/decimal.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/enum.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/factory.h DESTINATION include/clickhouse/columns/) +INSTALL(FILES columns/geo.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/ip4.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/ip6.h DESTINATION include/clickhouse/columns/) INSTALL(FILES columns/itemview.h DESTINATION include/clickhouse/columns/) diff --git a/clickhouse/client.h b/clickhouse/client.h index 2b315b00..63a51c2e 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -7,6 +7,7 @@ #include "columns/date.h" #include "columns/decimal.h" #include "columns/enum.h" +#include "columns/geo.h" #include "columns/ip4.h" #include "columns/ip6.h" #include "columns/lowcardinality.h" diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index 2ea7b6f5..fbd57889 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -4,17 +4,19 @@ #include "date.h" #include "decimal.h" #include "enum.h" +#include "geo.h" #include "ip4.h" #include "ip6.h" #include "lowcardinality.h" #include "lowcardinalityadaptor.h" +#include "map.h" #include "nothing.h" #include "nullable.h" #include "numeric.h" #include "string.h" #include "tuple.h" #include "uuid.h" -#include "map.h" + #include "../types/type_parser.h" @@ -97,6 +99,18 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) { case Type::UUID: return std::make_shared(); + case Type::Point: + return std::make_shared(); + + case Type::Ring: + return std::make_shared(); + + case Type::Polygon: + return std::make_shared(); + + case Type::MultiPolygon: + return std::make_shared(); + default: return nullptr; } diff --git a/clickhouse/columns/geo.cpp b/clickhouse/columns/geo.cpp new file mode 100644 index 00000000..5d749374 --- /dev/null +++ b/clickhouse/columns/geo.cpp @@ -0,0 +1,108 @@ +#include "geo.h" + +#include "utils.h" + +namespace { +using namespace ::clickhouse; + +template +TypeRef CreateGeoType() { + if constexpr (type_code == Type::Code::Point) { + return Type::CreatePoint(); + } else if constexpr (type_code == Type::Code::Ring) { + return Type::CreateRing(); + } else if constexpr (type_code == Type::Code::Polygon) { + return Type::CreatePolygon(); + } else if constexpr (type_code == Type::Code::MultiPolygon) { + return Type::CreateMultiPolygon(); + } +} + +template +std::shared_ptr CreateColumn() { + if constexpr (std::is_same_v>) { + return std::make_shared>( + std::make_tuple(std::make_shared(), std::make_shared())); + } else { + return std::make_shared(); + } +} + +} // namespace + +namespace clickhouse { + +template +ColumnGeo::ColumnGeo() + : Column(std::move(CreateGeoType())), + data_(std::move(CreateColumn())) { +} + +template +ColumnGeo::ColumnGeo(ColumnRef data) + : Column(std::move(CreateGeoType())) + , data_(std::move(WrapColumn(std::move(data)))) { +} + +template +void ColumnGeo::Clear() { + data_->Clear(); +} + +template +const typename ColumnGeo::ValueType ColumnGeo::At(size_t n) const { + return data_->At(n); +} + +template +const typename ColumnGeo::ValueType ColumnGeo::operator[](size_t n) const { + return data_->At(n); +} + +template +void ColumnGeo::Append(ColumnRef column) { + if (auto col = column->template As()) { + data_->Append(col->data_->template As()); + } +} + +template +bool ColumnGeo::LoadBody(InputStream* input, size_t rows) { + return data_->LoadBody(input, rows); +} + +template +void ColumnGeo::SaveBody(OutputStream* output) { + data_->SaveBody(output); +} + +template +size_t ColumnGeo::Size() const { + return data_->Size(); +} + +template +ColumnRef ColumnGeo::Slice(size_t begin, size_t len) const { + return std::make_shared(data_->Slice(begin, len)); +} + +template +ColumnRef ColumnGeo::CloneEmpty() const { + return std::make_shared(); +} + +template +void ColumnGeo::Swap(Column& other) { + auto& col = dynamic_cast(other); + data_.swap(col.data_); +} + +template class ColumnGeo, Type::Code::Point>; + +template class ColumnGeo, Type::Code::Ring>; + +template class ColumnGeo, Type::Code::Polygon>; + +template class ColumnGeo, Type::Code::MultiPolygon>; + +} // namespace clickhouse diff --git a/clickhouse/columns/geo.h b/clickhouse/columns/geo.h new file mode 100644 index 00000000..5f3db9b6 --- /dev/null +++ b/clickhouse/columns/geo.h @@ -0,0 +1,76 @@ +#pragma once + +#include "array.h" +#include "column.h" +#include "numeric.h" +#include "tuple.h" + +namespace clickhouse { + +template +class ColumnGeo : public Column { +public: + using ValueType = typename NestedColumnType::ValueType; + + ColumnGeo(); + + explicit ColumnGeo(ColumnRef data); + + /// Appends one element to the end of column. + template + void Append(const T& value) { + data_->Append(value); + } + + /// Returns element at given row number. + const ValueType At(size_t n) const; + + /// Returns element at given row number. + const ValueType operator[](size_t n) const; + +public: + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override; + + /// Loads column data from input stream. + bool LoadBody(InputStream* input, size_t rows) override; + + /// Saves column data to output stream. + void SaveBody(OutputStream* output) override; + + /// Clear column data . + void Clear() override; + + /// Returns count of rows in the column. + size_t Size() const override; + + /// Makes slice of the current column. + ColumnRef Slice(size_t begin, size_t len) const override; + ColumnRef CloneEmpty() const override; + void Swap(Column& other) override; + +private: + std::shared_ptr data_; +}; + +// /** +// * Represents a Point column. +// */ +using ColumnPoint = ColumnGeo, Type::Code::Point>; + +/** + * Represents a Ring column. + */ +using ColumnRing = ColumnGeo, Type::Code::Ring>; + +/** + * Represents a Polygon column. + */ +using ColumnPolygon = ColumnGeo, Type::Code::Polygon>; + +/** + * Represents a MultiPolygon column. + */ +using ColumnMultiPolygon = ColumnGeo, Type::Code::MultiPolygon>; + +} // namespace clickhouse diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index a5f3a25f..ff33ba02 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -72,16 +72,11 @@ class ColumnTupleT : public ColumnTuple { inline ValueType operator[](size_t index) const { return GetTupleOfValues(index); } - template > - inline void Append([[maybe_unused]] T value) { - static_assert(index <= std::tuple_size_v); - static_assert(std::tuple_size_v == std::tuple_size_v); - if constexpr (index == 0) { - return; - } else { - std::get(typed_columns_)->Append(std::move(std::get(value))); - Append(std::move(value)); - } + using ColumnTuple::Append; + + template + inline void Append(std::tuple value) { + AppendTuple(std::move(value)); } /** Create a ColumnTupleT from a ColumnTuple, without copying data and offsets, but by @@ -118,6 +113,18 @@ class ColumnTupleT : public ColumnTuple { } private: + template > + inline void AppendTuple([[maybe_unused]] T value) { + static_assert(index <= std::tuple_size_v); + static_assert(std::tuple_size_v == std::tuple_size_v); + if constexpr (index == 0) { + return; + } else { + std::get(typed_columns_)->Append(std::move(std::get(value))); + AppendTuple(std::move(value)); + } + } + template > inline static std::vector TupleToVector([[maybe_unused]] const T& value) { static_assert(index <= std::tuple_size_v); diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index ee05a244..e16aadb5 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -46,7 +46,11 @@ static const std::unordered_map kTypeCode = { { "Decimal64", Type::Decimal64 }, { "Decimal128", Type::Decimal128 }, { "LowCardinality", Type::LowCardinality }, - { "Map", Type::Map}, + { "Map", Type::Map}, + { "Point", Type::Point}, + { "Ring", Type::Ring}, + { "Polygon", Type::Polygon}, + { "MultiPolygon", Type::MultiPolygon}, }; static Type::Code GetTypeCode(const std::string& name) { diff --git a/clickhouse/types/types.cpp b/clickhouse/types/types.cpp index 370bc47b..d5093f10 100644 --- a/clickhouse/types/types.cpp +++ b/clickhouse/types/types.cpp @@ -47,6 +47,10 @@ const char* Type::TypeName(Type::Code code) { case Type::Code::DateTime64: return "DateTime64"; case Type::Code::Date32: return "Date32"; case Type::Code::Map: return "Map"; + case Type::Code::Point: return "Point"; + case Type::Code::Ring: return "Ring"; + case Type::Code::Polygon: return "Polygon"; + case Type::Code::MultiPolygon: return "MultiPolygon"; } return "Unknown type"; @@ -72,6 +76,10 @@ std::string Type::GetName() const { case IPv6: case Date: case Date32: + case Point: + case Ring: + case Polygon: + case MultiPolygon: return TypeName(code_); case FixedString: return As()->GetName(); @@ -126,6 +134,10 @@ uint64_t Type::GetTypeUniqueId() const { case IPv6: case Date: case Date32: + case Point: + case Ring: + case Polygon: + case MultiPolygon: // For simple types, unique ID is the same as Type::Code return code_; @@ -233,6 +245,22 @@ TypeRef Type::CreateMap(TypeRef key_type, TypeRef value_type) { return std::make_shared(key_type, value_type); } +TypeRef Type::CreatePoint() { + return TypeRef(new Type(Type::Point)); +} + +TypeRef Type::CreateRing() { + return TypeRef(new Type(Type::Ring)); +} + +TypeRef Type::CreatePolygon() { + return TypeRef(new Type(Type::Polygon)); +} + +TypeRef Type::CreateMultiPolygon() { + return TypeRef(new Type(Type::MultiPolygon)); +} + /// class ArrayType ArrayType::ArrayType(TypeRef item_type) : Type(Array), item_type_(item_type) { diff --git a/clickhouse/types/types.h b/clickhouse/types/types.h index 16570f34..423a6f70 100644 --- a/clickhouse/types/types.h +++ b/clickhouse/types/types.h @@ -50,7 +50,11 @@ class Type { LowCardinality, DateTime64, Date32, - Map + Map, + Point, + Ring, + Polygon, + MultiPolygon }; using EnumItem = std::pair; @@ -128,6 +132,14 @@ class Type { static TypeRef CreateMap(TypeRef key_type, TypeRef value_type); + static TypeRef CreatePoint(); + + static TypeRef CreateRing(); + + static TypeRef CreatePolygon(); + + static TypeRef CreateMultiPolygon(); + private: uint64_t GetTypeUniqueId() const; diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 58d6304b..0278d6d0 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -67,6 +67,21 @@ class ClientCase : public testing::TestWithParam { } } + std::string GetSettingValue(const std::string& name) { + std::string result; + client_->Select("SELECT value FROM system.settings WHERE name = \'" + name + "\'", + [&result](const Block& block) + { + if (block.GetRowCount() == 0) { + return; + } + result = block[0]->AsStrict()->At(0); + } + ); + return result; + } + + std::unique_ptr client_; const std::string table_name = "test_clickhouse_cpp_test_ut_table"; const std::string column_name = "test_column"; @@ -1051,6 +1066,79 @@ TEST_P(ClientCase, RoundtripMapUUID_Tuple_String_Array_Uint64) { EXPECT_TRUE(CompareRecursive(*map, *result_typed)); } +TEST_P(ClientCase, RoundtripPoint) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + col->Append({1.0, 2.0}); + col->Append({0.1, 0.2}); + + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(ClientCase, RoundtripRing) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector ring{{1.0, 2.0}, {3.0, 4.0}}; + col->Append(ring); + } + { + std::vector ring{{0.1, 0.2}, {0.3, 0.4}}; + col->Append(ring); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(ClientCase, RoundtripPolygon) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector> polygon + {{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}; + col->Append(polygon); + } + { + std::vector> polygon + {{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}; + col->Append(polygon); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(ClientCase, RoundtripMultiPolygon) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector>> multi_polygon + {{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}, + {{{1.1, 2.2}, {3.3, 4.4}}, {{5.5, 6.6}, {7.7, 8.8}}}}; + col->Append(multi_polygon); + } + { + std::vector>> multi_polygon + {{{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}, + {{{1.1, 1.2}, {1.3, 1.4}}, {{1.5, 1.6}, {1.7, 1.8}}}}; + col->Append(multi_polygon); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + TEST_P(ClientCase, OnProgress) { Block block; createTableWithOneColumn(block); diff --git a/ut/types_ut.cpp b/ut/types_ut.cpp index 8e2f943c..3f795269 100644 --- a/ut/types_ut.cpp +++ b/ut/types_ut.cpp @@ -109,7 +109,11 @@ TEST(TypesCase, IsEqual) { "Map(String, Tuple(String, Int8, Date, DateTime))", "Map(UUID, Array(Tuple(String, Int8, Date, DateTime)))", "Map(String, Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime))))))", - "Map(LowCardinality(FixedString(10)), Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC'))))))))" + "Map(LowCardinality(FixedString(10)), Array(Array(Array(Array(Nullable(Tuple(String, Int8, Date, DateTime('UTC'))))))))", + "Point", + "Ring", + "Polygon", + "MultiPolygon" }; // Check that Type::IsEqual returns true only if: diff --git a/ut/utils.cpp b/ut/utils.cpp index 9f8d9cab..1cb5978c 100644 --- a/ut/utils.cpp +++ b/ut/utils.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -47,10 +48,14 @@ std::ostream& operator<<(std::ostream & ostr, const DateTimeValue & time) { return ostr << buffer; } -template +template ().At(0)) > bool doPrintValue(const ColumnRef & c, const size_t row, std::ostream & ostr) { if (const auto & casted_c = c->As()) { - ostr << static_cast(casted_c->At(row)); + if constexpr (is_container_v>) { + ostr << PrintContainer{static_cast(casted_c->At(row))}; + } else { + ostr << static_cast(casted_c->At(row)); + } return true; } return false; @@ -173,7 +178,11 @@ std::ostream & printColumnValue(const ColumnRef& c, const size_t row, std::ostre || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) || doPrintValue(c, row, ostr) - || doPrintValue(c, row, ostr); + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr) + || doPrintValue(c, row, ostr); if (!r) ostr << "Unable to print value of type " << c->GetType().GetName(); From 209d3ea0b197dd51e677b7af230692e4a1bbb1c4 Mon Sep 17 00:00:00 2001 From: den818 Date: Tue, 6 Dec 2022 21:21:52 +0100 Subject: [PATCH 09/44] fix --- clickhouse/columns/array.h | 2 +- clickhouse/columns/map.h | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/clickhouse/columns/array.h b/clickhouse/columns/array.h index 06408fe7..0ea33d5a 100644 --- a/clickhouse/columns/array.h +++ b/clickhouse/columns/array.h @@ -272,7 +272,7 @@ class ColumnArrayT : public ColumnArray { size_t counter = 0; while (begin != end) { - nested_data.Append(*begin); + nested_data.Append(std::move(*begin)); ++begin; ++counter; } diff --git a/clickhouse/columns/map.h b/clickhouse/columns/map.h index 0249e97b..24a8b4ae 100644 --- a/clickhouse/columns/map.h +++ b/clickhouse/columns/map.h @@ -193,14 +193,17 @@ class ColumnMapT : public ColumnMap { } const auto make_index = [](const auto& data) { std::vector result{data.Size()}; - std::generate(result.begin(), result.end(), [i = 0] () mutable { return i++; }); + std::generate(result.begin(), result.end(), [i = 0] () mutable {return i++;}); std::sort(result.begin(), result.end(), [&data](size_t l, size_t r) {return data[l] < data[r];}); return result; }; - const auto l_index = make_index(data_); - const auto r_index = make_index(other.data_); - return std::equal(l_index.begin(), l_index.end(), r_index.begin(), r_index.end(), - [&l_data = data_, &r_data = other.data_](size_t l, size_t r) { return l_data[l] == r_data[r];}); + const auto index = make_index(data_); + for (const auto& val : other.data_) { + if (!std::binary_search(index.begin(), index.end(), val, + [&data = data_](const auto& l, size_t r) {return l < data[r];})) { + return false; + } + } return true; } @@ -221,14 +224,14 @@ class ColumnMapT : public ColumnMap { template inline void Append(const T& value) { - using BaseIter = typename T::const_iterator; + using BaseIter = decltype(value.begin()); using KeyOfT = decltype(std::declval()->first); using ValOfT = decltype(std::declval()->second); using Functor = std::function(const BaseIter&)>; using Iterator = ProjectedIterator; Functor functor = [](const BaseIter& i) { - return std::make_tuple(i->first, i->second); + return std::make_tuple(std::cref(i->first), std::cref(i->second)); }; typed_data_->Append(Iterator{value.begin(), functor}, Iterator{value.end(), functor}); From bebaac0c871246711e159d2ac60b0ba7c5e0880f Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 9 Dec 2022 18:39:34 +0400 Subject: [PATCH 10/44] Scheduling CI/CD runs on Monday midnight --- .github/workflows/linux.yml | 2 ++ .github/workflows/macos.yml | 2 ++ .github/workflows/windows_mingw.yml | 2 ++ .github/workflows/windows_msvc.yml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index caeb4497..5168e73e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,6 +1,8 @@ name: Linux on: + schedule: + - cron: '0 0 * * 1' push: branches: [ master ] pull_request: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index a2bb171f..007fc08d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,6 +1,8 @@ name: macOS on: + schedule: + - cron: '0 0 * * 1' push: branches: [ master ] pull_request: diff --git a/.github/workflows/windows_mingw.yml b/.github/workflows/windows_mingw.yml index 016b7a7e..52928370 100644 --- a/.github/workflows/windows_mingw.yml +++ b/.github/workflows/windows_mingw.yml @@ -1,6 +1,8 @@ name: Windows mingw on: + schedule: + - cron: '0 0 * * 1' push: branches: [ master ] pull_request: diff --git a/.github/workflows/windows_msvc.yml b/.github/workflows/windows_msvc.yml index 5ac5178b..55ae5faa 100644 --- a/.github/workflows/windows_msvc.yml +++ b/.github/workflows/windows_msvc.yml @@ -1,6 +1,8 @@ name: Windows on: + schedule: + - cron: '0 0 * * 1' push: branches: [ master ] pull_request: From 7c64a81f014a8213605b7c85b1e642375071a0fb Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 9 Dec 2022 18:47:16 +0400 Subject: [PATCH 11/44] Explicitly using ubuntu-20.04 instead of ubuntu-latest --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index caeb4497..dff66e0d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -12,7 +12,7 @@ env: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: compiler: [clang-6, gcc-7, gcc-8, gcc-9] From 14ee88262d39e58945806a82b7ac2e3c291162f0 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sat, 10 Dec 2022 19:41:24 +0400 Subject: [PATCH 12/44] Removed SELECT statement that is not supported on some remote endpoints --- ut/client_ut.cpp | 1 - ut/ssl_ut.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 6a0af56b..104da97b 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1196,7 +1196,6 @@ const auto QUERIES = std::vector{ "SELECT fqdn()", "SELECT buildId()", "SELECT uptime()", - "SELECT filesystemFree()", "SELECT now()" }; } diff --git a/ut/ssl_ut.cpp b/ut/ssl_ut.cpp index f68db08c..cd567422 100644 --- a/ut/ssl_ut.cpp +++ b/ut/ssl_ut.cpp @@ -16,7 +16,6 @@ namespace { "SELECT fqdn()", "SELECT buildId()", "SELECT uptime()", - "SELECT filesystemFree()", "SELECT now()" }; } From 022271369afaa3730c5bcbcebcdb07c983f07ef7 Mon Sep 17 00:00:00 2001 From: Huy Pham Date: Wed, 14 Dec 2022 18:27:58 +0700 Subject: [PATCH 13/44] Support get value at. --- clickhouse/columns/tuple.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index a5f3a25f..a70210bb 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -21,6 +21,10 @@ class ColumnTuple : public Column { return columns_[n]; } + ColumnRef At(size_t n) const { + return columns_[n]; + } + public: /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; From 76cc8eea56f5fb71ae19b94cfc291b223bd955e9 Mon Sep 17 00:00:00 2001 From: den818 Date: Sun, 18 Dec 2022 19:15:31 +0100 Subject: [PATCH 14/44] fix bug in ColumnLowCardinality::Load --- clickhouse/columns/lowcardinality.cpp | 2 +- ut/low_cardinality_nullable_tests.cpp | 39 ++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index 0ffef7e0..2269a351 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -281,7 +281,7 @@ auto Load(ColumnRef new_dictionary_column, InputStream& input, size_t rows) { if (auto nullable = new_dictionary_column->As()) { nullable->Append(true); - for(std::size_t i = 1; i < new_index_column->Size(); i++) { + for(std::size_t i = 1; i < dataColumn->Size(); i++) { nullable->Append(false); } } diff --git a/ut/low_cardinality_nullable_tests.cpp b/ut/low_cardinality_nullable_tests.cpp index 41c0d3c5..8918f251 100644 --- a/ut/low_cardinality_nullable_tests.cpp +++ b/ut/low_cardinality_nullable_tests.cpp @@ -78,6 +78,43 @@ TEST(LowCardinalityOfNullable, InsertAndQuery) { }); } +TEST(LowCardinalityOfNullable, InsertAndQueryOneRow) { + const auto rowsData = std::vector { + "eminem" + }; + + const auto nulls = std::vector { + false + }; + + auto column = buildTestColumn(rowsData, nulls); + + Block block; + block.AppendColumn("words", column); + + Client client(ClientOptions(localHostEndpoint) + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false) + .SetPingBeforeQuery(true)); + + createTable(client); + + client.Insert("lc_of_nullable", block); + + client.Select("SELECT * FROM lc_of_nullable", [&](const Block& bl) { + for (size_t row = 0; row < bl.GetRowCount(); row++) { + auto lc_col = bl[0]->As(); + auto item = lc_col->GetItem(row); + + if (nulls[row]) { + ASSERT_EQ(Type::Code::Void, item.type); + } else { + ASSERT_EQ(rowsData[row], item.get()); + } + } + }); +} + + TEST(LowCardinalityOfNullable, InsertAndQueryEmpty) { auto column = buildTestColumn({}, {}); @@ -113,4 +150,4 @@ TEST(LowCardinalityOfNullable, ThrowOnBackwardsCompatibleLCColumn) { client.Select("SELECT * FROM lc_of_nullable", [&](const Block& bl) { ASSERT_EQ(bl.GetRowCount(), 0u); }); -} \ No newline at end of file +} From 3359603d4dc013568edba0c72172ab5fd40c798b Mon Sep 17 00:00:00 2001 From: atom Date: Thu, 22 Dec 2022 17:39:46 +0300 Subject: [PATCH 15/44] Separate building static and shared libs, use cmake BUILD_SHARED_LIBS --- CMakeLists.txt | 7 +++++++ clickhouse/CMakeLists.txt | 30 +++++++++++++----------------- tests/simple/CMakeLists.txt | 2 +- ut/CMakeLists.txt | 2 +- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c7635c79..d79b5864 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ INCLUDE (cmake/openssl.cmake) OPTION (BUILD_BENCHMARK "Build benchmark" OFF) OPTION (BUILD_TESTS "Build tests" OFF) +OPTION (BUILD_SHARED_LIBS "Build shared libs" OFF) OPTION (WITH_OPENSSL "Use OpenSSL for TLS connections" OFF) PROJECT (CLICKHOUSE-CLIENT) @@ -27,6 +28,12 @@ PROJECT (CLICKHOUSE-CLIENT) SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wno-deprecated-declarations") ENDIF () + IF (APPLE OR MSVC) + IF(BUILD_SHARED_LIBS) + MESSAGE(FATAL "Does not support shared on this platform") + ENDIF() + ENDIF() + SUBDIRS ( clickhouse contrib/absl diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index dbe06b7f..9c5ad217 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -38,8 +38,8 @@ IF (WITH_OPENSSL) LIST(APPEND clickhouse-cpp-lib-src base/sslsocket.cpp) ENDIF () -ADD_LIBRARY (clickhouse-cpp-lib SHARED ${clickhouse-cpp-lib-src}) -SET_TARGET_PROPERTIES(clickhouse-cpp-lib PROPERTIES LINKER_LANGUAGE CXX) +ADD_LIBRARY (clickhouse-cpp-lib ${clickhouse-cpp-lib-src}) +SET_TARGET_PROPERTIES (clickhouse-cpp-lib PROPERTIES LINKER_LANGUAGE CXX) TARGET_LINK_LIBRARIES (clickhouse-cpp-lib absl-lib cityhash-lib @@ -49,15 +49,10 @@ TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib PUBLIC ${PROJECT_SOURCE_DIR} ) -ADD_LIBRARY (clickhouse-cpp-lib-static STATIC ${clickhouse-cpp-lib-src}) -TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static - absl-lib - cityhash-lib - lz4-lib -) -TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib-static - PUBLIC ${PROJECT_SOURCE_DIR} -) +IF (NOT BUILD_SHARED_LIBS) + ADD_LIBRARY (clickhouse-cpp-lib-static ALIAS clickhouse-cpp-lib) +ENDIF() + IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") INCLUDE (CheckCXXSourceCompiles) @@ -76,17 +71,20 @@ IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") # /usr/bin/ld: CMakeFiles/simple-test.dir/main.cpp.o: undefined reference to symbol '_Unwind_Resume@@GCC_3.0' # /usr/bin/ld: /lib/x86_64-linux-gnu/libgcc_s.so.1: error adding symbols: DSO missing from command line # FIXME: that workaround breaks clang build on mingw - TARGET_LINK_LIBRARIES (clickhouse-cpp-lib gcc_s) - TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static gcc_s) + IF (BUILD_SHARED_LIBS) + TARGET_LINK_LIBRARIES (clickhouse-cpp-lib gcc_s) + ELSE() + TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static gcc_s) + ENDIF() ENDIF () ENDIF () -INSTALL (TARGETS clickhouse-cpp-lib clickhouse-cpp-lib-static + +INSTALL (TARGETS clickhouse-cpp-lib ARCHIVE DESTINATION lib LIBRARY DESTINATION lib ) - # general INSTALL(FILES block.h DESTINATION include/clickhouse/) INSTALL(FILES client.h DESTINATION include/clickhouse/) @@ -137,10 +135,8 @@ INSTALL(FILES types/types.h DESTINATION include/clickhouse/types/) IF (WITH_OPENSSL) TARGET_LINK_LIBRARIES (clickhouse-cpp-lib OpenSSL::SSL) - TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static OpenSSL::SSL) ENDIF () IF (WIN32 OR MINGW) TARGET_LINK_LIBRARIES (clickhouse-cpp-lib wsock32 ws2_32) - TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static wsock32 ws2_32) ENDIF () diff --git a/tests/simple/CMakeLists.txt b/tests/simple/CMakeLists.txt index cd102ec3..728ce76d 100644 --- a/tests/simple/CMakeLists.txt +++ b/tests/simple/CMakeLists.txt @@ -4,7 +4,7 @@ ADD_EXECUTABLE (simple-test ) TARGET_LINK_LIBRARIES (simple-test - clickhouse-cpp-lib-static + clickhouse-cpp-lib ) IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index 86066965..b16393f0 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -35,7 +35,7 @@ ADD_EXECUTABLE (clickhouse-cpp-ut ) TARGET_LINK_LIBRARIES (clickhouse-cpp-ut - clickhouse-cpp-lib-static + clickhouse-cpp-lib gtest-lib ) IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") From ef2b9085c9a86f6c1a8db665daab88c799b5150c Mon Sep 17 00:00:00 2001 From: den818 Date: Thu, 22 Dec 2022 21:20:30 +0100 Subject: [PATCH 16/44] fix bug with load/save Array with empty arrays --- clickhouse/columns/array.cpp | 10 ++++++++-- ut/client_ut.cpp | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/clickhouse/columns/array.cpp b/clickhouse/columns/array.cpp index 9ef160b5..9f66b91f 100644 --- a/clickhouse/columns/array.cpp +++ b/clickhouse/columns/array.cpp @@ -84,7 +84,11 @@ bool ColumnArray::LoadBody(InputStream* input, size_t rows) { if (!offsets_->LoadBody(input, rows)) { return false; } - if (!data_->LoadBody(input, (*offsets_)[rows - 1])) { + const auto nested_rows = (*offsets_)[rows - 1]; + if (nested_rows == 0) { + return true; + } + if (!data_->LoadBody(input, nested_rows)) { return false; } return true; @@ -96,7 +100,9 @@ void ColumnArray::SavePrefix(OutputStream* output) { void ColumnArray::SaveBody(OutputStream* output) { offsets_->SaveBody(output); - data_->SaveBody(output); + if (data_->Size() > 0) { + data_->SaveBody(output); + } } void ColumnArray::Clear() { diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index aad7d882..62641c19 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1037,6 +1037,31 @@ TEST_P(ClientCase, RoundtripArrayTString) { EXPECT_TRUE(CompareRecursive(*array, *result_typed)); } +TEST_P(ClientCase, RoundtripArrayLowCardinalityTString) { + // TODO replase by Roundtrip test + using TestColumn = ColumnArrayT>; + + Block block; + auto array = createTableWithOneColumn(block); + array->Append(std::vector{}); + array->Append(std::vector{}); + + block.RefreshRowCount(); + client_->Insert(table_name, block); + + size_t total_rows = 0; + client_->Select(getOneColumnSelectQuery(), + [&total_rows](const Block& block) { + total_rows += block.GetRowCount(); + if (block.GetRowCount() == 0) { + return; + } + } + ); + + ASSERT_EQ(total_rows, 2u); +} + TEST_P(ClientCase, RoundtripMapTUint64String) { using Map = ColumnMapT; auto map = std::make_shared(std::make_shared(), std::make_shared()); From ad26fe9953bd799e10f273654e5014a4ec3680a4 Mon Sep 17 00:00:00 2001 From: den818 Date: Wed, 30 Nov 2022 20:09:07 +0400 Subject: [PATCH 17/44] ColumnNulableT --- clickhouse/columns/lowcardinality.h | 65 +++++- clickhouse/columns/nullable.h | 91 +++++++- ut/CMakeLists.txt | 1 + ut/Column_ut.cpp | 56 ++++- ut/client_ut.cpp | 202 ------------------ ut/roundtrip_tests.cpp | 313 ++++++++++++++++++++++++++++ ut/utils.h | 10 + 7 files changed, 520 insertions(+), 218 deletions(-) create mode 100644 ut/roundtrip_tests.cpp diff --git a/clickhouse/columns/lowcardinality.h b/clickhouse/columns/lowcardinality.h index 3d8581fc..afadae22 100644 --- a/clickhouse/columns/lowcardinality.h +++ b/clickhouse/columns/lowcardinality.h @@ -53,9 +53,16 @@ class ColumnLowCardinality : public Column { UniqueItems unique_items_map_; public: + ColumnLowCardinality(ColumnLowCardinality&& col) = default; // c-tor makes a deep copy of the dictionary_column. explicit ColumnLowCardinality(ColumnRef dictionary_column); explicit ColumnLowCardinality(std::shared_ptr dictionary_column); + + template + explicit ColumnLowCardinality(std::shared_ptr> dictionary_column) + : ColumnLowCardinality(dictionary_column->template As()) + {} + ~ColumnLowCardinality(); /// Appends another LowCardinality column to the end of this one, updating dictionary. @@ -117,6 +124,13 @@ class ColumnLowCardinalityT : public ColumnLowCardinality { // Type this column takes as argument of Append and returns with At() and operator[] using ValueType = typename DictionaryColumnType::ValueType; + explicit ColumnLowCardinalityT(ColumnLowCardinality&& col) + : ColumnLowCardinality(std::move(col)) + , typed_dictionary_(dynamic_cast(*GetDictionary())) + , type_(GetTypeCode(typed_dictionary_)) + { + } + template explicit ColumnLowCardinalityT(Args &&... args) : ColumnLowCardinalityT(std::make_shared(std::forward(args)...)) @@ -124,9 +138,9 @@ class ColumnLowCardinalityT : public ColumnLowCardinality { // Create LC column from existing T-column, making a deep copy of all contents. explicit ColumnLowCardinalityT(std::shared_ptr dictionary_col) - : ColumnLowCardinality(dictionary_col), - typed_dictionary_(dynamic_cast(*GetDictionary())), - type_(typed_dictionary_.Type()->GetCode()) + : ColumnLowCardinality(dictionary_col) + , typed_dictionary_(dynamic_cast(*GetDictionary())) + , type_(GetTypeCode(typed_dictionary_)) {} /// Extended interface to simplify reading/adding individual items. @@ -145,7 +159,15 @@ class ColumnLowCardinalityT : public ColumnLowCardinality { using ColumnLowCardinality::Append; inline void Append(const ValueType & value) { - AppendUnsafe(ItemView{type_, value}); + if constexpr (IsNullable) { + if (value.has_value()) { + AppendUnsafe(ItemView{type_, *value}); + } else { + AppendUnsafe(ItemView{}); + } + } else { + AppendUnsafe(ItemView{type_, value}); + } } template @@ -154,6 +176,41 @@ class ColumnLowCardinalityT : public ColumnLowCardinality { Append(item); } } + + /** Create a ColumnLowCardinalityT from a ColumnLowCardinality, without copying data and offsets, but by + * 'stealing' those from `col`. + * + * Ownership of column internals is transferred to returned object, original (argument) object + * MUST NOT BE USED IN ANY WAY, it is only safe to dispose it. + * + * Throws an exception if `col` is of wrong type, it is safe to use original col in this case. + * This is a static method to make such conversion verbose. + */ + static auto Wrap(ColumnLowCardinality&& col) { + return std::make_shared>(std::move(col)); + } + + static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast(col))); } + + // Helper to simplify integration with other APIs + static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict())); } + + ColumnRef Slice(size_t begin, size_t size) const override { + return Wrap(ColumnLowCardinality::Slice(begin, size)); + } + + ColumnRef CloneEmpty() const override { return Wrap(ColumnLowCardinality::CloneEmpty()); } + +private: + + template + static auto GetTypeCode(T& column) { + if constexpr (IsNullable) { + return GetTypeCode(*column.Nested()->template AsStrict()); + } else { + return column.Type()->GetCode(); + } + } }; } diff --git a/clickhouse/columns/nullable.h b/clickhouse/columns/nullable.h index 41806624..c1924af0 100644 --- a/clickhouse/columns/nullable.h +++ b/clickhouse/columns/nullable.h @@ -3,6 +3,8 @@ #include "column.h" #include "numeric.h" +#include + namespace clickhouse { /** @@ -42,7 +44,7 @@ class ColumnNullable : public Column { /// Clear column data . void Clear() override; - + /// Returns count of rows in the column. size_t Size() const override; @@ -58,4 +60,91 @@ class ColumnNullable : public Column { std::shared_ptr nulls_; }; +template +class ColumnNullableT : public ColumnNullable { +public: + using NestedColumnType = ColumnType; + using ValueType = std::optional().At(0))>>; + + ColumnNullableT(std::shared_ptr data, std::shared_ptr nulls) + : ColumnNullable(data, nulls) + , typed_nested_data_(data) + {} + + explicit ColumnNullableT(std::shared_ptr data) + : ColumnNullableT(data, FillNulls(data->Size())) + {} + + template + explicit ColumnNullableT(Args &&... args) + : ColumnNullableT(std::make_shared(std::forward(args)...)) + {} + + inline ValueType At(size_t index) const { + return IsNull(index) ? ValueType{} : ValueType{typed_nested_data_->At(index)}; + } + + inline ValueType operator[](size_t index) const { return At(index); } + + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override { + ColumnNullable::Append(std::move(column)); + } + + inline void Append(ValueType value) { + ColumnNullable::Append(!value.has_value()); + if (value.has_value()) { + typed_nested_data_->Append(std::move(*value)); + } else { + typed_nested_data_->Append(typename ValueType::value_type{}); + } + } + + /** Create a ColumnNullableT from a ColumnNullable, without copying data and offsets, but by + * 'stealing' those from `col`. + * + * Ownership of column internals is transferred to returned object, original (argument) object + * MUST NOT BE USED IN ANY WAY, it is only safe to dispose it. + * + * Throws an exception if `col` is of wrong type, it is safe to use original col in this case. + * This is a static method to make such conversion verbose. + */ + static auto Wrap(ColumnNullable&& col) { + return std::make_shared>( + col.Nested()->AsStrict(), + col.Nulls()->AsStrict()) ; + } + + static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast(col))); } + + // Helper to simplify integration with other APIs + static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict())); } + + ColumnRef Slice(size_t begin, size_t size) const override { + return Wrap(ColumnNullable::Slice(begin, size)); + } + + ColumnRef CloneEmpty() const override { return Wrap(ColumnNullable::CloneEmpty()); } + + void Swap(Column& other) override { + auto& col = dynamic_cast&>(other); + typed_nested_data_.swap(col.typed_nested_data_); + ColumnNullable::Swap(other); + } + +private: + static inline auto FillNulls(size_t n){ + auto result = std::make_shared(); + for (size_t i = 0; i < n; ++i) { + result->Append(0); + } + return result; + } + + std::shared_ptr typed_nested_data_; +}; + +template +constexpr bool IsNullable = std::is_base_of_v; + } diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index 86066965..cf19056d 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -20,6 +20,7 @@ SET ( clickhouse-cpp-ut-src CreateColumnByType_ut.cpp Column_ut.cpp roundtrip_column.cpp + roundtrip_tests.cpp utils.cpp value_generators.cpp diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp index 51cd4980..544be975 100644 --- a/ut/Column_ut.cpp +++ b/ut/Column_ut.cpp @@ -106,6 +106,28 @@ class GenericColumnTest : public testing::Test { return std::tuple{column, values}; } + + static std::optional SkipTest(clickhouse::Client& client) { + if constexpr (std::is_same_v) { + // Date32 first appeared in v21.9.2.17-stable + const auto server_info = client.GetServerInfo(); + if (versionNumber(server_info) < versionNumber(21, 9)) { + std::stringstream buffer; + buffer << "Date32 is available since v21.9.2.17-stable and can't be tested against server: " << server_info; + return buffer.str(); + } + } + + if constexpr (std::is_same_v) { + const auto server_info = client.GetServerInfo(); + if (versionNumber(server_info) < versionNumber(21, 7)) { + std::stringstream buffer; + buffer << "ColumnInt128 is available since v21.7.2.7-stable and can't be tested against server: " << server_info; + return buffer.str(); + } + } + return std::nullopt; + } }; using ValueColumns = ::testing::Types< @@ -279,21 +301,33 @@ TYPED_TEST(GenericColumnTest, RoundTrip) { clickhouse::Client client(LocalHostEndpoint); - if constexpr (std::is_same_v) { - // Date32 first appeared in v21.9.2.17-stable - const auto server_info = client.GetServerInfo(); - if (versionNumber(server_info) < versionNumber(21, 9)) { - GTEST_SKIP() << "Date32 is available since v21.9.2.17-stable and can't be tested against server: " << server_info; - } + if (auto message = this->SkipTest(client)) { + GTEST_SKIP() << *message; } - if constexpr (std::is_same_v) { - const auto server_info = client.GetServerInfo(); - if (versionNumber(server_info) < versionNumber(21, 7)) { - GTEST_SKIP() << "ColumnInt128 is available since v21.7.2.7-stable and can't be tested against server: " << server_info; + auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); + EXPECT_TRUE(CompareRecursive(*column, *result_typed)); +} + +TYPED_TEST(GenericColumnTest, NulableT_RoundTrip) { + using NullableType = ColumnNullableT; + auto column = std::make_shared(this->MakeColumn()); + auto values = this->GenerateValues(100); + FromVectorGenerator is_null({true, false}); + for (size_t i = 0; i < values.size(); ++i) { + if (is_null(i)) { + column->Append(std::nullopt); + } else { + column->Append(values[i]); } } - auto result_typed = RoundtripColumnValues(client, column)->template AsStrict(); + clickhouse::Client client(LocalHostEndpoint); + + if (auto message = this->SkipTest(client)) { + GTEST_SKIP() << *message; + } + + auto result_typed = WrapColumn(RoundtripColumnValues(client, column)); EXPECT_TRUE(CompareRecursive(*column, *result_typed)); } diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 62641c19..6e758f97 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -3,7 +3,6 @@ #include "readonly_client_test.h" #include "connection_failed_client_test.h" #include "utils.h" -#include "roundtrip_column.h" #include @@ -67,21 +66,6 @@ class ClientCase : public testing::TestWithParam { } } - std::string GetSettingValue(const std::string& name) { - std::string result; - client_->Select("SELECT value FROM system.settings WHERE name = \'" + name + "\'", - [&result](const Block& block) - { - if (block.GetRowCount() == 0) { - return; - } - result = block[0]->AsStrict()->At(0); - } - ); - return result; - } - - std::unique_ptr client_; const std::string table_name = "test_clickhouse_cpp_test_ut_table"; const std::string column_name = "test_column"; @@ -978,192 +962,6 @@ TEST_P(ClientCase, DISABLED_ArrayArrayUInt64) { } } -TEST_P(ClientCase, RoundtripArrayTUint64) { - auto array = std::make_shared>(); - array->Append({0, 1, 2}); - - auto result = RoundtripColumnValues(*client_, array)->AsStrict(); - auto row = result->GetAsColumn(0)->As(); - - EXPECT_EQ(0u, row->At(0)); - EXPECT_EQ(1u, (*row)[1]); - EXPECT_EQ(2u, (*row)[2]); -} - -TEST_P(ClientCase, RoundtripArrayTArrayTUint64) { - const std::vector> row_values = { - {1, 2, 3}, - {4, 5, 6}, - {7, 8, 9, 10} - }; - - auto array = std::make_shared>>(); - array->Append(row_values); - - auto result_typed = ColumnArrayT>::Wrap(RoundtripColumnValues(*client_, array)); - EXPECT_TRUE(CompareRecursive(*array, *result_typed)); -} - -TEST_P(ClientCase, RoundtripArrayTArrayTArrayTUint64) { - using ColumnType = ColumnArrayT>>; - const std::vector>> row_values = { - {{1, 2, 3}, {3, 2, 1}}, - {{4, 5, 6}, {6, 5, 4}}, - {{7, 8, 9, 10}, {}}, - {{}, {10, 9, 8, 7}} - }; - - auto array = std::make_shared(); - array->Append(row_values); - - auto result_typed = ColumnType::Wrap(RoundtripColumnValues(*client_, array)); - EXPECT_TRUE(CompareRecursive(*array, *result_typed)); -} - - -TEST_P(ClientCase, RoundtripArrayTFixedString) { - auto array = std::make_shared>(6); - array->Append({"hello", "world"}); - - auto result_typed = ColumnArrayT::Wrap(RoundtripColumnValues(*client_, array)); - EXPECT_TRUE(CompareRecursive(*array, *result_typed)); -} - -TEST_P(ClientCase, RoundtripArrayTString) { - auto array = std::make_shared>(); - array->Append({"hello", "world"}); - - auto result_typed = ColumnArrayT::Wrap(RoundtripColumnValues(*client_, array)); - EXPECT_TRUE(CompareRecursive(*array, *result_typed)); -} - -TEST_P(ClientCase, RoundtripArrayLowCardinalityTString) { - // TODO replase by Roundtrip test - using TestColumn = ColumnArrayT>; - - Block block; - auto array = createTableWithOneColumn(block); - array->Append(std::vector{}); - array->Append(std::vector{}); - - block.RefreshRowCount(); - client_->Insert(table_name, block); - - size_t total_rows = 0; - client_->Select(getOneColumnSelectQuery(), - [&total_rows](const Block& block) { - total_rows += block.GetRowCount(); - if (block.GetRowCount() == 0) { - return; - } - } - ); - - ASSERT_EQ(total_rows, 2u); -} - -TEST_P(ClientCase, RoundtripMapTUint64String) { - using Map = ColumnMapT; - auto map = std::make_shared(std::make_shared(), std::make_shared()); - - std::map row; - row[1] = "hello"; - row[2] = "world"; - map->Append(row); - - auto result_typed = Map::Wrap(RoundtripColumnValues(*client_, map)); - EXPECT_TRUE(CompareRecursive(*map, *result_typed)); -} - -TEST_P(ClientCase, RoundtripMapUUID_Tuple_String_Array_Uint64) { - using Tuple = ColumnTupleT>; - using Map = ColumnMapT; - auto map = std::make_shared(std::make_shared(), std::make_shared( - std::make_tuple(std::make_shared(), std::make_shared>()))); - - - std::map>> row; - row[UUID{1, 1}] = std::make_tuple("hello", std::vector{1, 2, 3}) ; - row[UUID{2, 2}] = std::make_tuple("world", std::vector{4, 5, 6}) ; - map->Append(row); - - auto result_typed = Map::Wrap(RoundtripColumnValues(*client_, map)); - EXPECT_TRUE(CompareRecursive(*map, *result_typed)); -} - -TEST_P(ClientCase, RoundtripPoint) { - if (GetSettingValue("allow_experimental_geo_types") != "1") { - GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; - } - - auto col = std::make_shared(); - col->Append({1.0, 2.0}); - col->Append({0.1, 0.2}); - - auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); - EXPECT_TRUE(CompareRecursive(*col, *result_typed)); -} - -TEST_P(ClientCase, RoundtripRing) { - if (GetSettingValue("allow_experimental_geo_types") != "1") { - GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; - } - - auto col = std::make_shared(); - { - std::vector ring{{1.0, 2.0}, {3.0, 4.0}}; - col->Append(ring); - } - { - std::vector ring{{0.1, 0.2}, {0.3, 0.4}}; - col->Append(ring); - } - auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); - EXPECT_TRUE(CompareRecursive(*col, *result_typed)); -} - -TEST_P(ClientCase, RoundtripPolygon) { - if (GetSettingValue("allow_experimental_geo_types") != "1") { - GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; - } - - auto col = std::make_shared(); - { - std::vector> polygon - {{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}; - col->Append(polygon); - } - { - std::vector> polygon - {{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}; - col->Append(polygon); - } - auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); - EXPECT_TRUE(CompareRecursive(*col, *result_typed)); -} - -TEST_P(ClientCase, RoundtripMultiPolygon) { - if (GetSettingValue("allow_experimental_geo_types") != "1") { - GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; - } - - auto col = std::make_shared(); - { - std::vector>> multi_polygon - {{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}, - {{{1.1, 2.2}, {3.3, 4.4}}, {{5.5, 6.6}, {7.7, 8.8}}}}; - col->Append(multi_polygon); - } - { - std::vector>> multi_polygon - {{{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}, - {{{1.1, 1.2}, {1.3, 1.4}}, {{1.5, 1.6}, {1.7, 1.8}}}}; - col->Append(multi_polygon); - } - auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); - EXPECT_TRUE(CompareRecursive(*col, *result_typed)); -} - TEST_P(ClientCase, OnProgress) { Block block; createTableWithOneColumn(block); diff --git a/ut/roundtrip_tests.cpp b/ut/roundtrip_tests.cpp new file mode 100644 index 00000000..5f4fce9c --- /dev/null +++ b/ut/roundtrip_tests.cpp @@ -0,0 +1,313 @@ +#include + +#include "utils.h" +#include "roundtrip_column.h" + +#include +#include + +using namespace clickhouse; + +// Use value-parameterized tests to run same tests with different client +// options. +class RoundtripCase : public testing::TestWithParam { +protected: + void SetUp() override { + client_ = std::make_unique(GetParam()); + } + + void TearDown() override { + client_.reset(); + } + + std::string GetSettingValue(const std::string& name) { + std::string result; + client_->Select("SELECT value FROM system.settings WHERE name = \'" + name + "\'", + [&result](const Block& block) + { + if (block.GetRowCount() == 0) { + return; + } + result = block[0]->AsStrict()->At(0); + } + ); + return result; + } + + + std::unique_ptr client_; +}; + +TEST_P(RoundtripCase, ArrayTUint64) { + auto array = std::make_shared>(); + array->Append({0, 1, 2}); + + auto result = RoundtripColumnValues(*client_, array)->AsStrict(); + auto row = result->GetAsColumn(0)->As(); + + EXPECT_EQ(0u, row->At(0)); + EXPECT_EQ(1u, (*row)[1]); + EXPECT_EQ(2u, (*row)[2]); +} + +TEST_P(RoundtripCase, ArrayTArrayTUint64) { + const std::vector> row_values = { + {1, 2, 3}, + {4, 5, 6}, + {7, 8, 9, 10} + }; + + auto array = std::make_shared>>(); + array->Append(row_values); + + auto result_typed = ColumnArrayT>::Wrap(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + +TEST_P(RoundtripCase, ArrayTArrayTArrayTUint64) { + using ColumnType = ColumnArrayT>>; + const std::vector>> row_values = { + {{1, 2, 3}, {3, 2, 1}}, + {{4, 5, 6}, {6, 5, 4}}, + {{7, 8, 9, 10}, {}}, + {{}, {10, 9, 8, 7}} + }; + + auto array = std::make_shared(); + array->Append(row_values); + + auto result_typed = ColumnType::Wrap(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + + +TEST_P(RoundtripCase, ArrayTFixedString) { + auto array = std::make_shared>(6); + array->Append({"hello", "world"}); + + auto result_typed = ColumnArrayT::Wrap(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + +TEST_P(RoundtripCase, ArrayTString) { + auto array = std::make_shared>(); + array->Append({"hello", "world"}); + + auto result_typed = ColumnArrayT::Wrap(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + +TEST_P(RoundtripCase, MapTUint64String) { + using Map = ColumnMapT; + auto map = std::make_shared(std::make_shared(), std::make_shared()); + + std::map row; + row[1] = "hello"; + row[2] = "world"; + map->Append(row); + + auto result_typed = Map::Wrap(RoundtripColumnValues(*client_, map)); + EXPECT_TRUE(CompareRecursive(*map, *result_typed)); +} + +TEST_P(RoundtripCase, MapUUID_Tuple_String_Array_Uint64) { + using Tuple = ColumnTupleT>; + using Map = ColumnMapT; + auto map = std::make_shared(std::make_shared(), std::make_shared( + std::make_tuple(std::make_shared(), std::make_shared>()))); + + + std::map>> row; + row[UUID{1, 1}] = std::make_tuple("hello", std::vector{1, 2, 3}) ; + row[UUID{2, 2}] = std::make_tuple("world", std::vector{4, 5, 6}) ; + map->Append(row); + + auto result_typed = Map::Wrap(RoundtripColumnValues(*client_, map)); + EXPECT_TRUE(CompareRecursive(*map, *result_typed)); +} + +TEST_P(RoundtripCase, Point) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + col->Append({1.0, 2.0}); + col->Append({0.1, 0.2}); + + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, Ring) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector ring{{1.0, 2.0}, {3.0, 4.0}}; + col->Append(ring); + } + { + std::vector ring{{0.1, 0.2}, {0.3, 0.4}}; + col->Append(ring); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, Polygon) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector> polygon + {{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}; + col->Append(polygon); + } + { + std::vector> polygon + {{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}; + col->Append(polygon); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, MultiPolygon) { + if (GetSettingValue("allow_experimental_geo_types") != "1") { + GTEST_SKIP() << "Test is skipped because experimental geo types are not allowed. Set setting allow_experimental_geo_types = 1 in order to allow it." << std::endl; + } + + auto col = std::make_shared(); + { + std::vector>> multi_polygon + {{{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}, + {{{1.1, 2.2}, {3.3, 4.4}}, {{5.5, 6.6}, {7.7, 8.8}}}}; + col->Append(multi_polygon); + } + { + std::vector>> multi_polygon + {{{{0.1, 0.2}, {0.3, 0.4}}, {{0.5, 0.6}, {0.7, 0.8}}}, + {{{1.1, 1.2}, {1.3, 1.4}}, {{1.5, 1.6}, {1.7, 1.8}}}}; + col->Append(multi_polygon); + } + auto result_typed = RoundtripColumnValues(*client_, col)->AsStrict(); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, LowCardinalityTString) { + using TestColumn = ColumnLowCardinalityT; + auto col = std::make_shared(); + col->Append("abc"); + col->Append("def"); + col->Append("abc"); + col->Append("abc"); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, LowCardinalityTNullableString) { + using TestColumn = ColumnLowCardinalityT>; + auto col = std::make_shared(); + col->Append("abc"); + col->Append("def"); + col->Append("abc"); + col->Append("abc"); + col->Append(std::nullopt); + col->Append(std::nullopt); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, ArrayTNullableString) { + using TestColumn = ColumnArrayT>; + auto col = std::make_shared(); + col->Append({std::nullopt, std::nullopt, std::nullopt}); + col->Append(std::vector>{"abc", std::nullopt}); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, TupleTNullableString) { + using TestColumn = ColumnTupleT>; + auto col = std::make_shared(std::make_tuple(std::make_shared>())); + col->Append(std::make_tuple(std::nullopt)); + col->Append(std::make_tuple("abc")); + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, Map_TString_TNullableString) { + using Key = ColumnString; + using Value = ColumnNullableT; + using TestColumn = ColumnMapT; + auto col = std::make_shared(std::make_shared(), std::make_shared()); + { + std::map> value; + value["1"]= "one"; + value["2"]= std::nullopt; + col->Append(value); + } + { + std::map> value; + value["4"]= "one"; + value["2"]= std::nullopt; + col->Append(value); + } + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, Map_LowCardinalityTString_LowCardinalityTNullableString) { + using Key = ColumnLowCardinalityT; + using Value = ColumnLowCardinalityT>; + using TestColumn = ColumnMapT; + auto col = std::make_shared(std::make_shared(), std::make_shared()); + { + std::map> value; + value["1"]= "one"; + value["2"]= std::nullopt; + col->Append(value); + } + { + std::map> value; + value["4"]= "one"; + value["2"]= std::nullopt; + col->Append(value); + } + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, col)); + EXPECT_TRUE(CompareRecursive(*col, *result_typed)); +} + +TEST_P(RoundtripCase, RoundtripArrayLowCardinalityTString) { + using TestColumn = ColumnArrayT>; + auto array = std::make_shared(); + array->Append(std::vector{}); + array->Append(std::vector{}); + + auto result_typed = WrapColumn(RoundtripColumnValues(*client_, array)); + EXPECT_TRUE(CompareRecursive(*array, *result_typed)); +} + +const auto LocalHostEndpoint = ClientOptions() + .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); + +INSTANTIATE_TEST_SUITE_P( + Roundtrip, RoundtripCase, + ::testing::Values( + ClientOptions(LocalHostEndpoint) + .SetPingBeforeQuery(true) + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false), + ClientOptions(LocalHostEndpoint) + .SetPingBeforeQuery(false) + .SetCompressionMethod(CompressionMethod::LZ4) + .SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn(false) + )); diff --git a/ut/utils.h b/ut/utils.h index a0d136fd..f3740fc7 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -6,6 +6,7 @@ #include "utils_meta.h" #include "utils_comparison.h" +#include #include #include #include @@ -112,6 +113,15 @@ template inline ostream & operator<<(ostream & ostr, const tuple & t) { return printTuple(ostr, t); } + +template +inline ostream & operator<<(ostream & ostr, const optional & t) { + if (t.has_value()) { + return ostr << *t; + } else { + return ostr << "NULL"; + } +} } From f6a4301adcafc07aede983a3571bdd10184fae40 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 23 Dec 2022 19:16:34 +0400 Subject: [PATCH 18/44] Test to verify issue #266 --- ut/client_ut.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 62641c19..daae955e 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1313,6 +1313,17 @@ TEST_P(ClientCase, OnProfileEvents) { } } +TEST_P(ClientCase, Issue266) { + client_->Execute("CREATE TEMPORARY TABLE IF NOT EXISTS tableplus_crash_example (col AggregateFunction(argMax, Int32, DateTime(3))) engine = Memory"); + client_->Execute("insert into tableplus_crash_example values (unhex('010000000001089170A883010000'))"); + + client_->Select("select col from tableplus_crash_example", + [&](const Block& block) { + std::cerr << PrettyPrintBlock{block} << std::endl; + }); +} + + const auto LocalHostEndpoint = ClientOptions() .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) From 0fbd7035b4aa3a783ff33f1322c8cb4adaee3df7 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 23 Dec 2022 19:30:39 +0400 Subject: [PATCH 19/44] Update client_ut.cpp Added version --- ut/client_ut.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index daae955e..61b60f53 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1317,6 +1317,11 @@ TEST_P(ClientCase, Issue266) { client_->Execute("CREATE TEMPORARY TABLE IF NOT EXISTS tableplus_crash_example (col AggregateFunction(argMax, Int32, DateTime(3))) engine = Memory"); client_->Execute("insert into tableplus_crash_example values (unhex('010000000001089170A883010000'))"); + client_->Select("select version()", + [&](const Block& block) { + std::cerr << PrettyPrintBlock{block} << std::endl; + }); + client_->Select("select col from tableplus_crash_example", [&](const Block& block) { std::cerr << PrettyPrintBlock{block} << std::endl; From 4116f74ffb7fe14380072a2d15c6c2d28ae4a56b Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 26 Dec 2022 16:48:05 +0400 Subject: [PATCH 20/44] Fixed unsafe access to AST elements --- clickhouse/columns/factory.cpp | 49 +++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index fbd57889..2e1fd6ad 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -23,10 +23,26 @@ #include "../exceptions.h" #include +#include namespace clickhouse { namespace { +// Like Python's list's []: +// * 0 - first element +// * 1 - second element +// * -1 - last element +// * -2 - one before last, etc. +const auto& GetASTChildElement(const TypeAst & ast, int position) { + if (abs(position) >= ast.elements.size()) + throw ValidationError("AST child element index out of bounds: " + std::to_string(position)); + + if (position < 0) + position = ast.elements.size() + position; + + return ast.elements[static_cast(position)]; +} + static ColumnRef CreateTerminalColumn(const TypeAst& ast) { switch (ast.code) { case Type::Void: @@ -58,24 +74,24 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) { return std::make_shared(); case Type::Decimal: - return std::make_shared(ast.elements.front().value, ast.elements.back().value); + return std::make_shared(GetASTChildElement(ast, 0).value, GetASTChildElement(ast, -1).value); case Type::Decimal32: - return std::make_shared(9, ast.elements.front().value); + return std::make_shared(9, GetASTChildElement(ast, 0).value); case Type::Decimal64: - return std::make_shared(18, ast.elements.front().value); + return std::make_shared(18, GetASTChildElement(ast, 0).value); case Type::Decimal128: - return std::make_shared(38, ast.elements.front().value); + return std::make_shared(38, GetASTChildElement(ast, 0).value); case Type::String: return std::make_shared(); case Type::FixedString: - return std::make_shared(ast.elements.front().value); + return std::make_shared(GetASTChildElement(ast, 0).value); case Type::DateTime: if (ast.elements.empty()) { return std::make_shared(); } else { - return std::make_shared(ast.elements[0].value_string); + return std::make_shared(GetASTChildElement(ast, 0).value_string); } case Type::DateTime64: if (ast.elements.empty()) { @@ -120,13 +136,13 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti switch (ast.meta) { case TypeAst::Array: { return std::make_shared( - CreateColumnFromAst(ast.elements.front(), settings) + CreateColumnFromAst(GetASTChildElement(ast, 0), settings) ); } case TypeAst::Nullable: { return std::make_shared( - CreateColumnFromAst(ast.elements.front(), settings), + CreateColumnFromAst(GetASTChildElement(ast, 0), settings), std::make_shared() ); } @@ -159,9 +175,10 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti enum_items.reserve(ast.elements.size() / 2); for (size_t i = 0; i < ast.elements.size(); i += 2) { - enum_items.push_back( - Type::EnumItem{ ast.elements[i].value_string, - (int16_t)ast.elements[i + 1].value }); + enum_items.push_back(Type::EnumItem{ + ast.elements[i].value_string, + static_cast(ast.elements[i + 1].value) + }); } if (ast.code == Type::Enum8) { @@ -176,14 +193,14 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti break; } case TypeAst::LowCardinality: { - const auto nested = ast.elements.front(); + const auto nested = GetASTChildElement(ast, 0); if (settings.low_cardinality_as_wrapped_column) { switch (nested.code) { // TODO (nemkov): update this to maximize code reuse. case Type::String: return std::make_shared>(); case Type::FixedString: - return std::make_shared>(nested.elements.front().value); + return std::make_shared>(GetASTChildElement(nested, 0).value); case Type::Nullable: throw UnimplementedError("LowCardinality(" + nested.name + ") is not supported with LowCardinalityAsWrappedColumn on"); default: @@ -196,11 +213,11 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti case Type::String: return std::make_shared>(); case Type::FixedString: - return std::make_shared>(nested.elements.front().value); + return std::make_shared>(GetASTChildElement(nested, 0).value); case Type::Nullable: return std::make_shared( std::make_shared( - CreateColumnFromAst(nested.elements.front(), settings), + CreateColumnFromAst(GetASTChildElement(nested, 0), settings), std::make_shared() ) ); @@ -210,7 +227,7 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti } } case TypeAst::SimpleAggregateFunction: { - return CreateTerminalColumn(ast.elements.back()); + return CreateTerminalColumn(GetASTChildElement(ast, -1)); } case TypeAst::Map: { From 7551a7d071408221a312ab5de5a72efee8e51e10 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 26 Dec 2022 16:49:27 +0400 Subject: [PATCH 21/44] Fixed token parsing, when empty input would produce invalid AST --- clickhouse/columns/factory.cpp | 2 +- clickhouse/types/type_parser.cpp | 7 +++++++ ut/type_parser_ut.cpp | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index 2e1fd6ad..e003b7f5 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -34,7 +34,7 @@ namespace { // * -1 - last element // * -2 - one before last, etc. const auto& GetASTChildElement(const TypeAst & ast, int position) { - if (abs(position) >= ast.elements.size()) + if (static_cast(abs(position)) >= ast.elements.size()) throw ValidationError("AST child element index out of bounds: " + std::to_string(position)); if (position < 0) diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index e16aadb5..23d93a3b 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -111,6 +111,7 @@ bool TypeParser::Parse(TypeAst* type) { type_ = type; open_elements_.push(type_); + size_t processed_tokens = 0; do { const Token & token = NextToken(); switch (token.type) { @@ -159,11 +160,17 @@ bool TypeParser::Parse(TypeAst* type) { // Ubalanced braces, brackets, etc is an error. if (open_elements_.size() != 1) return false; + + // Empty input string, no tokens produced + if (processed_tokens == 0) + return false; + return true; } case Token::Invalid: return false; } + ++processed_tokens; } while (true); } diff --git a/ut/type_parser_ut.cpp b/ut/type_parser_ut.cpp index ee1258b7..66e4ef4c 100644 --- a/ut/type_parser_ut.cpp +++ b/ut/type_parser_ut.cpp @@ -239,3 +239,22 @@ TEST(TypeParserCase, ParseMap) { ASSERT_EQ(ast.elements[1].meta, TypeAst::Terminal); ASSERT_EQ(ast.elements[1].name, "String"); } + +TEST(TypeParser, EmptyName) { + { + TypeAst ast; + EXPECT_EQ(false, TypeParser("").Parse(&ast)); + } + + { + TypeAst ast; + EXPECT_EQ(false, TypeParser(" ").Parse(&ast)); + } +} + +TEST(ParseTypeName, EmptyName) { + // Empty and invalid names shouldn't produce any AST and shoudn't crash + EXPECT_EQ(nullptr, ParseTypeName("")); + EXPECT_EQ(nullptr, ParseTypeName(" ")); + EXPECT_EQ(nullptr, ParseTypeName(std::string(5, '\0'))); +} From 262b15275790180524b7e5f173d5aa8403ac9269 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 27 Dec 2022 01:34:25 +0400 Subject: [PATCH 22/44] Hackish fix of parsing unknown type --- clickhouse/types/type_parser.cpp | 50 ++++++++++++++++++++++++++++---- ut/CreateColumnByType_ut.cpp | 6 ++++ ut/client_ut.cpp | 9 ++++-- ut/type_parser_ut.cpp | 12 ++++++++ ut/types_ut.cpp | 8 +++-- 5 files changed, 74 insertions(+), 11 deletions(-) diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index 23d93a3b..1ec7a7b7 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -1,10 +1,21 @@ #include "type_parser.h" +#include "clickhouse/exceptions.h" +#include "clickhouse/base/platform.h" // for _win_ + #include +#include #include #include #include +#if defined _win_ +#include +#else +#include +#endif + + namespace clickhouse { bool TypeAst::operator==(const TypeAst & other) const { @@ -16,6 +27,7 @@ bool TypeAst::operator==(const TypeAst & other) const { } static const std::unordered_map kTypeCode = { + { "Void", Type::Void }, { "Int8", Type::Int8 }, { "Int16", Type::Int16 }, { "Int32", Type::Int32 }, @@ -41,23 +53,38 @@ static const std::unordered_map kTypeCode = { { "IPv4", Type::IPv4 }, { "IPv6", Type::IPv6 }, { "Int128", Type::Int128 }, +// { "UInt128", Type::UInt128 }, { "Decimal", Type::Decimal }, { "Decimal32", Type::Decimal32 }, { "Decimal64", Type::Decimal64 }, { "Decimal128", Type::Decimal128 }, { "LowCardinality", Type::LowCardinality }, - { "Map", Type::Map}, - { "Point", Type::Point}, - { "Ring", Type::Ring}, - { "Polygon", Type::Polygon}, - { "MultiPolygon", Type::MultiPolygon}, + { "Map", Type::Map }, + { "Point", Type::Point }, + { "Ring", Type::Ring }, + { "Polygon", Type::Polygon }, + { "MultiPolygon", Type::MultiPolygon }, }; +template +inline int CompateStringsCaseInsensitive(const L& left, const R& right) { + int64_t size_diff = left.size() - right.size(); + if (size_diff != 0) + return size_diff > 0 ? 1 : -1; + +#if defined _win_ + return _strnicmp(left.data(), right.data(), left.size()); +#else + return strncasecmp(left.data(), right.data(), left.size()); +#endif +} + static Type::Code GetTypeCode(const std::string& name) { auto it = kTypeCode.find(name); if (it != kTypeCode.end()) { return it->second; } + return Type::Void; } @@ -97,6 +124,17 @@ static TypeAst::Meta GetTypeMeta(const StringView& name) { return TypeAst::Terminal; } +bool ValidateAST(const TypeAst& ast) { + // Void terminal that is not actually "void" produced when unknown type is encountered. + if (ast.meta == TypeAst::Terminal + && ast.code == Type::Void + && CompateStringsCaseInsensitive(ast.name, std::string_view("void")) != 0) + //throw UnimplementedError("Unsupported type: " + ast.name); + return false; + + return true; +} + TypeParser::TypeParser(const StringView& name) : cur_(name.data()) @@ -165,7 +203,7 @@ bool TypeParser::Parse(TypeAst* type) { if (processed_tokens == 0) return false; - return true; + return ValidateAST(*type); } case Token::Invalid: return false; diff --git a/ut/CreateColumnByType_ut.cpp b/ut/CreateColumnByType_ut.cpp index fb7ffd85..fecf0ea3 100644 --- a/ut/CreateColumnByType_ut.cpp +++ b/ut/CreateColumnByType_ut.cpp @@ -50,6 +50,12 @@ TEST(CreateColumnByType, DateTime) { ASSERT_EQ(CreateColumnByType("DateTime64(3, 'UTC')")->As()->Timezone(), "UTC"); } +TEST(CreateColumnByType, AggregateFunction) { + EXPECT_EQ(nullptr, CreateColumnByType("AggregateFunction(argMax, Int32, DateTime64(3))")); + EXPECT_EQ(nullptr, CreateColumnByType("AggregateFunction(argMax, FIxedString(10), DateTime64(3, 'UTC'))")); +} + + class CreateColumnByTypeWithName : public ::testing::TestWithParam {}; diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 61b60f53..f86f3ff7 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1313,7 +1313,9 @@ TEST_P(ClientCase, OnProfileEvents) { } } -TEST_P(ClientCase, Issue266) { +TEST_P(ClientCase, SelectAggregateFunction) { + // Verifies that perofing SELECT value of type AggregateFunction(...) doesn't crash the client. + // For details: https://github.com/ClickHouse/clickhouse-cpp/issues/266 client_->Execute("CREATE TEMPORARY TABLE IF NOT EXISTS tableplus_crash_example (col AggregateFunction(argMax, Int32, DateTime(3))) engine = Memory"); client_->Execute("insert into tableplus_crash_example values (unhex('010000000001089170A883010000'))"); @@ -1322,10 +1324,11 @@ TEST_P(ClientCase, Issue266) { std::cerr << PrettyPrintBlock{block} << std::endl; }); - client_->Select("select col from tableplus_crash_example", + // Column type `AggregateFunction` is not supported. + EXPECT_THROW(client_->Select("select toTypeName(col), col from tableplus_crash_example", [&](const Block& block) { std::cerr << PrettyPrintBlock{block} << std::endl; - }); + }), clickhouse::UnimplementedError); } diff --git a/ut/type_parser_ut.cpp b/ut/type_parser_ut.cpp index 66e4ef4c..b0193ded 100644 --- a/ut/type_parser_ut.cpp +++ b/ut/type_parser_ut.cpp @@ -258,3 +258,15 @@ TEST(ParseTypeName, EmptyName) { EXPECT_EQ(nullptr, ParseTypeName(" ")); EXPECT_EQ(nullptr, ParseTypeName(std::string(5, '\0'))); } + +TEST(TypeParser, AggregateFunction) { + { + TypeAst ast; + EXPECT_FALSE(TypeParser("AggregateFunction(argMax, Int32, DateTime(3))").Parse(&ast)); + } + + { + TypeAst ast; + EXPECT_FALSE(TypeParser("AggregateFunction(argMax, LowCardinality(Nullable(FixedString(4))), DateTime(3, 'UTC'))").Parse(&ast)); + } +} diff --git a/ut/types_ut.cpp b/ut/types_ut.cpp index 3f795269..7af343b5 100644 --- a/ut/types_ut.cpp +++ b/ut/types_ut.cpp @@ -80,7 +80,7 @@ TEST(TypesCase, IsEqual) { const std::string type_names[] = { "UInt8", "Int8", - "UInt128", +// "UInt128", "String", "FixedString(0)", "FixedString(10000)", @@ -128,7 +128,11 @@ TEST(TypesCase, IsEqual) { EXPECT_TRUE(type->IsEqual(*type)); for (const auto & other_type_name : type_names) { - const auto other_type = clickhouse::CreateColumnByType(other_type_name)->Type(); + SCOPED_TRACE(other_type_name); + const auto other_column = clickhouse::CreateColumnByType(other_type_name); + ASSERT_NE(nullptr, other_column); + + const auto other_type = other_column->Type(); const auto should_be_equal = type_name == other_type_name; EXPECT_EQ(should_be_equal, type->IsEqual(other_type)) From 3f3c7d72c3736adf12bcb5ca9c35d2b9d1b5ef9f Mon Sep 17 00:00:00 2001 From: Anton Date: Tue, 27 Dec 2022 10:25:19 +0000 Subject: [PATCH 23/44] Update CMakeLists.txt Do not use target_link_libraries with alias --- clickhouse/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 9c5ad217..870b1040 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -71,11 +71,7 @@ IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") # /usr/bin/ld: CMakeFiles/simple-test.dir/main.cpp.o: undefined reference to symbol '_Unwind_Resume@@GCC_3.0' # /usr/bin/ld: /lib/x86_64-linux-gnu/libgcc_s.so.1: error adding symbols: DSO missing from command line # FIXME: that workaround breaks clang build on mingw - IF (BUILD_SHARED_LIBS) - TARGET_LINK_LIBRARIES (clickhouse-cpp-lib gcc_s) - ELSE() - TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static gcc_s) - ENDIF() + TARGET_LINK_LIBRARIES (clickhouse-cpp-lib gcc_s) ENDIF () ENDIF () From 967a80cd5cf0943507e2685eb08a21a66dc83e2f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 2 Jan 2023 15:17:59 +0300 Subject: [PATCH 24/44] Update LICENSE --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 7327b333..9269404a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2018-2022 ClickHouse, Inc +Copyright 2018-2023 ClickHouse, Inc. Copyright 2017 Pavel Artemkin Apache License @@ -189,7 +189,7 @@ Copyright 2017 Pavel Artemkin same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2018-2020 Yandex LLC + Copyright 2018-2023 ClickHouse, Inc. Copyright 2017 Pavel Artemkin From 25968874fba19893e9c35c479c5a073bf99b0343 Mon Sep 17 00:00:00 2001 From: Andrey Bugaevskiy Date: Thu, 5 Jan 2023 16:44:45 +0300 Subject: [PATCH 25/44] Fix getaddrinfo error reporting --- clickhouse/base/socket.cpp | 21 +++++++++++++++++++++ clickhouse/base/socket.h | 12 ++++++++++++ ut/socket_ut.cpp | 10 ++++++++++ 3 files changed, 43 insertions(+) diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index e62e90df..296afdfb 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -43,6 +43,21 @@ windowsErrorCategory const& windowsErrorCategory::category() { } #endif +#if defined(_unix_) +char const* getaddrinfoErrorCategory::name() const noexcept { + return "getaddrinfoError"; +} + +std::string getaddrinfoErrorCategory::message(int c) const { + return gai_strerror(c); +} + +getaddrinfoErrorCategory const& getaddrinfoErrorCategory::category() { + static getaddrinfoErrorCategory c; + return c; +} +#endif + namespace { class LocalNames : public std::unordered_set { @@ -258,6 +273,12 @@ NetworkAddress::NetworkAddress(const std::string& host, const std::string& port) const int error = getaddrinfo(host.c_str(), port.c_str(), &hints, &info_); +#if defined(_unix_) + if (error != EAI_SYSTEM) { + throw std::system_error(error, getaddrinfoErrorCategory::category()); + } +#endif + if (error) { throw std::system_error(getSocketErrorCode(), getErrorCategory()); } diff --git a/clickhouse/base/socket.h b/clickhouse/base/socket.h index c68f250d..ed8f8a16 100644 --- a/clickhouse/base/socket.h +++ b/clickhouse/base/socket.h @@ -60,6 +60,18 @@ class windowsErrorCategory : public std::error_category { #endif +#if defined(_unix_) + +class getaddrinfoErrorCategory : public std::error_category { +public: + char const* name() const noexcept override final; + std::string message(int c) const override final; + + static getaddrinfoErrorCategory const& category(); +}; + +#endif + class SocketBase { public: diff --git a/ut/socket_ut.cpp b/ut/socket_ut.cpp index 5a263435..581366e6 100644 --- a/ut/socket_ut.cpp +++ b/ut/socket_ut.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -63,6 +64,15 @@ TEST(Socketcase, timeoutrecv) { server.stop(); } +TEST(Socketcase, gaierror) { + try { + NetworkAddress addr("host.invalid", "80"); // never resolves + FAIL(); + } catch (const std::system_error& e) { + ASSERT_EQ(EAI_NONAME, e.code().value()); + } +} + // Test to verify that reading from empty socket doesn't hangs. //TEST(Socketcase, ReadFromEmptySocket) { // const int port = 12345; From 88aba68c428c7c7248ba935d35646fe7d33a622d Mon Sep 17 00:00:00 2001 From: Andrey Bugaevskiy Date: Thu, 5 Jan 2023 15:25:02 +0300 Subject: [PATCH 26/44] Add connect timeout to socket --- clickhouse/base/socket.cpp | 4 ++-- clickhouse/base/socket.h | 1 + clickhouse/base/sslsocket.cpp | 2 +- clickhouse/client.h | 3 +++ ut/socket_ut.cpp | 26 +++++++++++++++++++++++++- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index e62e90df..a5f99d01 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -202,7 +202,7 @@ SOCKET SocketConnect(const NetworkAddress& addr, const SocketTimeoutParams& time fd.fd = *s; fd.events = POLLOUT; fd.revents = 0; - ssize_t rval = Poll(&fd, 1, 5000); + ssize_t rval = Poll(&fd, 1, timeout_params.connect_timeout.count()); if (rval == -1) { throw std::system_error(getSocketErrorCode(), getErrorCategory(), "fail to connect"); @@ -372,7 +372,7 @@ std::unique_ptr NonSecureSocketFactory::connect(const ClientOptions } std::unique_ptr NonSecureSocketFactory::doConnect(const NetworkAddress& address, const ClientOptions& opts) { - SocketTimeoutParams timeout_params { opts.connection_recv_timeout, opts.connection_send_timeout }; + SocketTimeoutParams timeout_params { opts.connection_connect_timeout, opts.connection_recv_timeout, opts.connection_send_timeout }; return std::make_unique(address, timeout_params); } diff --git a/clickhouse/base/socket.h b/clickhouse/base/socket.h index c68f250d..392fa824 100644 --- a/clickhouse/base/socket.h +++ b/clickhouse/base/socket.h @@ -83,6 +83,7 @@ class SocketFactory { struct SocketTimeoutParams { + std::chrono::milliseconds connect_timeout{ 5000 }; std::chrono::milliseconds recv_timeout{ 0 }; std::chrono::milliseconds send_timeout{ 0 }; }; diff --git a/clickhouse/base/sslsocket.cpp b/clickhouse/base/sslsocket.cpp index 03b064b1..8b1971e0 100644 --- a/clickhouse/base/sslsocket.cpp +++ b/clickhouse/base/sslsocket.cpp @@ -268,7 +268,7 @@ SSLSocketFactory::SSLSocketFactory(const ClientOptions& opts) SSLSocketFactory::~SSLSocketFactory() = default; std::unique_ptr SSLSocketFactory::doConnect(const NetworkAddress& address, const ClientOptions& opts) { - SocketTimeoutParams timeout_params { opts.connection_recv_timeout, opts.connection_send_timeout }; + SocketTimeoutParams timeout_params { opts.connection_connect_timeout, opts.connection_recv_timeout, opts.connection_send_timeout }; return std::make_unique(address, timeout_params, ssl_params_, *ssl_context_); } diff --git a/clickhouse/client.h b/clickhouse/client.h index 63a51c2e..854fd70c 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -88,6 +88,9 @@ struct ClientOptions { // TCP options DECLARE_FIELD(tcp_nodelay, bool, TcpNoDelay, true); + /// Connection socket connect timeout. If the timeout is negative then the connect operation will never timeout. + DECLARE_FIELD(connection_connect_timeout, std::chrono::milliseconds, SetConnectionConnectTimeout, std::chrono::seconds(5)); + /// Connection socket timeout. If the timeout is set to zero then the operation will never timeout. DECLARE_FIELD(connection_recv_timeout, std::chrono::milliseconds, SetConnectionRecvTimeout, std::chrono::milliseconds(0)); DECLARE_FIELD(connection_send_timeout, std::chrono::milliseconds, SetConnectionSendTimeout, std::chrono::milliseconds(0)); diff --git a/ut/socket_ut.cpp b/ut/socket_ut.cpp index 5a263435..c51308dc 100644 --- a/ut/socket_ut.cpp +++ b/ut/socket_ut.cpp @@ -43,7 +43,7 @@ TEST(Socketcase, timeoutrecv) { std::this_thread::sleep_for(std::chrono::seconds(1)); try { - Socket socket(addr, SocketTimeoutParams { Seconds(5), Seconds(5) }); + Socket socket(addr, SocketTimeoutParams { Seconds(5), Seconds(5), Seconds(5) }); std::unique_ptr ptr_input_stream = socket.makeInputStream(); char buf[1024]; @@ -63,6 +63,30 @@ TEST(Socketcase, timeoutrecv) { server.stop(); } +TEST(Socketcase, connecttimeout) { + using Clock = std::chrono::steady_clock; + + try { + NetworkAddress("::1", "19980"); + } catch (const std::system_error& e) { + GTEST_SKIP() << "missing IPv6 support"; + } + + NetworkAddress addr("100::1", "19980"); // "discard" IPv6 address + + const auto connect_start = Clock::now(); + try { + Socket socket(addr, SocketTimeoutParams{std::chrono::milliseconds(100)}); + FAIL(); + } catch (const std::system_error& e) { + if (e.code().value() == ENETUNREACH) { + GTEST_SKIP() << "missing IPv6 support"; + } + EXPECT_EQ(EINPROGRESS, e.code().value()); + EXPECT_LT(Clock::now() - connect_start, std::chrono::seconds(5)); + } +} + // Test to verify that reading from empty socket doesn't hangs. //TEST(Socketcase, ReadFromEmptySocket) { // const int port = 12345; From bda1c115d4e58ba01fec8bc1c9ade0bfe2d1d5d8 Mon Sep 17 00:00:00 2001 From: Andrey Bugaevskiy Date: Thu, 5 Jan 2023 17:17:39 +0300 Subject: [PATCH 27/44] typo --- clickhouse/base/socket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index 296afdfb..07e81329 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -274,7 +274,7 @@ NetworkAddress::NetworkAddress(const std::string& host, const std::string& port) const int error = getaddrinfo(host.c_str(), port.c_str(), &hints, &info_); #if defined(_unix_) - if (error != EAI_SYSTEM) { + if (error && error != EAI_SYSTEM) { throw std::system_error(error, getaddrinfoErrorCategory::category()); } #endif From a6643314813898c234a763ac996bda4a5ac58ea0 Mon Sep 17 00:00:00 2001 From: Andrey Bugaevskiy Date: Thu, 5 Jan 2023 23:29:50 +0300 Subject: [PATCH 28/44] fix test --- ut/socket_ut.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ut/socket_ut.cpp b/ut/socket_ut.cpp index 581366e6..709d21c3 100644 --- a/ut/socket_ut.cpp +++ b/ut/socket_ut.cpp @@ -4,11 +4,17 @@ #include #include -#include #include #include #include +// for EAI_* error codes +#if defined(_win_) +# include +#else +# include +#endif + using namespace clickhouse; TEST(Socketcase, connecterror) { @@ -69,7 +75,7 @@ TEST(Socketcase, gaierror) { NetworkAddress addr("host.invalid", "80"); // never resolves FAIL(); } catch (const std::system_error& e) { - ASSERT_EQ(EAI_NONAME, e.code().value()); + ASSERT_PRED1([](int error) { return error == EAI_NONAME || error == EAI_AGAIN || error == EAI_FAIL; }, e.code().value()); } } From 67a9f4d5c5c10182838114ec0bb6725c7258f0c7 Mon Sep 17 00:00:00 2001 From: Andrey Bugaevskiy Date: Fri, 6 Jan 2023 14:12:31 +0300 Subject: [PATCH 29/44] fix test and error code on windows Timed out `poll`/`WSAPoll` does not set an error. --- clickhouse/base/socket.cpp | 7 +++++++ ut/socket_ut.cpp | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/clickhouse/base/socket.cpp b/clickhouse/base/socket.cpp index a5f99d01..7e08e06a 100644 --- a/clickhouse/base/socket.cpp +++ b/clickhouse/base/socket.cpp @@ -207,6 +207,13 @@ SOCKET SocketConnect(const NetworkAddress& addr, const SocketTimeoutParams& time if (rval == -1) { throw std::system_error(getSocketErrorCode(), getErrorCategory(), "fail to connect"); } + if (rval == 0) { +#if defined(_win_) + last_err = WSAETIMEDOUT; +#else + last_err = ETIMEDOUT; +#endif + } if (rval > 0) { socklen_t len = sizeof(err); getsockopt(*s, SOL_SOCKET, SO_ERROR, (char*)&err, &len); diff --git a/ut/socket_ut.cpp b/ut/socket_ut.cpp index c51308dc..3ac1210f 100644 --- a/ut/socket_ut.cpp +++ b/ut/socket_ut.cpp @@ -79,10 +79,20 @@ TEST(Socketcase, connecttimeout) { Socket socket(addr, SocketTimeoutParams{std::chrono::milliseconds(100)}); FAIL(); } catch (const std::system_error& e) { - if (e.code().value() == ENETUNREACH) { + const int error = e.code().value(); + if (error == ENETUNREACH || error == EHOSTUNREACH +#if defined(_win_) + || error == WSAENETUNREACH +#endif + ) { GTEST_SKIP() << "missing IPv6 support"; } - EXPECT_EQ(EINPROGRESS, e.code().value()); +#if defined(_win_) + const auto expected = WSAETIMEDOUT; +#else + const auto expected = ETIMEDOUT; +#endif + EXPECT_EQ(expected, error); EXPECT_LT(Clock::now() - connect_start, std::chrono::seconds(5)); } } From 5592626ecacfb94cf88bf87e3fa604879f6284f7 Mon Sep 17 00:00:00 2001 From: Alexey Shakula Date: Thu, 8 Dec 2022 18:59:48 -0500 Subject: [PATCH 30/44] Handle "Bool" columns --- clickhouse/types/type_parser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index 1ec7a7b7..6142dec2 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -32,6 +32,7 @@ static const std::unordered_map kTypeCode = { { "Int16", Type::Int16 }, { "Int32", Type::Int32 }, { "Int64", Type::Int64 }, + { "Bool", Type::UInt8 }, { "UInt8", Type::UInt8 }, { "UInt16", Type::UInt16 }, { "UInt32", Type::UInt32 }, From 525a08bec60369f969107af2b8ea658d296e2292 Mon Sep 17 00:00:00 2001 From: Alexey Shakula Date: Fri, 9 Dec 2022 10:13:24 -0500 Subject: [PATCH 31/44] + unit test for Bool columns --- ut/CreateColumnByType_ut.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ut/CreateColumnByType_ut.cpp b/ut/CreateColumnByType_ut.cpp index fecf0ea3..b6794275 100644 --- a/ut/CreateColumnByType_ut.cpp +++ b/ut/CreateColumnByType_ut.cpp @@ -59,6 +59,12 @@ TEST(CreateColumnByType, AggregateFunction) { class CreateColumnByTypeWithName : public ::testing::TestWithParam {}; +TEST(CreateColumnByType, Bool) { + const auto col = CreateColumnByType("Bool"); + ASSERT_NE(nullptr, col); + EXPECT_EQ(col->GetType().GetName(), "UInt8"); +} + TEST_P(CreateColumnByTypeWithName, CreateColumnByType) { const auto col = CreateColumnByType(GetParam()); From 9fc86b932cb3e966ab88f575100da4159b1fc29a Mon Sep 17 00:00:00 2001 From: Alexey Shakula Date: Fri, 9 Dec 2022 10:27:04 -0500 Subject: [PATCH 32/44] + client unit test for Bool columns --- ut/client_ut.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index f86f3ff7..e4c55a2c 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -288,16 +288,17 @@ TEST_P(ClientCase, LowCardinalityString_AsString) { TEST_P(ClientCase, Generic) { client_->Execute( - "CREATE TEMPORARY TABLE IF NOT EXISTS test_clickhouse_cpp_client (id UInt64, name String) "); + "CREATE TEMPORARY TABLE IF NOT EXISTS test_clickhouse_cpp_client (id UInt64, name String, f Bool) "); const struct { uint64_t id; std::string name; + bool f; } TEST_DATA[] = { - { 1, "id" }, - { 3, "foo" }, - { 5, "bar" }, - { 7, "name" }, + { 1, "id", true }, + { 3, "foo", false }, + { 5, "bar", true }, + { 7, "name", false }, }; /// Insert some values. @@ -306,20 +307,23 @@ TEST_P(ClientCase, Generic) { auto id = std::make_shared(); auto name = std::make_shared(); + auto f = std::make_shared (); for (auto const& td : TEST_DATA) { id->Append(td.id); name->Append(td.name); + f->Append(td.f); } block.AppendColumn("id" , id); block.AppendColumn("name", name); + block.AppendColumn("f", f); client_->Insert("test_clickhouse_cpp_client", block); } /// Select values inserted in the previous step. size_t row = 0; - client_->Select("SELECT id, name FROM test_clickhouse_cpp_client", [TEST_DATA, &row](const Block& block) + client_->Select("SELECT id, name, f FROM test_clickhouse_cpp_client", [TEST_DATA, &row](const Block& block) { if (block.GetRowCount() == 0) { return; @@ -329,6 +333,7 @@ TEST_P(ClientCase, Generic) { for (size_t c = 0; c < block.GetRowCount(); ++c, ++row) { EXPECT_EQ(TEST_DATA[row].id, (*block[0]->As())[c]); EXPECT_EQ(TEST_DATA[row].name, (*block[1]->As())[c]); + EXPECT_EQ(TEST_DATA[row].f, (*block[2]->As())[c]); } } ); From 63b9356a6278f759bca858c6d506c8924d933152 Mon Sep 17 00:00:00 2001 From: David Keller Date: Tue, 5 Apr 2022 11:51:13 +0200 Subject: [PATCH 33/44] Add options to use system abseil, and lz4 This change solves #86, #99. Signed-off-by: David Keller --- .github/workflows/linux.yml | 70 +++++++++++++------ .github/workflows/macos.yml | 24 +++++-- CMakeLists.txt | 60 ++++++++++++++-- clickhouse/CMakeLists.txt | 15 +++- clickhouse/base/compressed.cpp | 4 +- clickhouse/columns/lowcardinality.cpp | 2 +- clickhouse/types/types.cpp | 2 +- cmake/Findlz4.cmake | 38 ++++++++++ contrib/absl/CMakeLists.txt | 6 -- contrib/absl/absl/CMakeLists.txt | 9 +++ contrib/absl/{ => absl}/base/attributes.h | 0 contrib/absl/{ => absl}/base/config.h | 0 contrib/absl/{ => absl}/base/internal/bits.h | 0 contrib/absl/{ => absl}/base/macros.h | 0 contrib/absl/{ => absl}/base/optimization.h | 0 contrib/absl/{ => absl}/base/options.h | 0 contrib/absl/{ => absl}/base/policy_checks.h | 0 contrib/absl/{ => absl}/base/port.h | 0 contrib/absl/{ => absl}/numeric/int128.cc | 0 contrib/absl/{ => absl}/numeric/int128.h | 0 .../numeric/int128_have_intrinsic.inc | 0 .../numeric/int128_no_intrinsic.inc | 0 contrib/cityhash/CMakeLists.txt | 5 -- contrib/cityhash/{ => cityhash}/BUCK | 0 contrib/cityhash/cityhash/CMakeLists.txt | 7 ++ contrib/cityhash/{ => cityhash}/COPYING | 0 contrib/cityhash/{ => cityhash}/city.cc | 0 contrib/cityhash/{ => cityhash}/city.h | 0 contrib/cityhash/{ => cityhash}/citycrc.h | 0 contrib/cityhash/{ => cityhash}/config.h | 0 contrib/lz4/CMakeLists.txt | 6 -- contrib/lz4/{ => lz4}/BUCK | 0 contrib/lz4/lz4/CMakeLists.txt | 8 +++ contrib/lz4/{ => lz4}/LICENSE | 0 contrib/lz4/{ => lz4}/lz4.c | 0 contrib/lz4/{ => lz4}/lz4.h | 0 contrib/lz4/{ => lz4}/lz4hc.c | 0 contrib/lz4/{ => lz4}/lz4hc.h | 0 tests/simple/CMakeLists.txt | 8 --- ut/CMakeLists.txt | 3 - 40 files changed, 199 insertions(+), 68 deletions(-) create mode 100644 cmake/Findlz4.cmake delete mode 100644 contrib/absl/CMakeLists.txt create mode 100644 contrib/absl/absl/CMakeLists.txt rename contrib/absl/{ => absl}/base/attributes.h (100%) rename contrib/absl/{ => absl}/base/config.h (100%) rename contrib/absl/{ => absl}/base/internal/bits.h (100%) rename contrib/absl/{ => absl}/base/macros.h (100%) rename contrib/absl/{ => absl}/base/optimization.h (100%) rename contrib/absl/{ => absl}/base/options.h (100%) rename contrib/absl/{ => absl}/base/policy_checks.h (100%) rename contrib/absl/{ => absl}/base/port.h (100%) rename contrib/absl/{ => absl}/numeric/int128.cc (100%) rename contrib/absl/{ => absl}/numeric/int128.h (100%) rename contrib/absl/{ => absl}/numeric/int128_have_intrinsic.inc (100%) rename contrib/absl/{ => absl}/numeric/int128_no_intrinsic.inc (100%) delete mode 100644 contrib/cityhash/CMakeLists.txt rename contrib/cityhash/{ => cityhash}/BUCK (100%) create mode 100644 contrib/cityhash/cityhash/CMakeLists.txt rename contrib/cityhash/{ => cityhash}/COPYING (100%) rename contrib/cityhash/{ => cityhash}/city.cc (100%) rename contrib/cityhash/{ => cityhash}/city.h (100%) rename contrib/cityhash/{ => cityhash}/citycrc.h (100%) rename contrib/cityhash/{ => cityhash}/config.h (100%) delete mode 100644 contrib/lz4/CMakeLists.txt rename contrib/lz4/{ => lz4}/BUCK (100%) create mode 100644 contrib/lz4/lz4/CMakeLists.txt rename contrib/lz4/{ => lz4}/LICENSE (100%) rename contrib/lz4/{ => lz4}/lz4.c (100%) rename contrib/lz4/{ => lz4}/lz4.h (100%) rename contrib/lz4/{ => lz4}/lz4hc.c (100%) rename contrib/lz4/{ => lz4}/lz4hc.h (100%) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3385582c..905c0868 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -14,44 +14,67 @@ env: jobs: build: - runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: - compiler: [clang-6, gcc-7, gcc-8, gcc-9] + os: [ubuntu-20.04] + compiler: [clang-6, clang-10-libc++, gcc-7, gcc-8, gcc-9] ssl: [ssl_ON, ssl_OFF] + dependencies: [dependencies_BUILT_IN] + include: - compiler: clang-6 - INSTALL: clang-6.0 + COMPILER_INSTALL: clang-6.0 libc++-dev C_COMPILER: clang-6.0 CXX_COMPILER: clang++-6.0 + - compiler: clang-10-libc++ + COMPILER_INSTALL: clang-10 libc++-dev + C_COMPILER: clang-10 + CXX_COMPILER: clang++-10 + - compiler: gcc-7 - INSTALL: gcc-7 g++-7 + COMPILER_INSTALL: gcc-7 g++-7 C_COMPILER: gcc-7 CXX_COMPILER: g++-7 - compiler: gcc-8 - INSTALL: gcc-8 g++-8 + COMPILER_INSTALL: gcc-8 g++-8 C_COMPILER: gcc-8 CXX_COMPILER: g++-8 - compiler: gcc-9 - INSTALL: gcc-9 g++-9 + COMPILER_INSTALL: gcc-9 g++-9 C_COMPILER: gcc-9 CXX_COMPILER: g++-9 - ssl: ssl_ON - INSTALL_SSL: libssl-dev - EXTRA_CMAKE_FLAGS: -DWITH_OPENSSL=ON + SSL_CMAKE_OPTION: -D WITH_OPENSSL=ON - - ssl: ssl_OFF - EXTRA_CMAKE_FLAGS: -DWITH_OPENSSL=OFF + - dependencies: dependencies_SYSTEM + compiler: compiler_SYSTEM + os: ubuntu-22.04 + COMPILER_INSTALL: gcc g++ + C_COMPILER: gcc + CXX_COMPILER: g++ + DEPENDENCIES_INSTALL: libabsl-dev liblz4-dev + DEPENDENCIES_CMAKE_OPTIONS: >- + -D WITH_SYSTEM_LZ4=ON + -D WITH_SYSTEM_ABSEIL=ON + + runs-on: ${{matrix.os}} steps: - uses: actions/checkout@v2 - name: Install dependencies - run: sudo apt-get install -y docker cmake ${{ matrix.INSTALL }} ${{ matrix.INSTALL_SSL }} + run: | + sudo apt-get update && \ + sudo apt-get install -y \ + docker \ + cmake \ + ${{matrix.COMPILER_INSTALL}} \ + ${{matrix.DEPENDENCIES_INSTALL}} - name: Install dependencies - Docker run: | @@ -62,17 +85,24 @@ jobs: sudo apt update -q sudo apt install docker-ce docker-ce-cli containerd.io - - name: Configure CMake + - name: Configure project + run: | + cmake \ + -D CMAKE_C_COMPILER=${{matrix.C_COMPILER}} \ + -D CMAKE_CXX_COMPILER=${{matrix.CXX_COMPILER}} \ + -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \ + -D BUILD_TESTS=ON \ + ${{matrix.SSL_CMAKE_OPTION}} \ + ${{matrix.DEPENDENCIES_CMAKE_OPTIONS}} \ + -S ${{github.workspace}} \ + -B ${{github.workspace}}/build + + - name: Build project run: | cmake \ - -DCMAKE_C_COMPILER=${{ matrix.C_COMPILER}} \ - -DCMAKE_CXX_COMPILER=${{ matrix.CXX_COMPILER}} \ - -B ${{github.workspace}}/build \ - -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBUILD_TESTS=ON \ - ${{ matrix.EXTRA_CMAKE_FLAGS }} - - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} --target all + --build ${{github.workspace}}/build \ + --config ${{env.BUILD_TYPE}} \ + --target all - name: Test - Start ClickHouse server in background run: | diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 007fc08d..2fe8ff75 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -23,24 +23,34 @@ jobs: build: [nossl, ssl] include: - build: nossl - extra_cmake_flags: -DWITH_OPENSSL=OFF - extra_install: - build: ssl - extra_cmake_flags: -DWITH_OPENSSL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl/ - extra_install: openssl + SSL_CMAKE_OPTION: >- + -D WITH_OPENSSL=ON + -D OPENSSL_ROOT_DIR=/usr/local/opt/openssl/ + SSL_INSTALL: openssl steps: - uses: actions/checkout@v2 - name: Install dependencies - run: brew install cmake ${{matrix.extra_install}} + run: brew install cmake ${{matrix.SSL_INSTALL}} - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBUILD_TESTS=ON ${{matrix.extra_cmake_flags}} + run: | + cmake \ + -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \ + -D BUILD_TESTS=ON \ + ${{matrix.SSL_CMAKE_OPTION}} \ + -S ${{github.workspace}} \ + -B ${{github.workspace}}/build - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} --target all + run: | + cmake \ + --build ${{github.workspace}}/build \ + --config ${{env.BUILD_TYPE}} \ + --target all - name: Start tls offoader proxy # that mimics non-secure clickhouse running on localhost diff --git a/CMakeLists.txt b/CMakeLists.txt index d79b5864..1ac72186 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,13 +1,18 @@ CMAKE_MINIMUM_REQUIRED (VERSION 3.0.2) -INCLUDE (cmake/cpp17.cmake) -INCLUDE (cmake/subdirs.cmake) -INCLUDE (cmake/openssl.cmake) +LIST (APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +INCLUDE (cpp17) +INCLUDE (subdirs) +INCLUDE (openssl) OPTION (BUILD_BENCHMARK "Build benchmark" OFF) OPTION (BUILD_TESTS "Build tests" OFF) OPTION (BUILD_SHARED_LIBS "Build shared libs" OFF) OPTION (WITH_OPENSSL "Use OpenSSL for TLS connections" OFF) +OPTION (WITH_SYSTEM_ABSEIL "Use system ABSEIL" OFF) +OPTION (WITH_SYSTEM_LZ4 "Use system LZ4" OFF) +OPTION (WITH_SYSTEM_CITYHASH "Use system cityhash" OFF) PROJECT (CLICKHOUSE-CLIENT) @@ -34,11 +39,54 @@ PROJECT (CLICKHOUSE-CLIENT) ENDIF() ENDIF() + IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + INCLUDE (CheckCXXSourceCompiles) + + CHECK_CXX_SOURCE_COMPILES("#include \nint main() { return __GLIBCXX__ != 0; }" + CLANG_WITH_LIB_STDCXX) + ENDIF () + + IF (CLANG_WITH_LIB_STDCXX) + # there is a problem with __builtin_mul_overflow call at link time + # the error looks like: ... undefined reference to `__muloti4' ... + # caused by clang bug https://bugs.llvm.org/show_bug.cgi?id=16404 + # explicit linking to compiler-rt allows to workaround the problem + SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --rtlib=compiler-rt") + SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --rtlib=compiler-rt") + + # some workaround for linking issues on linux: + # /usr/bin/ld: CMakeFiles/simple-test.dir/main.cpp.o: undefined reference to symbol '_Unwind_Resume@@GCC_3.0' + # /usr/bin/ld: /lib/x86_64-linux-gnu/libgcc_s.so.1: error adding symbols: DSO missing from command line + # FIXME: that workaround breaks clang build on mingw + SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lgcc_s") + SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lgcc_s") + ENDIF () + + INCLUDE_DIRECTORIES (.) + + IF (WITH_SYSTEM_ABSEIL) + FIND_PACKAGE(absl REQUIRED) + ELSE () + INCLUDE_DIRECTORIES (contrib/absl) + SUBDIRS (contrib/absl/absl) + ENDIF () + + IF (WITH_SYSTEM_LZ4) + FIND_PACKAGE(lz4 REQUIRED) + ELSE () + INCLUDE_DIRECTORIES (contrib/lz4/lz4) + SUBDIRS (contrib/lz4/lz4) + ENDIF () + + IF (WITH_SYSTEM_CITYHASH) + FIND_PACKAGE(cityhash REQUIRED) + ELSE () + INCLUDE_DIRECTORIES (contrib/cityhash/cityhash) + SUBDIRS (contrib/cityhash/cityhash) + ENDIF () + SUBDIRS ( clickhouse - contrib/absl - contrib/cityhash - contrib/lz4 ) IF (BUILD_BENCHMARK) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 870b1040..65baf2f7 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -41,9 +41,9 @@ ENDIF () ADD_LIBRARY (clickhouse-cpp-lib ${clickhouse-cpp-lib-src}) SET_TARGET_PROPERTIES (clickhouse-cpp-lib PROPERTIES LINKER_LANGUAGE CXX) TARGET_LINK_LIBRARIES (clickhouse-cpp-lib - absl-lib - cityhash-lib - lz4-lib + absl::int128 + cityhash::cityhash + lz4::lz4 ) TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib PUBLIC ${PROJECT_SOURCE_DIR} @@ -52,6 +52,15 @@ TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib IF (NOT BUILD_SHARED_LIBS) ADD_LIBRARY (clickhouse-cpp-lib-static ALIAS clickhouse-cpp-lib) ENDIF() +ADD_LIBRARY (clickhouse-cpp-lib-static STATIC ${clickhouse-cpp-lib-src}) +TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static + absl::int128 + cityhash::cityhash + lz4::lz4 +) +TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib-static + PUBLIC ${PROJECT_SOURCE_DIR} +) IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") diff --git a/clickhouse/base/compressed.cpp b/clickhouse/base/compressed.cpp index 8478ad28..4d5cc65d 100644 --- a/clickhouse/base/compressed.cpp +++ b/clickhouse/base/compressed.cpp @@ -3,8 +3,8 @@ #include "output.h" #include "../exceptions.h" -#include -#include +#include +#include #include #include diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index 2269a351..d3627038 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -4,7 +4,7 @@ #include "nullable.h" #include "../base/wire_format.h" -#include +#include #include #include diff --git a/clickhouse/types/types.cpp b/clickhouse/types/types.cpp index d5093f10..79e8be48 100644 --- a/clickhouse/types/types.cpp +++ b/clickhouse/types/types.cpp @@ -2,7 +2,7 @@ #include "../exceptions.h" -#include +#include #include diff --git a/cmake/Findlz4.cmake b/cmake/Findlz4.cmake new file mode 100644 index 00000000..ef8a366e --- /dev/null +++ b/cmake/Findlz4.cmake @@ -0,0 +1,38 @@ +find_path(lz4_INCLUDE_DIR + NAMES lz4.h + DOC "lz4 include directory") +mark_as_advanced(lz4_INCLUDE_DIR) +find_library(lz4_LIBRARY + NAMES lz4 liblz4 + DOC "lz4 library") +mark_as_advanced(lz4_LIBRARY) + +if (lz4_INCLUDE_DIR) + file(STRINGS "${lz4_INCLUDE_DIR}/lz4.h" _lz4_version_lines + REGEX "#define[ \t]+LZ4_VERSION_(MAJOR|MINOR|RELEASE)") + string(REGEX REPLACE ".*LZ4_VERSION_MAJOR *\([0-9]*\).*" "\\1" _lz4_version_major "${_lz4_version_lines}") + string(REGEX REPLACE ".*LZ4_VERSION_MINOR *\([0-9]*\).*" "\\1" _lz4_version_minor "${_lz4_version_lines}") + string(REGEX REPLACE ".*LZ4_VERSION_RELEASE *\([0-9]*\).*" "\\1" _lz4_version_release "${_lz4_version_lines}") + set(lz4_VERSION "${_lz4_version_major}.${_lz4_version_minor}.${_lz4_version_release}") + unset(_lz4_version_major) + unset(_lz4_version_minor) + unset(_lz4_version_release) + unset(_lz4_version_lines) +endif () + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(lz4 + REQUIRED_VARS lz4_LIBRARY lz4_INCLUDE_DIR + VERSION_VAR lz4_VERSION) + +if (lz4_FOUND) + set(lz4_INCLUDE_DIRS "${lz4_INCLUDE_DIR}") + set(lz4_LIBRARIES "${lz4_LIBRARY}") + + if (NOT TARGET lz4::lz4) + add_library(lz4::lz4 UNKNOWN IMPORTED) + set_target_properties(lz4::lz4 PROPERTIES + IMPORTED_LOCATION "${lz4_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}") + endif () +endif () diff --git a/contrib/absl/CMakeLists.txt b/contrib/absl/CMakeLists.txt deleted file mode 100644 index 2cd0f2be..00000000 --- a/contrib/absl/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -ADD_LIBRARY (absl-lib STATIC - numeric/int128.cc -) -TARGET_INCLUDE_DIRECTORIES (absl-lib - PUBLIC ${PROJECT_SOURCE_DIR}/contrib -) diff --git a/contrib/absl/absl/CMakeLists.txt b/contrib/absl/absl/CMakeLists.txt new file mode 100644 index 00000000..69f088f4 --- /dev/null +++ b/contrib/absl/absl/CMakeLists.txt @@ -0,0 +1,9 @@ +ADD_LIBRARY (absl_int128 STATIC + numeric/int128.cc +) + +TARGET_INCLUDE_DIRECTORIES (absl_int128 + PUBLIC ${PROJECT_SOURCE_DIR}/contrib/absl +) + +ADD_LIBRARY (absl::int128 ALIAS absl_int128) diff --git a/contrib/absl/base/attributes.h b/contrib/absl/absl/base/attributes.h similarity index 100% rename from contrib/absl/base/attributes.h rename to contrib/absl/absl/base/attributes.h diff --git a/contrib/absl/base/config.h b/contrib/absl/absl/base/config.h similarity index 100% rename from contrib/absl/base/config.h rename to contrib/absl/absl/base/config.h diff --git a/contrib/absl/base/internal/bits.h b/contrib/absl/absl/base/internal/bits.h similarity index 100% rename from contrib/absl/base/internal/bits.h rename to contrib/absl/absl/base/internal/bits.h diff --git a/contrib/absl/base/macros.h b/contrib/absl/absl/base/macros.h similarity index 100% rename from contrib/absl/base/macros.h rename to contrib/absl/absl/base/macros.h diff --git a/contrib/absl/base/optimization.h b/contrib/absl/absl/base/optimization.h similarity index 100% rename from contrib/absl/base/optimization.h rename to contrib/absl/absl/base/optimization.h diff --git a/contrib/absl/base/options.h b/contrib/absl/absl/base/options.h similarity index 100% rename from contrib/absl/base/options.h rename to contrib/absl/absl/base/options.h diff --git a/contrib/absl/base/policy_checks.h b/contrib/absl/absl/base/policy_checks.h similarity index 100% rename from contrib/absl/base/policy_checks.h rename to contrib/absl/absl/base/policy_checks.h diff --git a/contrib/absl/base/port.h b/contrib/absl/absl/base/port.h similarity index 100% rename from contrib/absl/base/port.h rename to contrib/absl/absl/base/port.h diff --git a/contrib/absl/numeric/int128.cc b/contrib/absl/absl/numeric/int128.cc similarity index 100% rename from contrib/absl/numeric/int128.cc rename to contrib/absl/absl/numeric/int128.cc diff --git a/contrib/absl/numeric/int128.h b/contrib/absl/absl/numeric/int128.h similarity index 100% rename from contrib/absl/numeric/int128.h rename to contrib/absl/absl/numeric/int128.h diff --git a/contrib/absl/numeric/int128_have_intrinsic.inc b/contrib/absl/absl/numeric/int128_have_intrinsic.inc similarity index 100% rename from contrib/absl/numeric/int128_have_intrinsic.inc rename to contrib/absl/absl/numeric/int128_have_intrinsic.inc diff --git a/contrib/absl/numeric/int128_no_intrinsic.inc b/contrib/absl/absl/numeric/int128_no_intrinsic.inc similarity index 100% rename from contrib/absl/numeric/int128_no_intrinsic.inc rename to contrib/absl/absl/numeric/int128_no_intrinsic.inc diff --git a/contrib/cityhash/CMakeLists.txt b/contrib/cityhash/CMakeLists.txt deleted file mode 100644 index e31bc245..00000000 --- a/contrib/cityhash/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -ADD_LIBRARY (cityhash-lib STATIC - city.cc -) - -set_property(TARGET cityhash-lib PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/contrib/cityhash/BUCK b/contrib/cityhash/cityhash/BUCK similarity index 100% rename from contrib/cityhash/BUCK rename to contrib/cityhash/cityhash/BUCK diff --git a/contrib/cityhash/cityhash/CMakeLists.txt b/contrib/cityhash/cityhash/CMakeLists.txt new file mode 100644 index 00000000..d1311562 --- /dev/null +++ b/contrib/cityhash/cityhash/CMakeLists.txt @@ -0,0 +1,7 @@ +ADD_LIBRARY (cityhash STATIC + city.cc +) + +set_property(TARGET cityhash PROPERTY POSITION_INDEPENDENT_CODE ON) + +ADD_LIBRARY (cityhash::cityhash ALIAS cityhash) diff --git a/contrib/cityhash/COPYING b/contrib/cityhash/cityhash/COPYING similarity index 100% rename from contrib/cityhash/COPYING rename to contrib/cityhash/cityhash/COPYING diff --git a/contrib/cityhash/city.cc b/contrib/cityhash/cityhash/city.cc similarity index 100% rename from contrib/cityhash/city.cc rename to contrib/cityhash/cityhash/city.cc diff --git a/contrib/cityhash/city.h b/contrib/cityhash/cityhash/city.h similarity index 100% rename from contrib/cityhash/city.h rename to contrib/cityhash/cityhash/city.h diff --git a/contrib/cityhash/citycrc.h b/contrib/cityhash/cityhash/citycrc.h similarity index 100% rename from contrib/cityhash/citycrc.h rename to contrib/cityhash/cityhash/citycrc.h diff --git a/contrib/cityhash/config.h b/contrib/cityhash/cityhash/config.h similarity index 100% rename from contrib/cityhash/config.h rename to contrib/cityhash/cityhash/config.h diff --git a/contrib/lz4/CMakeLists.txt b/contrib/lz4/CMakeLists.txt deleted file mode 100644 index 7b471da3..00000000 --- a/contrib/lz4/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -ADD_LIBRARY (lz4-lib STATIC - lz4.c - lz4hc.c -) - -set_property(TARGET lz4-lib PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/contrib/lz4/BUCK b/contrib/lz4/lz4/BUCK similarity index 100% rename from contrib/lz4/BUCK rename to contrib/lz4/lz4/BUCK diff --git a/contrib/lz4/lz4/CMakeLists.txt b/contrib/lz4/lz4/CMakeLists.txt new file mode 100644 index 00000000..1ec7fb07 --- /dev/null +++ b/contrib/lz4/lz4/CMakeLists.txt @@ -0,0 +1,8 @@ +ADD_LIBRARY (lz4 STATIC + lz4.c + lz4hc.c +) + +set_property(TARGET lz4 PROPERTY POSITION_INDEPENDENT_CODE ON) + +ADD_LIBRARY(lz4::lz4 ALIAS lz4) diff --git a/contrib/lz4/LICENSE b/contrib/lz4/lz4/LICENSE similarity index 100% rename from contrib/lz4/LICENSE rename to contrib/lz4/lz4/LICENSE diff --git a/contrib/lz4/lz4.c b/contrib/lz4/lz4/lz4.c similarity index 100% rename from contrib/lz4/lz4.c rename to contrib/lz4/lz4/lz4.c diff --git a/contrib/lz4/lz4.h b/contrib/lz4/lz4/lz4.h similarity index 100% rename from contrib/lz4/lz4.h rename to contrib/lz4/lz4/lz4.h diff --git a/contrib/lz4/lz4hc.c b/contrib/lz4/lz4/lz4hc.c similarity index 100% rename from contrib/lz4/lz4hc.c rename to contrib/lz4/lz4/lz4hc.c diff --git a/contrib/lz4/lz4hc.h b/contrib/lz4/lz4/lz4hc.h similarity index 100% rename from contrib/lz4/lz4hc.h rename to contrib/lz4/lz4/lz4hc.h diff --git a/tests/simple/CMakeLists.txt b/tests/simple/CMakeLists.txt index 728ce76d..3804fcec 100644 --- a/tests/simple/CMakeLists.txt +++ b/tests/simple/CMakeLists.txt @@ -6,11 +6,3 @@ ADD_EXECUTABLE (simple-test TARGET_LINK_LIBRARIES (simple-test clickhouse-cpp-lib ) - -IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - # there is a problem with __builtin_mul_overflow call at link time - # the error looks like: ... undefined reference to `__muloti4' ... - # caused by clang bug https://bugs.llvm.org/show_bug.cgi?id=16404 - # explicit linking to compiler-rt allows to workaround the problem - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --rtlib=compiler-rt") -ENDIF () diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index 888d3c33..c12707b9 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -39,6 +39,3 @@ TARGET_LINK_LIBRARIES (clickhouse-cpp-ut clickhouse-cpp-lib gtest-lib ) -IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --rtlib=compiler-rt") -ENDIF () From a2c17ecd7c0c6ff9c7fbffe056c3a23135a7ad38 Mon Sep 17 00:00:00 2001 From: Jihadist Date: Thu, 2 Feb 2023 23:24:06 +0300 Subject: [PATCH 34/44] Fix msvc ut build, fix due to alias to static --- CMakeLists.txt | 2 -- clickhouse/CMakeLists.txt | 9 --------- ut/CMakeLists.txt | 4 ++++ 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ac72186..aed0d85c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,8 +62,6 @@ PROJECT (CLICKHOUSE-CLIENT) SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lgcc_s") ENDIF () - INCLUDE_DIRECTORIES (.) - IF (WITH_SYSTEM_ABSEIL) FIND_PACKAGE(absl REQUIRED) ELSE () diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 65baf2f7..67663ec5 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -52,15 +52,6 @@ TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib IF (NOT BUILD_SHARED_LIBS) ADD_LIBRARY (clickhouse-cpp-lib-static ALIAS clickhouse-cpp-lib) ENDIF() -ADD_LIBRARY (clickhouse-cpp-lib-static STATIC ${clickhouse-cpp-lib-src}) -TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static - absl::int128 - cityhash::cityhash - lz4::lz4 -) -TARGET_INCLUDE_DIRECTORIES (clickhouse-cpp-lib-static - PUBLIC ${PROJECT_SOURCE_DIR} -) IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index c12707b9..2c9f6ee5 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -39,3 +39,7 @@ TARGET_LINK_LIBRARIES (clickhouse-cpp-ut clickhouse-cpp-lib gtest-lib ) + +IF (MSVC) + TARGET_COMPILE_OPTIONS(clickhouse-cpp-ut PRIVATE /bigobj) +ENDIF() From 30068ac4740a54c8b30a183d9298eff92724e242 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 15 Feb 2023 13:37:04 +0400 Subject: [PATCH 35/44] Fixed deprecation warning Client's that do not even call any deprecated APIs were getting warning/error: 'backward_compatibility_lowcardinality_as_wrapped_column' is deprecated --- clickhouse/client.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clickhouse/client.h b/clickhouse/client.h index 854fd70c..63177313 100644 --- a/clickhouse/client.h +++ b/clickhouse/client.h @@ -45,12 +45,13 @@ enum class CompressionMethod { }; struct ClientOptions { + // Setter goes first, so it is possible to apply 'deprecated' annotation safely. #define DECLARE_FIELD(name, type, setter, default_value) \ - type name = default_value; \ inline auto & setter(const type& value) { \ name = value; \ return *this; \ - } + } \ + type name = default_value /// Hostname of the server. DECLARE_FIELD(host, std::string, SetHost, std::string()); From 84f10f99526801bc0b51970162ad6fbe80492715 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 15 Feb 2023 14:22:52 +0400 Subject: [PATCH 36/44] Dependecies info --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index ce1f68fe..179e8c6f 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,17 @@ C++ client for [ClickHouse](https://clickhouse.com/). * Map * Point, Ring, Polygon, MultiPolygon +## Dependencies +In the most basic case one needs only: +- C++-17 complaint compiler (e.g. Clang-6, GCC-7) +- `cmake` (3.20 or newer) +- `ninja` + +Optional dependencies: +- openssl +- liblz4 +- libabsl + ## Building ```sh @@ -33,6 +44,13 @@ $ cmake .. [-DBUILD_TESTS=ON] $ make ``` +Plese refer to the workflows for the reference on dependencies/build options +- https://github.com/ClickHouse/clickhouse-cpp/blob/master/.github/workflows/linux.yml +- https://github.com/ClickHouse/clickhouse-cpp/blob/master/.github/workflows/windows_msvc.yml +- https://github.com/ClickHouse/clickhouse-cpp/blob/master/.github/workflows/windows_mingw.yml +- https://github.com/ClickHouse/clickhouse-cpp/blob/master/.github/workflows/macos.yml + + ## Example ```cpp From 26572c1218785ecf9b4b06d92319921b6b4608d4 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 21 Mar 2023 16:46:38 +0100 Subject: [PATCH 37/44] Added Query.onProfile and handling of Profile data - added unit tests - fixed minor issue with pretty-printing ColumnString as container --- clickhouse/query.h | 10 +++++++++- ut/client_ut.cpp | 32 ++++++++++++++++++++++++++++++++ ut/utils.cpp | 5 ++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/clickhouse/query.h b/clickhouse/query.h index 21d7231f..7129bbf4 100644 --- a/clickhouse/query.h +++ b/clickhouse/query.h @@ -79,6 +79,7 @@ using SelectCallback = std::function; using SelectCancelableCallback = std::function; using SelectServerLogCallback = std::function; using ProfileEventsCallback = std::function; +using ProfileCallbak = std::function; class Query : public QueryEvents { @@ -158,6 +159,11 @@ class Query : public QueryEvents { return *this; } + inline Query& OnProfile(ProfileCallbak cb) { + profile_callback_cb_ = std::move(cb); + return *this; + } + static const std::string default_query_id; private: @@ -182,7 +188,8 @@ class Query : public QueryEvents { } void OnProfile(const Profile& profile) override { - (void)profile; + if (profile_callback_cb_) + profile_callback_cb_(profile); } void OnProgress(const Progress& progress) override { @@ -217,6 +224,7 @@ class Query : public QueryEvents { SelectCancelableCallback select_cancelable_cb_; SelectServerLogCallback select_server_log_cb_; ProfileEventsCallback profile_events_callback_cb_; + ProfileCallbak profile_callback_cb_; }; } diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 2a7aac07..ba248b56 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -6,6 +6,7 @@ #include +#include #include #include @@ -1116,6 +1117,37 @@ TEST_P(ClientCase, OnProfileEvents) { } } +TEST_P(ClientCase, OnProfile) { + Query query("SELECT * FROM system.numbers LIMIT 10;"); + + std::optional profile; + query.OnProfile([&profile](const Profile & new_profile) { + profile = new_profile; + + std::cout << + "Profile:" << + "\n\trows: " << new_profile.rows << + "\n\tblocks: " << new_profile.blocks << + "\n\tbytes: " << new_profile.bytes << + "\n\trows_before_limit: " << new_profile.rows_before_limit << + "\n\tapplied_limit: " << new_profile.applied_limit << + "\n\tcalculated_rows_before_limit: " << new_profile.calculated_rows_before_limit << + std::endl; + }); + + client_->Execute(query); + + // Make sure that profile event came through + ASSERT_NE(profile, std::nullopt); + + EXPECT_GE(profile->rows, 10u); + EXPECT_GE(profile->blocks, 1u); + EXPECT_GT(profile->bytes, 1u); + EXPECT_GE(profile->rows_before_limit, 10u); + EXPECT_EQ(profile->applied_limit, true); + EXPECT_EQ(profile->calculated_rows_before_limit, true); +} + TEST_P(ClientCase, SelectAggregateFunction) { // Verifies that perofing SELECT value of type AggregateFunction(...) doesn't crash the client. // For details: https://github.com/ClickHouse/clickhouse-cpp/issues/266 diff --git a/ut/utils.cpp b/ut/utils.cpp index 1cb5978c..b5252310 100644 --- a/ut/utils.cpp +++ b/ut/utils.cpp @@ -21,6 +21,7 @@ #include #include #include +#include namespace { @@ -51,7 +52,9 @@ std::ostream& operator<<(std::ostream & ostr, const DateTimeValue & time) { template ().At(0)) > bool doPrintValue(const ColumnRef & c, const size_t row, std::ostream & ostr) { if (const auto & casted_c = c->As()) { - if constexpr (is_container_v>) { + if constexpr (is_container_v> + && !std::is_same_v + && !std::is_same_v) { ostr << PrintContainer{static_cast(casted_c->At(row))}; } else { ostr << static_cast(casted_c->At(row)); From 6cb84a19eb98e3b7e51051fcbb55c3b74266e206 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 21 Mar 2023 16:51:47 +0100 Subject: [PATCH 38/44] Removed debug logging --- ut/client_ut.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index ba248b56..dfabcba9 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1123,16 +1123,6 @@ TEST_P(ClientCase, OnProfile) { std::optional profile; query.OnProfile([&profile](const Profile & new_profile) { profile = new_profile; - - std::cout << - "Profile:" << - "\n\trows: " << new_profile.rows << - "\n\tblocks: " << new_profile.blocks << - "\n\tbytes: " << new_profile.bytes << - "\n\trows_before_limit: " << new_profile.rows_before_limit << - "\n\tapplied_limit: " << new_profile.applied_limit << - "\n\tcalculated_rows_before_limit: " << new_profile.calculated_rows_before_limit << - std::endl; }); client_->Execute(query); From 52277c6ab9d0d7f035948da7230290bc86aa72a1 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 23 Mar 2023 21:03:28 +0000 Subject: [PATCH 39/44] Expand application tutorial --- README.md | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 179e8c6f..79306d1c 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ C++ client for [ClickHouse](https://clickhouse.com/). ## Dependencies In the most basic case one needs only: -- C++-17 complaint compiler (e.g. Clang-6, GCC-7) -- `cmake` (3.20 or newer) +- a C++-17-complaint compiler, +- `cmake` (3.12 or newer), and - `ninja` Optional dependencies: @@ -51,7 +51,14 @@ Plese refer to the workflows for the reference on dependencies/build options - https://github.com/ClickHouse/clickhouse-cpp/blob/master/.github/workflows/macos.yml -## Example +## Example application build with clickhouse-cpp + +There are various ways to integrate clickhouse-cpp with the build system of an application. Below example uses the simple approach based on +submodules presented in https://www.youtube.com/watch?v=ED-WUk440qc . + +- `mkdir clickhouse-app && cd clickhouse-app && git init` +- `git submodule add https://github.com/ClickHouse/clickhouse-cpp.git contribs/clickhouse-cpp` +- `touch app.cpp`, then copy the following C++ code into that file ```cpp #include @@ -100,8 +107,27 @@ int main() return 0; } + +- `touch CMakeLists.txt`, then copy the following CMake code into that file + +```cmake +cmake_minimum_required(VERSION 3.12) +project(application-example) + +set(CMAKE_CXX_STANDARD 17) + +add_subdirectory(contribs/clickhouse-cpp) + +add_executable(${PROJECT_NAME} "app.cpp") + +target_include_directories(${PROJECT_NAME} PRIVATE contribs/clickhouse-cpp/ contribs/clickhouse-cpp/contrib/absl) + +target_link_libraries(${PROJECT_NAME} PRIVATE clickhouse-cpp-lib) ``` +- run `rm -rf build && cmake -B build -S . && cmake --build build -j32` to remove remainders of the previous builds, run CMake and build the + application. The generated binary is located in location `build/application-example`. + ## Thread-safety ⚠ Please note that `Client` instance is NOT thread-safe. I.e. you must create a separate `Client` for each thread or utilize some synchronization techniques. ⚠ From 52e787b13df6054d6657802ef9a812db6371e42b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Mar 2023 08:11:25 +0000 Subject: [PATCH 40/44] Fix formatting in README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 79306d1c..04fd2485 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ int main() return 0; } +``` - `touch CMakeLists.txt`, then copy the following CMake code into that file From f2e8f84b9379c7901f189c14b16c0ae9e7c28d9d Mon Sep 17 00:00:00 2001 From: Alexey Shakula Date: Wed, 29 Mar 2023 11:34:28 -0400 Subject: [PATCH 41/44] Fixed gcc13 warnings --- clickhouse/base/wire_format.h | 1 + clickhouse/columns/geo.cpp | 8 ++++---- clickhouse/columns/tuple.h | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/clickhouse/base/wire_format.h b/clickhouse/base/wire_format.h index 9bbf7959..6ff53528 100644 --- a/clickhouse/base/wire_format.h +++ b/clickhouse/base/wire_format.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace clickhouse { diff --git a/clickhouse/columns/geo.cpp b/clickhouse/columns/geo.cpp index 5d749374..ebea9895 100644 --- a/clickhouse/columns/geo.cpp +++ b/clickhouse/columns/geo.cpp @@ -34,14 +34,14 @@ namespace clickhouse { template ColumnGeo::ColumnGeo() - : Column(std::move(CreateGeoType())), - data_(std::move(CreateColumn())) { + : Column(CreateGeoType()), + data_(CreateColumn()) { } template ColumnGeo::ColumnGeo(ColumnRef data) - : Column(std::move(CreateGeoType())) - , data_(std::move(WrapColumn(std::move(data)))) { + : Column(CreateGeoType()) + , data_(WrapColumn(std::move(data))) { } template diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index decf8233..b1b5ad31 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -137,7 +137,7 @@ class ColumnTupleT : public ColumnTuple { result.reserve(std::tuple_size_v); return result; } else { - auto result = std::move(TupleToVector(value)); + auto result = TupleToVector(value); result.push_back(std::get(value)); return result; } From 2727d49da4ef30b5c862812d81eb1226ea3b291c Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sun, 16 Apr 2023 19:36:14 +0200 Subject: [PATCH 42/44] Not starting Client/ClientCase.Query_ID tests on non-Linux platforms Since on those we user remote clickhouse server, and it doesn't support 'system.query_log' for unpaind/anonymous users --- .github/workflows/macos.yml | 7 ++++--- .github/workflows/windows_mingw.yml | 7 ++++++- .github/workflows/windows_msvc.yml | 6 +++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2fe8ff75..1a267aa8 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -65,6 +65,7 @@ jobs: working-directory: ${{github.workspace}}/build/ut env: # It is impossible to start CH server in docker on macOS due to github actions limitations, - # so limit tests to ones that do no require server interaction. - GTEST_FILTER_ONLY_LOCAL: "-Client/*" - run: ./clickhouse-cpp-ut + # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: + # - system.query_log used by 'Client/ClientCase.Query_ID' + GTEST_FILTER: "-Client/ClientCase.Query_ID*" + run: ./clickhouse-cpp-ut ${GTEST_FILTER} diff --git a/.github/workflows/windows_mingw.yml b/.github/workflows/windows_mingw.yml index 52928370..d99c1ef7 100644 --- a/.github/workflows/windows_mingw.yml +++ b/.github/workflows/windows_mingw.yml @@ -81,7 +81,12 @@ jobs: ./go-tlsoffloader.exe -l localhost:9000 -b github.demo.trial.altinity.cloud:9440 & - name: Test - run: ./build/ut/clickhouse-cpp-ut.exe + env: + # It is impossible to start CH server in docker on Windows due to github actions limitations, + # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: + # - system.query_log used by 'Client/ClientCase.Query_ID' + GTEST_FILTER: "-Client/ClientCase.Query_ID*" + run: ./build/ut/clickhouse-cpp-ut.exe ${GTEST_FILTER} - name: Test (simple) run: ./build/tests/simple/simple-test.exe diff --git a/.github/workflows/windows_msvc.yml b/.github/workflows/windows_msvc.yml index 55ae5faa..149e3df4 100644 --- a/.github/workflows/windows_msvc.yml +++ b/.github/workflows/windows_msvc.yml @@ -10,7 +10,6 @@ on: env: BUILD_TYPE: Release - GTEST_FILTER: --gtest_filter=-"*" CLICKHOUSE_USER: clickhouse_cpp_cicd CLICKHOUSE_PASSWORD: clickhouse_cpp_cicd # @@ -58,5 +57,10 @@ jobs: ./go-tlsoffloader.exe -l localhost:9000 -b github.demo.trial.altinity.cloud:9440 & - name: Test + env: + # It is impossible to start CH server in docker on Windows due to github actions limitations, + # so we use remote server to execute tests, some do not allow some features for anonymoust/free users: + # - system.query_log used by 'Client/ClientCase.Query_ID' + GTEST_FILTER: "-Client/ClientCase.Query_ID*" working-directory: ${{github.workspace}}/build/ut run: Release\clickhouse-cpp-ut.exe "${{env.GTEST_FILTER}}" From 50a3817bc36d460c632230969090a3f9c82a5be0 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 17 Apr 2023 13:29:59 +0200 Subject: [PATCH 43/44] Fixing Windows build --- clickhouse/query.h | 1 + ut/utils.cpp | 19 +++++++++++++++++++ ut/utils.h | 6 ++++++ 3 files changed, 26 insertions(+) diff --git a/clickhouse/query.h b/clickhouse/query.h index 21d7231f..36ea85ed 100644 --- a/clickhouse/query.h +++ b/clickhouse/query.h @@ -19,6 +19,7 @@ struct QuerySettingsField { { IMPORTANT = 0x01, CUSTOM = 0x02, + OBSOLETE = 0x04, }; std::string value; uint64_t flags{0}; diff --git a/ut/utils.cpp b/ut/utils.cpp index 1cb5978c..c2d52dd6 100644 --- a/ut/utils.cpp +++ b/ut/utils.cpp @@ -310,6 +310,25 @@ std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info) { << " (" << server_info.revision << ")"; } +std::ostream & operator<<(std::ostream & ostr, const Profile & profile) { + return ostr + << "rows : " << profile.rows + << " blocks : " << profile.blocks + << " bytes : " << profile.bytes + << " rows_before_limit : " << profile.rows_before_limit + << " applied_limit : " << profile.applied_limit + << " calculated_rows_before_limit : " << profile.calculated_rows_before_limit; +} + +std::ostream & operator<<(std::ostream & ostr, const Progress & progress) { + return ostr + << "rows : " << progress.rows + << " bytes : " << progress.bytes + << " total_rows : " << progress.total_rows + << " written_rows : " << progress.written_rows + << " written_bytes : " << progress.written_bytes; +} + } uint64_t versionNumber(const ServerInfo & server_info) { diff --git a/ut/utils.h b/ut/utils.h index f3740fc7..b9484626 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -3,6 +3,7 @@ #include #include +#include "clickhouse/query.h" #include "utils_meta.h" #include "utils_comparison.h" @@ -24,6 +25,9 @@ namespace clickhouse { class Block; class Type; struct ServerInfo; + struct Profile; + struct QuerySettingsField; + struct Progress; } template @@ -136,6 +140,8 @@ namespace clickhouse { std::ostream& operator<<(std::ostream & ostr, const Block & block); std::ostream& operator<<(std::ostream & ostr, const Type & type); std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info); +std::ostream & operator<<(std::ostream & ostr, const Profile & profile); +std::ostream & operator<<(std::ostream & ostr, const Progress & progress); } std::ostream& operator<<(std::ostream & ostr, const PrettyPrintBlock & block); From 3c33fdaf5898d66ec91276eb4f152bb6be7b1eb7 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 17 Apr 2023 15:53:01 +0200 Subject: [PATCH 44/44] Deprecated LowCardinalitySerializationAdaptor --- clickhouse/columns/lowcardinalityadaptor.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clickhouse/columns/lowcardinalityadaptor.h b/clickhouse/columns/lowcardinalityadaptor.h index 8b579a0d..bcde1a9b 100644 --- a/clickhouse/columns/lowcardinalityadaptor.h +++ b/clickhouse/columns/lowcardinalityadaptor.h @@ -22,7 +22,9 @@ class CodedInputStream; * @see ClientOptions, CreateColumnByType */ template -class LowCardinalitySerializationAdaptor : public AdaptedColumnType +class +[[deprecated("Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) ")]] +LowCardinalitySerializationAdaptor : public AdaptedColumnType { public: using AdaptedColumnType::AdaptedColumnType;