Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Initial version of extension to allow creating operators outside of duckdb core lib #5144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Nov 12, 2022
+118 −13
Merged

Initial version of extension to allow creating operators outside of duckdb core lib #5144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
77e11da
Initial version of extension to allow creating operators outside of d…
rjatwal Oct 31, 2022
bcd613d
Changing extension operators planning to use a virtual method on the …
rjatwal Nov 7, 2022
3ab2d97
Moved extension bind hook up the call stack from binder.cpp into plan…
rjatwal Nov 7, 2022
e612716
Removing forward-declaration made redundant by refactoring
rjatwal Nov 7, 2022
42ba867
Formatting
rjatwal Nov 10, 2022
b2c92e8
Fixing ctidy check
rjatwal Nov 10, 2022
cc2eff0
Adding if clause that I staged incorrectly
rjatwal Nov 11, 2022
0bbd8b3
Fixing edge case brought up by fuzzer
rjatwal Nov 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/common/enums/logical_operator_type.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
return "LOAD";
case LogicalOperatorType::LOGICAL_INVALID:
break;
case LogicalOperatorType::LOGICAL_EXTENSION_OPERATOR:
return "CUSTOM_OP";
}
return "INVALID";
}
Expand Down
14 changes: 12 additions & 2 deletions src/execution/physical_plan_generator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
#include "duckdb/execution/physical_plan_generator.hpp"
#include "duckdb/main/query_profiler.hpp"

#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp"
#include "duckdb/common/types/column_data_collection.hpp"
#include "duckdb/execution/column_binding_resolver.hpp"
#include "duckdb/main/client_context.hpp"
#include "duckdb/main/config.hpp"
#include "duckdb/main/query_profiler.hpp"
#include "duckdb/planner/expression/bound_function_expression.hpp"
#include "duckdb/common/types/column_data_collection.hpp"
#include "duckdb/planner/operator/logical_extension_operator.hpp"

namespace duckdb {

Expand Down Expand Up @@ -191,6 +194,13 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
case LogicalOperatorType::LOGICAL_SET:
plan = CreatePlan((LogicalSet &)op);
break;
case LogicalOperatorType::LOGICAL_EXTENSION_OPERATOR:
plan = ((LogicalExtensionOperator &)op).CreatePlan(context, *this);

if (!plan) {
throw InternalException("Missing PhysicalOperator for Extension Operator");
}
break;
default: {
throw NotImplementedException("Unimplemented logical operator type!");
}
Expand Down
4 changes: 3 additions & 1 deletion src/include/duckdb/common/enums/logical_operator_type.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,9 @@ enum class LogicalOperatorType : uint8_t {
LOGICAL_EXPORT = 177,
LOGICAL_VACUUM = 178,
LOGICAL_SET = 179,
LOGICAL_LOAD = 180
LOGICAL_LOAD = 180,

LOGICAL_EXTENSION_OPERATOR = 255
};

DUCKDB_API string LogicalOperatorToString(LogicalOperatorType type);
Expand Down
21 changes: 12 additions & 9 deletions src/include/duckdb/main/config.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,22 @@
#include "duckdb/common/allocator.hpp"
#include "duckdb/common/case_insensitive_map.hpp"
#include "duckdb/common/common.hpp"
#include "duckdb/common/enums/compression_type.hpp"
#include "duckdb/common/enums/optimizer_type.hpp"
#include "duckdb/common/enums/order_type.hpp"
#include "duckdb/common/enums/set_scope.hpp"
#include "duckdb/common/enums/window_aggregation_mode.hpp"
#include "duckdb/common/file_system.hpp"
#include "duckdb/common/winapi.hpp"
#include "duckdb/common/set.hpp"
#include "duckdb/common/types/value.hpp"
#include "duckdb/common/vector.hpp"
#include "duckdb/function/replacement_scan.hpp"
#include "duckdb/function/replacement_open.hpp"
#include "duckdb/common/set.hpp"
#include "duckdb/common/enums/compression_type.hpp"
#include "duckdb/common/enums/optimizer_type.hpp"
#include "duckdb/common/enums/window_aggregation_mode.hpp"
#include "duckdb/common/enums/set_scope.hpp"
#include "duckdb/parser/parser_extension.hpp"
#include "duckdb/common/winapi.hpp"
#include "duckdb/function/cast/default_casts.hpp"
#include "duckdb/function/replacement_open.hpp"
#include "duckdb/function/replacement_scan.hpp"
#include "duckdb/optimizer/optimizer_extension.hpp"
#include "duckdb/parser/parser_extension.hpp"
#include "duckdb/planner/operator_extension.hpp"

namespace duckdb {
class CastFunctionSet;
Expand Down Expand Up @@ -164,6 +165,8 @@ struct DBConfig {
unique_ptr<ErrorManager> error_manager;
//! A reference to the (shared) default allocator (Allocator::DefaultAllocator)
shared_ptr<Allocator> default_allocator;
//! Extensions made to binder
vector<OperatorExtension> operator_extensions;

public:
DUCKDB_API static DBConfig &GetConfig(ClientContext &context);
Expand Down
2 changes: 2 additions & 0 deletions src/include/duckdb/parser/parsed_data/create_info.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "duckdb/common/enums/catalog_type.hpp"
#include "duckdb/common/field_writer.hpp"
#include "duckdb/parser/parsed_data/parse_info.hpp"
#include "duckdb/planner/plan_serialization.hpp"

namespace duckdb {
struct AlterInfo;
Expand Down Expand Up @@ -56,6 +57,7 @@ struct CreateInfo : public ParseInfo {
void Serialize(Serializer &serializer) const;

static unique_ptr<CreateInfo> Deserialize(Deserializer &deserializer);
static unique_ptr<CreateInfo> Deserialize(Deserializer &deserializer, PlanDeserializationState &state);

virtual unique_ptr<CreateInfo> Copy() const = 0;

Expand Down
26 changes: 26 additions & 0 deletions src/include/duckdb/planner/operator/logical_extension_operator.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
//===----------------------------------------------------------------------===//
// DuckDB
//
// duckdb/planner/operator/logical_extension.operator.hpp
//
//
//===----------------------------------------------------------------------===//

#pragma once

#include "duckdb/planner/logical_operator.hpp"
#include "duckdb/planner/operator_extension.hpp"

namespace duckdb {

struct LogicalExtensionOperator : public LogicalOperator {

LogicalExtensionOperator() : LogicalOperator(LogicalOperatorType::LOGICAL_EXTENSION_OPERATOR) {
}
LogicalExtensionOperator(vector<unique_ptr<Expression>> expressions)
: LogicalOperator(LogicalOperatorType::LOGICAL_EXTENSION_OPERATOR, move(expressions)) {
}

virtual unique_ptr<PhysicalOperator> CreatePlan(ClientContext &context, PhysicalPlanGenerator &generator) = 0;
};
} // namespace duckdb
37 changes: 37 additions & 0 deletions src/include/duckdb/planner/operator_extension.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
//===----------------------------------------------------------------------===//
// DuckDB
//
// duckdb/planner/operator_extension.hpp
//
//
//===----------------------------------------------------------------------===//

#pragma once

#include "duckdb/common/common.hpp"
#include "duckdb/execution/physical_plan_generator.hpp"
#include "duckdb/planner/binder.hpp"

namespace duckdb {

//! The OperatorExtensionInfo holds static information relevant to the operator extension
struct OperatorExtensionInfo {
DUCKDB_API virtual ~OperatorExtensionInfo() {
}
};

typedef BoundStatement (*bind_function_t)(ClientContext &context, Binder &binder, OperatorExtensionInfo *info,
SQLStatement &statement);

class OperatorExtension {
public:
bind_function_t Bind;

//! Additional info passed to the CreatePlan & Bind functions
shared_ptr<OperatorExtensionInfo> operator_info;

DUCKDB_API virtual ~OperatorExtension() {
}
};

} // namespace duckdb
4 changes: 4 additions & 0 deletions src/parser/parsed_data/create_info.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ unique_ptr<CreateInfo> CreateInfo::Deserialize(Deserializer &deserializer) {
}
}

unique_ptr<CreateInfo> CreateInfo::Deserialize(Deserializer &source, PlanDeserializationState &state) {
return Deserialize(source);
}

void CreateInfo::CopyProperties(CreateInfo &other) const {
other.type = type;
other.schema = schema;
Expand Down
1 change: 1 addition & 0 deletions src/planner/binder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
#include "duckdb/main/config.hpp"
#include "duckdb/parser/query_node/select_node.hpp"
#include "duckdb/parser/statement/list.hpp"
#include "duckdb/parser/tableref/table_function_ref.hpp"
Expand Down
2 changes: 2 additions & 0 deletions src/planner/logical_operator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,8 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
case LogicalOperatorType::LOGICAL_LOAD:
result = LogicalSimple::Deserialize(state, reader);
break;
case LogicalOperatorType::LOGICAL_EXTENSION_OPERATOR:
throw SerializationException("Invalid type for operator deserialization");
case LogicalOperatorType::LOGICAL_INVALID:
/* no default here to trigger a warning if we forget to implement deserialize for a new operator */
throw SerializationException("Invalid type for operator deserialization");
Expand Down
18 changes: 17 additions & 1 deletion src/planner/planner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,23 @@ void Planner::CreatePlan(SQLStatement &statement) {
this->plan = nullptr;
parameters_resolved = false;
} catch (const Exception &ex) {
throw;
auto &config = DBConfig::GetConfig(context);

this->plan = nullptr;
for (auto &extension_op : config.operator_extensions) {
auto bound_statement =
extension_op.Bind(context, *this->binder, extension_op.operator_info.get(), statement);
if (bound_statement.plan != nullptr) {
this->names = bound_statement.names;
this->types = bound_statement.types;
this->plan = move(bound_statement.plan);
break;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Won't this still throw the exception - as this break only breaks out of the loop and into the throw statement?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for taking close look at my PR. Indeed I staged my commit incorrectly and forgot the guard check. Doing another round of testing and will push the update.

}
}

if (!this->plan) {
throw;
}
} catch (std::exception &ex) {
throw;
}
Expand Down