From ad3b53f4366335ec7b310a7a2b4366a110be1c49 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 4 Mar 2025 07:28:42 -0800 Subject: [PATCH 1/2] wip Signed-off-by: Eduardo Silva --- cmake/plugins_options.cmake | 3 +- include/fluent-bit/flb_processor.h | 1 + plugins/CMakeLists.txt | 1 + plugins/processor_content_modifier/cm.c | 5 ++-- plugins/processor_content_modifier/cm.h | 1 + .../processor_content_modifier/cm_traces.c | 3 ++ src/flb_input_metric.c | 1 - src/flb_input_trace.c | 30 ++++++++++++------- src/flb_processor.c | 25 ++++++++++++++-- 9 files changed, 52 insertions(+), 18 deletions(-) diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index b104ebd1f00..cb00f2cb92f 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -70,7 +70,8 @@ DEFINE_OPTION(FLB_PROCESSOR_CONTENT_MODIFIER "Enable content modifier processor DEFINE_OPTION(FLB_PROCESSOR_LABELS "Enable metrics label manipulation processor" ON) DEFINE_OPTION(FLB_PROCESSOR_METRICS_SELECTOR "Enable metrics selector processor" ON) DEFINE_OPTION(FLB_PROCESSOR_SQL "Enable SQL processor" ON) -DEFINE_OPTION(FLB_PROCESSOR_OPENTELEMETRY_ENVELOPE "Enable OpenTelemetry envelope processor" ON) +DEFINE_OPTION(FLB_PROCESSOR_OPENTELEMETRY_ENVELOPE "Enable OpenTelemetry envelope processor" ON) +DEFINE_OPTION(FLB_PROCESSOR_BUFFERED_TRACE "Enable buffered trace processor" ON) # Filters # ======= diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index 5781d41e47e..085aa65936b 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -155,6 +155,7 @@ struct flb_processor_plugin { int (*cb_process_traces) (struct flb_processor_instance *, struct ctrace *, + struct ctrace **, const char *, int); diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index deb24958038..e3a57e0f6da 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -290,6 +290,7 @@ REGISTER_PROCESSOR_PLUGIN("processor_labels") REGISTER_PROCESSOR_PLUGIN("processor_metrics_selector") REGISTER_PROCESSOR_PLUGIN("processor_sql") REGISTER_PROCESSOR_PLUGIN("processor_opentelemetry_envelope") +REGISTER_PROCESSOR_PLUGIN("processor_buffered_trace") # OUTPUTS # ======= diff --git a/plugins/processor_content_modifier/cm.c b/plugins/processor_content_modifier/cm.c index 4c56d659ff6..097fd2dc0c7 100644 --- a/plugins/processor_content_modifier/cm.c +++ b/plugins/processor_content_modifier/cm.c @@ -81,7 +81,8 @@ static int cb_process_logs(struct flb_processor_instance *ins, } static int cb_process_traces(struct flb_processor_instance *ins, - struct ctrace *traces_context, + struct ctrace *in_ctr, + struct ctrace **out_ctr, const char *tag, int tag_len) { @@ -93,7 +94,7 @@ static int cb_process_traces(struct flb_processor_instance *ins, } ctx = ins->context; - ret = cm_traces_process(ins, ctx, traces_context, tag, tag_len); + ret = cm_traces_process(ins, ctx, in_ctr, out_ctr, tag, tag_len); return ret; } diff --git a/plugins/processor_content_modifier/cm.h b/plugins/processor_content_modifier/cm.h index 181872b0754..915528e65f2 100644 --- a/plugins/processor_content_modifier/cm.h +++ b/plugins/processor_content_modifier/cm.h @@ -114,6 +114,7 @@ int cm_logs_process(struct flb_processor_instance *ins, int cm_traces_process(struct flb_processor_instance *ins, struct content_modifier_ctx *ctx, struct ctrace *traces_context, + struct ctrace **out_traces_context, const char *tag, int tag_len); int cm_metrics_process(struct flb_processor_instance *ins, diff --git a/plugins/processor_content_modifier/cm_traces.c b/plugins/processor_content_modifier/cm_traces.c index 54d4469bcb6..fb58a18909d 100644 --- a/plugins/processor_content_modifier/cm_traces.c +++ b/plugins/processor_content_modifier/cm_traces.c @@ -560,6 +560,7 @@ static int traces_hash_attributes(struct content_modifier_ctx *ctx, struct ctrac int cm_traces_process(struct flb_processor_instance *ins, struct content_modifier_ctx *ctx, struct ctrace *traces_context, + struct ctrace **out_traces_context, const char *tag, int tag_len) { int ret = -1; @@ -587,6 +588,8 @@ int cm_traces_process(struct flb_processor_instance *ins, ret = traces_convert_attributes(ctx, traces_context, ctx->key, ctx->converted_type); } + *out_traces_context = traces_context; + if (ret != 0) { return FLB_PROCESSOR_FAILURE; } diff --git a/src/flb_input_metric.c b/src/flb_input_metric.c index 9daf21ba037..fa2a6edb349 100644 --- a/src/flb_input_metric.c +++ b/src/flb_input_metric.c @@ -71,7 +71,6 @@ static int input_metrics_append(struct flb_input_instance *ins, if (ret != 0) { flb_plg_error(ins, "could not encode metrics"); - return -1; } } diff --git a/src/flb_input_trace.c b/src/flb_input_trace.c index 169e9213296..79784e499af 100644 --- a/src/flb_input_trace.c +++ b/src/flb_input_trace.c @@ -32,9 +32,10 @@ static int input_trace_append(struct flb_input_instance *ins, struct ctrace *ctr) { int ret; - char *out_buf; - size_t out_size; + char *out_buf = NULL; + size_t out_size = 0; int processor_is_active; + struct ctrace *out_context = NULL; processor_is_active = flb_processor_is_active(ins->processor); if (processor_is_active) { @@ -53,19 +54,26 @@ static int input_trace_append(struct flb_input_instance *ins, processor_starting_stage, FLB_PROCESSOR_TRACES, tag, tag_len, - (char *) ctr, - 0, NULL, NULL); - + (char *) ctr, 0, + (void **) &out_context, NULL); if (ret == -1) { return -1; } - } - /* Convert trace context to msgpack */ - ret = ctr_encode_msgpack_create(ctr, &out_buf, &out_size); - if (ret != 0) { - flb_plg_error(ins, "could not encode traces"); - return -1; + if (out_context != NULL) { + ret = ctr_encode_msgpack_create(out_context, &out_buf, &out_size); + if (out_context != ctr) { + ctr_destroy(out_context); + } + if (ret != 0) { + flb_plg_error(ins, "could not encode traces"); + return -1; + } + } + else { + /* nothing to do */ + return 0; + } } /* Append packed metrics */ diff --git a/src/flb_processor.c b/src/flb_processor.c index 9322e297b1e..23831480beb 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -745,20 +745,39 @@ int flb_processor_run(struct flb_processor *proc, } } else if (type == FLB_PROCESSOR_TRACES) { - if (p_ins->p->cb_process_traces != NULL) { + tmp_buf = NULL; + out_size = NULL; ret = p_ins->p->cb_process_traces(p_ins, (struct ctrace *) cur_buf, + (struct ctrace **) &tmp_buf, tag, tag_len); - - if (ret != FLB_PROCESSOR_SUCCESS) { + if (ret == FLB_PROCESSOR_FAILURE) { release_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); return -1; } + else if (ret == FLB_PROCESSOR_SUCCESS) { + if (tmp_buf == NULL) { + /* + * the processsor ran successfuly but there is no + * trace output, that means that the invoked processor + * will enqueue the trace through a different mechanism, + * we just return saying nothing else is needed. + */ + release_lock(&pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + return 0; + } + else { + cur_buf = tmp_buf; + } + } + } } else if (type == FLB_PROCESSOR_PROFILES) { From 32cf579b77825d15e344e5cae1f666d39fe17f16 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 9 Mar 2025 15:04:32 -0700 Subject: [PATCH 2/2] wip Signed-off-by: Eduardo Silva --- .../processor_buffered_trace/CMakeLists.txt | 4 + plugins/processor_buffered_trace/bt.c | 173 ++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 plugins/processor_buffered_trace/CMakeLists.txt create mode 100644 plugins/processor_buffered_trace/bt.c diff --git a/plugins/processor_buffered_trace/CMakeLists.txt b/plugins/processor_buffered_trace/CMakeLists.txt new file mode 100644 index 00000000000..0d139408a47 --- /dev/null +++ b/plugins/processor_buffered_trace/CMakeLists.txt @@ -0,0 +1,4 @@ +set(src + bt.c) + +FLB_PLUGIN(processor_buffered_trace "${src}" "") diff --git a/plugins/processor_buffered_trace/bt.c b/plugins/processor_buffered_trace/bt.c new file mode 100644 index 00000000000..9fd7cda50be --- /dev/null +++ b/plugins/processor_buffered_trace/bt.c @@ -0,0 +1,173 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct trace_entry { + struct ctrace *ctr; + struct cfl_list _head; +}; + +struct buffered_trace { + /* config properties*/ + int flush; + + /* list of buffered traces */ + struct cfl_list list; + + /* internal */ + struct flb_processor_instance *ins; +}; + +static void cb_flush(struct flb_config *config, void *data) +{ + struct cfl_list *head; + struct buffered_trace *ctx; + struct trace_entry *entry; + + ctx = (struct buffered_trace *) data; + flb_plg_info(ctx->ins, "flush callback"); + + int i = 0; + cfl_list_foreach(head, &ctx->list) { + entry = cfl_list_entry(head, struct trace_entry, _head); + printf("[%i] entry ctr: %p\n", i, entry->ctr); + } +} + +static int cb_init(struct flb_processor_instance *processor_instance, + void *source_plugin_instance, + int source_plugin_type, + struct flb_config *config) +{ + int ret; + struct buffered_trace *ctx; + struct flb_sched *sched; + + /* processor context */ + ctx = flb_calloc(1, sizeof(struct buffered_trace)); + if (!ctx) { + flb_errno(); + return FLB_PROCESSOR_FAILURE; + } + processor_instance->context = ctx; + ctx->ins = processor_instance; + cfl_list_init(&ctx->list); + + /* get the scheduler context */ + sched = flb_sched_ctx_get(); + if (!sched) { + flb_plg_error(ctx->ins, "could not get scheduler context"); + return -1; + } + + /* load config map */ + flb_processor_instance_config_map_set(ctx->ins, ctx); + + /* set a timer callback */ + ret = flb_sched_timer_cb_create(sched, FLB_SCHED_TIMER_CB_PERM, + ctx->flush, cb_flush, + ctx, NULL); + printf("cb_create() = %i\n", ret); + + return FLB_PROCESSOR_SUCCESS; +} + +static int cb_exit(struct flb_processor_instance *ins, void *data) +{ + return FLB_PROCESSOR_SUCCESS; +} + +static int cb_process_traces(struct flb_processor_instance *ins, + struct ctrace *in_ctr, + struct ctrace **out_ctr, + const char *tag, + int tag_len) +{ + int ret; + off_t offset = 0; + char *out_buf = NULL; + size_t out_size = 0; + struct ctrace *ctr_copy; + struct buffered_trace *ctx; + struct trace_entry *entry; + + ctx = ins->context; + + /* copy the original ctrace in another buffer (the caller will destory it */ + ret = ctr_encode_msgpack_create(in_ctr, &out_buf, &out_size); + if (ret != 0) { + return FLB_PROCESSOR_FAILURE; + } + + ret = ctr_decode_msgpack_create(&ctr_copy, out_buf, out_size, &offset); + if (ret != 0) { + return FLB_PROCESSOR_FAILURE; + } + + entry = flb_malloc(sizeof(struct trace_entry)); + if (!entry) { + flb_errno(); + ctr_decode_msgpack_destroy(out_buf); + ctr_destroy(ctr_copy); + return -1; + } + entry->ctr = ctr_copy; + cfl_list_add(&entry->_head, &ctx->list); + + *out_ctr = NULL; + return FLB_PROCESSOR_SUCCESS; + +} + +/* Configuration properties map */ +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_TIME, "flush", "5000", + 0, FLB_TRUE, offsetof(struct buffered_trace, flush), + "Flush time for queued traces" + }, + + /* EOF */ + {0} +}; + +struct flb_processor_plugin processor_buffered_trace_plugin = { + .name = "buffered_trace", + .description = "Test buffered trace", + .cb_init = cb_init, + .cb_process_logs = NULL, + .cb_process_metrics = NULL, + .cb_process_traces = cb_process_traces, + .cb_exit = cb_exit, + .config_map = config_map, + .flags = 0 +};