Codestin Search App

milkrage · 2025-04-09T21:08:01Z

These changes resolve the following issue: #8464

Enter [N/A] in the box, if an item is not applicable to your change.

Testing
Before we can approve your change; please submit the following in a comment:

Example configuration file for the change

service:
  log_level: debug

pipeline:
  inputs:
    - name: go-input-plugin-with-context
      config-key: config-value-1
      tag: one
    - name: go-input-plugin-with-context
      config-key: config-value-1
      tag: one

  outputs:
    - name: stdout
      match: '*'

Go Plugin Example (without .h)

package main

/*
#include <stdlib.h>
*/
import "C"
import (
	"fmt"
	"log"
	"time"
	"unsafe"

	"fluent-bit-input-plugin/input"
)

//export FLBPluginRegister
func FLBPluginRegister(def unsafe.Pointer) int {
	return input.FLBPluginRegister(def, "go-input-plugin-with-context", "Go Input Plugin With Context!")
}

//export FLBPluginInit
func FLBPluginInit(plugin unsafe.Pointer) int {
	// Get config param
	param := input.FLBPluginConfigKey(plugin, "config-key")
	fmt.Printf("[flb-go][FLBPluginInit] Plugin parameter config-key=%s \n", param)

	// Set config param to context
	input.FLBPluginSetContext(plugin, param)

	return input.FLB_OK
}

//export FLBPluginInputCallbackCtx
func FLBPluginInputCallbackCtx(ctx unsafe.Pointer, data *unsafe.Pointer, size *C.size_t) int {
	// Get config param from context
	value := input.FLBPluginGetContext(ctx).(string)
	log.Printf("[flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=%s \n", value)

	now := time.Now()
	flb_time := input.FLBTime{now}
	message := map[string]string{"message": value}

	entry := []interface{}{flb_time, message}

	enc := input.NewEncoder()
	packed, err := enc.Encode(entry)
	if err != nil {
		fmt.Println("Can't convert to msgpack:", message, err)
		return input.FLB_ERROR
	}

	length := len(packed)
	*data = C.CBytes(packed)
	*size = C.size_t(length)
	// For emitting interval adjustment.
	time.Sleep(5000 * time.Millisecond)

	return input.FLB_OK
}

//export FLBPluginInputCleanupCallback
func FLBPluginInputCleanupCallback(data unsafe.Pointer) int {
	return input.FLB_OK
}

//export FLBPluginExit
func FLBPluginExit() int {
	return input.FLB_OK
}

func main() {
}

Debug log output from testing the change

root@35c092a175d8:/fluent-bit-input-plugin# fluent-bit -e ./input-plugin-with-context.so -c development/fluent-bit.yaml
Fluent Bit v4.0.1
* Copyright (C) 2015-2025 The Fluent Bit Authors
* Fluent Bit is a CNCF sub-project under the umbrella of Fluentd
* https://fluentbit.io

______ _                  _    ______ _ _             ___  _____ 
|  ___| |                | |   | ___ (_) |           /   ||  _  |
| |_  | |_   _  ___ _ __ | |_  | |_/ /_| |_  __   __/ /| || |/' |
|  _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| ||  /| |
| |   | | |_| |  __/ | | | |_  | |_/ / | |_   \ V /\___  |\ |_/ /
\_|   |_|\__,_|\___|_| |_|\__| \____/|_|\__|   \_/     |_(_)___/ 


[2025/04/09 20:14:02] [ info] Configuration:
[2025/04/09 20:14:02] [ info]  flush time     | 1.000000 seconds
[2025/04/09 20:14:02] [ info]  grace          | 5 seconds
[2025/04/09 20:14:02] [ info]  daemon         | 0
[2025/04/09 20:14:02] [ info] ___________
[2025/04/09 20:14:02] [ info]  inputs:
[2025/04/09 20:14:02] [ info]      go-input-plugin-with-context
[2025/04/09 20:14:02] [ info]      go-input-plugin-with-context
[2025/04/09 20:14:02] [ info] ___________
[2025/04/09 20:14:02] [ info]  filters:
[2025/04/09 20:14:02] [ info] ___________
[2025/04/09 20:14:02] [ info]  outputs:
[2025/04/09 20:14:02] [ info]      stdout.0
[2025/04/09 20:14:02] [ info] ___________
[2025/04/09 20:14:02] [ info]  collectors:
[2025/04/09 20:14:02] [ info] [fluent bit] version=4.0.1, commit=7505318ca3, pid=9174
[2025/04/09 20:14:02] [debug] [engine] coroutine stack size: 24576 bytes (24.0K)
[2025/04/09 20:14:02] [ info] [storage] ver=1.5.2, type=memory, sync=normal, checksum=off, max_chunks_up=128
[2025/04/09 20:14:02] [ info] [simd    ] disabled
[2025/04/09 20:14:02] [ info] [cmetrics] version=0.9.9
[2025/04/09 20:14:02] [ info] [ctraces ] version=0.6.2
[2025/04/09 20:14:02] [ info] [input:go-input-plugin-with-context:go-input-plugin-with-context.0] initializing
[2025/04/09 20:14:02] [ info] [input:go-input-plugin-with-context:go-input-plugin-with-context.0] storage_strategy='memory' (memory only)
[flb-go][FLBPluginInit] Plugin parameter config-key=config-value-1 
[2025/04/09 20:14:02] [debug] [input:go-input-plugin-with-context:go-input-plugin-with-context.0] [thread init] initialization OK
[2025/04/09 20:14:02] [ info] [input:go-input-plugin-with-context:go-input-plugin-with-context.0] thread instance initialized
[2025/04/09 20:14:02] [debug] [go-input-plugin-with-context:go-input-plugin-with-context.0] created event channels: read=38 write=39
[2025/04/09 20:14:02] [ info] [input:go-input-plugin-with-context:go-input-plugin-with-context.1] initializing
[2025/04/09 20:14:02] [ info] [input:go-input-plugin-with-context:go-input-plugin-with-context.1] storage_strategy='memory' (memory only)
[flb-go][FLBPluginInit] Plugin parameter config-key=config-value-2 
[2025/04/09 20:14:02] [debug] [input:go-input-plugin-with-context:go-input-plugin-with-context.1] [thread init] initialization OK
[2025/04/09 20:14:02] [ info] [input:go-input-plugin-with-context:go-input-plugin-with-context.1] thread instance initialized
[2025/04/09 20:14:02] [debug] [go-input-plugin-with-context:go-input-plugin-with-context.1] created event channels: read=55 write=56
[2025/04/09 20:14:02] [debug] [stdout:stdout.0] created event channels: read=59 write=60
[2025/04/09 20:14:02] [debug] [router] match rule go-input-plugin-with-context.0:stdout.0
[2025/04/09 20:14:02] [ info] [output:stdout:stdout.0] worker #0 started
[2025/04/09 20:14:02] [debug] [router] match rule go-input-plugin-with-context.1:stdout.0
[2025/04/09 20:14:02] [ info] [sp] stream processor started
2025/04/09 20:14:03 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-1 
2025/04/09 20:14:03 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-2 
2025/04/09 20:14:08 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-2 
2025/04/09 20:14:08 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-1 
[2025/04/09 20:14:09] [debug] [task] created task=0x7f90b802bed0 id=0 OK
[2025/04/09 20:14:09] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[2025/04/09 20:14:09] [debug] [task] created task=0x7f90b802bfa0 id=1 OK
[2025/04/09 20:14:09] [debug] [output:stdout:stdout.0] task_id=1 assigned to thread #0
[0] one: [[1744229643.399235929, {}], {"message"=>"config-value-1"}]
[0] one: [[1744229643.400736182, {}], {"message"=>"config-value-2"}]
[2025/04/09 20:14:09] [debug] [out flush] cb_destroy coro_id=0
[2025/04/09 20:14:09] [debug] [out flush] cb_destroy coro_id=1
[2025/04/09 20:14:09] [debug] [task] destroy task=0x7f90b802bed0 (task_id=0)
[2025/04/09 20:14:09] [debug] [task] destroy task=0x7f90b802bfa0 (task_id=1)
2025/04/09 20:14:13 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-2 
2025/04/09 20:14:13 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-1 
[2025/04/09 20:14:14] [debug] [task] created task=0x7f90b80ac390 id=0 OK
[2025/04/09 20:14:14] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[2025/04/09 20:14:14] [debug] [task] created task=0x7f90b80ac460 id=1 OK
[0] one: [[1744229648.404982589, {}], {"message"=>"config-value-1"}]
[2025/04/09 20:14:14] [debug] [output:stdout:stdout.0] task_id=1 assigned to thread #0
[0] one: [[1744229648.404970546, {}], {"message"=>"config-value-2"}]
[2025/04/09 20:14:14] [debug] [out flush] cb_destroy coro_id=2
[2025/04/09 20:14:14] [debug] [out flush] cb_destroy coro_id=3
[2025/04/09 20:14:14] [debug] [task] destroy task=0x7f90b80ac390 (task_id=0)
[2025/04/09 20:14:14] [debug] [task] destroy task=0x7f90b80ac460 (task_id=1)
^C[2025/04/09 20:14:15] [engine] caught signal (SIGINT)
[2025/04/09 20:14:15] [ warn] [engine] service will shutdown in max 5 seconds
[2025/04/09 20:14:15] [debug] [input:go-input-plugin-with-context:go-input-plugin-with-context.0] thread pause instance
[2025/04/09 20:14:15] [debug] [input:go-input-plugin-with-context:go-input-plugin-with-context.1] thread pause instance
[2025/04/09 20:14:16] [ info] [engine] service has stopped (0 pending tasks)
[2025/04/09 20:14:16] [debug] [input:go-input-plugin-with-context:go-input-plugin-with-context.0] thread pause instance
[2025/04/09 20:14:16] [debug] [input:go-input-plugin-with-context:go-input-plugin-with-context.1] thread pause instance
[2025/04/09 20:14:16] [ info] [output:stdout:stdout.0] thread worker #0 stopping...
[2025/04/09 20:14:16] [ info] [output:stdout:stdout.0] thread worker #0 stopped
2025/04/09 20:14:18 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-1 
2025/04/09 20:14:18 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-2 
2025/04/09 20:14:23 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-1 
[2025/04/09 20:14:23] [debug] [GO] running exit callback
[2025/04/09 20:14:23] [debug] [input:go-input-plugin-with-context:go-input-plugin-with-context.1] thread exit instance
2025/04/09 20:14:28 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-1 
2025/04/09 20:14:33 [flb-go][multiinstance][FLBPluginInputCallbackCtx] Plugin parameter config-key=config-value-1 
[2025/04/09 20:14:38] [debug] [GO] running exit callback
[2025/04/09 20:14:38] [debug] [input:go-input-plugin-with-context:go-input-plugin-with-context.0] thread exit instance
Segmentation fault

Attached Valgrind output that shows no leaks or memory corruption was found

If this is a change to packaging of containers or native binaries then please confirm it works for all targets.

Run local packaging test showing all targets (including any new ones) build.
Set ok-package-test label to test for all targets (requires maintainer to do).

Documentation

[N/A] Documentation required for this feature

Backporting

[N/A] Backport to latest stable release.

Fluent Bit is licensed under Apache 2.0, by submitting this pull request I understand that this code will be released under the terms of that license.

milkrage · 2025-04-10T09:26:05Z

@mp3monster, jfyi (as the author of the issue)

milkrage · 2025-04-11T11:07:50Z

Нello, @edsiper, @leonardo-albertovich! Could you please tell if there is an estimated start date of review of this PR?

This PR would be helpful for others, because before these changes, the context was only available in the output plugin. This significantly limited the capabilities of input plugins in GoLang, since working in multiinstance mode was impossible.

`flb_plugin_proxy_context` was replaced by `flb_plugin_input_proxy_context` Signed-off-by: milkrage <[email protected]>

This is necessary so that go input plugin can implement methods with context in the same way as it is done in go output plugin. Signed-off-by: milkrage <[email protected]>

The way to add context to input methods is completely identical to the approach used for output methods. Without this, the callback can only perform hardwired tasks, and the configuration values can not be accessed. As a result, the input becomes extremely constrained on what it can do. Signed-off-by: milkrage <[email protected]>

milkrage · 2025-04-27T08:10:20Z

I can't reproduce the error (flb-rt-filter_rewrite_tag) on my laptop.

➜  ~/Projects/fluent-bit/build cmake -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm/bin/clang++ -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On -DFLB_JEMALLOC=Off ..

➜  ~/Projects/fluent-bit/build make

➜  ~/Projects/fluent-bit/build/bin/flb-rt-filter_rewrite_tag
Test matched...                                 [2025/04/26 21:34:50] [ info] [input] pausing emitter_for_rewrite_tag.0
[ OK ]
Test not_matched...                             [2025/04/26 21:34:52] [ info] [input] pausing emitter_for_rewrite_tag.0
[ OK ]
Test keep_true...                               [2025/04/26 21:34:55] [ info] [input] pausing emitter_for_rewrite_tag.0
[ OK ]
Test heavy_input_pause_emitter...               [2025/04/26 21:34:58] [ info] [input] pausing emitter_for_rewrite_tag.0
[ OK ]
Test issue_4518...                              [2025/04/26 21:35:01] [ info] [input] pausing emitter_for_rewrite_tag.0
[2025/04/26 21:35:01] [ info] [input] pausing emitter_for_rewrite_tag.1
[ OK ]
Test issue_4793...                              [2025/04/26 21:35:03] [ info] [input] pausing emitter_for_rewrite_tag.0
[ OK ]
Test sigsegv_issue_5846...                      [2025/04/26 21:35:06] [ info] [input] pausing emitter_for_rewrite_tag.0
[ OK ]
SUCCESS: All unit tests have passed.

@edsiper, could you please rerun the matrix unit tests?

milkrage · 2025-04-29T16:49:00Z

This is a working example of the input plugin with the changes from this PR: fluent-bit-cloudwatch-input-plugin

niedbalski · 2025-04-30T09:50:13Z

Just to clarify the intention of this PR:

It adds new callback functions with context parameters

a. FLBPluginInputCallbackCtx - A collect function that receives context
b. FLBPluginInputCleanupCallbackCtx - A cleanup function that receives context

Adding a new flb_plugin_input_proxy_context with a remote_context field that can be set by the plugin itself.

This looks fine to me @milkrage as you are keeping backwards compatibility -- Would you happen to also check the input plugin interface? https://github.com/chronosphereio/calyptia-plugin/tree/main

cosmo0920

Seems good.
I was worried about necessities of context requirements for collect and cleanup callbacks.
Now I realized that they are optional. So, it looks good to me.

milkrage · 2025-04-30T11:59:15Z

@niedbalski, yes, of course. I'll check this evening.

milkrage · 2025-04-30T19:53:48Z

fluent-bit config:

service:
  log_level: debug

pipeline:
  inputs:
    - name: gdummy

  outputs:
    - name: stdout
      match: '*'

run in_gdummy from calyptia-plugin/examples:

➜  ~/Projects/calyptia-plugin/examples/in_gdummy go install
➜  ~/Projects/calyptia-plugin/examples/in_gdummy make all
go build -buildmode=c-shared -o in_gdummy.so .

# check: git commit from my branch 
➜  ~/Projects/calyptia-plugin/examples/in_gdummy /usr/local/bin/fluent-bit --version
Fluent Bit v4.0.2
Git commit: 3ed8c97b2142d4c319fd02786905ceb019d379e4

# run fluent-bit with gdummy plugin
➜  ~/Projects/calyptia-plugin/examples/in_gdummy /usr/local/bin/fluent-bit -e in_gdummy.so -c config.yaml
Fluent Bit v4.0.2
* Copyright (C) 2015-2025 The Fluent Bit Authors
* Fluent Bit is a CNCF sub-project under the umbrella of Fluentd
* https://fluentbit.io

______ _                  _    ______ _ _             ___  _____
|  ___| |                | |   | ___ (_) |           /   ||  _  |
| |_  | |_   _  ___ _ __ | |_  | |_/ /_| |_  __   __/ /| || |/' |
|  _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| ||  /| |
| |   | | |_| |  __/ | | | |_  | |_/ / | |_   \ V /\___  |\ |_/ /
\_|   |_|\__,_|\___|_| |_|\__| \____/|_|\__|   \_/     |_(_)___/


[2025/04/30 22:51:13] [ info] Configuration:
[2025/04/30 22:51:13] [ info]  flush time     | 1.000000 seconds
[2025/04/30 22:51:13] [ info]  grace          | 5 seconds
[2025/04/30 22:51:13] [ info]  daemon         | 0
[2025/04/30 22:51:13] [ info] ___________
[2025/04/30 22:51:13] [ info]  inputs:
[2025/04/30 22:51:13] [ info]      gdummy
[2025/04/30 22:51:13] [ info] ___________
[2025/04/30 22:51:13] [ info]  filters:
[2025/04/30 22:51:13] [ info] ___________
[2025/04/30 22:51:13] [ info]  outputs:
[2025/04/30 22:51:13] [ info]      stdout.0
[2025/04/30 22:51:13] [ info] ___________
[2025/04/30 22:51:13] [ info]  collectors:
[2025/04/30 22:51:13] [ info] [fluent bit] version=4.0.2, commit=3ed8c97b21, pid=59827
[2025/04/30 22:51:13] [debug] [engine] coroutine stack size: 24576 bytes (24.0K)
[2025/04/30 22:51:13] [ info] [storage] ver=1.5.3, type=memory, sync=normal, checksum=off, max_chunks_up=128
[2025/04/30 22:51:13] [ info] [simd    ] disabled
[2025/04/30 22:51:13] [ info] [cmetrics] version=1.0.0
[2025/04/30 22:51:13] [ info] [ctraces ] version=0.6.4
[2025/04/30 22:51:13] [ info] [input:gdummy:gdummy.0] initializing
[2025/04/30 22:51:13] [ info] [input:gdummy:gdummy.0] storage_strategy='memory' (memory only)
[2025/04/30 22:51:13] [debug] [input:gdummy:gdummy.0] [thread init] initialization OK
[2025/04/30 22:51:13] [ info] [input:gdummy:gdummy.0] thread instance initialized
[2025/04/30 22:51:13] [debug] [gdummy:gdummy.0] created event channels: read=38 write=39
[2025/04/30 22:51:13] [debug] [stdout:stdout.0] created event channels: read=42 write=43
[2025/04/30 22:51:13] [ info] [output:stdout:stdout.0] worker #0 started
[2025/04/30 22:51:13] [ info] [sp] stream processor started
[2025/04/30 22:51:14] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:15] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:16] [debug] [task] created task=0x6000022440b0 id=0 OK
[2025/04/30 22:51:16] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] gdummy.0: [[1746042674.417130000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:16] [debug] [out flush] cb_destroy coro_id=0
[2025/04/30 22:51:16] [debug] [task] destroy task=0x6000022440b0 (task_id=0)
[2025/04/30 22:51:16] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:17] [debug] [task] created task=0x60000224c000 id=0 OK
[2025/04/30 22:51:17] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] gdummy.0: [[1746042675.416635000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:17] [debug] [out flush] cb_destroy coro_id=1
[2025/04/30 22:51:17] [debug] [task] destroy task=0x60000224c000 (task_id=0)
[2025/04/30 22:51:17] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:18] [debug] [task] created task=0x600002250000 id=0 OK
[2025/04/30 22:51:18] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] gdummy.0: [[1746042676.417520000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:18] [debug] [out flush] cb_destroy coro_id=2
[2025/04/30 22:51:18] [debug] [task] destroy task=0x600002250000 (task_id=0)
[2025/04/30 22:51:18] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:19] [debug] [task] created task=0x600002248160 id=0 OK
[2025/04/30 22:51:19] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] gdummy.0: [[1746042677.417811000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:19] [debug] [out flush] cb_destroy coro_id=3
[2025/04/30 22:51:19] [debug] [task] destroy task=0x600002248160 (task_id=0)
[2025/04/30 22:51:19] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:20] [debug] [task] created task=0x600002248160 id=0 OK
[0] gdummy.0: [[1746042678.416987000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:20] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[2025/04/30 22:51:20] [debug] [out flush] cb_destroy coro_id=4
[2025/04/30 22:51:20] [debug] [task] destroy task=0x600002248160 (task_id=0)
[2025/04/30 22:51:20] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:21] [debug] [task] created task=0x600002254000 id=0 OK
[2025/04/30 22:51:21] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] gdummy.0: [[1746042679.417916000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:21] [debug] [out flush] cb_destroy coro_id=5
[2025/04/30 22:51:21] [debug] [task] destroy task=0x600002254000 (task_id=0)
[2025/04/30 22:51:21] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:22] [debug] [task] created task=0x600002254000 id=0 OK
[2025/04/30 22:51:22] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] gdummy.0: [[1746042680.418499000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:22] [debug] [out flush] cb_destroy coro_id=6
[2025/04/30 22:51:22] [debug] [task] destroy task=0x600002254000 (task_id=0)
[2025/04/30 22:51:22] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:23] [debug] [task] created task=0x600002248160 id=0 OK
[2025/04/30 22:51:23] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] gdummy.0: [[1746042681.418806000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:23] [debug] [out flush] cb_destroy coro_id=7
[2025/04/30 22:51:23] [debug] [task] destroy task=0x600002248160 (task_id=0)
[2025/04/30 22:51:23] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:24] [debug] [task] created task=0x600002254000 id=0 OK
[2025/04/30 22:51:24] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] gdummy.0: [[1746042682.418883000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:24] [debug] [out flush] cb_destroy coro_id=8
[2025/04/30 22:51:24] [debug] [task] destroy task=0x600002254000 (task_id=0)
[2025/04/30 22:51:24] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:25] [debug] [task] created task=0x600002254000 id=0 OK
[2025/04/30 22:51:25] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] gdummy.0: [[1746042683.418970000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:25] [debug] [out flush] cb_destroy coro_id=9
[2025/04/30 22:51:25] [debug] [task] destroy task=0x600002254000 (task_id=0)
[2025/04/30 22:51:25] [debug] [gdummy] operation succeeded
[2025/04/30 22:51:26] [debug] [task] created task=0x600002250000 id=0 OK
[0] gdummy.0: [[1746042684.418798000, {}], {"message"=>"dummy"}]
[2025/04/30 22:51:26] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[2025/04/30 22:51:26] [debug] [out flush] cb_destroy coro_id=10
[2025/04/30 22:51:26] [debug] [task] destroy task=0x600002250000 (task_id=0)
[2025/04/30 22:51:26] [debug] [gdummy] operation succeeded
^C[2025/04/30 22:51:27] [engine] caught signal (SIGINT)
[2025/04/30 22:51:27] [debug] [input:gdummy:gdummy.0] thread pause instance
[2025/04/30 22:51:27] [ info] [output:stdout:stdout.0] thread worker #0 stopping...
[2025/04/30 22:51:27] [ info] [output:stdout:stdout.0] thread worker #0 stopped
[2025/04/30 22:51:27] [debug] [GO] running exit callback
[2025/04/30 22:51:27] [debug] [input:gdummy:gdummy.0] thread exit instance
2025/04/30 22:51:27 goroutine will be stopping: name="gdummy"

Everything works

edsiper · 2025-04-30T23:53:39Z

@cosmo0920 @niedbalski are there any potential breaking changes ?

cc: @pwhelan

edsiper · 2025-04-30T23:53:56Z

moved to other milestone in the meanwhile

cosmo0920 · 2025-05-01T01:27:14Z

AFAIK, there's no potential breaking changes.
This is because both of newly added callbacks are optional:
https://github.com/fluent/fluent-bit/pull/10193/files#diff-ea7dbddc45de64b1e6a2afaa4f4013a472bf8b81ad9ac605b108c0120d417d2fR244-R249
https://github.com/fluent/fluent-bit/pull/10193/files#diff-ea7dbddc45de64b1e6a2afaa4f4013a472bf8b81ad9ac605b108c0120d417d2fR262-R265

shaj13 · 2025-08-11T16:52:42Z

Is there a plan to proceed with this PR ? Currently, the proxy causes issues when running multiple instances of the plugin. If I add multiple plugins, there’s no way to identify which instance to call when FLBPluginInputCallback is triggered. Calling all plugins causes Fluent Bit to wrongly tag the records.

[INPUT]
    Name              example
    Tag               example1

[INPUT]
    Name              example
    Tag               example2

[INPUT]
    Name              example
    Tag               example3

milkrage · 2025-08-11T23:13:47Z

From my side, the PR is completely ready for merging. I should note that PR doesn't contain breaking changes, it just extends the current interface by adding optional methods with context. If this PR is merged in the near future, I can update the examples at https://github.com/fluent/fluent-bit-go.

The current changes can be added to Fluent Bit v4.1, since they do not contain breaking changes and just extend the current interface. This PR is very useful for Go developers, and @shaj13's question confirms this.

@edsiper, @niedbalski, @cosmo0920, is it possible to change the milestone (Fluent Bit Next -> Fluent Bit v4.1) to merge this change in September?

milkrage · 2025-09-14T14:44:48Z

@edsiper, @cosmo0920, please merge this branch into master so that the changes are included in the v4.1 release

edsiper · 2025-09-19T04:22:03Z

thank you!

milkrage requested review from edsiper, fujimotos, koleini and leonardo-albertovich as code owners April 9, 2025 21:08

github-actions bot added the docs-required label Apr 9, 2025

milkrage force-pushed the ctx_go_input_plugin branch from 7505318 to 27a727b Compare April 11, 2025 10:23

milkrage force-pushed the ctx_go_input_plugin branch from 27a727b to aa810b0 Compare April 22, 2025 18:12

milkrage temporarily deployed to pr April 23, 2025 19:42 — with GitHub Actions Inactive

milkrage temporarily deployed to pr April 23, 2025 20:05 — with GitHub Actions Inactive

milkrage temporarily deployed to pr April 23, 2025 20:06 — with GitHub Actions Inactive

milkrage mentioned this pull request Apr 24, 2025

Support Input plugin too aws/amazon-cloudwatch-logs-for-fluent-bit#48

Open

milkrage force-pushed the ctx_go_input_plugin branch from aa810b0 to eba088c Compare April 26, 2025 20:38

milkrage added 3 commits April 26, 2025 23:45

input: change instance context type for plugin proxy

db09528

`flb_plugin_proxy_context` was replaced by `flb_plugin_input_proxy_context` Signed-off-by: milkrage <[email protected]>

proxy: send proxy context to go input plugin

4713f44

This is necessary so that go input plugin can implement methods with context in the same way as it is done in go output plugin. Signed-off-by: milkrage <[email protected]>

milkrage force-pushed the ctx_go_input_plugin branch from eba088c to 3ed8c97 Compare April 26, 2025 20:48

niedbalski self-requested a review April 29, 2025 08:25

milkrage temporarily deployed to pr April 30, 2025 09:46 — with GitHub Actions Inactive

niedbalski approved these changes Apr 30, 2025

View reviewed changes

cosmo0920 approved these changes Apr 30, 2025

View reviewed changes

milkrage temporarily deployed to pr April 30, 2025 10:10 — with GitHub Actions Inactive

cosmo0920 added this to the Fluent Bit v4.0.2 milestone Apr 30, 2025

edsiper modified the milestones: Fluent Bit v4.0.2, Fluent Bit Next Apr 30, 2025

cosmo0920 modified the milestones: Fluent Bit Next, Fluent Bit v4.1 Aug 12, 2025

edsiper merged commit 730e7b0 into fluent:master Sep 19, 2025
49 checks passed

Comments

Conversation

milkrage commented Apr 9, 2025

Uh oh!

milkrage commented Apr 10, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

milkrage commented Apr 11, 2025

Uh oh!

milkrage commented Apr 27, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

milkrage commented Apr 29, 2025

Uh oh!

niedbalski commented Apr 30, 2025

Uh oh!

cosmo0920 left a comment • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

milkrage commented Apr 30, 2025

Uh oh!

milkrage commented Apr 30, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

edsiper commented Apr 30, 2025

Uh oh!

edsiper commented Apr 30, 2025

Uh oh!

cosmo0920 commented May 1, 2025

Uh oh!

shaj13 commented Aug 11, 2025

Uh oh!

milkrage commented Aug 11, 2025

Uh oh!

milkrage commented Sep 14, 2025

Uh oh!

Uh oh!

edsiper commented Sep 19, 2025

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

5 participants

milkrage commented Apr 10, 2025 •

edited

Loading

milkrage commented Apr 27, 2025 •

edited

Loading

cosmo0920 left a comment •

edited

Loading

milkrage commented Apr 30, 2025 •

edited

Loading