Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 87512a8

Browse files
AndersonQAndersonQ
authored
input metrics refactor and add per-input metrics to libbeat pipeline client (#42618)
This PR introduces a new way for inputs to register and publish their metrics. The `inputmon` package has been refactored to remove the need for global state. `NewInputRegistry` is now deprecated in favour of `NewMetricsRegistry` and `CancelMetricsRegistry`. The other methods were adjusted to also receive a local `*monitoring.Registry` and use it in addition to the global 'dataset' registry. Filebeat's `v2.Context` now includes a `*monitoring.Registry`, accessed through `v2.Context.MetricRegistry`. It should be the parent registry for any input metric. Now that the pipeline client can accept a metrics registry, it will collect per-input metrics. The pipeline client will collect `events_pipeline_total`, `events_pipeline_filtered_total`, and `events_pipeline_published_total`. Integration tests added for `filestream`, `httpjson`, and `cel` to verify new metrics.
1 parent 8165a58 commit 87512a8

35 files changed

Lines changed: 1392 additions & 333 deletions

‎CHANGELOG-developer.next.asciidoc‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,14 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
220220
- Add CEL input program evaluation coverage collection support. {pull}41884[41884]
221221
- Fix test framework and python tests saving config file with wrong permission. {issue}42779[42779]
222222
- Tests reading testdata config file now set flag `strict.perms=false`. They use the new `testflag` package helper function MustSetStrictPermsFalse. {issue}42779[42779]
223+
- Add a metric registry to Filebeat's v2.Context. This should be the registry
224+
used by inputs to publish metrics. This registry is then passed to the pipeline
225+
client by the input/v2. When receiving a non-nil metrics registry the pipeline
226+
client will aggregate metrics by input on this registry. It'll collect
227+
`events_pipeline_total`, `events_pipeline_filtered_total`,
228+
`events_pipeline_published_total`.{pull}42618[42618] {issue}42761[42761]
229+
- Add new API to libbeat/monitoring/inputmon. The API allows to register and
230+
unregister input metrics without relaying on the global 'dataset' namespace.{pull}42618[42618] {issue}42761[42761]
223231

224232
==== Deprecated
225233

‎filebeat/beater/channels.go‎

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -56,10 +56,6 @@ type countingClientListener struct {
5656
wgEvents *eventCounter
5757
}
5858

59-
type combinedClientListener struct {
60-
a, b beat.ClientListener
61-
}
62-
6359
func newRegistrarLogger(reg *registrar.Registrar) *registrarLogger {
6460
return &registrarLogger{
6561
done: make(chan struct{}),
@@ -115,7 +111,8 @@ func withPipelineEventCounter(pipeline beat.PipelineConnector, counter *eventCou
115111

116112
pipeline = pipetool.WithClientConfigEdit(pipeline, func(config beat.ClientConfig) (beat.ClientConfig, error) {
117113
if evts := config.ClientListener; evts != nil {
118-
config.ClientListener = &combinedClientListener{evts, counterListener}
114+
config.ClientListener = &beat.CombinedClientListener{
115+
A: evts, B: counterListener}
119116
} else {
120117
config.ClientListener = counterListener
121118
}
@@ -147,28 +144,10 @@ func (c *countingClient) Close() error {
147144

148145
func (*countingClientListener) Closing() {}
149146
func (*countingClientListener) Closed() {}
147+
func (*countingClientListener) NewEvent() {}
148+
func (*countingClientListener) Filtered() {}
150149
func (*countingClientListener) Published() {}
151150

152151
func (c *countingClientListener) DroppedOnPublish(_ beat.Event) {
153152
c.wgEvents.Done()
154153
}
155-
156-
func (c *combinedClientListener) Closing() {
157-
c.a.Closing()
158-
c.b.Closing()
159-
}
160-
161-
func (c *combinedClientListener) Closed() {
162-
c.a.Closed()
163-
c.b.Closed()
164-
}
165-
166-
func (c *combinedClientListener) Published() {
167-
c.a.Published()
168-
c.b.Published()
169-
}
170-
171-
func (c *combinedClientListener) DroppedOnPublish(event beat.Event) {
172-
c.a.DroppedOnPublish(event)
173-
c.b.DroppedOnPublish(event)
174-
}

‎filebeat/beater/filebeat.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,15 +123,15 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea
123123
}
124124

125125
if b.API != nil {
126-
if err = inputmon.AttachHandler(b.API.Router()); err != nil {
126+
if err = inputmon.AttachHandler(b.API.Router(), b.Info.Monitoring.NamespaceRegistry()); err != nil {
127127
return nil, fmt.Errorf("failed attach inputs api to monitoring endpoint server: %w", err)
128128
}
129129
}
130130

131131
if b.Manager != nil {
132132
b.Manager.RegisterDiagnosticHook("input_metrics", "Metrics from active inputs.",
133133
"input_metrics.json", "application/json", func() []byte {
134-
data, err := inputmon.MetricSnapshotJSON()
134+
data, err := inputmon.MetricSnapshotJSON(b.Info.Monitoring.NamespaceRegistry())
135135
if err != nil {
136136
b.Info.Logger.Warnw("Failed to collect input metric snapshot for Agent diagnostics.", "error", err)
137137
return []byte(err.Error())

‎filebeat/input/filestream/environment_test.go‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,12 @@ import (
3838
"github.com/elastic/beats/v7/libbeat/common/acker"
3939
"github.com/elastic/beats/v7/libbeat/common/file"
4040
"github.com/elastic/beats/v7/libbeat/common/transform/typeconv"
41+
"github.com/elastic/beats/v7/libbeat/monitoring/inputmon"
4142
"github.com/elastic/beats/v7/libbeat/statestore"
4243
"github.com/elastic/beats/v7/libbeat/statestore/storetest"
4344
conf "github.com/elastic/elastic-agent-libs/config"
4445
"github.com/elastic/elastic-agent-libs/logp"
46+
"github.com/elastic/elastic-agent-libs/monitoring"
4547
"github.com/elastic/go-concert/unison"
4648
)
4749

@@ -123,12 +125,30 @@ func (e *inputTestingEnvironment) getManager() v2.InputManager {
123125
return e.plugin.Manager
124126
}
125127

126-
func (e *inputTestingEnvironment) startInput(ctx context.Context, inp v2.Input) {
128+
func (e *inputTestingEnvironment) startInput(ctx context.Context, id string, inp v2.Input) {
127129
e.wg.Add(1)
128130
go func(wg *sync.WaitGroup, grp *unison.TaskGroup) {
129131
defer wg.Done()
130132
defer func() { _ = grp.Stop() }()
131-
inputCtx := v2.Context{Logger: logp.L(), Cancelation: ctx, ID: "fake-ID"}
133+
134+
info := beat.Info{Monitoring: beat.Monitoring{
135+
Namespace: monitoring.GetNamespace("dataset")},
136+
}
137+
reg := inputmon.NewMetricsRegistry(
138+
id, inp.Name(), info.Monitoring.NamespaceRegistry(), logp.L())
139+
defer inputmon.CancelMetricsRegistry(
140+
id, inp.Name(), info.Monitoring.NamespaceRegistry(), logp.L())
141+
142+
inputCtx := v2.Context{
143+
ID: id,
144+
IDWithoutName: id,
145+
Name: inp.Name(),
146+
Agent: info,
147+
Cancelation: ctx,
148+
StatusReporter: nil,
149+
MetricsRegistry: reg,
150+
Logger: logp.L(),
151+
}
132152
_ = inp.Run(inputCtx, e.pipeline)
133153
}(&e.wg, &e.grp)
134154
}

‎filebeat/input/filestream/input_integration_test.go‎

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ func TestFilestreamCloseRenamed(t *testing.T) {
6565
env.mustWriteToFile(testlogName, testlines)
6666

6767
ctx, cancelInput := context.WithCancel(context.Background())
68-
env.startInput(ctx, inp)
68+
env.startInput(ctx, id, inp)
6969

7070
// first event has made it successfully
7171
env.waitUntilEventCount(1)
@@ -124,7 +124,7 @@ func TestFilestreamMetadataUpdatedOnRename(t *testing.T) {
124124
env.mustWriteToFile(testlogName, testline)
125125

126126
ctx, cancelInput := context.WithCancel(context.Background())
127-
env.startInput(ctx, inp)
127+
env.startInput(ctx, id, inp)
128128

129129
env.waitUntilEventCount(1)
130130
env.waitUntilMetaInRegistry(testlogName, id, fileMeta{Source: env.abspath(testlogName), IdentifierName: "native"})
@@ -166,7 +166,7 @@ func TestFilestreamCloseRemoved(t *testing.T) {
166166
env.mustWriteToFile(testlogName, testlines)
167167

168168
ctx, cancelInput := context.WithCancel(context.Background())
169-
env.startInput(ctx, inp)
169+
env.startInput(ctx, id, inp)
170170

171171
// first event has made it successfully
172172
env.waitUntilEventCount(1)
@@ -209,7 +209,7 @@ func TestFilestreamCloseEOF(t *testing.T) {
209209
env.mustWriteToFile(testlogName, testlines)
210210

211211
ctx, cancelInput := context.WithCancel(context.Background())
212-
env.startInput(ctx, inp)
212+
env.startInput(ctx, id, inp)
213213

214214
// first event has made it successfully
215215
env.waitUntilEventCount(1)
@@ -242,7 +242,7 @@ func TestFilestreamEmptyLine(t *testing.T) {
242242
})
243243

244244
ctx, cancelInput := context.WithCancel(context.Background())
245-
env.startInput(ctx, inp)
245+
env.startInput(ctx, id, inp)
246246

247247
testlines := []byte("first log line\nnext is an empty line\n")
248248
env.mustWriteToFile(testlogName, testlines)
@@ -283,7 +283,7 @@ func TestFilestreamEmptyLinesOnly(t *testing.T) {
283283
})
284284

285285
ctx, cancelInput := context.WithCancel(context.Background())
286-
env.startInput(ctx, inp)
286+
env.startInput(ctx, id, inp)
287287

288288
testlines := []byte("\n\n\n")
289289
env.mustWriteToFile(testlogName, testlines)
@@ -319,7 +319,7 @@ func TestFilestreamBOMUTF8(t *testing.T) {
319319
env.mustWriteToFile(testlogName, lines)
320320

321321
ctx, cancelInput := context.WithCancel(context.Background())
322-
env.startInput(ctx, inp)
322+
env.startInput(ctx, id, inp)
323323

324324
env.waitUntilEventCount(7)
325325

@@ -362,7 +362,7 @@ func TestFilestreamUTF16BOMs(t *testing.T) {
362362
env.mustWriteToFile(testlogName, buf.Bytes())
363363

364364
ctx, cancelInput := context.WithCancel(context.Background())
365-
env.startInput(ctx, inp)
365+
env.startInput(ctx, id, inp)
366366

367367
env.waitUntilEventCount(1)
368368

@@ -394,7 +394,7 @@ func TestFilestreamCloseTimeout(t *testing.T) {
394394
env.mustWriteToFile(testlogName, testlines)
395395

396396
ctx, cancelInput := context.WithCancel(context.Background())
397-
env.startInput(ctx, inp)
397+
env.startInput(ctx, id, inp)
398398

399399
env.waitUntilEventCount(1)
400400
env.requireOffsetInRegistry(testlogName, id, len(testlines))
@@ -430,7 +430,7 @@ func TestFilestreamCloseAfterInterval(t *testing.T) {
430430
env.mustWriteToFile(testlogName, testlines)
431431

432432
ctx, cancelInput := context.WithCancel(context.Background())
433-
env.startInput(ctx, inp)
433+
env.startInput(ctx, id, inp)
434434

435435
env.waitUntilEventCount(3)
436436
env.requireOffsetInRegistry(testlogName, id, len(testlines))
@@ -463,7 +463,7 @@ func TestFilestreamCloseAfterIntervalRemoved(t *testing.T) {
463463
env.mustWriteToFile(testlogName, testlines)
464464

465465
ctx, cancelInput := context.WithCancel(context.Background())
466-
env.startInput(ctx, inp)
466+
env.startInput(ctx, id, inp)
467467

468468
env.waitUntilEventCount(3)
469469
env.requireOffsetInRegistry(testlogName, id, len(testlines))
@@ -498,7 +498,7 @@ func TestFilestreamCloseAfterIntervalRenamed(t *testing.T) {
498498
env.mustWriteToFile(testlogName, testlines)
499499

500500
ctx, cancelInput := context.WithCancel(context.Background())
501-
env.startInput(ctx, inp)
501+
env.startInput(ctx, id, inp)
502502

503503
env.waitUntilEventCount(3)
504504
env.requireOffsetInRegistry(testlogName, id, len(testlines))
@@ -535,7 +535,7 @@ func TestFilestreamCloseAfterIntervalRotatedAndRemoved(t *testing.T) {
535535
env.mustWriteToFile(testlogName, testlines)
536536

537537
ctx, cancelInput := context.WithCancel(context.Background())
538-
env.startInput(ctx, inp)
538+
env.startInput(ctx, id, inp)
539539

540540
env.waitUntilEventCount(3)
541541
env.requireOffsetInRegistry(testlogName, id, len(testlines))
@@ -573,7 +573,7 @@ func TestFilestreamCloseAfterIntervalRotatedAndNewRemoved(t *testing.T) {
573573
env.mustWriteToFile(testlogName, testlines)
574574

575575
ctx, cancelInput := context.WithCancel(context.Background())
576-
env.startInput(ctx, inp)
576+
env.startInput(ctx, id, inp)
577577

578578
env.waitUntilEventCount(3)
579579
env.requireOffsetInRegistry(testlogName, id, len(testlines))
@@ -612,7 +612,7 @@ func TestFilestreamTruncatedFileOpen(t *testing.T) {
612612
})
613613

614614
ctx, cancelInput := context.WithCancel(context.Background())
615-
env.startInput(ctx, inp)
615+
env.startInput(ctx, id, inp)
616616

617617
testlines := []byte("first line\nsecond line\nthird line\n")
618618
env.mustWriteToFile(testlogName, testlines)
@@ -649,7 +649,7 @@ func TestFilestreamTruncatedFileClosed(t *testing.T) {
649649
})
650650

651651
ctx, cancelInput := context.WithCancel(context.Background())
652-
env.startInput(ctx, inp)
652+
env.startInput(ctx, id, inp)
653653

654654
testlines := []byte("first line\nsecond line\nthird line\n")
655655
env.mustWriteToFile(testlogName, testlines)
@@ -697,7 +697,7 @@ func TestFilestreamTruncateWithSymlink(t *testing.T) {
697697
env.mustSymlink(testlogName, symlinkName)
698698

699699
ctx, cancelInput := context.WithCancel(context.Background())
700-
env.startInput(ctx, inp)
700+
env.startInput(ctx, id, inp)
701701

702702
env.waitUntilEventCount(3)
703703

@@ -735,7 +735,7 @@ func TestFilestreamTruncateBigScannerInterval(t *testing.T) {
735735
})
736736

737737
ctx, cancelInput := context.WithCancel(context.Background())
738-
env.startInput(ctx, inp)
738+
env.startInput(ctx, id, inp)
739739

740740
testlines := []byte("first line\nsecond line\nthird line\n")
741741
env.mustWriteToFile(testlogName, testlines)
@@ -769,7 +769,7 @@ func TestFilestreamTruncateCheckOffset(t *testing.T) {
769769
})
770770

771771
ctx, cancelInput := context.WithCancel(context.Background())
772-
env.startInput(ctx, inp)
772+
env.startInput(ctx, id, inp)
773773

774774
testlines := []byte("first line\nsecond line\nthird line\n")
775775
env.mustWriteToFile(testlogName, testlines)
@@ -803,7 +803,7 @@ func TestFilestreamTruncateBlockedOutput(t *testing.T) {
803803
env.mustWriteToFile(testlogName, testlines)
804804

805805
ctx, cancelInput := context.WithCancel(context.Background())
806-
env.startInput(ctx, inp)
806+
env.startInput(ctx, id, inp)
807807

808808
for env.pipeline.clientsCount() != 1 {
809809
time.Sleep(10 * time.Millisecond)
@@ -860,7 +860,7 @@ func TestFilestreamSymlinksEnabled(t *testing.T) {
860860
env.mustSymlink(testlogName, symlinkName)
861861

862862
ctx, cancelInput := context.WithCancel(context.Background())
863-
env.startInput(ctx, inp)
863+
env.startInput(ctx, id, inp)
864864

865865
env.waitUntilEventCount(1)
866866

@@ -899,7 +899,7 @@ func TestFilestreamSymlinkRotated(t *testing.T) {
899899
env.mustSymlink(firstTestlogName, symlinkName)
900900

901901
ctx, cancelInput := context.WithCancel(context.Background())
902-
env.startInput(ctx, inp)
902+
env.startInput(ctx, id, inp)
903903

904904
env.waitUntilEventCount(1)
905905

@@ -949,7 +949,7 @@ func TestFilestreamSymlinkRemoved(t *testing.T) {
949949
env.mustSymlink(testlogName, symlinkName)
950950

951951
ctx, cancelInput := context.WithCancel(context.Background())
952-
env.startInput(ctx, inp)
952+
env.startInput(ctx, id, inp)
953953

954954
env.waitUntilEventCount(1)
955955

@@ -994,7 +994,7 @@ func TestFilestreamTruncate(t *testing.T) {
994994
env.mustSymlink(testlogName, symlinkName)
995995

996996
ctx, cancelInput := context.WithCancel(context.Background())
997-
env.startInput(ctx, inp)
997+
env.startInput(ctx, id, inp)
998998

999999
env.waitUntilEventCount(3)
10001000

@@ -1036,8 +1036,9 @@ func TestFilestreamHarvestAllFilesWhenHarvesterLimitExceeded(t *testing.T) {
10361036
lf.path, []byte(strings.Join(lf.lines, "\n")+"\n"))
10371037
}
10381038

1039+
id := "TestFilestreamHarvestAllFilesWhenHarvesterLimitExceeded"
10391040
inp := env.mustCreateInput(map[string]interface{}{
1040-
"id": "TestFilestreamHarvestAllFilesWhenHarvesterLimitExceeded",
1041+
"id": id,
10411042
"harvester_limit": 1,
10421043
"close.reader.on_eof": true,
10431044
"paths": []string{
@@ -1049,7 +1050,7 @@ func TestFilestreamHarvestAllFilesWhenHarvesterLimitExceeded(t *testing.T) {
10491050

10501051
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
10511052

1052-
env.startInput(ctx, inp)
1053+
env.startInput(ctx, id, inp)
10531054

10541055
env.waitUntilEventCountCtx(ctx, 4)
10551056

@@ -1091,7 +1092,7 @@ func TestRotatingCloseInactiveLargerWriteRate(t *testing.T) {
10911092
})
10921093

10931094
ctx, cancelInput := context.WithCancel(context.Background())
1094-
env.startInput(ctx, inp)
1095+
env.startInput(ctx, id, inp)
10951096

10961097
time.Sleep(1 * time.Second)
10971098

@@ -1140,7 +1141,7 @@ func TestRotatingCloseInactiveLowWriteRate(t *testing.T) {
11401141
})
11411142

11421143
ctx, cancelInput := context.WithCancel(context.Background())
1143-
env.startInput(ctx, inp)
1144+
env.startInput(ctx, id, inp)
11441145

11451146
time.Sleep(1 * time.Second)
11461147

0 commit comments

Comments
 (0)