From 2eb3930dd0e5a21c695e61e3df19d553c42eee55 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Wed, 30 Jul 2025 12:01:13 +0200 Subject: [PATCH 1/2] chore: use consistent copyright headers --- config/config.go | 5 ++--- extkafka/alter_test.go | 5 ++--- extkafka/check_brokers.go | 4 ---- extkafka/check_brokers_test.go | 5 ++--- extkafka/check_consumer_group.go | 6 +----- extkafka/check_consumer_group_test.go | 5 ++--- extkafka/check_partitions.go | 6 +----- extkafka/check_topic_lag_for_consumer.go | 6 +----- extkafka/check_topic_lag_for_consumer_test.go | 5 ++--- extkafka/produce.go | 5 ++--- extkafka/produceFixAmount.go | 5 ++--- extkafka/produceFixAmount_test.go | 5 ++--- extkafka/producePeriodically.go | 5 ++--- extkafka/producePeriodically_test.go | 5 ++--- extkafka/produce_test.go | 5 ++--- extkafka/util.go | 5 ++--- extkafka/util_test.go | 3 +++ 17 files changed, 30 insertions(+), 55 deletions(-) diff --git a/config/config.go b/config/config.go index aedb914..5a7ab29 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,5 @@ -/* - * Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package config diff --git a/extkafka/alter_test.go b/extkafka/alter_test.go index d2d1807..315364e 100644 --- a/extkafka/alter_test.go +++ b/extkafka/alter_test.go @@ -1,6 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/check_brokers.go b/extkafka/check_brokers.go index 97eb7a5..457c33b 100644 --- a/extkafka/check_brokers.go +++ b/extkafka/check_brokers.go @@ -1,7 +1,3 @@ -/* -* Copyright 2025 steadybit GmbH. All rights reserved. - */ - // SPDX-License-Identifier: MIT // SPDX-FileCopyrightText: 2022 Steadybit GmbH diff --git a/extkafka/check_brokers_test.go b/extkafka/check_brokers_test.go index b35874e..9127101 100644 --- a/extkafka/check_brokers_test.go +++ b/extkafka/check_brokers_test.go @@ -1,6 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/check_consumer_group.go b/extkafka/check_consumer_group.go index f20707a..fe87331 100644 --- a/extkafka/check_consumer_group.go +++ b/extkafka/check_consumer_group.go @@ -1,9 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ - // SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: 2022 Steadybit GmbH +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/check_consumer_group_test.go b/extkafka/check_consumer_group_test.go index d5d235b..e70c841 100644 --- a/extkafka/check_consumer_group_test.go +++ b/extkafka/check_consumer_group_test.go @@ -1,6 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/check_partitions.go b/extkafka/check_partitions.go index a2fd2b5..cd17fbf 100644 --- a/extkafka/check_partitions.go +++ b/extkafka/check_partitions.go @@ -1,9 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ - // SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: 2022 Steadybit GmbH +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/check_topic_lag_for_consumer.go b/extkafka/check_topic_lag_for_consumer.go index c4b5c8a..326cd23 100644 --- a/extkafka/check_topic_lag_for_consumer.go +++ b/extkafka/check_topic_lag_for_consumer.go @@ -1,9 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ - // SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: 2022 Steadybit GmbH +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/check_topic_lag_for_consumer_test.go b/extkafka/check_topic_lag_for_consumer_test.go index 1d710af..af3780b 100644 --- a/extkafka/check_topic_lag_for_consumer_test.go +++ b/extkafka/check_topic_lag_for_consumer_test.go @@ -1,6 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/produce.go b/extkafka/produce.go index 037cdf1..bc358b5 100644 --- a/extkafka/produce.go +++ b/extkafka/produce.go @@ -1,6 +1,5 @@ -/* - * Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/produceFixAmount.go b/extkafka/produceFixAmount.go index bdc72d9..343a862 100644 --- a/extkafka/produceFixAmount.go +++ b/extkafka/produceFixAmount.go @@ -1,6 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/produceFixAmount_test.go b/extkafka/produceFixAmount_test.go index 6f02208..1ba128c 100644 --- a/extkafka/produceFixAmount_test.go +++ b/extkafka/produceFixAmount_test.go @@ -1,6 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/producePeriodically.go b/extkafka/producePeriodically.go index d2c9fcb..53fbc2d 100644 --- a/extkafka/producePeriodically.go +++ b/extkafka/producePeriodically.go @@ -1,6 +1,5 @@ -/* - * Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/producePeriodically_test.go b/extkafka/producePeriodically_test.go index 7487ec7..889188a 100644 --- a/extkafka/producePeriodically_test.go +++ b/extkafka/producePeriodically_test.go @@ -1,6 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/produce_test.go b/extkafka/produce_test.go index 5c218c3..32f48ae 100644 --- a/extkafka/produce_test.go +++ b/extkafka/produce_test.go @@ -1,6 +1,5 @@ -/* -* Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/util.go b/extkafka/util.go index 763ab88..3966979 100644 --- a/extkafka/util.go +++ b/extkafka/util.go @@ -1,6 +1,5 @@ -/* - * Copyright 2024 steadybit GmbH. All rights reserved. - */ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka diff --git a/extkafka/util_test.go b/extkafka/util_test.go index 651ce60..8e5f7e3 100644 --- a/extkafka/util_test.go +++ b/extkafka/util_test.go @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH + package extkafka import ( From 5b384e87460803d3fb713d3053b82014964ec19d Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Wed, 30 Jul 2025 10:56:51 +0200 Subject: [PATCH 2/2] fix: support changing IO and network thread counts --- CHANGELOG.md | 62 ++++++++++++++++++- e2e/integration_test.go | 89 +++++++++++++++++++++++++-- extkafka/alter_num_io_threads.go | 6 +- extkafka/alter_num_network_threads.go | 5 +- extkafka/common.go | 33 +++++++--- 5 files changed, 174 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e36d989..d96c729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,67 @@ # Changelog -## v1.0.5 (next) +## v1.2.4 + +- Support changing IO and network thread count values with huge increments or decrements +- Update dependencies + +## v1.2.3 + +- Bump app version + +## v1.2.2 + +- Bump app version + +## v1.2.1 + +- Add cluster name to topic and consumergroup targets + +## v1.2.0 + +- Add cluster name to broker target attributes +- Better target ID for brokers in case of multiple clusters +- Add min/max validations +- Update dependencies + +## v1.1.1 + +- Make extension-kafka compatible with AWS MSK SCRAM-SHA-512 Auth +- Add TLS for compatibility with SASL_SSL security protocol +- Update to go 1.24 +- Update dependencies + +## v1.1.0 + +- Fix log line for check error +- Change metric colors behavior +- Change name of kafka config for certs + +## v1.0.9 + +- Fix log line for check error +- Fix metric ID for broker check + +## v1.0.8 + +- Add pod and container enrichment + +## v1.0.7 + +- Fix action ID + +## v1.0.6 + +- Add controller information to target attributes +- Add new broker check +- Add TLS connection support +- Update dependencies + +## v1.0.5 -- update dependencies - Use uid instead of name for user statement in Dockerfile +- Fix data race issue +- Update dependencies ## v1.0.0 diff --git a/e2e/integration_test.go b/e2e/integration_test.go index 5fd9ffd..f3c3248 100644 --- a/e2e/integration_test.go +++ b/e2e/integration_test.go @@ -1,15 +1,12 @@ -/* - * Copyright 2024 steadybit GmbH. All rights reserved. - */ - // SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: 2024 Steadybit GmbH +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package e2e import ( "context" "fmt" + "github.com/steadybit/action-kit/go/action_kit_api/v2" "github.com/steadybit/action-kit/go/action_kit_test/e2e" actValidate "github.com/steadybit/action-kit/go/action_kit_test/validate" "github.com/steadybit/discovery-kit/go/discovery_kit_api" @@ -51,6 +48,14 @@ func TestWithMinikube(t *testing.T) { Name: "validate Actions", Test: validateActions, }, + { + Name: "alter num io threads", + Test: testAlterNumIoThreads, + }, + { + Name: "alter num network threads", + Test: testAlterNumNetworkThreads, + }, }) } @@ -59,7 +64,7 @@ func validateDiscovery(t *testing.T, _ *e2e.Minikube, e *e2e.Extension) { } func testDiscovery(t *testing.T, _ *e2e.Minikube, e *e2e.Extension) { - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 120*time.Second) defer cancel() target, err := e2e.PollForTarget(ctx, e, "com.steadybit.extension_kafka.broker", func(target discovery_kit_api.Target) bool { @@ -76,6 +81,78 @@ func validateActions(t *testing.T, _ *e2e.Minikube, e *e2e.Extension) { assert.NoError(t, actValidate.ValidateEndpointReferences("/", e.Client)) } +func testAlterNumIoThreads(t *testing.T, _ *e2e.Minikube, e *e2e.Extension) { + target := &action_kit_api.Target{ + Name: "test_broker", + Attributes: map[string][]string{ + "kafka.broker.node-id": {"0"}, + }, + } + + config := struct { + Duration int `json:"duration"` + IoThreads float32 `json:"io_threads"` + }{ + Duration: 5000, + IoThreads: 1.0, + } + + // Reduce + increaseThreadsAction, err := e.RunAction("com.steadybit.extension_kafka.broker.limit-io-threads", target, config, &action_kit_api.ExecutionContext{}) + require.NoError(t, err) + defer func() { _ = increaseThreadsAction.Cancel() }() + + require.NoError(t, increaseThreadsAction.Wait()) + require.NotEmpty(t, t, increaseThreadsAction.Messages()) + require.NotEmpty(t, t, increaseThreadsAction.Metrics()) + + // Increase + config.IoThreads = 100.0 + decreaseThreadsAction, err := e.RunAction("com.steadybit.extension_kafka.broker.limit-io-threads", target, config, &action_kit_api.ExecutionContext{}) + require.NoError(t, err) + defer func() { _ = decreaseThreadsAction.Cancel() }() + + require.NoError(t, increaseThreadsAction.Wait()) + require.NotEmpty(t, t, decreaseThreadsAction.Messages()) + require.NotEmpty(t, t, decreaseThreadsAction.Metrics()) +} + +func testAlterNumNetworkThreads(t *testing.T, _ *e2e.Minikube, e *e2e.Extension) { + target := &action_kit_api.Target{ + Name: "test_broker", + Attributes: map[string][]string{ + "kafka.broker.node-id": {"0"}, + }, + } + + config := struct { + Duration int `json:"duration"` + NetworkThreads float32 `json:"network_threads"` + }{ + Duration: 5000, + NetworkThreads: 1.0, + } + + // Reduce + increaseThreadsAction, err := e.RunAction("com.steadybit.extension_kafka.broker.limit-network-threads", target, config, &action_kit_api.ExecutionContext{}) + require.NoError(t, err) + defer func() { _ = increaseThreadsAction.Cancel() }() + + require.NoError(t, increaseThreadsAction.Wait()) + require.NotEmpty(t, t, increaseThreadsAction.Messages()) + require.NotEmpty(t, t, increaseThreadsAction.Metrics()) + + // Increase + config.NetworkThreads = 100.0 + decreaseThreadsAction, err := e.RunAction("com.steadybit.extension_kafka.broker.limit-network-threads", target, config, &action_kit_api.ExecutionContext{}) + require.NoError(t, err) + defer func() { _ = decreaseThreadsAction.Cancel() }() + + require.NoError(t, increaseThreadsAction.Wait()) + require.NotEmpty(t, t, decreaseThreadsAction.Messages()) + require.NotEmpty(t, t, decreaseThreadsAction.Metrics()) +} + func helmInstallLocalStack(minikube *e2e.Minikube) error { out, err := exec.Command("helm", "repo", "add", "bitnami", "https://charts.bitnami.com/bitnami").CombinedOutput() if err != nil { diff --git a/extkafka/alter_num_io_threads.go b/extkafka/alter_num_io_threads.go index 9e3048c..9b1b578 100644 --- a/extkafka/alter_num_io_threads.go +++ b/extkafka/alter_num_io_threads.go @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: 2024 Steadybit GmbH +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka @@ -66,11 +66,10 @@ func (k *AlterNumberIOThreadsAttack) Describe() action_kit_api.ActionDescription } } -func (k *AlterNumberIOThreadsAttack) Prepare(_ context.Context, state *AlterState, request action_kit_api.PrepareActionRequestBody) (*action_kit_api.PrepareResult, error) { +func (k *AlterNumberIOThreadsAttack) Prepare(ctx context.Context, state *AlterState, request action_kit_api.PrepareActionRequestBody) (*action_kit_api.PrepareResult, error) { state.BrokerID = extutil.ToInt32(request.Target.Attributes["kafka.broker.node-id"][0]) state.BrokerConfigValue = fmt.Sprintf("%.0f", request.Config["io_threads"]) state.BrokerHosts = strings.Split(config.Config.SeedBrokers, ",") - return nil, nil } @@ -88,6 +87,7 @@ func (k *AlterNumberIOThreadsAttack) Start(ctx context.Context, state *AlterStat if err := adjustThreads(ctx, state.BrokerHosts, NumberIOThreads, targetValue, state.BrokerID); err != nil { return nil, err } + return &action_kit_api.StartResult{ Messages: &[]action_kit_api.Message{{ Level: extutil.Ptr(action_kit_api.Info), diff --git a/extkafka/alter_num_network_threads.go b/extkafka/alter_num_network_threads.go index 6c76242..16d0999 100644 --- a/extkafka/alter_num_network_threads.go +++ b/extkafka/alter_num_network_threads.go @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: 2023 Steadybit GmbH +// SPDX-FileCopyrightText: 2025 Steadybit GmbH package extkafka @@ -66,11 +66,10 @@ func (k *AlterNumberNetworkThreadsAttack) Describe() action_kit_api.ActionDescri } } -func (k *AlterNumberNetworkThreadsAttack) Prepare(_ context.Context, state *AlterState, request action_kit_api.PrepareActionRequestBody) (*action_kit_api.PrepareResult, error) { +func (k *AlterNumberNetworkThreadsAttack) Prepare(ctx context.Context, state *AlterState, request action_kit_api.PrepareActionRequestBody) (*action_kit_api.PrepareResult, error) { state.BrokerID = extutil.ToInt32(request.Target.Attributes["kafka.broker.node-id"][0]) state.BrokerConfigValue = fmt.Sprintf("%.0f", request.Config["network_threads"]) state.BrokerHosts = strings.Split(config.Config.SeedBrokers, ",") - return nil, nil } diff --git a/extkafka/common.go b/extkafka/common.go index fb0a708..18fdb8d 100644 --- a/extkafka/common.go +++ b/extkafka/common.go @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 Steadybit GmbH + package extkafka import ( @@ -186,31 +189,32 @@ func createNewAdminClient(brokers []string) (*kadm.Client, error) { } func describeConfig(ctx context.Context, brokers []string, configName string, brokerID int32) (string, error) { - var initialValue string adminClient, err := createNewAdminClient(brokers) if err != nil { return "", err } - // Get the initial value + return describeConfigOf(ctx, adminClient, configName, brokerID) +} + +func describeConfigOf(ctx context.Context, adminClient *kadm.Client, configName string, brokerID int32) (initialValue string, err error) { configs, err := adminClient.DescribeBrokerConfigs(ctx, brokerID) if err != nil { return "", err } - _, err = configs.On(strconv.FormatInt(int64(brokerID), 10), func(resourceConfig *kadm.ResourceConfig) error { + _, err = configs.On(strconv.FormatInt(int64(brokerID), 10), func(resourceConfig *kadm.ResourceConfig) error { for i := range resourceConfig.Configs { if resourceConfig.Configs[i].Key == configName { initialValue = resourceConfig.Configs[i].MaybeValue() - // Found! break } } - - return err + return nil }) if err != nil { return "", err } + if initialValue == "" { log.Warn().Msgf("No value found for configuration key: %s, for broker node-id: %d", configName, brokerID) } @@ -239,7 +243,22 @@ func alterConfig(ctx context.Context, brokers []string, configName string, confi if len(errs) > 0 { return errors.Join(errs...) } - return nil + + // Changes may take time to be applied, wait accordingly + now := time.Now() + for { + if time.Since(now).Seconds() > 5 { + return fmt.Errorf("configuration change of %s to %s was not applied in time", configName, configValue) + } + value, err := describeConfigOf(ctx, adminClient, configName, brokerID) + if err != nil { + return err + } + if value == configValue { + return nil + } + log.Debug().Msgf("Configuration change of %s to %s was not applied yet, waiting", configName, configValue) + } } func adjustThreads(ctx context.Context, hosts []string, configName string, targetValue int, brokerId int32) error {