From 7ef8fac1aa60ea84453b368ee9d6a5fdb583607c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 14 Jul 2022 11:49:40 +0200 Subject: [PATCH 1/7] Move default runtime validation to its own function Since we want to validate the default runtime separately when reloading the runtime config we factor out default runtime validation into a ValidateDefaultRuntime method. Signed-off-by: Evan Lezar --- pkg/config/config.go | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index ed1eaf2aaac..674c47d1b21 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -963,21 +963,8 @@ func (c *RuntimeConfig) Validate(systemContext *types.SystemContext, onExecution return err } - // check we do have at least a runtime - if _, ok := c.Runtimes[c.DefaultRuntime]; !ok { - // Set the default runtime to "runc" if default_runtime is not set - if c.DefaultRuntime == "" { - logrus.Debugf("Defaulting to %q as the runtime since default_runtime is not set", defaultRuntime) - // The default config sets runc and its path in the runtimes map, so check for that - // first. If it does not exist then we add runc + its path to the runtimes map. - if _, ok := c.Runtimes[defaultRuntime]; !ok { - c.Runtimes[defaultRuntime] = defaultRuntimeHandler() - } - // Set the DefaultRuntime to runc so we don't fail further along in the code - c.DefaultRuntime = defaultRuntime - } else { - return fmt.Errorf("default_runtime set to %q, but no runtime entry table [crio.runtime.runtimes.%s] was found", c.DefaultRuntime, c.DefaultRuntime) - } + if err := c.ValidateDefaultRuntime(); err != nil { + return err } if c.LogSizeMax >= 0 && c.LogSizeMax < OCIBufSize { @@ -1108,6 +1095,27 @@ func (c *RuntimeConfig) Validate(systemContext *types.SystemContext, onExecution return nil } +// ValidateDefaultRuntime ensures that the default runtime is set and valid. +func (c *RuntimeConfig) ValidateDefaultRuntime() error { + // check we do have at least a runtime + if _, ok := c.Runtimes[c.DefaultRuntime]; !ok { + // Set the default runtime to "runc" if default_runtime is not set + if c.DefaultRuntime == "" { + logrus.Debugf("Defaulting to %q as the runtime since default_runtime is not set", defaultRuntime) + // The default config sets runc and its path in the runtimes map, so check for that + // first. If it does not exist then we add runc + its path to the runtimes map. + if _, ok := c.Runtimes[defaultRuntime]; !ok { + c.Runtimes[defaultRuntime] = defaultRuntimeHandler() + } + // Set the DefaultRuntime to runc so we don't fail further along in the code + c.DefaultRuntime = defaultRuntime + } else { + return fmt.Errorf("default_runtime set to %q, but no runtime entry table [crio.runtime.runtimes.%s] was found", c.DefaultRuntime, c.DefaultRuntime) + } + } + return nil +} + func defaultRuntimeHandler() *RuntimeHandler { return &RuntimeHandler{ RuntimeType: DefaultRuntimeType, From f06c012314c8d3530a4b8e371ac0d055817923ad Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 18 Jul 2022 13:28:11 +0200 Subject: [PATCH 2/7] Invert conditional check in ValidateDefaultRuntime This change cleans up the conditional checks in ValidateDefaultRuntime for readability. Signed-off-by: Evan Lezar --- pkg/config/config.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 674c47d1b21..6f954bf9c2d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1097,22 +1097,26 @@ func (c *RuntimeConfig) Validate(systemContext *types.SystemContext, onExecution // ValidateDefaultRuntime ensures that the default runtime is set and valid. func (c *RuntimeConfig) ValidateDefaultRuntime() error { - // check we do have at least a runtime - if _, ok := c.Runtimes[c.DefaultRuntime]; !ok { - // Set the default runtime to "runc" if default_runtime is not set - if c.DefaultRuntime == "" { - logrus.Debugf("Defaulting to %q as the runtime since default_runtime is not set", defaultRuntime) - // The default config sets runc and its path in the runtimes map, so check for that - // first. If it does not exist then we add runc + its path to the runtimes map. - if _, ok := c.Runtimes[defaultRuntime]; !ok { - c.Runtimes[defaultRuntime] = defaultRuntimeHandler() - } - // Set the DefaultRuntime to runc so we don't fail further along in the code - c.DefaultRuntime = defaultRuntime - } else { - return fmt.Errorf("default_runtime set to %q, but no runtime entry table [crio.runtime.runtimes.%s] was found", c.DefaultRuntime, c.DefaultRuntime) - } + // If the default runtime is defined in the runtime entry table, then it is valid + if _, ok := c.Runtimes[c.DefaultRuntime]; ok { + return nil } + + // If a non-empty runtime does not exist in the runtime entry table, this is an error. + if c.DefaultRuntime != "" { + return fmt.Errorf("default_runtime set to %q, but no runtime entry table [crio.runtime.runtimes.%s] was found", c.DefaultRuntime, c.DefaultRuntime) + } + + // Set the default runtime to "runc" if default_runtime is not set + logrus.Debugf("Defaulting to %q as the runtime since default_runtime is not set", defaultRuntime) + // The default config sets runc and its path in the runtimes map, so check for that + // first. If it does not exist then we add runc + its path to the runtimes map. + if _, ok := c.Runtimes[defaultRuntime]; !ok { + c.Runtimes[defaultRuntime] = defaultRuntimeHandler() + } + // Set the DefaultRuntime to runc so we don't fail further along in the code + c.DefaultRuntime = defaultRuntime + return nil } From d51a01ad30bd8d51cf07c41bed06a176c91ca6e6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 14 Jul 2022 11:50:55 +0200 Subject: [PATCH 3/7] Reload runtime configs on reload This change includes the update of runtime configs on a config reload. This is currently limited to: * Changing the default runtime * Adding new runtimes Signed-off-by: Evan Lezar --- pkg/config/reload.go | 36 ++++++++++++++++++ pkg/config/reload_test.go | 80 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/pkg/config/reload.go b/pkg/config/reload.go index 196590bdbaa..2dd7352c7ea 100644 --- a/pkg/config/reload.go +++ b/pkg/config/reload.go @@ -88,6 +88,9 @@ func (c *Config) Reload() error { if err := c.ReloadRdtConfig(newConfig); err != nil { return err } + if err := c.ReloadRuntimes(newConfig); err != nil { + return err + } cdi.GetRegistry(cdi.WithSpecDirs(newConfig.CDISpecDirs...)) return nil @@ -237,3 +240,36 @@ func (c *Config) ReloadRdtConfig(newConfig *Config) error { } return nil } + +// ReloadRuntimes reloads the runtimes configuration if changed +func (c *Config) ReloadRuntimes(newConfig *Config) error { + var updated bool + for runtime := range newConfig.Runtimes { + if _, ok := c.Runtimes[runtime]; ok { + logrus.Warnf("Skipping existing runtime %q", runtime) + continue + } + c.Runtimes[runtime] = newConfig.Runtimes[runtime] + logrus.Infof("Registered new runtime %q", runtime) + updated = true + } + + if c.DefaultRuntime != newConfig.DefaultRuntime { + c.DefaultRuntime = newConfig.DefaultRuntime + if err := c.ValidateDefaultRuntime(); err != nil { + return fmt.Errorf("unable to reload runtimes: %w", err) + } + logConfig("default_runtime", c.DefaultRuntime) + updated = true + } + + if !updated { + return nil + } + + if err := c.ValidateRuntimes(); err != nil { + return fmt.Errorf("unabled to reload runtimes: %w", err) + } + + return nil +} diff --git a/pkg/config/reload_test.go b/pkg/config/reload_test.go index e8e7de16724..e3f48b06f57 100644 --- a/pkg/config/reload_test.go +++ b/pkg/config/reload_test.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/containers/common/pkg/apparmor" + "github.com/cri-o/cri-o/pkg/config" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -325,4 +326,83 @@ var _ = t.Describe("Config", func() { Expect(sut.ApparmorProfile).To(Equal(profile)) }) }) + + t.Describe("ReloadRuntimes", func() { + It("should succeed without any config change", func() { + // Given + // When + err := sut.ReloadRuntimes(sut) + + // Then + Expect(err).To(BeNil()) + }) + + It("should fail for invalid default_runtime", func() { + // Given + newConfig := &config.Config{} + newConfig.DefaultRuntime = "invalid" + + // When + err := sut.ReloadRuntimes(newConfig) + + // Then + Expect(err).NotTo(BeNil()) + }) + + It("should add a new runtime", func() { + // Given + newRuntimeHandler := &config.RuntimeHandler{ + RuntimePath: "/usr/bin/runc", + PrivilegedWithoutHostDevices: true, + } + newConfig := &config.Config{} + newConfig.Runtimes = make(config.Runtimes) + newConfig.Runtimes["new"] = newRuntimeHandler + + // When + err := sut.ReloadRuntimes(newConfig) + + // Then + Expect(err).To(BeNil()) + Expect(sut.Runtimes).To(HaveKeyWithValue("new", newRuntimeHandler)) + }) + + It("should change the default runtime", func() { + // Given + sut.Runtimes["existing"] = &config.RuntimeHandler{ + RuntimePath: "/usr/bin/runc", + } + newConfig := &config.Config{} + newConfig.DefaultRuntime = "existing" + + // When + err := sut.ReloadRuntimes(newConfig) + + // Then + Expect(err).To(BeNil()) + Expect(sut.DefaultRuntime).To(Equal("existing")) + }) + + It("should not add existing runtime", func() { + // Given + existingRuntime := &config.RuntimeHandler{ + RuntimePath: "/usr/bin/runc", + } + sut.Runtimes["existing"] = existingRuntime + newConfig := &config.Config{} + newConfig.Runtimes = make(config.Runtimes) + newConfig.Runtimes["existing"] = &config.RuntimeHandler{ + RuntimePath: "/usr/bin/runc", + PrivilegedWithoutHostDevices: true, + } + + // When + err := sut.ReloadRuntimes(newConfig) + + // Then + Expect(err).To(BeNil()) + Expect(sut.Runtimes).To(HaveKeyWithValue("existing", existingRuntime)) + Expect(sut.Runtimes["existing"].PrivilegedWithoutHostDevices).To(BeFalse()) + }) + }) }) From 7fcef1dbd0b5de1eda4f94c9d05b9f50c0bf058d Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 15 Jul 2022 13:27:56 +0200 Subject: [PATCH 4/7] Add notes on runtime reload support to documentation Signed-off-by: Evan Lezar --- docs/crio.conf.5.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index 42911b7b0a1..d80617c0878 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -101,7 +101,7 @@ The `crio.api` table contains settings for the kubelet/gRPC interface. The `crio.runtime` table contains settings pertaining to the OCI runtime used and options for how to set up and manage the OCI runtime. **default_runtime**="runc" - The _name_ of the OCI runtime to be used as the default. + The _name_ of the OCI runtime to be used as the default. This option supports live configuration reload. **default_ulimits**=[] A list of ulimits to be set in containers by default, specified as "=:", for example:"nofile=1024:2048". If nothing is set here, settings will be inherited from the CRI-O daemon. @@ -291,7 +291,7 @@ the container runtime configuration. Enable CRIU integration, requires that the criu binary is available in $PATH. (default: false) ### CRIO.RUNTIME.RUNTIMES TABLE -The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. The runtime to use is picked based on the runtime handler provided by the CRI. If no runtime handler is provided, the runtime will be picked based on the level of trust of the workload. +The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. The runtime to use is picked based on the runtime handler provided by the CRI. If no runtime handler is provided, the runtime will be picked based on the level of trust of the workload. This option supports live configuration reload. Note that on reload runtimes can only be added to this table but not modified or removed. **runtime_path**="" Path to the OCI compatible runtime used for this runtime handler. From 9dc1a70b4daf7ec8360b6c4dd19733ce254188c7 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 18 Jul 2022 13:11:28 +0200 Subject: [PATCH 5/7] Add basic integration tests for runtime reload Signed-off-by: Evan Lezar --- test/reload_config.bats | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/reload_config.bats b/test/reload_config.bats index 4dac607ad47..a97ff8793b8 100644 --- a/test/reload_config.bats +++ b/test/reload_config.bats @@ -13,6 +13,7 @@ function setup() { } function teardown() { + rm -f "$CRIO_CONFIG_DIR/00-new*Runtime.conf" cleanup_test } @@ -215,3 +216,17 @@ function expect_log_failure() { # then expect_log_failure "unable to reload apparmor_profile" } + +@test "reload config should add new runtime" { + # given + cat << EOF > "$CRIO_CONFIG_DIR/00-newRuntime.conf" +[crio.runtime.runtimes.new] +runtime_path = "$RUNTIME_BINARY_PATH" +EOF + + # when + reload_crio + + #then + wait_for_log '"registered new runtime \\"new\\""' +} From 556d8523191145e65926f59dc4239dad7090e193 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 22 Sep 2022 12:00:06 +0200 Subject: [PATCH 6/7] Allow complete Runtimes config to change Signed-off-by: Evan Lezar --- pkg/config/reload.go | 10 +++------- pkg/config/reload_test.go | 15 +++++++++------ test/reload_config.bats | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/pkg/config/reload.go b/pkg/config/reload.go index 2dd7352c7ea..7520641871f 100644 --- a/pkg/config/reload.go +++ b/pkg/config/reload.go @@ -244,13 +244,9 @@ func (c *Config) ReloadRdtConfig(newConfig *Config) error { // ReloadRuntimes reloads the runtimes configuration if changed func (c *Config) ReloadRuntimes(newConfig *Config) error { var updated bool - for runtime := range newConfig.Runtimes { - if _, ok := c.Runtimes[runtime]; ok { - logrus.Warnf("Skipping existing runtime %q", runtime) - continue - } - c.Runtimes[runtime] = newConfig.Runtimes[runtime] - logrus.Infof("Registered new runtime %q", runtime) + if !RuntimesEqual(c.Runtimes, newConfig.Runtimes) { + logrus.Infof("Updating runtime configuration") + c.Runtimes = newConfig.Runtimes updated = true } diff --git a/pkg/config/reload_test.go b/pkg/config/reload_test.go index e3f48b06f57..292be07b5bb 100644 --- a/pkg/config/reload_test.go +++ b/pkg/config/reload_test.go @@ -373,6 +373,7 @@ var _ = t.Describe("Config", func() { RuntimePath: "/usr/bin/runc", } newConfig := &config.Config{} + newConfig.Runtimes = sut.Runtimes newConfig.DefaultRuntime = "existing" // When @@ -383,26 +384,28 @@ var _ = t.Describe("Config", func() { Expect(sut.DefaultRuntime).To(Equal("existing")) }) - It("should not add existing runtime", func() { + It("should overwrite existing runtime", func() { // Given existingRuntime := &config.RuntimeHandler{ RuntimePath: "/usr/bin/runc", } sut.Runtimes["existing"] = existingRuntime - newConfig := &config.Config{} - newConfig.Runtimes = make(config.Runtimes) - newConfig.Runtimes["existing"] = &config.RuntimeHandler{ + + newRuntime := &config.RuntimeHandler{ RuntimePath: "/usr/bin/runc", PrivilegedWithoutHostDevices: true, } + newConfig := &config.Config{} + newConfig.Runtimes = make(config.Runtimes) + newConfig.Runtimes["existing"] = newRuntime // When err := sut.ReloadRuntimes(newConfig) // Then Expect(err).To(BeNil()) - Expect(sut.Runtimes).To(HaveKeyWithValue("existing", existingRuntime)) - Expect(sut.Runtimes["existing"].PrivilegedWithoutHostDevices).To(BeFalse()) + Expect(sut.Runtimes).To(HaveKeyWithValue("existing", newRuntime)) + Expect(sut.Runtimes["existing"].PrivilegedWithoutHostDevices).To(BeTrue()) }) }) }) diff --git a/test/reload_config.bats b/test/reload_config.bats index a97ff8793b8..ea360b194e7 100644 --- a/test/reload_config.bats +++ b/test/reload_config.bats @@ -228,5 +228,5 @@ EOF reload_crio #then - wait_for_log '"registered new runtime \\"new\\""' + wait_for_log '"updating runtime configuration"' } From aa329a1e3d8135f3ea32096a604051826fd884bb Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 22 Sep 2022 16:16:48 +0200 Subject: [PATCH 7/7] Update config README Signed-off-by: Evan Lezar --- docs/crio.conf.5.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index d80617c0878..433bf71c50d 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -291,7 +291,7 @@ the container runtime configuration. Enable CRIU integration, requires that the criu binary is available in $PATH. (default: false) ### CRIO.RUNTIME.RUNTIMES TABLE -The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. The runtime to use is picked based on the runtime handler provided by the CRI. If no runtime handler is provided, the runtime will be picked based on the level of trust of the workload. This option supports live configuration reload. Note that on reload runtimes can only be added to this table but not modified or removed. +The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. The runtime to use is picked based on the runtime handler provided by the CRI. If no runtime handler is provided, the runtime will be picked based on the level of trust of the workload. This option supports live configuration reload. This option supports live configuration reload. **runtime_path**="" Path to the OCI compatible runtime used for this runtime handler.