diff --git a/completions/bash/crio b/completions/bash/crio index d2cfbbc18b3..69a365cef28 100755 --- a/completions/bash/crio +++ b/completions/bash/crio @@ -61,6 +61,7 @@ h --insecure-registry --internal-wipe --irqbalance-config-file +--irqbalance-config-restore-file --listen --log --log-dir diff --git a/completions/fish/crio.fish b/completions/fish/crio.fish index 36d394ce23a..00508f3df91 100644 --- a/completions/fish/crio.fish +++ b/completions/fish/crio.fish @@ -98,6 +98,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l insecure-registry -r -d 'E \'--insecure-registry\'.' complete -c crio -n '__fish_crio_no_subcommand' -f -l internal-wipe -d 'Whether CRI-O should wipe containers after a reboot and images after an upgrade when the server starts. If set to false, one must run `crio wipe` to wipe the containers and images in these situations. This option is deprecated, and will be removed in the future.' complete -c crio -n '__fish_crio_no_subcommand' -f -l irqbalance-config-file -r -d 'The irqbalance service config file which is used by CRI-O.' +complete -c crio -n '__fish_crio_no_subcommand' -f -l irqbalance-config-restore-file -r -d 'Determines if CRI-O should attempt to restore the irqbalance config at startup with the mask in this file. Use the 'disable' value to disable the restore flow entirely.' complete -c crio -n '__fish_crio_no_subcommand' -l listen -r -d 'Path to the CRI-O socket.' complete -c crio -n '__fish_crio_no_subcommand' -l log -r -d 'Set the log file path where internal debug information is written.' complete -c crio -n '__fish_crio_no_subcommand' -l log-dir -r -d 'Default log directory where all logs will go unless directly specified by the kubelet.' diff --git a/completions/zsh/_crio b/completions/zsh/_crio index 23ab91ab7c7..26d6f42567a 100644 --- a/completions/zsh/_crio +++ b/completions/zsh/_crio @@ -68,6 +68,7 @@ it later with **--config**. Global options will modify the output.' '--insecure-registry' '--internal-wipe' '--irqbalance-config-file' + '--irqbalance-config-restore-file' '--listen' '--log' '--log-dir' diff --git a/docs/crio.8.md b/docs/crio.8.md index 7f5cda127dc..9552d827ebc 100644 --- a/docs/crio.8.md +++ b/docs/crio.8.md @@ -60,6 +60,7 @@ crio [--insecure-registry]=[value] [--internal-wipe] [--irqbalance-config-file]=[value] +[--irqbalance-config-restore-file]=[value] [--listen]=[value] [--log-dir]=[value] [--log-filter]=[value] @@ -281,6 +282,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...] **--irqbalance-config-file**="": The irqbalance service config file which is used by CRI-O. (default: /etc/sysconfig/irqbalance) +**--irqbalance-config-restore-file**="": Determines if CRI-O should attempt to restore the irqbalance config at startup with the mask in this file. Use the 'disable' value to disable the restore flow entirely. (default: /etc/sysconfig/orig_irq_banned_cpus) + **--listen**="": Path to the CRI-O socket. (default: /var/run/crio/crio.sock) **--log**="": Set the log file path where internal debug information is written. diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index bd97b2d5f16..c5d95b319d6 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -159,6 +159,9 @@ the container runtime configuration. Used to change irqbalance service config file which is used by CRI-O. For CentOS/SUSE, this file is located at /etc/sysconfig/irqbalance. For Ubuntu, this file is located at /etc/default/irqbalance. +**irqbalance_config_restore_file**="/etc/sysconfig/orig_irq_banned_cpus" + Used to set the irqbalance banned cpu mask to restore at CRI-O startup. If set to 'disable', no restoration attempt will be done. + **rdt_config_file**="" Path to the RDT configuration file for configuring the resctrl pseudo-filesystem. diff --git a/internal/criocli/criocli.go b/internal/criocli/criocli.go index 464eee06fe4..02b9dba142e 100644 --- a/internal/criocli/criocli.go +++ b/internal/criocli/criocli.go @@ -332,6 +332,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error { if ctx.IsSet("absent-mount-sources-to-reject") { config.AbsentMountSourcesToReject = StringSliceTrySplit(ctx, "absent-mount-sources-to-reject") } + if ctx.IsSet("irqbalance-config-restore-file") { + config.IrqBalanceConfigRestoreFile = ctx.String("irqbalance-config-restore-file") + } if ctx.IsSet("internal-wipe") { config.InternalWipe = ctx.Bool("internal-wipe") } @@ -1083,6 +1086,11 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag { Usage: "If true, CRI-O starts sending the container events to the kubelet", EnvVars: []string{"ENABLE_POD_EVENTS"}, }, + &cli.StringFlag{ + Name: "irqbalance-config-restore-file", + Value: defConf.IrqBalanceConfigRestoreFile, + Usage: "Determines if CRI-O should attempt to restore the irqbalance config at startup with the mask in this file. Use the 'disable' value to disable the restore flow entirely.", + }, } } diff --git a/internal/runtimehandlerhooks/high_performance_hooks.go b/internal/runtimehandlerhooks/high_performance_hooks.go index f4945b0082a..2da5a66091a 100644 --- a/internal/runtimehandlerhooks/high_performance_hooks.go +++ b/internal/runtimehandlerhooks/high_performance_hooks.go @@ -20,7 +20,6 @@ import ( "github.com/cri-o/cri-o/utils/cmdrunner" "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/cgroups/systemd" - "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -29,8 +28,6 @@ import ( const ( // HighPerformance contains the high-performance runtime handler name HighPerformance = "high-performance" - // IrqBannedCPUConfigFile contains the original banned cpu mask configuration - IrqBannedCPUConfigFile = "/etc/sysconfig/orig_irq_banned_cpus" // IrqSmpAffinityProcFile contains the default smp affinity mask configuration IrqSmpAffinityProcFile = "/proc/irq/default_smp_affinity" ) @@ -78,7 +75,7 @@ func (h *HighPerformanceHooks) PreStart(ctx context.Context, c *oci.Container, s // disable the IRQ smp load balancing for the container CPUs if shouldIRQLoadBalancingBeDisabled(s.Annotations()) { log.Infof(ctx, "Disable irq smp balancing for container %q", c.ID()) - if err := setIRQLoadBalancing(c, false, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil { + if err := setIRQLoadBalancing(ctx, c, false, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil { return fmt.Errorf("set IRQ load balancing: %w", err) } } @@ -153,7 +150,7 @@ func (h *HighPerformanceHooks) PreStop(ctx context.Context, c *oci.Container, s // enable the IRQ smp balancing for the container CPUs if shouldIRQLoadBalancingBeDisabled(s.Annotations()) { - if err := setIRQLoadBalancing(c, true, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil { + if err := setIRQLoadBalancing(ctx, c, true, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil { return fmt.Errorf("set IRQ load balancing: %w", err) } } @@ -308,7 +305,7 @@ func setCPUSLoadBalancing(c *oci.Container, enable bool, schedDomainDir string) return nil } -func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqBalanceConfigFile string) error { +func setIRQLoadBalancing(ctx context.Context, c *oci.Container, enable bool, irqSmpAffinityFile, irqBalanceConfigFile string) error { lspec := c.Spec().Linux if lspec == nil || lspec.Resources == nil || @@ -341,7 +338,7 @@ func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqB if !isServiceEnabled(irqBalancedName) || !isIrqConfigExists { if _, err := exec.LookPath(irqBalancedName); err != nil { // irqbalance is not installed, skip the rest; pod should still start, so return nil instead - logrus.Warnf("Irqbalance binary not found: %v", err) + log.Warnf(ctx, "Irqbalance binary not found: %v", err) return nil } // run irqbalance in daemon mode, so this won't cause delay @@ -352,7 +349,7 @@ func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqB } if err := restartIrqBalanceService(); err != nil { - logrus.Warnf("Irqbalance service restart failed: %v", err) + log.Warnf(ctx, "Irqbalance service restart failed: %v", err) } return nil } @@ -600,7 +597,7 @@ func doSetCPUFreqGovernor(c *oci.Container, governor, cpuDir, cpuSaveDir string) } // RestoreIrqBalanceConfig restores irqbalance service with original banned cpu mask settings -func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityProcFile string) error { +func RestoreIrqBalanceConfig(ctx context.Context, irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityProcFile string) error { content, err := os.ReadFile(irqSmpAffinityProcFile) if err != nil { return err @@ -614,15 +611,19 @@ func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSm } if !isAllBitSet(currentMaskArray) { // not system reboot scenario, just return it. + log.Infof(ctx, "Restore irqbalance config: not system reboot, ignoring") return nil } bannedCPUMasks, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile) if err != nil { // Ignore returning err as given irqBalanceConfigFile may not exist. + log.Infof(ctx, "Restore irqbalance config: failed to get current CPU ban list, ignoring") return nil } + if !fileExists(irqBannedCPUConfigFile) { + log.Infof(ctx, "Creating banned CPU list file %q", irqBannedCPUConfigFile) irqBannedCPUsConfig, err := os.Create(irqBannedCPUConfigFile) if err != nil { return err @@ -632,6 +633,7 @@ func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSm if err != nil { return err } + log.Infof(ctx, "Restore irqbalance config: created backup file") return nil } @@ -642,14 +644,17 @@ func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSm origBannedCPUMasks := strings.TrimSpace(string(content)) if bannedCPUMasks == origBannedCPUMasks { + log.Infof(ctx, "Restore irqbalance config: nothing to do") return nil } + + log.Infof(ctx, "Restore irqbalance banned CPU list in %q to %q", irqBalanceConfigFile, origBannedCPUMasks) if err := updateIrqBalanceConfigFile(irqBalanceConfigFile, origBannedCPUMasks); err != nil { return err } if isServiceEnabled(irqBalancedName) { if err := restartIrqBalanceService(); err != nil { - logrus.Warnf("Irqbalance service restart failed: %v", err) + log.Warnf(ctx, "Irqbalance service restart failed: %v", err) } } return nil diff --git a/internal/runtimehandlerhooks/high_performance_hooks_test.go b/internal/runtimehandlerhooks/high_performance_hooks_test.go index fbf7e6bfcad..7c76043603c 100644 --- a/internal/runtimehandlerhooks/high_performance_hooks_test.go +++ b/internal/runtimehandlerhooks/high_performance_hooks_test.go @@ -111,7 +111,7 @@ var _ = Describe("high_performance_hooks", func() { irqSmpAffinityFile := filepath.Join(fixturesDir, "irq_smp_affinity") irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance") verifySetIRQLoadBalancing := func(enabled bool, expected string) { - err := setIRQLoadBalancing(container, enabled, irqSmpAffinityFile, irqBalanceConfigFile) + err := setIRQLoadBalancing(context.TODO(), container, enabled, irqSmpAffinityFile, irqBalanceConfigFile) Expect(err).To(BeNil()) content, err := os.ReadFile(irqSmpAffinityFile) @@ -164,7 +164,7 @@ var _ = Describe("high_performance_hooks", func() { irqSmpAffinityFile := filepath.Join(fixturesDir, "irq_smp_affinity") irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance") verifySetIRQLoadBalancing := func(enabled bool, expectedSmp, expectedBan string) { - err = setIRQLoadBalancing(container, enabled, irqSmpAffinityFile, irqBalanceConfigFile) + err = setIRQLoadBalancing(context.TODO(), container, enabled, irqSmpAffinityFile, irqBalanceConfigFile) Expect(err).To(BeNil()) content, err := os.ReadFile(irqSmpAffinityFile) @@ -511,16 +511,16 @@ var _ = Describe("high_performance_hooks", func() { irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance") irqBannedCPUConfigFile := filepath.Join(fixturesDir, "orig_irq_banned_cpus") verifyRestoreIrqBalanceConfig := func(expectedOrigBannedCPUs, expectedBannedCPUs string) { - err = RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityFile) - Expect(err).To(BeNil()) + err = RestoreIrqBalanceConfig(context.TODO(), irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityFile) + ExpectWithOffset(1, err).To(BeNil()) content, err := os.ReadFile(irqBannedCPUConfigFile) - Expect(err).To(BeNil()) - Expect(strings.Trim(string(content), "\n")).To(Equal(expectedOrigBannedCPUs)) + ExpectWithOffset(1, err).To(BeNil()) + ExpectWithOffset(1, strings.Trim(string(content), "\n")).To(Equal(expectedOrigBannedCPUs)) bannedCPUs, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile) - Expect(err).To(BeNil()) - Expect(bannedCPUs).To(Equal(expectedBannedCPUs)) + ExpectWithOffset(1, err).To(BeNil()) + ExpectWithOffset(1, bannedCPUs).To(Equal(expectedBannedCPUs)) } JustBeforeEach(func() { diff --git a/pkg/config/config.go b/pkg/config/config.go index 4cd60284536..ef63ab2afda 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -124,6 +124,8 @@ const ( const ( // DefaultIrqBalanceConfigFile default irqbalance service configuration file path DefaultIrqBalanceConfigFile = "/etc/sysconfig/irqbalance" + // DefaultIrqBalanceConfigRestoreFile contains the banned cpu mask configuration to restore. Name due to backward compatibility. + DefaultIrqBalanceConfigRestoreFile = "/etc/sysconfig/orig_irq_banned_cpus" ) // This structure is necessary to fake the TOML tables when parsing, @@ -410,6 +412,10 @@ type RuntimeConfig struct { // EnablePodEvents specifies if the container pod-level events should be generated to optimize the PLEG at Kubelet. EnablePodEvents bool `toml:"enable_pod_events"` + // IrqBalanceConfigRestoreFile is the irqbalance service banned CPU list to restore. + // If empty, no restoration attempt will be done. + IrqBalanceConfigRestoreFile string `toml:"irqbalance_config_restore_file"` + // seccompConfig is the internal seccomp configuration seccompConfig *seccomp.Config @@ -792,34 +798,35 @@ func DefaultConfig() (*Config, error) { Runtimes: Runtimes{ defaultRuntime: defaultRuntimeHandler(), }, - SELinux: selinuxEnabled(), - ApparmorProfile: apparmor.DefaultProfile, - BlockIOConfigFile: DefaultBlockIOConfigFile, - IrqBalanceConfigFile: DefaultIrqBalanceConfigFile, - RdtConfigFile: rdt.DefaultRdtConfigFile, - CgroupManagerName: cgroupManager.Name(), - PidsLimit: DefaultPidsLimit, - ContainerExitsDir: containerExitsDir, - ContainerAttachSocketDir: conmonconfig.ContainerAttachSocketDir, - MinimumMappableUID: -1, - MinimumMappableGID: -1, - LogSizeMax: DefaultLogSizeMax, - CtrStopTimeout: defaultCtrStopTimeout, - DefaultCapabilities: capabilities.Default(), - LogLevel: "info", - HooksDir: []string{hooks.DefaultDir}, - CDISpecDirs: cdi.DefaultSpecDirs, - NamespacesDir: defaultNamespacesDir, - DropInfraCtr: true, - SeccompUseDefaultWhenEmpty: seccompConfig.UseDefaultWhenEmpty(), - seccompConfig: seccomp.New(), - apparmorConfig: apparmor.New(), - blockioConfig: blockio.New(), - cgroupManager: cgroupManager, - deviceConfig: device.New(), - namespaceManager: nsmgr.New(defaultNamespacesDir, ""), - rdtConfig: rdt.New(), - ulimitsConfig: ulimits.New(), + SELinux: selinuxEnabled(), + ApparmorProfile: apparmor.DefaultProfile, + BlockIOConfigFile: DefaultBlockIOConfigFile, + IrqBalanceConfigFile: DefaultIrqBalanceConfigFile, + RdtConfigFile: rdt.DefaultRdtConfigFile, + CgroupManagerName: cgroupManager.Name(), + PidsLimit: DefaultPidsLimit, + ContainerExitsDir: containerExitsDir, + ContainerAttachSocketDir: conmonconfig.ContainerAttachSocketDir, + MinimumMappableUID: -1, + MinimumMappableGID: -1, + LogSizeMax: DefaultLogSizeMax, + CtrStopTimeout: defaultCtrStopTimeout, + DefaultCapabilities: capabilities.Default(), + LogLevel: "info", + HooksDir: []string{hooks.DefaultDir}, + CDISpecDirs: cdi.DefaultSpecDirs, + NamespacesDir: defaultNamespacesDir, + DropInfraCtr: true, + SeccompUseDefaultWhenEmpty: seccompConfig.UseDefaultWhenEmpty(), + IrqBalanceConfigRestoreFile: DefaultIrqBalanceConfigRestoreFile, + seccompConfig: seccomp.New(), + apparmorConfig: apparmor.New(), + blockioConfig: blockio.New(), + cgroupManager: cgroupManager, + deviceConfig: device.New(), + namespaceManager: nsmgr.New(defaultNamespacesDir, ""), + rdtConfig: rdt.New(), + ulimitsConfig: ulimits.New(), }, ImageConfig: ImageConfig{ DefaultTransport: "docker://", diff --git a/pkg/config/template.go b/pkg/config/template.go index 9d08cc7a21d..a6d699adfd4 100644 --- a/pkg/config/template.go +++ b/pkg/config/template.go @@ -265,6 +265,11 @@ func initCrioTemplateConfig(c *Config) ([]*templateConfigValue, error) { group: crioRuntimeConfig, isDefaultValue: simpleEqual(dc.IrqBalanceConfigFile, c.IrqBalanceConfigFile), }, + { + templateString: templateStringCrioRuntimeIrqBalanceConfigRestoreFile, + group: crioRuntimeConfig, + isDefaultValue: simpleEqual(dc.IrqBalanceConfigRestoreFile, c.IrqBalanceConfigRestoreFile), + }, { templateString: templateStringCrioRuntimeRdtConfigFile, group: crioRuntimeConfig, @@ -1066,6 +1071,13 @@ const templateStringCrioRuntimeDropInfraCtr = `# drop_infra_ctr determines wheth ` +const templateStringCrioRuntimeIrqBalanceConfigRestoreFile = `# irqbalance_config_restore_file allows to set a cpu mask CRI-O should +# restore as irqbalance config at startup. Set to empty string to disable this flow entirely. +# By default, CRI-O manages the irqbalance configuration to enable dynamic IRQ pinning. +{{ $.Comment }}irqbalance_config_restore_file = "{{ .IrqBalanceConfigRestoreFile }}" + +` + const templateStringCrioRuntimeInfraCtrCpuset = `# infra_ctr_cpuset determines what CPUs will be used to run infra containers. # You can use linux CPU list format to specify desired CPUs. # To get better isolation for guaranteed pods, set this parameter to be equal to kubelet reserved-cpus. diff --git a/server/server.go b/server/server.go index 7c789848345..b3f5f3411d4 100644 --- a/server/server.go +++ b/server/server.go @@ -49,8 +49,9 @@ import ( ) const ( - certRefreshInterval = time.Minute * 5 - rootlessEnvName = "_CRIO_ROOTLESS" + certRefreshInterval = time.Minute * 5 + rootlessEnvName = "_CRIO_ROOTLESS" + irqBalanceConfigRestoreDisable = "disable" ) var errSandboxNotCreated = errors.New("sandbox not created") @@ -419,9 +420,12 @@ func New( return nil, err } - err = runtimehandlerhooks.RestoreIrqBalanceConfig(config.IrqBalanceConfigFile, runtimehandlerhooks.IrqBannedCPUConfigFile, runtimehandlerhooks.IrqSmpAffinityProcFile) - if err != nil { - return nil, err + if strings.ToLower(strings.TrimSpace(config.IrqBalanceConfigRestoreFile)) != irqBalanceConfigRestoreDisable { + log.Infof(ctx, "Attempting to restore irqbalance config from %s", config.IrqBalanceConfigRestoreFile) + err = runtimehandlerhooks.RestoreIrqBalanceConfig(context.TODO(), config.IrqBalanceConfigFile, config.IrqBalanceConfigRestoreFile, runtimehandlerhooks.IrqSmpAffinityProcFile) + if err != nil { + return nil, err + } } hostportManager := hostport.NewMetaHostportManager() diff --git a/test/irqbalance.bats b/test/irqbalance.bats new file mode 100644 index 00000000000..511b73b73bb --- /dev/null +++ b/test/irqbalance.bats @@ -0,0 +1,250 @@ +#!/usr/bin/env bats +# vim: set syntax=sh: + +load helpers + +function detect_irqbalance_config() { + # debian/ubuntu + [ -f /etc/default/irqbalance ] && echo "/etc/default/irqbalance" + # fedora/centos/RHEL + [ -f /etc/sysconfig/irqbalance ] && echo "/etc/sysconfig/irqbalance" + # default + echo "" +} + +function setup_file() { + if ! command -v irqbalance > /dev/null; then + skip "irqbalance not found." + fi + IRQBALANCE_CONF=$(detect_irqbalance_config) + if [ -z "$IRQBALANCE_CONF" ]; then + echo "error: unable to find irqbalance config file" + return 1 + fi + CONFIGLET="$CRIO_CONFIG_DIR/99-irqbalance.conf" + + mkdir -p "/etc/sysconfig" + BANNEDCPUS_CONF="/etc/sysconfig/orig_irq_banned_cpus" + + export IRQBALANCE_CONF + export CONFIGLET + export BANNEDCPUS_CONF + + export BATS_NO_PARALLELIZE_WITHIN_FILE=true +} + +function setup() { + setup_test + stop_crio + # we don't unconditionally restore because the irqbalance package may be missing + # from the test system. if this is the case, one less thing to worry about, we can't + # pollute the system state anyway :) + if [ -f "$IRQBALANCE_CONF" ]; then + cp -v "$IRQBALANCE_CONF" "$IRQBALANCE_CONF".bkp + else + # placeholder to make the rest of the suite simpler. + touch "$IRQBALANCE_CONF" + touch /tmp/.test_owns_irqbalance_conf + fi + if [ -f "$BANNEDCPUS_CONF" ]; then + cp -v "$BANNEDCPUS_CONF" "$BANNEDCPUS_CONF".bkp + else + # empty file is fine and meaningful + touch "$BANNEDCPUS_CONF" + touch /tmp/.test_owns_bannedcpus_conf + fi +} + +function teardown() { + cleanup_test + stop_crio + # see setup about why we have these conditionals + if [ -f "$IRQBALANCE_CONF".bkp ]; then + mv -v "$IRQBALANCE_CONF".bkp "$IRQBALANCE_CONF" + elif [ -f /tmp/.test_owns_irqbalance_conf ]; then + rm -f "$IRQBALANCE_CONF" + rm -f /tmp/.test_owns_irqbalance_conf + fi + + if [ -f "$BANNEDCPUS_CONF".bkp ]; then + mv -v "$BANNEDCPUS_CONF".bkp "$BANNEDCPUS_CONF" + elif [ -f /tmp/.test_owns_bannedcpus_conf ]; then + rm -f "$BANNEDCPUS_CONF" + rm -f /tmp/.test_owns_bannedcpus_conf + fi +} + +# given +# there is no previous status of banned cpus +# when +# cri-o starts using the proper irqbalance service configuration file +# then +# we expect cri-o to save the irqbalance banned cpus mask in a file +# pointed by the "$BANNEDCPUS_CONF" env var +# and the mask must have value stated in "IRQBALANCE_BANNED_CPUS" field +# from irqbalance service configuration file. +@test "irqbalance cpu ban list save" { + # given + if ! grep -Eq '^[1,3,7,f]{1,}$' /proc/irq/default_smp_affinity; then + skip "requires default IRQ smp affinity (not banned CPUs)" + fi + [ -f "$CONFIGLET" ] && rm -f "$CONFIGLET" + + [ -f "$BANNEDCPUS_CONF" ] && rm -f "$BANNEDCPUS_CONF" + + local expected_banned_cpus + expected_banned_cpus=$(sed -n 's/^IRQBALANCE_BANNED_CPUS=\"\?\([^\"]*\)\"\?/\1/p' "$IRQBALANCE_CONF") + + # when + OVERRIDE_OPTIONS="--irqbalance-config-file ${IRQBALANCE_CONF}" start_crio + + # then + if [ ! -f "$BANNEDCPUS_CONF" ]; then + echo "error: ${BANNEDCPUS_CONF} file should have been created by CRI-o" + return 2 + fi + + local banned_cpus + banned_cpus=$(cat "$BANNEDCPUS_CONF") + + [ "$expected_banned_cpus" == "$banned_cpus" ] +} + +# given +# there is a previous status of banned cpus saved by cri-o +# when +# cri-o starts using the proper irqbalance service configuration file +# then +# we expect cri-o to read the irqbalance banned cpus mask from a file +# pointed by the "$BANNEDCPUS_CONF" env var +# and save the mask value in "IRQBALANCE_BANNED_CPUS" field +# of irqbalance service configuration file. +@test "irqbalance cpu ban list restore - default" { + # given + if ! grep -Eq '^[1,3,7,f]{1,}$' /proc/irq/default_smp_affinity; then + skip "requires default IRQ smp affinity (not banned CPUs)" + fi + [ -f "$CONFIGLET" ] && rm -f "$CONFIGLET" + + echo "IRQBALANCE_BANNED_CPUS=\"0\"" > "$IRQBALANCE_CONF" + + local banned_cpus_for_conf + banned_cpus_for_conf=$(cat /proc/irq/default_smp_affinity) + echo "$banned_cpus_for_conf" > "$BANNEDCPUS_CONF" + + # when + OVERRIDE_OPTIONS="--irqbalance-config-file ${IRQBALANCE_CONF}" start_crio + + # then + local banned_cpus + banned_cpus=$(sed -n 's/^IRQBALANCE_BANNED_CPUS=\"\?\([^\"]*\)\"\?/\1/p' "$IRQBALANCE_CONF") + + [ "$banned_cpus_for_conf" == "$banned_cpus" ] +} + +# restore option disabled. Check it does not disturb previous behaviour. +# given +# there is no previous status of banned cpus saved by cri-o +# when +# cri-o starts using the proper irqbalance service configuration file +# and we explictly disable the restore file option +# then +# restore option does not disturb cri-o behaviour +# so it reads banned cpus mask from "IRQBALANCE_BANNED_CPUS" field +# and save it in a file pointer by "BANNEDCPUS_CONF" env var +@test "irqbalance cpu ban list restore - disable and file missing" { + # given + if ! grep -Eq '^[1,3,7,f]{1,}$' /proc/irq/default_smp_affinity; then + skip "requires default IRQ smp affinity (not banned CPUs)" + fi + [ -f "$CONFIGLET" ] && rm -f "$CONFIGLET" + + local expected_banned_cpus + expected_banned_cpus=$(sed -n 's/^IRQBALANCE_BANNED_CPUS=\"\?\([^\"]*\)\"\?/\1/p' "$IRQBALANCE_CONF") + + [ -f "$BANNEDCPUS_CONF" ] && rm -f "$BANNEDCPUS_CONF" + + # when + OVERRIDE_OPTIONS="--irqbalance-config-file ${IRQBALANCE_CONF} --irqbalance-config-restore-file disable" start_crio + + # then + local banned_cpus + banned_cpus=$(sed -n 's/^IRQBALANCE_BANNED_CPUS=\"\?\([^\"]*\)\"\?/\1/p' "$IRQBALANCE_CONF") + + [ "$expected_banned_cpus" == "$banned_cpus" ] && [ ! -f "$BANNEDCPUS_CONF" ] +} + +# restore option disabled. Check it does not disturb previous behaviour. +# given +# there is a previous status of banned cpus saved by cri-o in a file +# pointer by "BANNEDCPUS_CONF" env var +# when +# cri-o starts using the proper irqbalance service configuration file +# and we explictly disable the restore file option +# then +# restore option does not disturb cri-o behaviour +# so cri-o reads banned cpus mask from "IRQBALANCE_BANNED_CPUS" field +# and save it in a file pointer by "BANNEDCPUS_CONF" env var +@test "irqbalance cpu ban list restore - disable" { + # given + if ! grep -Eq '^[1,3,7,f]{1,}$' /proc/irq/default_smp_affinity; then + skip "requires default IRQ smp affinity (not banned CPUs)" + fi + [ -f "$CONFIGLET" ] && rm -f "$CONFIGLET" + + local expected_banned_cpus + expected_banned_cpus=$(sed -n 's/^IRQBALANCE_BANNED_CPUS=\"\?\([^\"]*\)\"\?/\1/p' "$IRQBALANCE_CONF") + + local banned_cpus_for_conf + banned_cpus_for_conf=$(cat /proc/irq/default_smp_affinity) + echo "$banned_cpus_for_conf" > "$BANNEDCPUS_CONF" + + # when + OVERRIDE_OPTIONS="--irqbalance-config-file ${IRQBALANCE_CONF} --irqbalance-config-restore-file disable" start_crio + + # then + local banned_cpus + banned_cpus=$(sed -n 's/^IRQBALANCE_BANNED_CPUS=\"\?\([^\"]*\)\"\?/\1/p' "$IRQBALANCE_CONF") + + [ "$expected_banned_cpus" == "$banned_cpus" ] +} + +# explicit restore file. Check it uses it instead of any other file. +# given +# there is no previous status of banned cpus saved by cri-o +# when +# cri-o starts using the proper irqbalance service configuration file +# and the restore file option pointing to an existing file. +# then +# cri-o should read banned cpus mask from restore file +# and save it in "IRQBALANCE_BANNED_CPUS" field. +@test "irqbalance cpu ban list restore - explicit file" { + # given + if ! grep -Eq '^[1,3,7,f]{1,}$' /proc/irq/default_smp_affinity; then + skip "requires default IRQ smp affinity (not banned CPUs)" + fi + [ -f "$CONFIGLET" ] && rm -f "$CONFIGLET" + + [ -f "$BANNEDCPUS_CONF" ] && rm -f "$BANNEDCPUS_CONF" + + echo "IRQBALANCE_BANNED_CPUS=\"0\"" > "$IRQBALANCE_CONF" + + local irqbalance_restore_file + irqbalance_restore_file="$(mktemp /tmp/irq-restore.XXXXXXXXX)" + + cat /proc/irq/default_smp_affinity > "$irqbalance_restore_file" + + local banned_cpus_for_restore + banned_cpus_for_restore=$(cat "$irqbalance_restore_file") + + # when + OVERRIDE_OPTIONS="--irqbalance-config-file ${IRQBALANCE_CONF} --irqbalance-config-restore-file ${irqbalance_restore_file}" start_crio + + # then + local banned_cpus + banned_cpus=$(sed -n 's/^IRQBALANCE_BANNED_CPUS=\"\?\([^\"]*\)\"\?/\1/p' "$IRQBALANCE_CONF") + + # when a explicit file is used to restore default one is completely ignored, + # and so, it should not be created.( as it did not existed before) + [ "$banned_cpus_for_restore" == "$banned_cpus" ] && [ ! -f "$BANNEDCPUS_CONF" ] +}