cri-o · ffromani · Jul 29, 2022 · Jul 29, 2022 · Jul 29, 2022 · Aug 2, 2022
@@ -57,6 +57,7 @@ h
 --insecure-registry
 --internal-wipe
 --irqbalance-config-file
+--irqbalance-config-restore-file
 --listen
 --log
 --log-dir

@@ -94,6 +94,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l insecure-registry -r -d 'E
        \'--insecure-registry\'.'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l internal-wipe -d 'Whether CRI-O should wipe containers after a reboot and images after an upgrade when the server starts. If set to false, one must run `crio wipe` to wipe the containers and images in these situations. This option is deprecated, and will be removed in the future.'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l irqbalance-config-file -r -d 'The irqbalance service config file which is used by CRI-O.'
+complete -c crio -n '__fish_crio_no_subcommand' -f -l irqbalance-config-restore-file -r -d 'Determines if CRI-O should attempt to restore the irqbalance config at startup with the mask in this file. Use empty value to disable the restore flow entirely.'
 complete -c crio -n '__fish_crio_no_subcommand' -l listen -r -d 'Path to the CRI-O socket'
 complete -c crio -n '__fish_crio_no_subcommand' -l log -r -d 'Set the log file path where internal debug information is written'
 complete -c crio -n '__fish_crio_no_subcommand' -l log-dir -r -d 'Default log directory where all logs will go unless directly specified by the kubelet'

@@ -64,6 +64,7 @@ it later with **--config**. Global options will modify the output.'
         '--insecure-registry'
         '--internal-wipe'
         '--irqbalance-config-file'
+        '--irqbalance-config-restore-file'
         '--listen'
         '--log'
         '--log-dir'

@@ -56,6 +56,7 @@ crio
 [--insecure-registry]=[value]
 [--internal-wipe]
 [--irqbalance-config-file]=[value]
+[--irqbalance-config-restore-file]=[value]
 [--listen]=[value]
 [--log-dir]=[value]
 [--log-filter]=[value]
@@ -266,6 +267,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
 
 **--irqbalance-config-file**="": The irqbalance service config file which is used by CRI-O. (default: /etc/sysconfig/irqbalance)
 
+**--irqbalance-config-restore-file**="": Determines if CRI-O should attempt to restore the irqbalance config at startup with the mask in this file. Use empty value to disable the restore flow entirely. (default: /etc/sysconfig/orig_irq_banned_cpus)
+
 **--listen**="": Path to the CRI-O socket (default: /var/run/crio/crio.sock)
 
 **--log**="": Set the log file path where internal debug information is written

@@ -283,6 +283,9 @@ the container runtime configuration.
 **device_ownership_from_security_context**=false
   Changes the default behavior of setting container devices uid/gid from CRI's SecurityContext (RunAsUser/RunAsGroup) instead of taking host's uid/gid.
 
+**irqbalance_config_restore_file**="/etc/sysconfig/orig_irq_banned_cpus"
+  Used to set the irqbalance banned cpu mask to restore at CRI-O startup. If empty, no restoration attempt will be done.
+
 ### CRIO.RUNTIME.RUNTIMES TABLE
 The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.  The runtime to use is picked based on the runtime handler provided by the CRI.  If no runtime handler is provided, the runtime will be picked based on the level of trust of the workload.
 

@@ -326,6 +326,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
 	if ctx.IsSet("absent-mount-sources-to-reject") {
 		config.AbsentMountSourcesToReject = StringSliceTrySplit(ctx, "absent-mount-sources-to-reject")
 	}
+	if ctx.IsSet("irqbalance-config-restore-file") {
+		config.IrqBalanceConfigRestoreFile = ctx.String("irqbalance-config-restore-file")
+	}
 	if ctx.IsSet("internal-wipe") {
 		config.InternalWipe = ctx.Bool("internal-wipe")
 	}
@@ -1012,6 +1015,11 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
 			Usage:   "The number of seconds between collecting pod and container stats. If set to 0, the stats are collected on-demand instead.",
 			EnvVars: []string{"CONTAINER_STATS_COLLECTION_PERIOD"},
 		},
+		&cli.StringFlag{
+			Name:  "irqbalance-config-restore-file",
+			Value: defConf.IrqBalanceConfigRestoreFile,
+			Usage: "Determines if CRI-O should attempt to restore the irqbalance config at startup with the mask in this file. Use empty value to disable the restore flow entirely.",
+		},
 	}
 }
 

@@ -20,7 +20,6 @@ import (
 	"github.com/cri-o/cri-o/utils/cmdrunner"
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
-	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@@ -29,8 +28,6 @@ import (
 const (
 	// HighPerformance contains the high-performance runtime handler name
 	HighPerformance = "high-performance"
-	// IrqBannedCPUConfigFile contains the original banned cpu mask configuration
-	IrqBannedCPUConfigFile = "/etc/sysconfig/orig_irq_banned_cpus"
 	// IrqSmpAffinityProcFile contains the default smp affinity mask configuration
 	IrqSmpAffinityProcFile = "/proc/irq/default_smp_affinity"
 )
@@ -78,7 +75,7 @@ func (h *HighPerformanceHooks) PreStart(ctx context.Context, c *oci.Container, s
 	// disable the IRQ smp load balancing for the container CPUs
 	if shouldIRQLoadBalancingBeDisabled(s.Annotations()) {
 		log.Infof(ctx, "Disable irq smp balancing for container %q", c.ID())
-		if err := setIRQLoadBalancing(c, false, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil {
+		if err := setIRQLoadBalancing(ctx, c, false, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil {
 			return fmt.Errorf("set IRQ load balancing: %w", err)
 		}
 	}
@@ -151,7 +148,7 @@ func (h *HighPerformanceHooks) PreStop(ctx context.Context, c *oci.Container, s
 
 	// enable the IRQ smp balancing for the container CPUs
 	if shouldIRQLoadBalancingBeDisabled(s.Annotations()) {
-		if err := setIRQLoadBalancing(c, true, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil {
+		if err := setIRQLoadBalancing(ctx, c, true, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil {
 			return fmt.Errorf("set IRQ load balancing: %w", err)
 		}
 	}
@@ -306,7 +303,7 @@ func setCPUSLoadBalancing(c *oci.Container, enable bool, schedDomainDir string)
 	return nil
 }
 
-func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqBalanceConfigFile string) error {
+func setIRQLoadBalancing(ctx context.Context, c *oci.Container, enable bool, irqSmpAffinityFile, irqBalanceConfigFile string) error {
 	lspec := c.Spec().Linux
 	if lspec == nil ||
 		lspec.Resources == nil ||
@@ -339,7 +336,7 @@ func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqB
 	if !isServiceEnabled(irqBalancedName) || !isIrqConfigExists {
 		if _, err := exec.LookPath(irqBalancedName); err != nil {
 			// irqbalance is not installed, skip the rest; pod should still start, so return nil instead
-			logrus.Warnf("Irqbalance binary not found: %v", err)
+			log.Warnf(ctx, "Irqbalance binary not found: %v", err)
 			return nil
 		}
 		// run irqbalance in daemon mode, so this won't cause delay
@@ -350,7 +347,7 @@ func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqB
 	}
 
 	if err := restartIrqBalanceService(); err != nil {
-		logrus.Warnf("Irqbalance service restart failed: %v", err)
+		log.Warnf(ctx, "Irqbalance service restart failed: %v", err)
 	}
 	return nil
 }
@@ -598,7 +595,7 @@ func doSetCPUFreqGovernor(c *oci.Container, governor, cpuDir, cpuSaveDir string)
 }
 
 // RestoreIrqBalanceConfig restores irqbalance service with original banned cpu mask settings
-func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityProcFile string) error {
+func RestoreIrqBalanceConfig(ctx context.Context, irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityProcFile string) error {
 	content, err := os.ReadFile(irqSmpAffinityProcFile)
 	if err != nil {
 		return err
@@ -612,15 +609,19 @@ func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSm
 	}
 	if !isAllBitSet(currentMaskArray) {
 		// not system reboot scenario, just return it.
+		log.Infof(ctx, "Restore irqbalance config: not system reboot, ignoring")
 		return nil
 	}
 
 	bannedCPUMasks, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile)
 	if err != nil {
 		// Ignore returning err as given irqBalanceConfigFile may not exist.
+		log.Infof(ctx, "Restore irqbalance config: failed to get current CPU ban list, ignoring")
 		return nil
 	}
+
 	if !fileExists(irqBannedCPUConfigFile) {
+		log.Infof(ctx, "Creating banned CPU list file %q", irqBannedCPUConfigFile)
 		irqBannedCPUsConfig, err := os.Create(irqBannedCPUConfigFile)
 		if err != nil {
 			return err
@@ -630,6 +631,7 @@ func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSm
 		if err != nil {
 			return err
 		}
+		log.Infof(ctx, "Restore irqbalance config: created backup file")
 		return nil
 	}
 
@@ -640,14 +642,17 @@ func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSm
 	origBannedCPUMasks := strings.TrimSpace(string(content))
 
 	if bannedCPUMasks == origBannedCPUMasks {
+		log.Infof(ctx, "Restore irqbalance config: nothing to do")
 		return nil
 	}
+
+	log.Infof(ctx, "Restore irqbalance banned CPU list in %q to %q", irqBalanceConfigFile, origBannedCPUMasks)
 	if err := updateIrqBalanceConfigFile(irqBalanceConfigFile, origBannedCPUMasks); err != nil {
 		return err
 	}
 	if isServiceEnabled(irqBalancedName) {
 		if err := restartIrqBalanceService(); err != nil {
-			logrus.Warnf("Irqbalance service restart failed: %v", err)
+			log.Warnf(ctx, "Irqbalance service restart failed: %v", err)
 		}
 	}
 	return nil

@@ -111,7 +111,7 @@ var _ = Describe("high_performance_hooks", func() {
 		irqSmpAffinityFile := filepath.Join(fixturesDir, "irq_smp_affinity")
 		irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance")
 		verifySetIRQLoadBalancing := func(enabled bool, expected string) {
-			err := setIRQLoadBalancing(container, enabled, irqSmpAffinityFile, irqBalanceConfigFile)
+			err := setIRQLoadBalancing(context.TODO(), container, enabled, irqSmpAffinityFile, irqBalanceConfigFile)
 			Expect(err).To(BeNil())
 
 			content, err := os.ReadFile(irqSmpAffinityFile)
@@ -164,7 +164,7 @@ var _ = Describe("high_performance_hooks", func() {
 		irqSmpAffinityFile := filepath.Join(fixturesDir, "irq_smp_affinity")
 		irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance")
 		verifySetIRQLoadBalancing := func(enabled bool, expectedSmp, expectedBan string) {
-			err = setIRQLoadBalancing(container, enabled, irqSmpAffinityFile, irqBalanceConfigFile)
+			err = setIRQLoadBalancing(context.TODO(), container, enabled, irqSmpAffinityFile, irqBalanceConfigFile)
 			Expect(err).To(BeNil())
 
 			content, err := os.ReadFile(irqSmpAffinityFile)
@@ -511,16 +511,16 @@ var _ = Describe("high_performance_hooks", func() {
 		irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance")
 		irqBannedCPUConfigFile := filepath.Join(fixturesDir, "orig_irq_banned_cpus")
 		verifyRestoreIrqBalanceConfig := func(expectedOrigBannedCPUs, expectedBannedCPUs string) {
-			err = RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityFile)
-			Expect(err).To(BeNil())
+			err = RestoreIrqBalanceConfig(context.TODO(), irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityFile)
+			ExpectWithOffset(1, err).To(BeNil())
 
 			content, err := os.ReadFile(irqBannedCPUConfigFile)
-			Expect(err).To(BeNil())
-			Expect(strings.Trim(string(content), "\n")).To(Equal(expectedOrigBannedCPUs))
+			ExpectWithOffset(1, err).To(BeNil())
+			ExpectWithOffset(1, strings.Trim(string(content), "\n")).To(Equal(expectedOrigBannedCPUs))
 
 			bannedCPUs, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile)
-			Expect(err).To(BeNil())
-			Expect(bannedCPUs).To(Equal(expectedBannedCPUs))
+			ExpectWithOffset(1, err).To(BeNil())
+			ExpectWithOffset(1, bannedCPUs).To(Equal(expectedBannedCPUs))
 		}
 
 		JustBeforeEach(func() {

@@ -122,6 +122,8 @@ const (
 const (
 	// DefaultIrqBalanceConfigFile default irqbalance service configuration file path
 	DefaultIrqBalanceConfigFile = "/etc/sysconfig/irqbalance"
+	// DefaultIrqBalanceConfigRestoreFile contains the banned cpu mask configuration to restore. Name due to backward compatibility.
+	DefaultIrqBalanceConfigRestoreFile = "/etc/sysconfig/orig_irq_banned_cpus"
 )
 
 // This structure is necessary to fake the TOML tables when parsing,
@@ -397,6 +399,10 @@ type RuntimeConfig struct {
 	// will cause a container creation to fail (as opposed to the current behavior of creating a directory).
 	AbsentMountSourcesToReject []string `toml:"absent_mount_sources_to_reject"`
 
+	// IrqBalanceConfigRestoreFile is the irqbalance service banned CPU list to restore.
+	// If empty, no restoration attempt will be done.
+	IrqBalanceConfigRestoreFile string `toml:"irqbalance_config_restore_file"`
+
 	// seccompConfig is the internal seccomp configuration
 	seccompConfig *seccomp.Config
 
@@ -782,34 +788,35 @@ func DefaultConfig() (*Config, error) {
 			Runtimes: Runtimes{
 				defaultRuntime: defaultRuntimeHandler(),
 			},
-			SELinux:                    selinuxEnabled(),
-			ApparmorProfile:            apparmor.DefaultProfile,
-			BlockIOConfigFile:          DefaultBlockIOConfigFile,
-			IrqBalanceConfigFile:       DefaultIrqBalanceConfigFile,
-			RdtConfigFile:              rdt.DefaultRdtConfigFile,
-			CgroupManagerName:          cgroupManager.Name(),
-			PidsLimit:                  DefaultPidsLimit,
-			ContainerExitsDir:          containerExitsDir,
-			ContainerAttachSocketDir:   conmonconfig.ContainerAttachSocketDir,
-			MinimumMappableUID:         -1,
-			MinimumMappableGID:         -1,
-			LogSizeMax:                 DefaultLogSizeMax,
-			CtrStopTimeout:             defaultCtrStopTimeout,
-			DefaultCapabilities:        capabilities.Default(),
-			LogLevel:                   "info",
-			HooksDir:                   []string{hooks.DefaultDir},
-			CDISpecDirs:                cdi.DefaultSpecDirs,
-			NamespacesDir:              defaultNamespacesDir,
-			DropInfraCtr:               true,
-			SeccompUseDefaultWhenEmpty: seccompConfig.UseDefaultWhenEmpty(),
-			seccompConfig:              seccomp.New(),
-			apparmorConfig:             apparmor.New(),
-			blockioConfig:              blockio.New(),
-			cgroupManager:              cgroupManager,
-			deviceConfig:               device.New(),
-			namespaceManager:           nsmgr.New(defaultNamespacesDir, ""),
-			rdtConfig:                  rdt.New(),
-			ulimitsConfig:              ulimits.New(),
+			SELinux:                     selinuxEnabled(),
+			ApparmorProfile:             apparmor.DefaultProfile,
+			BlockIOConfigFile:           DefaultBlockIOConfigFile,
+			IrqBalanceConfigFile:        DefaultIrqBalanceConfigFile,
+			RdtConfigFile:               rdt.DefaultRdtConfigFile,
+			CgroupManagerName:           cgroupManager.Name(),
+			PidsLimit:                   DefaultPidsLimit,
+			ContainerExitsDir:           containerExitsDir,
+			ContainerAttachSocketDir:    conmonconfig.ContainerAttachSocketDir,
+			MinimumMappableUID:          -1,
+			MinimumMappableGID:          -1,
+			LogSizeMax:                  DefaultLogSizeMax,
+			CtrStopTimeout:              defaultCtrStopTimeout,
+			DefaultCapabilities:         capabilities.Default(),
+			LogLevel:                    "info",
+			HooksDir:                    []string{hooks.DefaultDir},
+			CDISpecDirs:                 cdi.DefaultSpecDirs,
+			NamespacesDir:               defaultNamespacesDir,
+			DropInfraCtr:                true,
+			SeccompUseDefaultWhenEmpty:  seccompConfig.UseDefaultWhenEmpty(),
+			IrqBalanceConfigRestoreFile: DefaultIrqBalanceConfigRestoreFile,
+			seccompConfig:               seccomp.New(),
+			apparmorConfig:              apparmor.New(),
+			blockioConfig:               blockio.New(),
+			cgroupManager:               cgroupManager,
+			deviceConfig:                device.New(),
+			namespaceManager:            nsmgr.New(defaultNamespacesDir, ""),
+			rdtConfig:                   rdt.New(),
+			ulimitsConfig:               ulimits.New(),
 		},
 		ImageConfig: ImageConfig{
 			DefaultTransport: "docker://",

@@ -427,6 +427,11 @@ func initCrioTemplateConfig(c *Config) ([]*templateConfigValue, error) {
 			group:          crioRuntimeConfig,
 			isDefaultValue: WorkloadsEqual(dc.Workloads, c.Workloads),
 		},
+		{
+			templateString: templateStringIrqBalanceConfigRestoreFile,
+			group:          crioRuntimeConfig,
+			isDefaultValue: simpleEqual(dc.IrqBalanceConfigRestoreFile, c.IrqBalanceConfigRestoreFile),
+		},
 		{
 			templateString: templateStringCrioImageDefaultTransport,
 			group:          crioImageConfig,
@@ -1022,6 +1027,13 @@ const templateStringCrioRuntimeDropInfraCtr = `# drop_infra_ctr determines wheth
 
 `
 
+const templateStringIrqBalanceConfigRestoreFile = `# irqbalance_config_restore_file allows to set a cpu mask CRI-O should
+# restore as irqbalance config at startup. Set to empty string to disable this flow entirely.
+# By default, CRI-O manages the irqbalance configuration to enable dynamic IRQ pinning.
+{{ $.Comment }}irqbalance_config_restore_file = "{{ .IrqBalanceConfigRestoreFile }}"
+
+`
+
 const templateStringCrioRuntimeInfraCtrCpuset = `# infra_ctr_cpuset determines what CPUs will be used to run infra containers.
 # You can use linux CPU list format to specify desired CPUs.
 # To get better isolation for guaranteed pods, set this parameter to be equal to kubelet reserved-cpus.

@@ -395,9 +395,12 @@ func New(
 		return nil, err
 	}
 
-	err = runtimehandlerhooks.RestoreIrqBalanceConfig(config.IrqBalanceConfigFile, runtimehandlerhooks.IrqBannedCPUConfigFile, runtimehandlerhooks.IrqSmpAffinityProcFile)
-	if err != nil {
-		return nil, err
+	if config.IrqBalanceConfigRestoreFile != "" {
+		log.Infof(ctx, "Attempting to restore irqbalance config from %s", config.IrqBalanceConfigRestoreFile)
+		err = runtimehandlerhooks.RestoreIrqBalanceConfig(context.TODO(), config.IrqBalanceConfigFile, config.IrqBalanceConfigRestoreFile, runtimehandlerhooks.IrqSmpAffinityProcFile)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	hostportManager := hostport.NewMetaHostportManager()