diff --git a/completions/bash/crio b/completions/bash/crio index e3d348ed19b..64f05d70aef 100755 --- a/completions/bash/crio +++ b/completions/bash/crio @@ -46,6 +46,7 @@ h --image-volumes --infra-ctr-cpuset --insecure-registry +--irqbalance-config-file --listen --log --log-dir diff --git a/completions/fish/crio.fish b/completions/fish/crio.fish index 4c0ff459e90..d66290251ff 100644 --- a/completions/fish/crio.fish +++ b/completions/fish/crio.fish @@ -84,6 +84,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l insecure-registry -r -d 'E be enabled for testing purposes**. For increased security, users should add their CA to their system\'s list of trusted CAs instead of using \'--insecure-registry\'.' +complete -c crio -n '__fish_crio_no_subcommand' -f -l irqbalance-config-file -r -d 'The irqbalance service config file which is used by CRI-O.' complete -c crio -n '__fish_crio_no_subcommand' -l listen -r -d 'Path to the CRI-O socket' complete -c crio -n '__fish_crio_no_subcommand' -l log -r -d 'Set the log file path where internal debug information is written' complete -c crio -n '__fish_crio_no_subcommand' -l log-dir -r -d 'Default log directory where all logs will go unless directly specified by the kubelet' diff --git a/completions/zsh/_crio b/completions/zsh/_crio index a328c294e5b..e0dd7a28844 100644 --- a/completions/zsh/_crio +++ b/completions/zsh/_crio @@ -7,7 +7,7 @@ it later with **--config**. Global options will modify the output.' 'version:dis _describe 'commands' cmds local -a opts - opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-metrics' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--infra-ctr-cpuset' '--insecure-registry' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version') + opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-metrics' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--infra-ctr-cpuset' '--insecure-registry' '--irqbalance-config-file' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version') _describe 'global options' opts return diff --git a/docs/crio.8.md b/docs/crio.8.md index bb6721b2a3a..aec216c1961 100644 --- a/docs/crio.8.md +++ b/docs/crio.8.md @@ -46,6 +46,7 @@ crio [--image-volumes]=[value] [--infra-ctr-cpuset]=[value] [--insecure-registry]=[value] +[--irqbalance-config-file]=[value] [--listen]=[value] [--log-dir]=[value] [--log-filter]=[value] @@ -224,6 +225,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...] their CA to their system's list of trusted CAs instead of using '--insecure-registry'. (default: []) +**--irqbalance-config-file**="": The irqbalance service config file which is used by CRI-O. (default: /etc/sysconfig/irqbalance) + **--listen**="": Path to the CRI-O socket (default: /var/run/crio/crio.sock) **--log**="": Set the log file path where internal debug information is written diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index 685b82a58b1..8a7a80cbfb6 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -125,6 +125,10 @@ the container runtime configuration. **apparmor_profile**="" Used to change the name of the default AppArmor profile of CRI-O. The default profile name is "crio-default". +**irqbalance_config_file**="/etc/sysconfig/irqbalance" + Used to change irqbalance service config file which is used by CRI-O. + For CentOS/SUSE, this file is located at /etc/sysconfig/irqbalance. For Ubuntu, this file is located at /etc/default/irqbalance. + **cgroup_manager**="systemd" Cgroup management implementation used for the runtime. diff --git a/internal/criocli/criocli.go b/internal/criocli/criocli.go index 6be40bfa4cd..7768053b204 100644 --- a/internal/criocli/criocli.go +++ b/internal/criocli/criocli.go @@ -158,6 +158,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error { if ctx.IsSet("apparmor-profile") { config.ApparmorProfile = ctx.String("apparmor-profile") } + if ctx.IsSet("irqbalance-config-file") { + config.IrqBalanceConfigFile = ctx.String("irqbalance-config-file") + } if ctx.IsSet("cgroup-manager") { config.CgroupManagerName = ctx.String("cgroup-manager") } @@ -524,6 +527,11 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag { Value: defConf.ApparmorProfile, EnvVars: []string{"CONTAINER_APPARMOR_PROFILE"}, }, + &cli.StringFlag{ + Name: "irqbalance-config-file", + Usage: "The irqbalance service config file which is used by CRI-O.", + Value: defConf.IrqBalanceConfigFile, + }, &cli.BoolFlag{ Name: "selinux", Usage: fmt.Sprintf("Enable selinux support (default: %t)", defConf.SELinux), diff --git a/internal/runtimehandlerhooks/high_performance_hooks.go b/internal/runtimehandlerhooks/high_performance_hooks.go index 8d3ef8c93cc..db3ea5eba12 100644 --- a/internal/runtimehandlerhooks/high_performance_hooks.go +++ b/internal/runtimehandlerhooks/high_performance_hooks.go @@ -26,18 +26,25 @@ import ( const ( // HighPerformance contains the high-performance runtime handler name HighPerformance = "high-performance" + // IrqBannedCPUConfigFile contains the original banned cpu mask configuration + IrqBannedCPUConfigFile = "/etc/sysconfig/orig_irq_banned_cpus" + // IrqSmpAffinityProcFile contains the default smp affinity mask configuration + IrqSmpAffinityProcFile = "/proc/irq/default_smp_affinity" ) const ( - annotationTrue = "true" - annotationDisable = "disable" - schedDomainDir = "/proc/sys/kernel/sched_domain" - irqSmpAffinityProcFile = "/proc/irq/default_smp_affinity" - cgroupMountPoint = "/sys/fs/cgroup" + annotationTrue = "true" + annotationDisable = "disable" + schedDomainDir = "/proc/sys/kernel/sched_domain" + cgroupMountPoint = "/sys/fs/cgroup" + irqBalanceBannedCpus = "IRQBALANCE_BANNED_CPUS" + irqBalancedName = "irqbalance" ) // HighPerformanceHooks used to run additional hooks that will configure a system for the latency sensitive workloads -type HighPerformanceHooks struct{} +type HighPerformanceHooks struct { + irqBalanceConfigFile string +} func (h *HighPerformanceHooks) PreStart(ctx context.Context, c *oci.Container, s *sandbox.Sandbox) error { log.Infof(ctx, "Run %q runtime handler pre-start hook for the container %q", HighPerformance, c.ID()) @@ -66,7 +73,7 @@ func (h *HighPerformanceHooks) PreStart(ctx context.Context, c *oci.Container, s // disable the IRQ smp load balancing for the container CPUs if shouldIRQLoadBalancingBeDisabled(s.Annotations()) { log.Infof(ctx, "Disable irq smp balancing for container %q", c.ID()) - if err := setIRQLoadBalancing(c, false, irqSmpAffinityProcFile); err != nil { + if err := setIRQLoadBalancing(c, false, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil { return errors.Wrap(err, "set IRQ load balancing") } } @@ -111,7 +118,7 @@ func (h *HighPerformanceHooks) PreStop(ctx context.Context, c *oci.Container, s // enable the IRQ smp balancing for the container CPUs if shouldIRQLoadBalancingBeDisabled(s.Annotations()) { - if err := setIRQLoadBalancing(c, true, irqSmpAffinityProcFile); err != nil { + if err := setIRQLoadBalancing(c, true, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil { return errors.Wrap(err, "set IRQ load balancing") } } @@ -219,7 +226,7 @@ func setCPUSLoadBalancing(c *oci.Container, enable bool, schedDomainDir string) return nil } -func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile string) error { +func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqBalanceConfigFile string) error { lspec := c.Spec().Linux if lspec == nil || lspec.Resources == nil || @@ -240,16 +247,32 @@ func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile strin if err := ioutil.WriteFile(irqSmpAffinityFile, []byte(newIRQSMPSetting), 0o644); err != nil { return err } - if _, err := exec.LookPath("irqbalance"); err != nil { - // irqbalance is not installed, skip the rest; pod should still start, so return nil instead - logrus.Warnf("irqbalance binary not found: %v", err) - return nil + + isIrqConfigExists := fileExists(irqBalanceConfigFile) + + if isIrqConfigExists { + if err := updateIrqBalanceConfigFile(irqBalanceConfigFile, newIRQBalanceSetting); err != nil { + return err + } + } + + if !isServiceEnabled(irqBalancedName) || !isIrqConfigExists { + if _, err := exec.LookPath(irqBalancedName); err != nil { + // irqbalance is not installed, skip the rest; pod should still start, so return nil instead + logrus.Warnf("irqbalance binary not found: %v", err) + return nil + } + // run irqbalance in daemon mode, so this won't cause delay + cmd := exec.Command(irqBalancedName, "--oneshot") + additionalEnv := irqBalanceBannedCpus + "=" + newIRQBalanceSetting + cmd.Env = append(os.Environ(), additionalEnv) + return cmd.Run() + } + + if err := restartIrqBalanceService(); err != nil { + logrus.Warnf("irqbalance service restart failed: %v", err) } - // run irqbalance in daemon mode, so this won't cause delay - cmd := exec.Command("irqbalance", "--oneshot") - additionalEnv := "IRQBALANCE_BANNED_CPUS=" + newIRQBalanceSetting - cmd.Env = append(os.Environ(), additionalEnv) - return cmd.Run() + return nil } func setCPUQuota(cpuMountPoint, parentDir string, c *oci.Container, enable bool) error { @@ -311,3 +334,59 @@ func setCPUQuota(cpuMountPoint, parentDir string, c *oci.Container, enable bool) return nil } + +// RestoreIrqBalanceConfig restores irqbalance service with original banned cpu mask settings +func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityProcFile string) error { + content, err := ioutil.ReadFile(irqSmpAffinityProcFile) + if err != nil { + return err + } + current := strings.TrimSpace(string(content)) + // remove ","; now each element is "0-9,a-f" + s := strings.ReplaceAll(current, ",", "") + currentMaskArray, err := mapHexCharToByte(s) + if err != nil { + return err + } + if !isAllBitSet(currentMaskArray) { + // not system reboot scenario, just return it. + return nil + } + + bannedCPUMasks, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile) + if err != nil { + // Ignore returning err as given irqBalanceConfigFile may not exist. + return nil + } + if !fileExists(irqBannedCPUConfigFile) { + irqBannedCPUsConfig, err := os.Create(irqBannedCPUConfigFile) + if err != nil { + return err + } + defer irqBannedCPUsConfig.Close() + _, err = irqBannedCPUsConfig.WriteString(bannedCPUMasks) + if err != nil { + return err + } + return nil + } + + content, err = ioutil.ReadFile(irqBannedCPUConfigFile) + if err != nil { + return err + } + origBannedCPUMasks := strings.TrimSpace(string(content)) + + if bannedCPUMasks == origBannedCPUMasks { + return nil + } + if err := updateIrqBalanceConfigFile(irqBalanceConfigFile, origBannedCPUMasks); err != nil { + return err + } + if isServiceEnabled(irqBalancedName) { + if err := restartIrqBalanceService(); err != nil { + logrus.Warnf("irqbalance service restart failed: %v", err) + } + } + return nil +} diff --git a/internal/runtimehandlerhooks/high_performance_hooks_test.go b/internal/runtimehandlerhooks/high_performance_hooks_test.go index f8022d6f034..406e2f56e68 100644 --- a/internal/runtimehandlerhooks/high_performance_hooks_test.go +++ b/internal/runtimehandlerhooks/high_performance_hooks_test.go @@ -28,7 +28,7 @@ var _ = Describe("high_performance_hooks", func() { false, "", "", time.Now(), "") Expect(err).To(BeNil()) - var flags string + var flags, bannedCPUFlags string BeforeEach(func() { err := os.MkdirAll(fixturesDir, os.ModePerm) @@ -107,10 +107,11 @@ var _ = Describe("high_performance_hooks", func() { }) }) - Describe("setIRQLoadBalancing", func() { + Describe("setIRQLoadBalancingUsingDaemonCommand", func() { irqSmpAffinityFile := filepath.Join(fixturesDir, "irq_smp_affinity") + irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance") verifySetIRQLoadBalancing := func(enabled bool, expected string) { - err := setIRQLoadBalancing(container, enabled, irqSmpAffinityFile) + err := setIRQLoadBalancing(container, enabled, irqSmpAffinityFile, irqBalanceConfigFile) Expect(err).To(BeNil()) content, err := ioutil.ReadFile(irqSmpAffinityFile) @@ -159,6 +160,130 @@ var _ = Describe("high_performance_hooks", func() { }) }) + Describe("setIRQLoadBalancingUsingServiceRestart", func() { + irqSmpAffinityFile := filepath.Join(fixturesDir, "irq_smp_affinity") + irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance") + verifySetIRQLoadBalancing := func(enabled bool, expectedSmp, expectedBan string) { + err = setIRQLoadBalancing(container, enabled, irqSmpAffinityFile, irqBalanceConfigFile) + Expect(err).To(BeNil()) + + content, err := ioutil.ReadFile(irqSmpAffinityFile) + Expect(err).To(BeNil()) + + Expect(strings.Trim(string(content), "\n")).To(Equal(expectedSmp)) + + bannedCPUs, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile) + Expect(err).To(BeNil()) + + Expect(bannedCPUs).To(Equal(expectedBan)) + } + + JustBeforeEach(func() { + // set irqbalanace config file with no banned cpus + err = ioutil.WriteFile(irqBalanceConfigFile, []byte(""), 0o644) + Expect(err).To(BeNil()) + err = updateIrqBalanceConfigFile(irqBalanceConfigFile, bannedCPUFlags) + Expect(err).To(BeNil()) + bannedCPUs, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile) + Expect(err).To(BeNil()) + Expect(bannedCPUs).To(Equal(bannedCPUFlags)) + // set container CPUs + container.SetSpec( + &specs.Spec{ + Linux: &specs.Linux{ + Resources: &specs.LinuxResources{ + CPU: &specs.LinuxCPU{ + Cpus: "4,5", + }, + }, + }, + }, + ) + + // create tests affinity file + err = ioutil.WriteFile(irqSmpAffinityFile, []byte(flags), 0o644) + Expect(err).To(BeNil()) + }) + + Context("with enabled equals to true", func() { + BeforeEach(func() { + flags = "00000000,00003003" + bannedCPUFlags = "ffffffff,ffffcffc" + }) + + It("should set the irq bit mask", func() { + verifySetIRQLoadBalancing(true, "00000000,00003033", "ffffffff,ffffcfcc") + }) + }) + + Context("with enabled equals to false", func() { + BeforeEach(func() { + flags = "00000000,00003033" + bannedCPUFlags = "ffffffff,ffffcfcc" + }) + + It("should clear the irq bit mask", func() { + verifySetIRQLoadBalancing(false, "00000000,00003003", "ffffffff,ffffcffc") + }) + }) + }) + + Describe("restoreIrqBalanceConfig", func() { + irqSmpAffinityFile := filepath.Join(fixturesDir, "irq_smp_affinity") + irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance") + irqBannedCPUConfigFile := filepath.Join(fixturesDir, "orig_irq_banned_cpus") + verifyRestoreIrqBalanceConfig := func(expectedOrigBannedCPUs, expectedBannedCPUs string) { + err = RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityFile) + Expect(err).To(BeNil()) + + content, err := ioutil.ReadFile(irqBannedCPUConfigFile) + Expect(err).To(BeNil()) + Expect(strings.Trim(string(content), "\n")).To(Equal(expectedOrigBannedCPUs)) + + bannedCPUs, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile) + Expect(err).To(BeNil()) + Expect(bannedCPUs).To(Equal(expectedBannedCPUs)) + } + + JustBeforeEach(func() { + // create tests affinity file + err = ioutil.WriteFile(irqSmpAffinityFile, []byte("ffffffff,ffffffff"), 0o644) + Expect(err).To(BeNil()) + // set irqbalanace config file with banned cpus mask + err = ioutil.WriteFile(irqBalanceConfigFile, []byte(""), 0o644) + Expect(err).To(BeNil()) + err = updateIrqBalanceConfigFile(irqBalanceConfigFile, "0000ffff,ffffcfcc") + Expect(err).To(BeNil()) + bannedCPUs, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile) + Expect(err).To(BeNil()) + Expect(bannedCPUs).To(Equal("0000ffff,ffffcfcc")) + }) + + Context("when banned cpu config file doesn't exist", func() { + BeforeEach(func() { + // ensure banned cpu config file doesn't exist + os.Remove(irqBannedCPUConfigFile) + }) + + It("should set banned cpu config file from irq balance config", func() { + verifyRestoreIrqBalanceConfig("0000ffff,ffffcfcc", "0000ffff,ffffcfcc") + }) + }) + + Context("when banned cpu config file exists", func() { + BeforeEach(func() { + // create banned cpu config file + os.Remove(irqBannedCPUConfigFile) + err = ioutil.WriteFile(irqBannedCPUConfigFile, []byte("00000000,00000000"), 0o644) + Expect(err).To(BeNil()) + }) + + It("should restore irq balance config with content from banned cpu config file", func() { + verifyRestoreIrqBalanceConfig("00000000,00000000", "00000000,00000000") + }) + }) + }) + Describe("setCPUQuota", func() { containerID := container.ID() parent := "parent.slice" diff --git a/internal/runtimehandlerhooks/runtime_handler_hooks.go b/internal/runtimehandlerhooks/runtime_handler_hooks.go index e4f4dd7ce85..be0d50d5051 100644 --- a/internal/runtimehandlerhooks/runtime_handler_hooks.go +++ b/internal/runtimehandlerhooks/runtime_handler_hooks.go @@ -7,6 +7,7 @@ import ( "github.com/cri-o/cri-o/internal/lib/sandbox" "github.com/cri-o/cri-o/internal/log" "github.com/cri-o/cri-o/internal/oci" + libconfig "github.com/cri-o/cri-o/pkg/config" ) type RuntimeHandlerHooks interface { @@ -15,19 +16,19 @@ type RuntimeHandlerHooks interface { } // GetRuntimeHandlerHooks returns RuntimeHandlerHooks implementation by the runtime handler name -func GetRuntimeHandlerHooks(ctx context.Context, handler string, r *oci.Runtime) (RuntimeHandlerHooks, error) { +func GetRuntimeHandlerHooks(ctx context.Context, config *libconfig.Config, handler string, r *oci.Runtime) (RuntimeHandlerHooks, error) { allowAnnotations, err := allowHighPerformanceAnnotations(handler, r) if err != nil { return nil, err } if allowAnnotations { - return &HighPerformanceHooks{}, nil + return &HighPerformanceHooks{config.IrqBalanceConfigFile}, nil } if !allowAnnotations && strings.Contains(handler, HighPerformance) { log.Warnf(ctx, "The usage of the handler %q without adding high-performance feature annotations under allowed_annotations will be deprecated under 1.21", HighPerformance) - return &HighPerformanceHooks{}, nil + return &HighPerformanceHooks{config.IrqBalanceConfigFile}, nil } return nil, nil diff --git a/internal/runtimehandlerhooks/utils.go b/internal/runtimehandlerhooks/utils.go index 7ee4f18d222..21e1285f0a5 100644 --- a/internal/runtimehandlerhooks/utils.go +++ b/internal/runtimehandlerhooks/utils.go @@ -3,9 +3,13 @@ package runtimehandlerhooks import ( "encoding/hex" "fmt" + "io/ioutil" + "os" + "os/exec" "strings" "unicode" + "github.com/sirupsen/logrus" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) @@ -75,6 +79,17 @@ func invertByteArray(in []byte) (out []byte) { return } +// take a byte array and returns true when bits of every byte element +// set to 1, otherwise returns false. +func isAllBitSet(in []byte) bool { + for _, b := range in { + if b&(b+1) != 0 { + return false + } + } + return true +} + // UpdateIRQSmpAffinityMask take input cpus that need to change irq affinity mask and // the current mask string, return an update mask string and inverted mask, with those cpus // enabled or disable in the mask. @@ -121,3 +136,65 @@ func UpdateIRQSmpAffinityMask(cpus, current string, set bool) (cpuMask, bannedCP } return maskStringWithComma, invertedMaskStringWithComma, nil } + +func restartIrqBalanceService() error { + return exec.Command("systemctl", "restart", "irqbalance").Run() +} + +func isServiceEnabled(serviceName string) bool { + cmd := exec.Command("systemctl", "is-enabled", serviceName) + status, err := cmd.CombinedOutput() + if err != nil { + logrus.Infof("service %s is-enabled check returned with: %v", serviceName, err) + return false + } + if strings.TrimSpace(string(status)) == "enabled" { + return true + } + return false +} + +func updateIrqBalanceConfigFile(irqBalanceConfigFile, newIRQBalanceSetting string) error { + input, err := ioutil.ReadFile(irqBalanceConfigFile) + if err != nil { + return err + } + lines := strings.Split(string(input), "\n") + found := false + for i, line := range lines { + if strings.HasPrefix(line, irqBalanceBannedCpus+"=") { + lines[i] = irqBalanceBannedCpus + "=" + "\"" + newIRQBalanceSetting + "\"" + "\n" + found = true + } + } + output := strings.Join(lines, "\n") + if !found { + output = output + "\n" + irqBalanceBannedCpus + "=" + "\"" + newIRQBalanceSetting + "\"" + "\n" + } + if err := ioutil.WriteFile(irqBalanceConfigFile, []byte(output), 0644); err != nil { + return err + } + return nil +} + +func retrieveIrqBannedCPUMasks(irqBalanceConfigFile string) (string, error) { + input, err := ioutil.ReadFile(irqBalanceConfigFile) + if err != nil { + return "", err + } + lines := strings.Split(string(input), "\n") + for _, line := range lines { + if strings.HasPrefix(line, irqBalanceBannedCpus+"=") { + return strings.Trim(strings.Split(line, "=")[1], "\""), nil + } + } + return "", nil +} + +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} diff --git a/pkg/config/config.go b/pkg/config/config.go index cda59626cd1..00e416d077c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -108,6 +108,11 @@ const ( DefaultLogToJournald = false ) +const ( + // DefaultIrqBalanceConfigFile default irqbalance service configuration file path + DefaultIrqBalanceConfigFile = "/etc/sysconfig/irqbalance" +) + // This structure is necessary to fake the TOML tables when parsing, // while also not requiring a bunch of layered structs for no good // reason. @@ -234,6 +239,10 @@ type RuntimeConfig struct { // default for the runtime. ApparmorProfile string `toml:"apparmor_profile"` + // IrqBalanceConfigFile is the irqbalance service config file which is used + // for configuring irqbalance daemon. + IrqBalanceConfigFile string `toml:"irqbalance_config_file"` + // CgroupManagerName is the manager implementation name which is used to // handle cgroups for containers. CgroupManagerName string `toml:"cgroup_manager"` @@ -580,6 +589,7 @@ func DefaultConfig() (*Config, error) { ConmonCgroup: "system.slice", SELinux: selinuxEnabled(), ApparmorProfile: apparmor.DefaultProfile, + IrqBalanceConfigFile: DefaultIrqBalanceConfigFile, CgroupManagerName: cgroupManager.Name(), PidsLimit: DefaultPidsLimit, ContainerExitsDir: containerExitsDir, diff --git a/pkg/config/template.go b/pkg/config/template.go index 4b654006c6a..9a9032422f2 100644 --- a/pkg/config/template.go +++ b/pkg/config/template.go @@ -155,6 +155,10 @@ seccomp_use_default_when_empty = {{ .SeccompUseDefaultWhenEmpty }} # This option supports live configuration reload. apparmor_profile = "{{ .ApparmorProfile }}" +# Used to change irqbalance service config file path which is used for configuring +# irqbalance daemon. +irqbalance_config_file = "{{ .IrqBalanceConfigFile }}" + # Cgroup management implementation used for the runtime. cgroup_manager = "{{ .CgroupManagerName }}" diff --git a/server/container_start.go b/server/container_start.go index 22f2d3d6ed6..baa03abde5f 100644 --- a/server/container_start.go +++ b/server/container_start.go @@ -25,7 +25,7 @@ func (s *Server) StartContainer(ctx context.Context, req *types.StartContainerRe } sandbox := s.getSandbox(c.Sandbox()) - hooks, err := runtimehandlerhooks.GetRuntimeHandlerHooks(ctx, sandbox.RuntimeHandler(), s.Runtime()) + hooks, err := runtimehandlerhooks.GetRuntimeHandlerHooks(ctx, &s.config, sandbox.RuntimeHandler(), s.Runtime()) if err != nil { return fmt.Errorf("failed to get runtime handler %q hooks", sandbox.RuntimeHandler()) } diff --git a/server/container_stop.go b/server/container_stop.go index 0e14d572ffe..a2b08eccd09 100644 --- a/server/container_stop.go +++ b/server/container_stop.go @@ -21,7 +21,7 @@ func (s *Server) StopContainer(ctx context.Context, req *types.StopContainerRequ } sandbox := s.getSandbox(c.Sandbox()) - hooks, err := runtimehandlerhooks.GetRuntimeHandlerHooks(ctx, sandbox.RuntimeHandler(), s.Runtime()) + hooks, err := runtimehandlerhooks.GetRuntimeHandlerHooks(ctx, &s.config, sandbox.RuntimeHandler(), s.Runtime()) if err != nil { return fmt.Errorf("failed to get runtime handler %q hooks", sandbox.RuntimeHandler()) } diff --git a/server/sandbox_stop_linux.go b/server/sandbox_stop_linux.go index ce667897514..236166e53dc 100644 --- a/server/sandbox_stop_linux.go +++ b/server/sandbox_stop_linux.go @@ -45,7 +45,7 @@ func (s *Server) stopPodSandbox(ctx context.Context, req *types.StopPodSandboxRe } // Get high-performance runtime hook to trigger preStop step for each container - hooks, err := runtimehandlerhooks.GetRuntimeHandlerHooks(ctx, sb.RuntimeHandler(), s.Runtime()) + hooks, err := runtimehandlerhooks.GetRuntimeHandlerHooks(ctx, &s.config, sb.RuntimeHandler(), s.Runtime()) if err != nil { return fmt.Errorf("failed to get runtime handler %q hooks", sb.RuntimeHandler()) } diff --git a/server/server.go b/server/server.go index 9df90141f4e..00cdee1f6cc 100644 --- a/server/server.go +++ b/server/server.go @@ -23,6 +23,7 @@ import ( "github.com/cri-o/cri-o/internal/lib/sandbox" "github.com/cri-o/cri-o/internal/oci" "github.com/cri-o/cri-o/internal/resourcestore" + "github.com/cri-o/cri-o/internal/runtimehandlerhooks" "github.com/cri-o/cri-o/internal/storage" libconfig "github.com/cri-o/cri-o/pkg/config" "github.com/cri-o/cri-o/server/cri/types" @@ -338,6 +339,11 @@ func New( return nil, err } + err = runtimehandlerhooks.RestoreIrqBalanceConfig(config.IrqBalanceConfigFile, runtimehandlerhooks.IrqBannedCPUConfigFile, runtimehandlerhooks.IrqSmpAffinityProcFile) + if err != nil { + return nil, err + } + hostportManager := hostport.NewMetaHostportManager() idMappings, err := getIDMappings(config)