Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions completions/bash/crio
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ h
--insecure-registry
--internal-wipe
--irqbalance-config-file
--irqbalance-config-restore-file
--listen
--log
--log-dir
Expand Down
1 change: 1 addition & 0 deletions completions/fish/crio.fish
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l insecure-registry -r -d 'E
\'--insecure-registry\'.'
complete -c crio -n '__fish_crio_no_subcommand' -f -l internal-wipe -d 'Whether CRI-O should wipe containers after a reboot and images after an upgrade when the server starts. If set to false, one must run `crio wipe` to wipe the containers and images in these situations. This option is deprecated, and will be removed in the future.'
complete -c crio -n '__fish_crio_no_subcommand' -f -l irqbalance-config-file -r -d 'The irqbalance service config file which is used by CRI-O.'
complete -c crio -n '__fish_crio_no_subcommand' -f -l irqbalance-config-restore-file -r -d 'Determines if CRI-O should attempt to restore the irqbalance config at startup with the mask in this file. Use the 'disable' value to disable the restore flow entirely.'
complete -c crio -n '__fish_crio_no_subcommand' -l listen -r -d 'Path to the CRI-O socket.'
complete -c crio -n '__fish_crio_no_subcommand' -l log -r -d 'Set the log file path where internal debug information is written.'
complete -c crio -n '__fish_crio_no_subcommand' -l log-dir -r -d 'Default log directory where all logs will go unless directly specified by the kubelet.'
Expand Down
1 change: 1 addition & 0 deletions completions/zsh/_crio
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ it later with **--config**. Global options will modify the output.'
'--insecure-registry'
'--internal-wipe'
'--irqbalance-config-file'
'--irqbalance-config-restore-file'
'--listen'
'--log'
'--log-dir'
Expand Down
3 changes: 3 additions & 0 deletions docs/crio.8.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ crio
[--insecure-registry]=[value]
[--internal-wipe]
[--irqbalance-config-file]=[value]
[--irqbalance-config-restore-file]=[value]
[--listen]=[value]
[--log-dir]=[value]
[--log-filter]=[value]
Expand Down Expand Up @@ -281,6 +282,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]

**--irqbalance-config-file**="": The irqbalance service config file which is used by CRI-O. (default: /etc/sysconfig/irqbalance)

**--irqbalance-config-restore-file**="": Determines if CRI-O should attempt to restore the irqbalance config at startup with the mask in this file. Use the 'disable' value to disable the restore flow entirely. (default: /etc/sysconfig/orig_irq_banned_cpus)

**--listen**="": Path to the CRI-O socket. (default: /var/run/crio/crio.sock)

**--log**="": Set the log file path where internal debug information is written.
Expand Down
3 changes: 3 additions & 0 deletions docs/crio.conf.5.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,9 @@ the container runtime configuration.
Used to change irqbalance service config file which is used by CRI-O.
For CentOS/SUSE, this file is located at /etc/sysconfig/irqbalance. For Ubuntu, this file is located at /etc/default/irqbalance.

**irqbalance_config_restore_file**="/etc/sysconfig/orig_irq_banned_cpus"
Used to set the irqbalance banned cpu mask to restore at CRI-O startup. If set to 'disable', no restoration attempt will be done.

**rdt_config_file**=""
Path to the RDT configuration file for configuring the resctrl pseudo-filesystem.

Expand Down
8 changes: 8 additions & 0 deletions internal/criocli/criocli.go
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
if ctx.IsSet("absent-mount-sources-to-reject") {
config.AbsentMountSourcesToReject = StringSliceTrySplit(ctx, "absent-mount-sources-to-reject")
}
if ctx.IsSet("irqbalance-config-restore-file") {
config.IrqBalanceConfigRestoreFile = ctx.String("irqbalance-config-restore-file")
}
if ctx.IsSet("internal-wipe") {
config.InternalWipe = ctx.Bool("internal-wipe")
}
Expand Down Expand Up @@ -1083,6 +1086,11 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
Usage: "If true, CRI-O starts sending the container events to the kubelet",
EnvVars: []string{"ENABLE_POD_EVENTS"},
},
&cli.StringFlag{
Name: "irqbalance-config-restore-file",
Value: defConf.IrqBalanceConfigRestoreFile,
Usage: "Determines if CRI-O should attempt to restore the irqbalance config at startup with the mask in this file. Use the 'disable' value to disable the restore flow entirely.",
},
}
}

Expand Down
25 changes: 15 additions & 10 deletions internal/runtimehandlerhooks/high_performance_hooks.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"github.com/cri-o/cri-o/utils/cmdrunner"
"github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
Expand All @@ -29,8 +28,6 @@ import (
const (
// HighPerformance contains the high-performance runtime handler name
HighPerformance = "high-performance"
// IrqBannedCPUConfigFile contains the original banned cpu mask configuration
IrqBannedCPUConfigFile = "/etc/sysconfig/orig_irq_banned_cpus"
// IrqSmpAffinityProcFile contains the default smp affinity mask configuration
IrqSmpAffinityProcFile = "/proc/irq/default_smp_affinity"
)
Expand Down Expand Up @@ -78,7 +75,7 @@ func (h *HighPerformanceHooks) PreStart(ctx context.Context, c *oci.Container, s
// disable the IRQ smp load balancing for the container CPUs
if shouldIRQLoadBalancingBeDisabled(s.Annotations()) {
log.Infof(ctx, "Disable irq smp balancing for container %q", c.ID())
if err := setIRQLoadBalancing(c, false, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil {
if err := setIRQLoadBalancing(ctx, c, false, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil {
return fmt.Errorf("set IRQ load balancing: %w", err)
}
}
Expand Down Expand Up @@ -153,7 +150,7 @@ func (h *HighPerformanceHooks) PreStop(ctx context.Context, c *oci.Container, s

// enable the IRQ smp balancing for the container CPUs
if shouldIRQLoadBalancingBeDisabled(s.Annotations()) {
if err := setIRQLoadBalancing(c, true, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil {
if err := setIRQLoadBalancing(ctx, c, true, IrqSmpAffinityProcFile, h.irqBalanceConfigFile); err != nil {
return fmt.Errorf("set IRQ load balancing: %w", err)
}
}
Expand Down Expand Up @@ -308,7 +305,7 @@ func setCPUSLoadBalancing(c *oci.Container, enable bool, schedDomainDir string)
return nil
}

func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqBalanceConfigFile string) error {
func setIRQLoadBalancing(ctx context.Context, c *oci.Container, enable bool, irqSmpAffinityFile, irqBalanceConfigFile string) error {
lspec := c.Spec().Linux
if lspec == nil ||
lspec.Resources == nil ||
Expand Down Expand Up @@ -341,7 +338,7 @@ func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqB
if !isServiceEnabled(irqBalancedName) || !isIrqConfigExists {
if _, err := exec.LookPath(irqBalancedName); err != nil {
// irqbalance is not installed, skip the rest; pod should still start, so return nil instead
logrus.Warnf("Irqbalance binary not found: %v", err)
log.Warnf(ctx, "Irqbalance binary not found: %v", err)
return nil
}
// run irqbalance in daemon mode, so this won't cause delay
Expand All @@ -352,7 +349,7 @@ func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqB
}

if err := restartIrqBalanceService(); err != nil {
logrus.Warnf("Irqbalance service restart failed: %v", err)
log.Warnf(ctx, "Irqbalance service restart failed: %v", err)
}
return nil
}
Expand Down Expand Up @@ -600,7 +597,7 @@ func doSetCPUFreqGovernor(c *oci.Container, governor, cpuDir, cpuSaveDir string)
}

// RestoreIrqBalanceConfig restores irqbalance service with original banned cpu mask settings
func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityProcFile string) error {
func RestoreIrqBalanceConfig(ctx context.Context, irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityProcFile string) error {
content, err := os.ReadFile(irqSmpAffinityProcFile)
if err != nil {
return err
Expand All @@ -614,15 +611,19 @@ func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSm
}
if !isAllBitSet(currentMaskArray) {
// not system reboot scenario, just return it.
log.Infof(ctx, "Restore irqbalance config: not system reboot, ignoring")
return nil
}

bannedCPUMasks, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile)
if err != nil {
// Ignore returning err as given irqBalanceConfigFile may not exist.
log.Infof(ctx, "Restore irqbalance config: failed to get current CPU ban list, ignoring")
return nil
}

if !fileExists(irqBannedCPUConfigFile) {
log.Infof(ctx, "Creating banned CPU list file %q", irqBannedCPUConfigFile)
irqBannedCPUsConfig, err := os.Create(irqBannedCPUConfigFile)
if err != nil {
return err
Expand All @@ -632,6 +633,7 @@ func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSm
if err != nil {
return err
}
log.Infof(ctx, "Restore irqbalance config: created backup file")
return nil
}

Expand All @@ -642,14 +644,17 @@ func RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSm
origBannedCPUMasks := strings.TrimSpace(string(content))

if bannedCPUMasks == origBannedCPUMasks {
log.Infof(ctx, "Restore irqbalance config: nothing to do")
return nil
}

log.Infof(ctx, "Restore irqbalance banned CPU list in %q to %q", irqBalanceConfigFile, origBannedCPUMasks)
if err := updateIrqBalanceConfigFile(irqBalanceConfigFile, origBannedCPUMasks); err != nil {
return err
}
if isServiceEnabled(irqBalancedName) {
if err := restartIrqBalanceService(); err != nil {
logrus.Warnf("Irqbalance service restart failed: %v", err)
log.Warnf(ctx, "Irqbalance service restart failed: %v", err)
}
}
return nil
Expand Down
16 changes: 8 additions & 8 deletions internal/runtimehandlerhooks/high_performance_hooks_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ var _ = Describe("high_performance_hooks", func() {
irqSmpAffinityFile := filepath.Join(fixturesDir, "irq_smp_affinity")
irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance")
verifySetIRQLoadBalancing := func(enabled bool, expected string) {
err := setIRQLoadBalancing(container, enabled, irqSmpAffinityFile, irqBalanceConfigFile)
err := setIRQLoadBalancing(context.TODO(), container, enabled, irqSmpAffinityFile, irqBalanceConfigFile)
Expect(err).To(BeNil())

content, err := os.ReadFile(irqSmpAffinityFile)
Expand Down Expand Up @@ -164,7 +164,7 @@ var _ = Describe("high_performance_hooks", func() {
irqSmpAffinityFile := filepath.Join(fixturesDir, "irq_smp_affinity")
irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance")
verifySetIRQLoadBalancing := func(enabled bool, expectedSmp, expectedBan string) {
err = setIRQLoadBalancing(container, enabled, irqSmpAffinityFile, irqBalanceConfigFile)
err = setIRQLoadBalancing(context.TODO(), container, enabled, irqSmpAffinityFile, irqBalanceConfigFile)
Expect(err).To(BeNil())

content, err := os.ReadFile(irqSmpAffinityFile)
Expand Down Expand Up @@ -511,16 +511,16 @@ var _ = Describe("high_performance_hooks", func() {
irqBalanceConfigFile := filepath.Join(fixturesDir, "irqbalance")
irqBannedCPUConfigFile := filepath.Join(fixturesDir, "orig_irq_banned_cpus")
verifyRestoreIrqBalanceConfig := func(expectedOrigBannedCPUs, expectedBannedCPUs string) {
err = RestoreIrqBalanceConfig(irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityFile)
Expect(err).To(BeNil())
err = RestoreIrqBalanceConfig(context.TODO(), irqBalanceConfigFile, irqBannedCPUConfigFile, irqSmpAffinityFile)
ExpectWithOffset(1, err).To(BeNil())

content, err := os.ReadFile(irqBannedCPUConfigFile)
Expect(err).To(BeNil())
Expect(strings.Trim(string(content), "\n")).To(Equal(expectedOrigBannedCPUs))
ExpectWithOffset(1, err).To(BeNil())
ExpectWithOffset(1, strings.Trim(string(content), "\n")).To(Equal(expectedOrigBannedCPUs))

bannedCPUs, err := retrieveIrqBannedCPUMasks(irqBalanceConfigFile)
Expect(err).To(BeNil())
Expect(bannedCPUs).To(Equal(expectedBannedCPUs))
ExpectWithOffset(1, err).To(BeNil())
ExpectWithOffset(1, bannedCPUs).To(Equal(expectedBannedCPUs))
}

JustBeforeEach(func() {
Expand Down
63 changes: 35 additions & 28 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ const (
const (
// DefaultIrqBalanceConfigFile default irqbalance service configuration file path
DefaultIrqBalanceConfigFile = "/etc/sysconfig/irqbalance"
// DefaultIrqBalanceConfigRestoreFile contains the banned cpu mask configuration to restore. Name due to backward compatibility.
DefaultIrqBalanceConfigRestoreFile = "/etc/sysconfig/orig_irq_banned_cpus"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would expect this config option would mean we remove the hard-coded one in internal/runtimehandlerhooks/high_performance_hooks.go

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also, given the name of the file, I wouldn't expect it to be a restore file. Is this file's name/format documented anywhere?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this PR provides an alternative param for the user to have more control over the restore process while keeping the already working way to do it, so it will keep the exact same behaviour if the user do not give the new option.
At least that is how I have understood the change but I could be wrong.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's true. We keep the filename originally added back in time for full backward compatibility.

)

// This structure is necessary to fake the TOML tables when parsing,
Expand Down Expand Up @@ -410,6 +412,10 @@ type RuntimeConfig struct {
// EnablePodEvents specifies if the container pod-level events should be generated to optimize the PLEG at Kubelet.
EnablePodEvents bool `toml:"enable_pod_events"`

// IrqBalanceConfigRestoreFile is the irqbalance service banned CPU list to restore.
// If empty, no restoration attempt will be done.
IrqBalanceConfigRestoreFile string `toml:"irqbalance_config_restore_file"`

// seccompConfig is the internal seccomp configuration
seccompConfig *seccomp.Config

Expand Down Expand Up @@ -792,34 +798,35 @@ func DefaultConfig() (*Config, error) {
Runtimes: Runtimes{
defaultRuntime: defaultRuntimeHandler(),
},
SELinux: selinuxEnabled(),
ApparmorProfile: apparmor.DefaultProfile,
BlockIOConfigFile: DefaultBlockIOConfigFile,
IrqBalanceConfigFile: DefaultIrqBalanceConfigFile,
RdtConfigFile: rdt.DefaultRdtConfigFile,
CgroupManagerName: cgroupManager.Name(),
PidsLimit: DefaultPidsLimit,
ContainerExitsDir: containerExitsDir,
ContainerAttachSocketDir: conmonconfig.ContainerAttachSocketDir,
MinimumMappableUID: -1,
MinimumMappableGID: -1,
LogSizeMax: DefaultLogSizeMax,
CtrStopTimeout: defaultCtrStopTimeout,
DefaultCapabilities: capabilities.Default(),
LogLevel: "info",
HooksDir: []string{hooks.DefaultDir},
CDISpecDirs: cdi.DefaultSpecDirs,
NamespacesDir: defaultNamespacesDir,
DropInfraCtr: true,
SeccompUseDefaultWhenEmpty: seccompConfig.UseDefaultWhenEmpty(),
seccompConfig: seccomp.New(),
apparmorConfig: apparmor.New(),
blockioConfig: blockio.New(),
cgroupManager: cgroupManager,
deviceConfig: device.New(),
namespaceManager: nsmgr.New(defaultNamespacesDir, ""),
rdtConfig: rdt.New(),
ulimitsConfig: ulimits.New(),
SELinux: selinuxEnabled(),
ApparmorProfile: apparmor.DefaultProfile,
BlockIOConfigFile: DefaultBlockIOConfigFile,
IrqBalanceConfigFile: DefaultIrqBalanceConfigFile,
RdtConfigFile: rdt.DefaultRdtConfigFile,
CgroupManagerName: cgroupManager.Name(),
PidsLimit: DefaultPidsLimit,
ContainerExitsDir: containerExitsDir,
ContainerAttachSocketDir: conmonconfig.ContainerAttachSocketDir,
MinimumMappableUID: -1,
MinimumMappableGID: -1,
LogSizeMax: DefaultLogSizeMax,
CtrStopTimeout: defaultCtrStopTimeout,
DefaultCapabilities: capabilities.Default(),
LogLevel: "info",
HooksDir: []string{hooks.DefaultDir},
CDISpecDirs: cdi.DefaultSpecDirs,
NamespacesDir: defaultNamespacesDir,
DropInfraCtr: true,
SeccompUseDefaultWhenEmpty: seccompConfig.UseDefaultWhenEmpty(),
IrqBalanceConfigRestoreFile: DefaultIrqBalanceConfigRestoreFile,
seccompConfig: seccomp.New(),
apparmorConfig: apparmor.New(),
blockioConfig: blockio.New(),
cgroupManager: cgroupManager,
deviceConfig: device.New(),
namespaceManager: nsmgr.New(defaultNamespacesDir, ""),
rdtConfig: rdt.New(),
ulimitsConfig: ulimits.New(),
},
ImageConfig: ImageConfig{
DefaultTransport: "docker://",
Expand Down
12 changes: 12 additions & 0 deletions pkg/config/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,11 @@ func initCrioTemplateConfig(c *Config) ([]*templateConfigValue, error) {
group: crioRuntimeConfig,
isDefaultValue: simpleEqual(dc.IrqBalanceConfigFile, c.IrqBalanceConfigFile),
},
{
templateString: templateStringCrioRuntimeIrqBalanceConfigRestoreFile,
group: crioRuntimeConfig,
isDefaultValue: simpleEqual(dc.IrqBalanceConfigRestoreFile, c.IrqBalanceConfigRestoreFile),
},
{
templateString: templateStringCrioRuntimeRdtConfigFile,
group: crioRuntimeConfig,
Expand Down Expand Up @@ -1066,6 +1071,13 @@ const templateStringCrioRuntimeDropInfraCtr = `# drop_infra_ctr determines wheth

`

const templateStringCrioRuntimeIrqBalanceConfigRestoreFile = `# irqbalance_config_restore_file allows to set a cpu mask CRI-O should
# restore as irqbalance config at startup. Set to empty string to disable this flow entirely.
# By default, CRI-O manages the irqbalance configuration to enable dynamic IRQ pinning.
{{ $.Comment }}irqbalance_config_restore_file = "{{ .IrqBalanceConfigRestoreFile }}"

`

const templateStringCrioRuntimeInfraCtrCpuset = `# infra_ctr_cpuset determines what CPUs will be used to run infra containers.
# You can use linux CPU list format to specify desired CPUs.
# To get better isolation for guaranteed pods, set this parameter to be equal to kubelet reserved-cpus.
Expand Down
14 changes: 9 additions & 5 deletions server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,9 @@ import (
)

const (
certRefreshInterval = time.Minute * 5
rootlessEnvName = "_CRIO_ROOTLESS"
certRefreshInterval = time.Minute * 5
rootlessEnvName = "_CRIO_ROOTLESS"
irqBalanceConfigRestoreDisable = "disable"
)

var errSandboxNotCreated = errors.New("sandbox not created")
Expand Down Expand Up @@ -419,9 +420,12 @@ func New(
return nil, err
}

err = runtimehandlerhooks.RestoreIrqBalanceConfig(config.IrqBalanceConfigFile, runtimehandlerhooks.IrqBannedCPUConfigFile, runtimehandlerhooks.IrqSmpAffinityProcFile)
if err != nil {
return nil, err
if strings.ToLower(strings.TrimSpace(config.IrqBalanceConfigRestoreFile)) != irqBalanceConfigRestoreDisable {
log.Infof(ctx, "Attempting to restore irqbalance config from %s", config.IrqBalanceConfigRestoreFile)
err = runtimehandlerhooks.RestoreIrqBalanceConfig(context.TODO(), config.IrqBalanceConfigFile, config.IrqBalanceConfigRestoreFile, runtimehandlerhooks.IrqSmpAffinityProcFile)
if err != nil {
return nil, err
}
}

hostportManager := hostport.NewMetaHostportManager()
Expand Down
Loading