cri-o · openshift-merge-robot · Jan 14, 2021 · Jan 6, 2021 · Jan 6, 2021 · Jan 6, 2021
@@ -44,6 +44,7 @@ h
 --grpc-max-send-msg-size
 --hooks-dir
 --image-volumes
+--infra-ctr-cpuset
 --insecure-registry
 --listen
 --log

@@ -73,6 +73,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l image-volumes -r -d 'Image
     2. bind: A directory is created inside container state directory and bind
        mounted into the container for the volumes.
 	3. ignore: All volumes are just ignored and no action is taken.'
+complete -c crio -n '__fish_crio_no_subcommand' -f -l infra-ctr-cpuset -r -d 'CPU set to run infra containers, if not specified CRI-O will use all online CPUs to run infra containers (default: \'\').'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l insecure-registry -r -d 'Enable insecure registry communication, i.e., enable un-encrypted and/or untrusted communication.
     1. List of insecure registries can contain an element with CIDR notation to
        specify a whole subnet.

@@ -7,7 +7,7 @@ it later with **--config**. Global options will modify the output.' 'version:dis
   _describe 'commands' cmds
 
   local -a opts
-  opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-metrics' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--insecure-registry' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--manage-ns-lifecycle' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
+  opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-metrics' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--infra-ctr-cpuset' '--insecure-registry' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--manage-ns-lifecycle' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
   _describe 'global options' opts
 
   return

@@ -44,6 +44,7 @@ crio
 [--help|-h]
 [--hooks-dir]=[value]
 [--image-volumes]=[value]
+[--infra-ctr-cpuset]=[value]
 [--insecure-registry]=[value]
 [--listen]=[value]
 [--log-dir]=[value]
@@ -211,6 +212,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
        mounted into the container for the volumes.
 	3. ignore: All volumes are just ignored and no action is taken. (default: mkdir)
 
+**--infra-ctr-cpuset**="": CPU set to run infra containers, if not specified CRI-O will use all online CPUs to run infra containers (default: '').
+
 **--insecure-registry**="": Enable insecure registry communication, i.e., enable un-encrypted and/or untrusted communication.
     1. List of insecure registries can contain an element with CIDR notation to
        specify a whole subnet.

@@ -225,6 +225,11 @@ the container runtime configuration.
 **drop_infra_ctr**=false
   Determines whether we drop the infra container when a pod does not have a private PID namespace, and does not use a kernel separating runtime (like kata).
   Requies **manage_ns_lifecycle** to be true.
+
+**infra_ctr_cpuset**=""
+    Determines the CPU set to run infra containers. If not specified, the CRI-O will use all online CPUs to run infra containers.
+    You can specify CPUs in the Linux CPU list format.
+    To get better isolation for guaranteed pods, set this parameter to be equal to kubelet reserved-cpus.
 
 **namespaces_dir**="/var/run"
   The directory where the state of the managed namespaces gets tracked. Only used when manage_ns_lifecycle is true

@@ -294,6 +294,10 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
 		config.SeparatePullCgroup = ctx.String("separate-pull-cgroup")
 	}
 
+	if ctx.IsSet("infra-ctr-cpuset") {
+		config.InfraCtrCPUSet = ctx.String("infra-ctr-cpuset")
+	}
+
 	return nil
 }
 
@@ -802,6 +806,11 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
 			EnvVars:   []string{"CONTAINER_VERSION_FILE_PERSIST"},
 			TakesFile: true,
 		},
+		&cli.StringFlag{
+			Name:    "infra-ctr-cpuset",
+			Usage:   "CPU set to run infra containers, if not specified CRI-O will use all online CPUs to run infra containers (default: '').",
+			EnvVars: []string{"CONTAINER_INFRA_CTR_CPUSET"},
+		},
 	}
 }
 

@@ -12,6 +12,7 @@ import (
 	"github.com/cri-o/cri-o/pkg/config"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/net/context"
+
 	"k8s.io/client-go/tools/remotecommand"
 )
 

@@ -31,6 +31,8 @@ import (
 	selinux "github.com/opencontainers/selinux/go-selinux"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+
+	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
 )
 
 // Defaults if none are specified
@@ -304,6 +306,9 @@ type RuntimeConfig struct {
 	// SeparatePullCgroup specifies whether an image pull must be performed in a separate cgroup
 	SeparatePullCgroup string `toml:"separate_pull_cgroup"`
 
+	// InfraCtrCPUSet is the CPUs set that will be used to run infra containers
+	InfraCtrCPUSet string `toml:"infra_ctr_cpuset"`
+
 	// seccompConfig is the internal seccomp configuration
 	seccompConfig *seccomp.Config
 
@@ -791,6 +796,12 @@ func (c *RuntimeConfig) Validate(systemContext *types.SystemContext, onExecution
 		return errors.Wrapf(err, "invalid capabilities")
 	}
 
+	if c.InfraCtrCPUSet != "" {
+		if _, err := cpuset.Parse(c.InfraCtrCPUSet); err != nil {
+			return errors.Wrap(err, "invalid infra_ctr_cpuset")
+		}
+	}
+
 	// check for validation on execution
 	if onExecution {
 		if err := c.ValidateRuntimes(); err != nil {

@@ -258,6 +258,11 @@ manage_ns_lifecycle = {{ .ManageNSLifecycle }}
 # It requires manage_ns_lifecycle to be true.
 drop_infra_ctr = {{ .DropInfraCtr }}
 
+# infra_ctr_cpuset determines what CPUs will be used to run infra containers.
+# You can use linux CPU list format to specify desired CPUs.
+# To get better isolation for guaranteed pods, set this parameter to be equal to kubelet reserved-cpus.
+# infra_ctr_cpuset = "{{ .InfraCtrCPUSet }}"
+
 # The directory where the state of the managed namespaces gets tracked.
 # Only used when manage_ns_lifecycle is true.
 namespaces_dir = "{{ .NamespacesDir }}"

@@ -747,6 +747,12 @@ func (s *Server) runPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 
 	g.SetLinuxResourcesCPUShares(PodInfraCPUshares)
 
+	// When infra-ctr-cpuset specified, set the infra container CPU set
+	if s.config.InfraCtrCPUSet != "" {
+		log.Debugf(ctx, "Set the infra container cpuset to %q", s.config.InfraCtrCPUSet)
+		g.SetLinuxResourcesCPUCpus(s.config.InfraCtrCPUSet)
+	}
+
 	saveOptions := generate.ExportOptions{}
 	mountPoint, err := s.StorageRuntimeServer().StartContainer(sbox.ID())
 	if err != nil {

@@ -0,0 +1,19 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+	setup_test
+	CONTAINER_INFRA_CTR_CPUSET="0" start_crio
+}
+
+function teardown() {
+	cleanup_test
+}
+
+@test "test infra ctr cpuset" {
+	pod_id=$(crictl runp "$TESTDATA"/sandbox_config.json)
+
+	output=$(crictl inspectp -o yaml "$pod_id")
+	[[ "$output" = *"cpus: \"0\""* ]]
+}