cri-o · openshift-merge-robot · Jun 8, 2021 · Jun 2, 2021 · May 20, 2021 · May 21, 2021
diff --git a/go.mod b/go.mod
@@ -40,7 +40,7 @@ require (
 	github.com/onsi/gomega v1.13.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2-0.20200206005212-79b036d80240
-	github.com/opencontainers/runc v1.0.0-rc95
+	github.com/opencontainers/runc v1.0.0-rc95.0.20210521141834-a95237f81684
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
 	github.com/opencontainers/runtime-tools v0.9.1-0.20200121211434-d1bf3e66ff0a
 	github.com/opencontainers/selinux v1.8.2

diff --git a/go.sum b/go.sum
@@ -1077,8 +1077,9 @@ github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h
 github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc91/go.mod h1:3Sm6Dt7OT8z88EbdQqqcRN2oCT54jbi72tT/HqgflT8=
 github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=
-github.com/opencontainers/runc v1.0.0-rc95 h1:RMuWVfY3E1ILlVsC3RhIq38n4sJtlOFwU9gfFZSqrd0=
 github.com/opencontainers/runc v1.0.0-rc95/go.mod h1:z+bZxa/+Tz/FmYVWkhUajJdzFeOqjc5vrqskhVyHGUM=
+github.com/opencontainers/runc v1.0.0-rc95.0.20210521141834-a95237f81684 h1:lxWmdjKd6ohpRh4G2ogFNS4EAyAbwWZnlOcfYfpu22s=
+github.com/opencontainers/runc v1.0.0-rc95.0.20210521141834-a95237f81684/go.mod h1:z+bZxa/+Tz/FmYVWkhUajJdzFeOqjc5vrqskhVyHGUM=
 github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1 h1:UAfI7SOCo1CNIu3RevW9B4HQyf7SY5aSzcSeoC7OPs0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20201121164853-7413a7f753e1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=

diff --git a/internal/config/cgmgr/cgmgr.go b/internal/config/cgmgr/cgmgr.go
@@ -58,11 +58,11 @@ type CgroupManager interface {
 	// returns the cgroup parent, cgroup path, and error. For systemd cgroups,
 	// it also checks there is enough memory in the given cgroup
 	SandboxCgroupPath(string, string) (string, string, error)
-	// MoveConmonToCgroup takes the container ID, cgroup parent, conmon's cgroup (from the config) and conmon's PID
-	// It attempts to move conmon to the correct cgroup.
+	// MoveConmonToCgroup takes the container ID, cgroup parent, conmon's cgroup (from the config), conmon's PID, and some customized resources
+	// It attempts to move conmon to the correct cgroup, and set the resources for that cgroup.
 	// It returns the cgroupfs parent that conmon was put into
 	// so that CRI-O can clean the parent cgroup of the newly added conmon once the process terminates (systemd handles this for us)
-	MoveConmonToCgroup(cid, cgroupParent, conmonCgroup string, pid int) (string, error)
+	MoveConmonToCgroup(cid, cgroupParent, conmonCgroup string, pid int, resources *rspec.LinuxResources) (string, error)
 	// CreateSandboxCgroup takes the sandbox parent, and sandbox ID.
 	// It creates a new cgroup for that sandbox, which is useful when spoofing an infra container.
 	CreateSandboxCgroup(sbParent, containerID string) error

diff --git a/internal/config/cgmgr/cgmgr_test.go b/internal/config/cgmgr/cgmgr_test.go
@@ -160,7 +160,7 @@ var _ = t.Describe("Config", func() {
 				// Given
 				conmonCgroup := "notPodOrEmpty"
 				// When
-				cgPath, err := sut.MoveConmonToCgroup("", "", conmonCgroup, 0)
+				cgPath, err := sut.MoveConmonToCgroup("", "", conmonCgroup, 0, nil)
 
 				// Then
 				Expect(cgPath).To(BeEmpty())
@@ -245,7 +245,7 @@ var _ = t.Describe("Config", func() {
 				// Given
 				conmonCgroup := "notPodOrEmpty"
 				// When
-				cgPath, err := sut.MoveConmonToCgroup("", "", conmonCgroup, -1)
+				cgPath, err := sut.MoveConmonToCgroup("", "", conmonCgroup, -1, nil)
 
 				// Then
 				Expect(cgPath).To(BeEmpty())

diff --git a/internal/config/cgmgr/cgroupfs.go b/internal/config/cgmgr/cgroupfs.go
@@ -9,6 +9,13 @@ import (
 	"strings"
 
 	"github.com/containers/podman/v3/pkg/cgroups"
+	"github.com/containers/podman/v3/pkg/rootless"
+	"github.com/cri-o/cri-o/internal/config/node"
+	libctr "github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fs"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fs2"
+	cgcfgs "github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/devices"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -68,11 +75,15 @@ func (m *CgroupfsManager) SandboxCgroupPath(sbParent, sbID string) (cgParent, cg
 // It attempts to move conmon to the correct cgroup.
 // It returns the cgroupfs parent that conmon was put into
 // so that CRI-O can clean the cgroup path of the newly added conmon once the process terminates (systemd handles this for us)
-func (*CgroupfsManager) MoveConmonToCgroup(cid, cgroupParent, conmonCgroup string, pid int) (string, error) {
+func (*CgroupfsManager) MoveConmonToCgroup(cid, cgroupParent, conmonCgroup string, pid int, resources *rspec.LinuxResources) (cgroupPathToClean string, _ error) {
 	if conmonCgroup != "pod" && conmonCgroup != "" {
 		return "", errors.Errorf("conmon cgroup %s invalid for cgroupfs", conmonCgroup)
 	}
 
+	if resources == nil {
+		resources = &rspec.LinuxResources{}
+	}
+
 	cgroupPath := fmt.Sprintf("%s/crio-conmon-%s", cgroupParent, cid)
 	control, err := cgroups.New(cgroupPath, &rspec.LinuxResources{})
 	if err != nil {
@@ -82,6 +93,10 @@ func (*CgroupfsManager) MoveConmonToCgroup(cid, cgroupParent, conmonCgroup strin
 		return cgroupPath, nil
 	}
 
+	if err := setWorkloadSettings(cgroupPath, resources); err != nil {
+		return cgroupPath, err
+	}
+
 	// Record conmon's cgroup path in the container, so we can properly
 	// clean it up when removing the container.
 	// Here we should defer a crio-connmon- cgroup hierarchy deletion, but it will
@@ -96,6 +111,51 @@ func (*CgroupfsManager) MoveConmonToCgroup(cid, cgroupParent, conmonCgroup strin
 	return cgroupPath, nil
 }
 
+func setWorkloadSettings(cgPath string, resources *rspec.LinuxResources) error {
+	var mgr libctr.Manager
+	if resources.CPU == nil {
+		return nil
+	}
+
+	paths := map[string]string{
+		"cpuset":  filepath.Join("/sys/fs/cgroup", "cpuset", cgPath),
+		"cpu":     filepath.Join("/sys/fs/cgroup", "cpu", cgPath),
+		"freezer": filepath.Join("/sys/fs/cgroup", "freezer", cgPath),
+		"devices": filepath.Join("/sys/fs/cgroup", "devices", cgPath),
+	}
+
+	cg := &cgcfgs.Cgroup{
+		Name:      cgPath,
+		Resources: &cgcfgs.Resources{},
+	}
+	if resources.CPU.Cpus != "" {
+		cg.Resources.CpusetCpus = resources.CPU.Cpus
+	}
+	if resources.CPU.Shares != nil {
+		cg.Resources.CpuShares = *resources.CPU.Shares
+	}
+
+	// We need to white list all devices
+	// so containers created underneath won't fail
+	cg.Resources.Devices = []*devices.Rule{
+		{
+			Type:  devices.WildcardDevice,
+			Allow: true,
+		},
+	}
+
+	if node.CgroupIsV2() {
+		var err error
+		mgr, err = fs2.NewManager(cg, cgPath, rootless.IsRootless())
+		if err != nil {
+			return err
+		}
+	} else {
+		mgr = fs.NewManager(cg, paths, rootless.IsRootless())
+	}
+	return mgr.Set(cg.Resources)
+}
+
 // CreateSandboxCgroup calls the helper function createSandboxCgroup for this manager.
 func (m *CgroupfsManager) CreateSandboxCgroup(sbParent, containerID string) error {
 	return createSandboxCgroup(sbParent, containerID, m)

diff --git a/internal/config/cgmgr/systemd.go b/internal/config/cgmgr/systemd.go
@@ -9,9 +9,11 @@ import (
 	"strings"
 
 	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
+	"github.com/cri-o/cri-o/internal/config/node"
 	"github.com/cri-o/cri-o/utils"
 	"github.com/godbus/dbus/v5"
 	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -68,7 +70,7 @@ func (*SystemdManager) ContainerCgroupAbsolutePath(sbParent, containerID string)
 // cgroupPathToClean should always be returned empty. It is part of the interface to return the cgroup path
 // that cri-o is responsible for cleaning up upon the container's death.
 // Systemd takes care of this cleaning for us, so return an empty string
-func (*SystemdManager) MoveConmonToCgroup(cid, cgroupParent, conmonCgroup string, pid int) (cgroupPathToClean string, _ error) {
+func (*SystemdManager) MoveConmonToCgroup(cid, cgroupParent, conmonCgroup string, pid int, resources *rspec.LinuxResources) (cgroupPathToClean string, _ error) {
 	if strings.HasSuffix(conmonCgroup, ".slice") {
 		cgroupParent = conmonCgroup
 	}
@@ -77,12 +79,39 @@ func (*SystemdManager) MoveConmonToCgroup(cid, cgroupParent, conmonCgroup string
 	// Set the systemd KillSignal to SIGPIPE that conmon ignores.
 	// This helps during node shutdown so that conmon waits for the container
 	// to exit and doesn't forward the SIGTERM that it gets.
-	killSignalProp := systemdDbus.Property{
-		Name:  "KillSignal",
-		Value: dbus.MakeVariant(int(unix.SIGPIPE)),
+	props := []systemdDbus.Property{
+		{
+			Name:  "KillSignal",
+			Value: dbus.MakeVariant(int(unix.SIGPIPE)),
+		},
+		systemdDbus.PropAfter("crio.service"),
 	}
+
+	if resources != nil && resources.CPU != nil {
+		if resources.CPU.Cpus != "" {
+			if !node.SystemdHasAllowedCPUs() {
+				logrus.Errorf("Systemd does not support AllowedCPUs; skipping setting for workload")
+			} else {
+				bits, err := systemd.RangeToBits(resources.CPU.Cpus)
+				if err != nil {
+					return "", errors.Wrapf(err, "cpuset conversion error")
+				}
+				props = append(props, systemdDbus.Property{
+					Name:  "AllowedCPUs",
+					Value: dbus.MakeVariant(bits),
+				})
+			}
+		}
+		if resources.CPU.Shares != nil {
+			props = append(props, systemdDbus.Property{
+				Name:  "CPUShares",
+				Value: dbus.MakeVariant(resources.CPU.Shares),
+			})
+		}
+	}
+
 	logrus.Debugf("Running conmon under slice %s and unitName %s", cgroupParent, conmonUnitName)
-	if err := utils.RunUnderSystemdScope(pid, cgroupParent, conmonUnitName, killSignalProp, systemdDbus.PropAfter("crio.service")); err != nil {
+	if err := utils.RunUnderSystemdScope(pid, cgroupParent, conmonUnitName, props...); err != nil {
 		return "", errors.Wrapf(err, "failed to add conmon to systemd sandbox cgroup")
 	}
 	// return empty string as path because cgroup cleanup is done by systemd

diff --git a/internal/config/node/node.go b/internal/config/node/node.go
@@ -55,6 +55,13 @@ func ValidateConfig() error {
 			activated: &systemdHasCollectMode,
 			fatal:     false,
 		},
+		{
+			name:      "systemd AllowedCPUs",
+			init:      SystemdHasAllowedCPUs,
+			err:       &systemdHasAllowedCPUsErr,
+			activated: &systemdHasAllowedCPUs,
+			fatal:     false,
+		},
 		{
 			name:      "fs.may_detach_mounts sysctl",
 			init:      checkFsMayDetachMounts,

diff --git a/internal/config/node/systemd.go b/internal/config/node/systemd.go
@@ -13,17 +13,35 @@ var (
 	systemdHasCollectModeOnce sync.Once
 	systemdHasCollectMode     bool
 	systemdHasCollectModeErr  error
+
+	systemdHasAllowedCPUsOnce sync.Once
+	systemdHasAllowedCPUs     bool
+	systemdHasAllowedCPUsErr  error
 )
 
 func SystemdHasCollectMode() bool {
 	systemdHasCollectModeOnce.Do(func() {
-		// This will show whether the currently running systemd supports CollectMode
-		_, err := exec.Command("systemctl", "show", "-p", "CollectMode", "systemd").Output()
-		if err != nil {
-			systemdHasCollectModeErr = errors.Wrapf(err, "check systemd CollectMode")
-			return
-		}
-		systemdHasCollectMode = true
+		systemdHasCollectMode, systemdHasCollectModeErr = systemdSupportsProperty("CollectMode")
 	})
 	return systemdHasCollectMode
 }
+
+func SystemdHasAllowedCPUs() bool {
+	systemdHasAllowedCPUsOnce.Do(func() {
+		systemdHasAllowedCPUs, systemdHasAllowedCPUsErr = systemdSupportsProperty("AllowedCPUs")
+	})
+	return systemdHasAllowedCPUs
+}
+
+// systemdSupportsProperty checks whether systemd supports a property
+// It returns an error if it does not.
+func systemdSupportsProperty(property string) (bool, error) {
+	output, err := exec.Command("systemctl", "show", "-p", property, "systemd").Output()
+	if err != nil {
+		return false, errors.Wrapf(err, "check systemd %s", property)
+	}
+	if len(output) == 0 {
+		return false, nil
+	}
+	return true, nil
+}
diff --git a/internal/oci/oci_linux.go b/internal/oci/oci_linux.go
@@ -14,6 +14,9 @@ import (
 
 	"github.com/containers/podman/v3/pkg/cgroups"
 	"github.com/cri-o/cri-o/internal/config/node"
+	"github.com/cri-o/cri-o/server/cri/types"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/generate"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -23,8 +26,20 @@ func (r *runtimeOCI) createContainerPlatform(c *Container, cgroupParent string,
 	if c.Spoofed() {
 		return nil
 	}
+	g := &generate.Generator{
+		Config: &rspec.Spec{
+			Linux: &rspec.Linux{
+				Resources: &rspec.LinuxResources{},
+			},
+		},
+	}
+	// Mutate our newly created spec to find the customizations that are needed for conmon
+	if err := r.config.Workloads.MutateSpecGivenAnnotations(types.InfraContainerName, g, c.Annotations()); err != nil {
+		return err
+	}
+
 	// Move conmon to specified cgroup
-	conmonCgroupfsPath, err := r.config.CgroupManager().MoveConmonToCgroup(c.id, cgroupParent, r.config.ConmonCgroup, pid)
+	conmonCgroupfsPath, err := r.config.CgroupManager().MoveConmonToCgroup(c.id, cgroupParent, r.config.ConmonCgroup, pid, g.Config.Linux.Resources)
 	if err != nil {
 		return err
 	}

diff --git a/internal/storage/runtime.go b/internal/storage/runtime.go
@@ -383,6 +383,10 @@ func (r *runtimeService) DeleteContainer(idOrName string) error {
 		return ErrInvalidContainerID
 	}
 	container, err := r.storageImageServer.GetStore().Container(idOrName)
+	// Already deleted
+	if errors.Is(err, storage.ErrContainerUnknown) {
+		return nil
+	}
 	if err != nil {
 		return err
 	}

diff --git a/server/cri/types/types.go b/server/cri/types/types.go
@@ -22,6 +22,8 @@ const (
 
 	PodSandboxStateSandboxReady    PodSandboxState = 0
 	PodSandboxStateSandboxNotReady PodSandboxState = 1
+
+	InfraContainerName = "POD"
 )
 
 type VersionRequest struct {

diff --git a/server/naming.go b/server/naming.go
@@ -10,14 +10,13 @@ import (
 
 const (
 	kubePrefix    = "k8s"
-	infraName     = "POD"
 	nameDelimiter = "_"
 )
 
 func makeSandboxContainerName(sandboxConfig *types.PodSandboxConfig) string {
 	return strings.Join([]string{
 		kubePrefix,
-		infraName,
+		types.InfraContainerName,
 		sandboxConfig.Metadata.Name,
 		sandboxConfig.Metadata.Namespace,
 		sandboxConfig.Metadata.UID,

diff --git a/server/sandbox_run_linux.go b/server/sandbox_run_linux.go
@@ -38,7 +38,6 @@ import (
 	"golang.org/x/net/context"
 	"golang.org/x/sys/unix"
 	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/kubernetes/pkg/kubelet/leaky"
 	kubeletTypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
 
@@ -281,7 +280,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ
 	pathsToChown := []string{}
 
 	// we need to fill in the container name, as it is not present in the request. Luckily, it is a constant.
-	log.Infof(ctx, "Running pod sandbox: %s%s", translateLabelsToDescription(sbox.Config().Labels), leaky.PodInfraContainerName)
+	log.Infof(ctx, "Running pod sandbox: %s%s", translateLabelsToDescription(sbox.Config().Labels), types.InfraContainerName)
 
 	kubeName := sbox.Config().Metadata.Name
 	namespace := sbox.Config().Metadata.Namespace
@@ -431,7 +430,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ
 
 	// Add special container name label for the infra container
 	if labels != nil {
-		labels[kubeletTypes.KubernetesContainerNameLabel] = leaky.PodInfraContainerName
+		labels[kubeletTypes.KubernetesContainerNameLabel] = types.InfraContainerName
 	}
 	labelsJSON, err := json.Marshal(labels)
 	if err != nil {

diff --git a/server/server.go b/server/server.go
@@ -206,7 +206,7 @@ func (s *Server) restore(ctx context.Context) []string {
 				log.Warnf(ctx, "Unable to delete container %s: %v", n, err)
 			}
 			// Release the infra container name and the pod name for future use
-			if strings.Contains(n, infraName) {
+			if strings.Contains(n, types.InfraContainerName) {
 				s.ReleaseContainerName(n)
 			} else {
 				s.ReleasePodName(n)