diff --git a/.golangci.yml b/.golangci.yml index 484a04449f7..232b4c1b9d6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -163,6 +163,6 @@ linters-settings: - unnamedResult - unnecessaryBlock gocyclo: - min-complexity: 122 + min-complexity: 127 nakedret: max-func-lines: 15 diff --git a/Makefile b/Makefile index 191ffae52c8..a571b44c9cf 100644 --- a/Makefile +++ b/Makefile @@ -311,6 +311,7 @@ testunit-bin: done mockgen: \ + mock-cmdrunner \ mock-containerstorage \ mock-criostorage \ mock-lib-config \ diff --git a/contrib/test/ci/cri-o.spec b/contrib/test/ci/cri-o.spec index d50475cf626..9ed9c4ebe2f 100644 --- a/contrib/test/ci/cri-o.spec +++ b/contrib/test/ci/cri-o.spec @@ -32,7 +32,7 @@ %global service_name crio Name: %{repo} -Version: 1.22.0 +Version: 1.22.1 Release: 1.ci%{?dist} Summary: Kubernetes Container Runtime Interface for OCI-based containers License: ASL 2.0 diff --git a/contrib/test/integration/build/parallel.yml b/contrib/test/integration/build/parallel.yml index 38bd198f2db..efe23e4015f 100644 --- a/contrib/test/integration/build/parallel.yml +++ b/contrib/test/integration/build/parallel.yml @@ -17,6 +17,7 @@ src: https://ftp.gnu.org/gnu/parallel/parallel-20190322.tar.bz2 dest: "{{ ansible_env.HOME }}" remote_src: yes + validate_certs: False when: ansible_distribution in ['RedHat', 'CentOS'] - name: install parallel from sources diff --git a/go.mod b/go.mod index 4b1b6c17542..1c719292e05 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/opencontainers/runc v1.0.2 github.com/opencontainers/runtime-spec v1.0.3-0.20210709190330-896175883324 github.com/opencontainers/runtime-tools v0.9.1-0.20210326182921-59cdde06764b - github.com/opencontainers/selinux v1.8.4 + github.com/opencontainers/selinux v1.9.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 github.com/psampaz/go-mod-outdated v0.8.0 diff --git a/go.sum b/go.sum index 79b473c8ba0..8124e729511 100644 --- a/go.sum +++ b/go.sum @@ -1047,8 +1047,9 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.8.3/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= -github.com/opencontainers/selinux v1.8.4 h1:krlgQ6/j9CkCXT5oW0yVXdQFOME3NjKuuAZXuR6O7P4= github.com/opencontainers/selinux v1.8.4/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= +github.com/opencontainers/selinux v1.9.1 h1:b4VPEF3O5JLZgdTDBmGepaaIbAo0GqoF6EBRq5f/g3Y= +github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656 h1:WaxyNFpmIDu4i6so9r6LVFIbSaXqsj8oitMitt86ae4= github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= diff --git a/images/os/Dockerfile b/images/os/Dockerfile index 91e9d3032bc..a828d894b36 100644 --- a/images/os/Dockerfile +++ b/images/os/Dockerfile @@ -15,4 +15,6 @@ RUN set -x && yum install -y ostree yum-utils selinux-policy-targeted && \ -s "cri-o-ci-dev overlay RPMs" --branch=cri-o-ci-dev FROM scratch -COPY --from=build /srv/ /srv/ \ No newline at end of file +COPY --from=build /srv/ /srv/ +LABEL io.openshift.build.version-display-names="machine-os=rhcos image for testing CRI-O only- if you see this outside of PR runs for CRI-O- you found an urgent blocker bug" \ + io.openshift.build.versions="machine-os=1.2.3-testing-if-you-see-this-outside-of-PR-runs-for-cri-o-cri-o-you-found-an-urgent-blocker-bug" diff --git a/internal/config/conmonmgr/conmonmgr.go b/internal/config/conmonmgr/conmonmgr.go index 094b4c818f4..857437c3f54 100644 --- a/internal/config/conmonmgr/conmonmgr.go +++ b/internal/config/conmonmgr/conmonmgr.go @@ -19,14 +19,10 @@ type ConmonManager struct { // this function is heavily based on github.com/containers/common#probeConmon func New(conmonPath string) (*ConmonManager, error) { - return newWithCommandRunner(conmonPath, &cmdrunner.RealCommandRunner{}) -} - -func newWithCommandRunner(conmonPath string, runner cmdrunner.CommandRunner) (*ConmonManager, error) { if !path.IsAbs(conmonPath) { return nil, errors.Errorf("conmon path is not absolute: %s", conmonPath) } - out, err := runner.CombinedOutput(conmonPath, "--version") + out, err := cmdrunner.CombinedOutput(conmonPath, "--version") if err != nil { return nil, errors.Wrapf(err, "get conmon version") } diff --git a/internal/config/conmonmgr/conmonmgr_test.go b/internal/config/conmonmgr/conmonmgr_test.go index d8e2b5fa389..a09731281af 100644 --- a/internal/config/conmonmgr/conmonmgr_test.go +++ b/internal/config/conmonmgr/conmonmgr_test.go @@ -2,6 +2,7 @@ package conmonmgr import ( runnerMock "github.com/cri-o/cri-o/test/mocks/cmdrunner" + "github.com/cri-o/cri-o/utils/cmdrunner" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -16,6 +17,7 @@ var _ = t.Describe("ConmonManager", func() { t.Describe("New", func() { BeforeEach(func() { runner = runnerMock.NewMockCommandRunner(mockCtrl) + cmdrunner.SetMocked(runner) }) It("should fail when path not absolute", func() { // Given @@ -24,7 +26,7 @@ var _ = t.Describe("ConmonManager", func() { ) // When - mgr, err := newWithCommandRunner("", runner) + mgr, err := New("") // Then Expect(err).NotTo(BeNil()) @@ -37,7 +39,7 @@ var _ = t.Describe("ConmonManager", func() { ) // When - mgr, err := newWithCommandRunner(validPath, runner) + mgr, err := New(validPath) // Then Expect(err).NotTo(BeNil()) @@ -50,7 +52,7 @@ var _ = t.Describe("ConmonManager", func() { ) // When - mgr, err := newWithCommandRunner(validPath, runner) + mgr, err := New(validPath) // Then Expect(err).NotTo(BeNil()) @@ -63,7 +65,7 @@ var _ = t.Describe("ConmonManager", func() { ) // When - mgr, err := newWithCommandRunner(validPath, runner) + mgr, err := New(validPath) // Then Expect(err).To(BeNil()) @@ -76,7 +78,7 @@ var _ = t.Describe("ConmonManager", func() { ) // When - mgr, err := newWithCommandRunner(validPath, runner) + mgr, err := New(validPath) // Then Expect(err).To(BeNil()) diff --git a/internal/config/node/systemd.go b/internal/config/node/systemd.go index 666776e2780..a95ad859f0f 100644 --- a/internal/config/node/systemd.go +++ b/internal/config/node/systemd.go @@ -3,9 +3,9 @@ package node import ( - "os/exec" "sync" + "github.com/cri-o/cri-o/utils/cmdrunner" "github.com/pkg/errors" ) @@ -36,7 +36,7 @@ func SystemdHasAllowedCPUs() bool { // systemdSupportsProperty checks whether systemd supports a property // It returns an error if it does not. func systemdSupportsProperty(property string) (bool, error) { - output, err := exec.Command("systemctl", "show", "-p", property, "systemd").Output() + output, err := cmdrunner.Command("systemctl", "show", "-p", property, "systemd").Output() if err != nil { return false, errors.Wrapf(err, "check systemd %s", property) } diff --git a/internal/config/nsmgr/nsmgr.go b/internal/config/nsmgr/nsmgr.go index bab0f91fa6e..d35dea9054c 100644 --- a/internal/config/nsmgr/nsmgr.go +++ b/internal/config/nsmgr/nsmgr.go @@ -4,13 +4,13 @@ import ( "bytes" "fmt" "os" - "os/exec" "path/filepath" "strings" "syscall" "github.com/containers/storage/pkg/idtools" "github.com/cri-o/cri-o/utils" + "github.com/cri-o/cri-o/utils/cmdrunner" "github.com/google/uuid" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -119,7 +119,7 @@ func (mgr *NamespaceManager) NewPodNamespaces(cfg *PodNamespacesConfig) ([]Names } logrus.Debugf("Calling pinns with %v", pinnsArgs) - output, err := exec.Command(mgr.pinnsPath, pinnsArgs...).CombinedOutput() + output, err := cmdrunner.Command(mgr.pinnsPath, pinnsArgs...).CombinedOutput() if err != nil { logrus.Warnf("Pinns %v failed: %s (%v)", pinnsArgs, string(output), err) // cleanup the mounts diff --git a/internal/config/nsmgr/types.go b/internal/config/nsmgr/types.go index c7c7aaa08bd..72f665415b4 100644 --- a/internal/config/nsmgr/types.go +++ b/internal/config/nsmgr/types.go @@ -51,7 +51,10 @@ type Namespace interface { // Type returns the namespace type (net, ipc, user, pid or uts). Type() NSType - // Remove ensures this namespace is closed and removed. + // Close ensures this namespace is closed. + Close() error + + // Remove ensures this namespace is removed. Remove() error } @@ -84,13 +87,13 @@ func (n *namespace) Type() NSType { return n.nsType } -// Remove ensures this namespace is closed and removed. -func (n *namespace) Remove() error { +// Close ensures this namespace is closed. +func (n *namespace) Close() error { n.Lock() defer n.Unlock() if n.closed { - // Remove() can be called multiple + // Close() can be called multiple // times without returning an error. return nil } @@ -101,16 +104,29 @@ func (n *namespace) Remove() error { n.closed = true - fp := n.Path() - if fp == "" { + if n.nsPath == "" { return nil } // try to unmount, ignoring "not mounted" (EINVAL) error. - if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil && err != unix.EINVAL { - return errors.Wrapf(err, "unable to unmount %s", fp) + if err := unix.Unmount(n.nsPath, unix.MNT_DETACH); err != nil && err != unix.EINVAL { + return errors.Wrapf(err, "unable to unmount %s", n.nsPath) + } + return nil +} + +// Remove ensures this namespace is closed and removed. +func (n *namespace) Remove() error { + n.Lock() + defer n.Unlock() + if !n.closed { + return errors.New("Namespace must be closed before it can be removed") + } + + if n.nsPath == "" { + return nil } - return os.RemoveAll(fp) + return os.Remove(n.nsPath) } // GetNamespace takes a path and type, checks if it is a namespace, and if so @@ -118,6 +134,10 @@ func (n *namespace) Remove() error { func GetNamespace(nsPath string, nsType NSType) (Namespace, error) { ns, err := nspkg.GetNS(nsPath) if err != nil { + // path exists but is not an NS, the namespace must have been closed + if _, ok := err.(nspkg.NSPathNotNSErr); ok { + return &namespace{nsType: nsType, nsPath: nsPath, closed: true}, nil + } return nil, err } diff --git a/internal/config/rdt/rdt.go b/internal/config/rdt/rdt.go index d9a439cc202..209e2eba199 100644 --- a/internal/config/rdt/rdt.go +++ b/internal/config/rdt/rdt.go @@ -34,7 +34,6 @@ func New() *Config { rdt.SetLogger(logrus.StandardLogger()) if err := rdt.Initialize(ResctrlPrefix); err != nil { - logrus.Infof("RDT is not enabled: %v", err) c.supported = false } return c diff --git a/internal/dbusmgr/user.go b/internal/dbusmgr/user.go index 0e0520e9708..98100742f68 100644 --- a/internal/dbusmgr/user.go +++ b/internal/dbusmgr/user.go @@ -6,12 +6,12 @@ import ( "bufio" "bytes" "os" - "os/exec" "path/filepath" "strconv" "strings" systemdDbus "github.com/coreos/go-systemd/v22/dbus" + "github.com/cri-o/cri-o/utils/cmdrunner" dbus "github.com/godbus/dbus/v5" "github.com/opencontainers/runc/libcontainer/userns" "github.com/pkg/errors" @@ -55,7 +55,7 @@ func DetectUID() (int, error) { if !userns.RunningInUserNS() { return os.Getuid(), nil } - b, err := exec.Command("busctl", "--user", "--no-pager", "status").CombinedOutput() + b, err := cmdrunner.Command("busctl", "--user", "--no-pager", "status").CombinedOutput() if err != nil { return -1, errors.Wrapf(err, "could not execute `busctl --user --no-pager status`: %q", string(b)) } @@ -91,7 +91,7 @@ func DetectUserDbusSessionBusAddress() (string, error) { return busAddress, nil } } - b, err := exec.Command("systemctl", "--user", "--no-pager", "show-environment").CombinedOutput() + b, err := cmdrunner.Command("systemctl", "--user", "--no-pager", "show-environment").CombinedOutput() if err != nil { return "", errors.Wrapf(err, "could not execute `systemctl --user --no-pager show-environment`, output=%q", string(b)) } diff --git a/internal/lib/container_server.go b/internal/lib/container_server.go index 6a3146958de..a93a8bf396b 100644 --- a/internal/lib/container_server.go +++ b/internal/lib/container_server.go @@ -209,26 +209,6 @@ func (c *ContainerServer) LoadSandbox(ctx context.Context, id string) (sb *sandb } } }() - // We add an NS only if we can load a permanent one. - // Otherwise, the sandbox will live in the host namespace. - namespacesToJoin := []struct { - rspecNS rspec.LinuxNamespaceType - joinFunc func(string) error - }{ - {rspecNS: rspec.NetworkNamespace, joinFunc: sb.NetNsJoin}, - {rspecNS: rspec.IPCNamespace, joinFunc: sb.IpcNsJoin}, - {rspecNS: rspec.UTSNamespace, joinFunc: sb.UtsNsJoin}, - {rspecNS: rspec.UserNamespace, joinFunc: sb.UserNsJoin}, - } - for _, namespaceToJoin := range namespacesToJoin { - path, err := configNsPath(&m, namespaceToJoin.rspecNS) - if err == nil { - if nsErr := namespaceToJoin.joinFunc(path); nsErr != nil { - return sb, nsErr - } - } - } - if err := c.AddSandbox(sb); err != nil { return sb, err } @@ -277,20 +257,45 @@ func (c *ContainerServer) LoadSandbox(ctx context.Context, id string) (sb *sandb if err != nil { return sb, err } - scontainer.SetSpec(&m) - scontainer.SetMountPoint(m.Annotations[annotations.MountPoint]) + } else { + scontainer = oci.NewSpoofedContainer(cID, cname, labels, id, created, sandboxPath) + } + scontainer.SetSpec(&m) + scontainer.SetMountPoint(m.Annotations[annotations.MountPoint]) + + if m.Annotations[annotations.Volumes] != "" { + containerVolumes := []oci.ContainerVolume{} + if err = json.Unmarshal([]byte(m.Annotations[annotations.Volumes]), &containerVolumes); err != nil { + return sb, fmt.Errorf("failed to unmarshal container volumes: %v", err) + } + for _, cv := range containerVolumes { + scontainer.AddVolume(cv) + } + } - if m.Annotations[annotations.Volumes] != "" { - containerVolumes := []oci.ContainerVolume{} - if err = json.Unmarshal([]byte(m.Annotations[annotations.Volumes]), &containerVolumes); err != nil { - return sb, fmt.Errorf("failed to unmarshal container volumes: %v", err) - } - for _, cv := range containerVolumes { - scontainer.AddVolume(cv) + if err := sb.SetInfraContainer(scontainer); err != nil { + return sb, err + } + + sb.RestoreStopped() + // We add an NS only if we can load a permanent one. + // Otherwise, the sandbox will live in the host namespace. + namespacesToJoin := []struct { + rspecNS rspec.LinuxNamespaceType + joinFunc func(string) error + }{ + {rspecNS: rspec.NetworkNamespace, joinFunc: sb.NetNsJoin}, + {rspecNS: rspec.IPCNamespace, joinFunc: sb.IpcNsJoin}, + {rspecNS: rspec.UTSNamespace, joinFunc: sb.UtsNsJoin}, + {rspecNS: rspec.UserNamespace, joinFunc: sb.UserNsJoin}, + } + for _, namespaceToJoin := range namespacesToJoin { + path, err := configNsPath(&m, namespaceToJoin.rspecNS) + if err == nil { + if nsErr := namespaceToJoin.joinFunc(path); nsErr != nil { + return sb, nsErr } } - } else { - scontainer = oci.NewSpoofedContainer(cID, cname, labels, id, created, sandboxPath) } if err := c.ContainerStateFromDisk(ctx, scontainer); err != nil { @@ -303,17 +308,11 @@ func (c *ContainerServer) LoadSandbox(ctx context.Context, id string) (sb *sandb return sb, fmt.Errorf("failed to write container %q state to disk: %v", scontainer.ID(), err) } - if err := sb.SetInfraContainer(scontainer); err != nil { - return sb, err - } - sb.SetCreated() if err := label.ReserveLabel(processLabel); err != nil { return sb, err } - sb.RestoreStopped() - if err := c.ctrIDIndex.Add(scontainer.ID()); err != nil { return sb, err } diff --git a/internal/lib/sandbox/namespaces.go b/internal/lib/sandbox/namespaces.go index 68a4fc411ce..b906f513737 100644 --- a/internal/lib/sandbox/namespaces.go +++ b/internal/lib/sandbox/namespaces.go @@ -91,29 +91,34 @@ func (s *Sandbox) NamespacePaths() []*ManagedNamespace { return typesAndPaths } -// RemoveManagedNamespaces cleans up after managing the namespaces. It removes all of the namespaces -// and the parent directory in which they lived. +// CloseManagedNamespaces cleans up after managing the namespaces. +// It unmounts all of the namespaces, but does not remove their parent directory. +func (s *Sandbox) CloseManagedNamespaces() error { + return s.runFunctionOnNamespaces(func(ns nsmgr.Namespace) error { + return ns.Close() + }) +} + +// RemoveManagedNamespaces removes the formerly mounted namespace. +// Must be stopped first or this will fail. func (s *Sandbox) RemoveManagedNamespaces() error { + return s.runFunctionOnNamespaces(func(ns nsmgr.Namespace) error { + if err := ns.Close(); err != nil { + return err + } + return ns.Remove() + }) +} + +func (s *Sandbox) runFunctionOnNamespaces(toRun func(nsmgr.Namespace) error) error { errs := make([]error, 0) - // use a map as a set to delete each parent directory just once - if s.utsns != nil { - if err := s.utsns.Remove(); err != nil { - errs = append(errs, err) - } - } - if s.ipcns != nil { - if err := s.ipcns.Remove(); err != nil { - errs = append(errs, err) - } - } - if s.netns != nil { - if err := s.netns.Remove(); err != nil { - errs = append(errs, err) + allNamespaces := []nsmgr.Namespace{s.utsns, s.ipcns, s.netns, s.userns} + for _, ns := range allNamespaces { + if ns == nil { + continue } - } - if s.userns != nil { - if err := s.userns.Remove(); err != nil { + if err := toRun(ns); err != nil { errs = append(errs, err) } } diff --git a/internal/lib/sandbox/namespaces_test.go b/internal/lib/sandbox/namespaces_test.go index 1434ee9d43b..f997eb29a23 100644 --- a/internal/lib/sandbox/namespaces_test.go +++ b/internal/lib/sandbox/namespaces_test.go @@ -23,6 +23,10 @@ func (s *spoofedIface) Type() nsmgr.NSType { return s.nsType } +func (s *spoofedIface) Close() error { + return nil +} + func (s *spoofedIface) Remove() error { s.removed = true return nil @@ -211,39 +215,6 @@ var _ = t.Describe("SandboxManagedNamespaces", func() { // When err := testSandbox.UserNsJoin("/proc/self/ns/user") - // Then - Expect(err).NotTo(BeNil()) - }) - It("should fail when asked to join a non-namespace", func() { - // Given - // When - err := testSandbox.NetNsJoin("/tmp") - - // Then - Expect(err).NotTo(BeNil()) - }) - It("should fail when asked to join a non-namespace", func() { - // Given - - // When - err := testSandbox.IpcNsJoin("/tmp") - - // Then - Expect(err).NotTo(BeNil()) - }) - It("should fail when asked to join a non-namespace", func() { - // Given - // When - err := testSandbox.UtsNsJoin("/tmp") - - // Then - Expect(err).NotTo(BeNil()) - }) - It("should fail when asked to join a non-namespace", func() { - // Given - // When - err := testSandbox.UserNsJoin("/tmp") - // Then Expect(err).NotTo(BeNil()) }) diff --git a/internal/oci/container.go b/internal/oci/container.go index 3e954702b50..b8b09771840 100644 --- a/internal/oci/container.go +++ b/internal/oci/container.go @@ -75,6 +75,7 @@ type Container struct { stopping bool stopTimeoutChan chan time.Duration stoppedChan chan struct{} + stopStoppingChan chan struct{} stopLock sync.Mutex } @@ -117,27 +118,28 @@ func NewContainer(id, name, bundlePath, logPath string, labels, crioAnnotations, state := &ContainerState{} state.Created = created c := &Container{ - id: id, - name: name, - bundlePath: bundlePath, - logPath: logPath, - labels: labels, - sandbox: sandbox, - terminal: terminal, - stdin: stdin, - stdinOnce: stdinOnce, - runtimeHandler: runtimeHandler, - metadata: metadata, - annotations: annotations, - crioAnnotations: crioAnnotations, - image: image, - imageName: imageName, - imageRef: imageRef, - dir: dir, - state: state, - stopSignal: stopSignal, - stopTimeoutChan: make(chan time.Duration, 1), - stoppedChan: make(chan struct{}, 1), + id: id, + name: name, + bundlePath: bundlePath, + logPath: logPath, + labels: labels, + sandbox: sandbox, + terminal: terminal, + stdin: stdin, + stdinOnce: stdinOnce, + runtimeHandler: runtimeHandler, + metadata: metadata, + annotations: annotations, + crioAnnotations: crioAnnotations, + image: image, + imageName: imageName, + imageRef: imageRef, + dir: dir, + state: state, + stopSignal: stopSignal, + stopTimeoutChan: make(chan time.Duration, 1), + stoppedChan: make(chan struct{}, 1), + stopStoppingChan: make(chan struct{}, 1), } return c, nil } @@ -570,11 +572,14 @@ func (c *Container) SetAsStopping(timeout int64) { select { case c.stopTimeoutChan <- time.Duration(timeout) * time.Second: case <-c.stoppedChan: // This case is to avoid waiting forever once another routine has finished. - return + case <-c.stopStoppingChan: // This case is to avoid deadlocking with SetAsNotStopping. } + return } // Regardless, set the container as actively stopping. c.stopping = true + // And reset the stopStoppingChan + c.stopStoppingChan = make(chan struct{}, 1) } // SetAsNotStopping unsets the stopping field indicating to new callers that the container diff --git a/internal/oci/oci.go b/internal/oci/oci.go index b59811eed1b..6c4efa93623 100644 --- a/internal/oci/oci.go +++ b/internal/oci/oci.go @@ -126,8 +126,9 @@ func (r *Runtime) WaitContainerStateStopped(ctx context.Context, c *Container) e return nil } - done := make(chan error) - chControl := make(chan struct{}) + done := make(chan error, 1) + chControl := make(chan struct{}, 1) + defer close(chControl) go func() { defer close(done) for { @@ -152,10 +153,8 @@ func (r *Runtime) WaitContainerStateStopped(ctx context.Context, c *Container) e case err = <-done: break case <-ctx.Done(): - close(chControl) return ctx.Err() case <-time.After(time.Duration(r.config.CtrStopTimeout) * time.Second): - close(chControl) return fmt.Errorf( "failed to get container stopped status: %ds timeout reached", r.config.CtrStopTimeout, diff --git a/internal/oci/runtime_oci.go b/internal/oci/runtime_oci.go index 11122587c7c..93f288111dc 100644 --- a/internal/oci/runtime_oci.go +++ b/internal/oci/runtime_oci.go @@ -21,6 +21,7 @@ import ( "github.com/cri-o/cri-o/server/cri/types" "github.com/cri-o/cri-o/server/metrics" "github.com/cri-o/cri-o/utils" + "github.com/cri-o/cri-o/utils/cmdrunner" "github.com/fsnotify/fsnotify" json "github.com/json-iterator/go" rspec "github.com/opencontainers/runtime-spec/specs-go" @@ -137,7 +138,7 @@ func (r *runtimeOCI) CreateContainer(ctx context.Context, c *Container, cgroupPa "args": args, }).Debugf("running conmon: %s", r.config.Conmon) - cmd := exec.Command(r.config.Conmon, args...) // nolint: gosec + cmd := cmdrunner.Command(r.config.Conmon, args...) // nolint: gosec cmd.Dir = c.bundlePath cmd.SysProcAttr = sysProcAttrPlatform() cmd.Stdin = os.Stdin @@ -167,21 +168,37 @@ func (r *runtimeOCI) CreateContainer(ctx context.Context, c *Container, cgroupPa childPipe.Close() childStartPipe.Close() - // Platform specific container setup - if err := r.createContainerPlatform(c, cgroupParent, cmd.Process.Pid); err != nil { - return err - } - - /* We set the cgroup, now the child can start creating children */ - someData := []byte{0} - _, err = parentStartPipe.Write(someData) - if err != nil { - if waitErr := cmd.Wait(); waitErr != nil { - return errors.Wrap(err, waitErr.Error()) + // Create new scope to reduce cleanup code. + if err := func() (retErr error) { + defer func() { + if retErr != nil { + // We need to always kill and wait on this process. + // Failing to do so will cause us to leak a zombie. + killErr := cmd.Process.Kill() + waitErr := cmd.Wait() + if killErr != nil { + retErr = errors.Wrapf(retErr, "failed to kill %+v after failing with", killErr) + } + // Per https://pkg.go.dev/os#ProcessState.ExitCode, the exit code is -1 when the process died because + // of a signal. We expect this in this case, as we've just killed it with a signal. Don't append the + // error in this case to reduce noise. + if exitErr, ok := waitErr.(*exec.ExitError); !ok || exitErr.ExitCode() != -1 { + retErr = errors.Wrapf(retErr, "failed to wait %+v after failing with", waitErr) + } + } + }() + // Platform specific container setup + if err := r.createContainerPlatform(c, cgroupParent, cmd.Process.Pid); err != nil { + return err } + + /* We set the cgroup, now the child can start creating children */ + someData := []byte{0} + _, err = parentStartPipe.Write(someData) + return err + }(); err != nil { return err } - /* Wait for initial setup and fork, and reap child */ err = cmd.Wait() if err != nil { @@ -337,7 +354,7 @@ func (r *runtimeOCI) ExecContainer(ctx context.Context, c *Container, cmd []stri args := []string{rootFlag, r.root, "exec"} args = append(args, "--process", processFile, c.ID()) - execCmd := exec.Command(r.path, args...) // nolint: gosec + execCmd := cmdrunner.Command(r.path, args...) // nolint: gosec if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found { execCmd.Env = append(execCmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v)) } @@ -377,6 +394,9 @@ func (r *runtimeOCI) ExecContainer(ctx context.Context, c *Container, cmd []stri // The read side of the pipe should be closed after the container process has been started. if r != nil { if err := r.Close(); err != nil { + if waitErr := execCmd.Wait(); waitErr != nil { + return errors.Wrap(err, waitErr.Error()) + } return err } } @@ -461,7 +481,7 @@ func (r *runtimeOCI) ExecSyncContainer(ctx context.Context, c *Container, comman "--exec-process-spec", processFile, "--runtime-arg", fmt.Sprintf("%s=%s", rootFlag, r.root)) - cmd := exec.Command(r.config.Conmon, args...) // nolint: gosec + cmd := cmdrunner.Command(r.config.Conmon, args...) // nolint: gosec var stdoutBuf, stderrBuf bytes.Buffer cmd.Stdout = &stdoutBuf @@ -572,7 +592,7 @@ func (r *runtimeOCI) UpdateContainer(ctx context.Context, c *Container, res *rsp return nil } - cmd := exec.Command(r.path, rootFlag, r.root, "update", "--resources", "-", c.id) // nolint: gosec + cmd := cmdrunner.Command(r.path, rootFlag, r.root, "update", "--resources", "-", c.ID()) // nolint: gosec var stdout bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &stdout @@ -679,6 +699,9 @@ func (r *runtimeOCI) StopContainer(ctx context.Context, c *Container, timeout in // Otherwise, we won't actually // attempt to stop when a new request comes in, // even though we're not actively stopping anymore. + // Also, close the stopStoppingChan to tell + // routines waiting to change the stop timeout to give up. + close(c.stopStoppingChan) c.SetAsNotStopping() } }() @@ -782,7 +805,7 @@ func (r *runtimeOCI) UpdateContainerStatus(ctx context.Context, c *Container) er } stateCmd := func() (*ContainerState, bool, error) { - cmd := exec.Command(r.path, rootFlag, r.root, "state", c.id) // nolint: gosec + cmd := cmdrunner.Command(r.path, rootFlag, r.root, "state", c.ID()) // nolint: gosec if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found { cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v)) } diff --git a/internal/runtimehandlerhooks/high_performance_hooks.go b/internal/runtimehandlerhooks/high_performance_hooks.go index 209ff4f3de4..f270cd722b0 100644 --- a/internal/runtimehandlerhooks/high_performance_hooks.go +++ b/internal/runtimehandlerhooks/high_performance_hooks.go @@ -15,6 +15,7 @@ import ( "github.com/cri-o/cri-o/internal/log" "github.com/cri-o/cri-o/internal/oci" crioannotations "github.com/cri-o/cri-o/pkg/annotations" + "github.com/cri-o/cri-o/utils/cmdrunner" "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/cgroups/systemd" "github.com/pkg/errors" @@ -263,7 +264,7 @@ func setIRQLoadBalancing(c *oci.Container, enable bool, irqSmpAffinityFile, irqB return nil } // run irqbalance in daemon mode, so this won't cause delay - cmd := exec.Command(irqBalancedName, "--oneshot") + cmd := cmdrunner.Command(irqBalancedName, "--oneshot") additionalEnv := irqBalanceBannedCpus + "=" + newIRQBalanceSetting cmd.Env = append(os.Environ(), additionalEnv) return cmd.Run() diff --git a/internal/runtimehandlerhooks/utils.go b/internal/runtimehandlerhooks/utils.go index 3728db25e20..e2fd6c6f43f 100644 --- a/internal/runtimehandlerhooks/utils.go +++ b/internal/runtimehandlerhooks/utils.go @@ -5,10 +5,10 @@ import ( "fmt" "io/ioutil" "os" - "os/exec" "strings" "unicode" + "github.com/cri-o/cri-o/utils/cmdrunner" "github.com/sirupsen/logrus" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) @@ -138,11 +138,11 @@ func UpdateIRQSmpAffinityMask(cpus, current string, set bool) (cpuMask, bannedCP } func restartIrqBalanceService() error { - return exec.Command("systemctl", "restart", "irqbalance").Run() + return cmdrunner.Command("systemctl", "restart", "irqbalance").Run() } func isServiceEnabled(serviceName string) bool { - cmd := exec.Command("systemctl", "is-enabled", serviceName) + cmd := cmdrunner.Command("systemctl", "is-enabled", serviceName) status, err := cmd.CombinedOutput() if err != nil { logrus.Infof("Service %s is-enabled check returned with: %v", serviceName, err) diff --git a/internal/storage/image.go b/internal/storage/image.go index 812a3037960..26775e1d3d3 100644 --- a/internal/storage/image.go +++ b/internal/storage/image.go @@ -586,6 +586,9 @@ func (svc *imageService) copyImage(systemContext *types.SystemContext, imageName } if err := json.NewEncoder(stdin).Encode(&stdinArguments); err != nil { stdin.Close() + if waitErr := cmd.Wait(); waitErr != nil { + return errors.Wrap(err, waitErr.Error()) + } return errors.Wrap(err, "json encode to pipe failed") } stdin.Close() diff --git a/internal/version/version.go b/internal/version/version.go index 41b786c135f..6f5ed90917c 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -21,7 +21,7 @@ import ( ) // Version is the version of the build. -const Version = "1.22.0" +const Version = "1.22.1" // Variables injected during build-time var ( diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 99a6df8296a..51920ebca0f 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -31,6 +31,10 @@ const ( // OCISeccompBPFHookAnnotation is the annotation used by the OCI seccomp BPF hook for tracing container syscalls OCISeccompBPFHookAnnotation = "io.containers.trace-syscall" + + // TrySkipVolumeSELinuxLabelAnnotation is the annotation used for optionally skipping relabeling a volume + // with the specified SELinux label. The relabeling will be skipped if the top layer is already labeled correctly. + TrySkipVolumeSELinuxLabelAnnotation = "io.kubernetes.cri-o.TrySkipVolumeSELinuxLabel" ) var AllAllowedAnnotations = []string{ @@ -43,4 +47,5 @@ var AllAllowedAnnotations = []string{ IRQLoadBalancingAnnotation, OCISeccompBPFHookAnnotation, rdt.RdtContainerAnnotation, + TrySkipVolumeSELinuxLabelAnnotation, } diff --git a/pkg/config/config.go b/pkg/config/config.go index 3c8a184b7e1..06d5cfa8ceb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -34,6 +34,7 @@ import ( "github.com/cri-o/cri-o/server/metrics/collectors" "github.com/cri-o/cri-o/server/useragent" "github.com/cri-o/cri-o/utils" + "github.com/cri-o/cri-o/utils/cmdrunner" "github.com/cri-o/ocicni/pkg/ocicni" selinux "github.com/opencontainers/selinux/go-selinux" "github.com/pkg/errors" @@ -53,6 +54,7 @@ const ( defaultCtrStopTimeout = 30 // seconds defaultNamespacesDir = "/var/run" RuntimeTypeVMBinaryPattern = "containerd-shim-([a-zA-Z0-9\\-\\+])+-v2" + tasksetBinary = "taskset" ) // Config represents the entire set of configuration values that can be set for @@ -556,6 +558,10 @@ func (c *Config) UpdateFromFile(path string) error { // Returns errors encountered when reading or parsing the files, or nil // otherwise. func (c *Config) UpdateFromDropInFile(path string) error { + // keeps the storage options from storage.conf and merge it to crio config + var storageOpts []string + storageOpts = append(storageOpts, c.StorageOptions...) + data, err := ioutil.ReadFile(path) if err != nil { return err @@ -575,6 +581,10 @@ func (c *Config) UpdateFromDropInFile(path string) error { delete(c.Runtimes, defaultRuntime) } + storageOpts = append(storageOpts, t.Crio.RootConfig.StorageOptions...) + storageOpts = removeDupStorageOpts(storageOpts) + t.Crio.RootConfig.StorageOptions = storageOpts + // Registries are deprecated in cri-o.conf and turned into a NOP. // Users should use registries.conf instead, so let's log it. if len(t.Crio.Image.Registries) > 0 { @@ -586,6 +596,24 @@ func (c *Config) UpdateFromDropInFile(path string) error { return nil } +// removeDupStorageOpts removes duplicated storage option from the list +// keeps the last appearance +func removeDupStorageOpts(storageOpts []string) []string { + var resOpts []string + opts := make(map[string]bool) + for i := len(storageOpts) - 1; i >= 0; i-- { + if ok := opts[storageOpts[i]]; ok { + continue + } + opts[storageOpts[i]] = true + resOpts = append(resOpts, storageOpts[i]) + } + for i, j := 0, len(resOpts)-1; i < j; i, j = i+1, j-1 { + resOpts[i], resOpts[j] = resOpts[j], resOpts[i] + } + return resOpts +} + // UpdateFromPath recursively iterates the provided path and updates the // configuration for it func (c *Config) UpdateFromPath(path string) error { @@ -898,9 +926,16 @@ func (c *RuntimeConfig) Validate(systemContext *types.SystemContext, onExecution } if c.InfraCtrCPUSet != "" { - if _, err := cpuset.Parse(c.InfraCtrCPUSet); err != nil { + set, err := cpuset.Parse(c.InfraCtrCPUSet) + if err != nil { return errors.Wrap(err, "invalid infra_ctr_cpuset") } + + executable, err := exec.LookPath(tasksetBinary) + if err != nil { + return errors.Wrapf(err, "%q not found in $PATH", tasksetBinary) + } + cmdrunner.PrependCommandsWith(executable, "--cpu-list", set.String()) } if err := c.Workloads.Validate(); err != nil { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 4023b79008c..11aad3ba6b2 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -835,6 +835,37 @@ var _ = t.Describe("Config", func() { Expect(sut.PidsLimit).To(BeEquivalentTo(2048)) }) + It("should inherit storage_options from storage.conf and remove duplicates", func() { + f := t.MustTempFile("config") + // Given + Expect(ioutil.WriteFile(f, + []byte(` + [crio] + storage_option = [ + "foo=bar", + ]`, + ), 0), + ).To(BeNil()) + for _, tc := range []struct { + opts []string + expect []string + }{ + {[]string{"option1=v1", "option2=v2", "option3=v3"}, []string{"option1=v1", "option2=v2", "option3=v3", "foo=bar"}}, + {[]string{"option1=v1", "option3=v3", "option2=v2", "option3=v3"}, []string{"option1=v1", "option2=v2", "option3=v3", "foo=bar"}}, + {[]string{"option1=v1", "option2=v2", "option3=v3", "option1=v1"}, []string{"option2=v2", "option3=v3", "option1=v1", "foo=bar"}}, + {[]string{"option1=v1", "option2=v2", "option3=v3", "option4=v4", "option3=v3", "option1=v1"}, []string{"option2=v2", "option4=v4", "option3=v3", "option1=v1", "foo=bar"}}, + } { + // When + defaultcfg := defaultConfig() + defaultcfg.StorageOptions = tc.opts + err := defaultcfg.UpdateFromFile(f) + + // Then + Expect(err).To(BeNil()) + Expect(defaultcfg.RootConfig.StorageOptions).To(Equal(tc.expect)) + } + }) + It("should succeed with custom runtime", func() { // Given f := t.MustTempFile("config") diff --git a/pkg/config/workloads.go b/pkg/config/workloads.go index 5ecfa6453bb..92afb46db16 100644 --- a/pkg/config/workloads.go +++ b/pkg/config/workloads.go @@ -96,6 +96,9 @@ func resourcesFromAnnotation(prefix, ctrName string, annotations map[string]stri } func (r *Resources) ValidateDefaults() error { + if r == nil { + return nil + } if r.CPUSet == "" { return nil } @@ -104,6 +107,9 @@ func (r *Resources) ValidateDefaults() error { } func (r *Resources) MutateSpec(specgen *generate.Generator) { + if r == nil { + return + } if r.CPUSet != "" { specgen.SetLinuxResourcesCPUCpus(r.CPUSet) } diff --git a/scripts/github-actions-packages b/scripts/github-actions-packages index 3c517a7cd40..8f1d712b0dd 100755 --- a/scripts/github-actions-packages +++ b/scripts/github-actions-packages @@ -18,5 +18,6 @@ sudo apt install -y \ libseccomp-dev \ libsystemd-dev \ libudev-dev \ + sed \ socat \ uuid-dev diff --git a/scripts/release-notes/release_notes.go b/scripts/release-notes/release_notes.go index 3a4e0998579..c8a7be700d5 100644 --- a/scripts/release-notes/release_notes.go +++ b/scripts/release-notes/release_notes.go @@ -19,8 +19,9 @@ import ( ) const ( - branch = "gh-pages" - tokenKey = "GITHUB_TOKEN" + branch = "gh-pages" + tokenKey = "GITHUB_TOKEN" + defaultBranch = "main" ) var outputPath string @@ -73,13 +74,13 @@ func run() error { } logrus.Infof("Using HEAD commit %s", head) - targetBranch := git.DefaultBranch + targetBranch := defaultBranch currentBranch, err := repo.CurrentBranch() if err != nil { return errors.Wrap(err, "get current branch") } logrus.Infof("Found current branch %s", currentBranch) - if git.IsReleaseBranch(currentBranch) && currentBranch != git.DefaultBranch { + if git.IsReleaseBranch(currentBranch) && currentBranch != defaultBranch { targetBranch = currentBranch } logrus.Infof("Using target branch %s", targetBranch) diff --git a/server/container_create.go b/server/container_create.go index df08616358a..3c1c1e3d973 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -131,7 +131,7 @@ func addImageVolumes(ctx context.Context, rootfs string, s *Server, containerInf return nil, err1 } if mountLabel != "" { - if err1 := securityLabel(fp, mountLabel, true); err1 != nil { + if err1 := securityLabel(fp, mountLabel, true, false); err1 != nil { return nil, err1 } } @@ -143,7 +143,7 @@ func addImageVolumes(ctx context.Context, rootfs string, s *Server, containerInf } // Label the source with the sandbox selinux mount label if mountLabel != "" { - if err1 := securityLabel(src, mountLabel, true); err1 != nil { + if err1 := securityLabel(src, mountLabel, true, false); err1 != nil { return nil, err1 } } @@ -235,7 +235,7 @@ func setupContainerUser(ctx context.Context, specgen *generate.Generator, rootfs return err } if passwdPath != "" { - if err := securityLabel(passwdPath, mountLabel, false); err != nil { + if err := securityLabel(passwdPath, mountLabel, false, false); err != nil { return err } @@ -399,15 +399,6 @@ func setupCapabilities(specgen *generate.Generator, caps *types.Capability, defa return nil } -func hostNetwork(containerConfig *types.ContainerConfig) bool { - securityContext := containerConfig.Linux.SecurityContext - if securityContext == nil || securityContext.NamespaceOptions == nil { - return false - } - - return securityContext.NamespaceOptions.Network == types.NamespaceModeNODE -} - // CreateContainer creates a new container in specified PodSandbox func (s *Server) CreateContainer(ctx context.Context, req *types.CreateContainerRequest) (res *types.CreateContainerResponse, retErr error) { log.Infof(ctx, "Creating container: %s", translateLabelsToDescription(req.Config.Labels)) diff --git a/server/container_create_linux.go b/server/container_create_linux.go index bbca6fa9a1e..cb2a97ca907 100644 --- a/server/container_create_linux.go +++ b/server/container_create_linux.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package server @@ -33,6 +34,7 @@ import ( rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/runtime-tools/generate" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "github.com/intel/goresctrl/pkg/blockio" @@ -135,6 +137,14 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai // eventually, we'd like to access all of these variables through the interface themselves, and do most // of the translation between CRI config -> oci/storage container in the container package + + // TODO: eventually, this should be in the container package, but it's going through a lot of churn + // and SpecAddAnnotations is already being passed too many arguments + // Filter early so any use of the annotations don't use the wrong values + if err := s.Runtime().FilterDisallowedAnnotations(sb.RuntimeHandler(), ctr.Config().Annotations); err != nil { + return nil, err + } + containerID := ctr.ID() containerName := ctr.Name() containerConfig := ctr.Config() @@ -268,7 +278,20 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai processLabel = "" } - containerVolumes, ociMounts, err := addOCIBindMounts(ctx, mountLabel, containerConfig, specgen, s.config.RuntimeConfig.BindMountPrefix, s.config.AbsentMountSourcesToReject) + maybeRelabel := false + if val, present := sb.Annotations()[crioann.TrySkipVolumeSELinuxLabelAnnotation]; present && val == "true" { + maybeRelabel = true + } + + skipRelabel := false + const superPrivilegedType = "spc_t" + if securityContext.SelinuxOptions.Type == superPrivilegedType || // super privileged container + (ctr.SandboxConfig().Linux.SecurityContext.SelinuxOptions.Type == superPrivilegedType && // super privileged pod + securityContext.SelinuxOptions.Type == "") { + skipRelabel = true + } + + containerVolumes, ociMounts, err := addOCIBindMounts(ctx, ctr, mountLabel, s.config.RuntimeConfig.BindMountPrefix, s.config.AbsentMountSourcesToReject, maybeRelabel, skipRelabel) if err != nil { return nil, err } @@ -452,19 +475,24 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai } // If the sandbox is configured to run in the host network, do not create a new network namespace - if sb.HostNetwork() { + if hostNet { if err := specgen.RemoveLinuxNamespace(string(rspec.NetworkNamespace)); err != nil { return nil, err } if !isInCRIMounts("/sys", containerConfig.Mounts) { - specgen.RemoveMount("/sys") ctr.SpecAddMount(rspec.Mount{ Destination: "/sys", Type: "sysfs", Source: "sysfs", Options: []string{"nosuid", "noexec", "nodev", "ro"}, }) + ctr.SpecAddMount(rspec.Mount{ + Destination: "/sys/fs/cgroup", + Type: "cgroup", + Source: "cgroup", + Options: []string{"nosuid", "noexec", "nodev", "relatime", "ro"}, + }) } } @@ -520,7 +548,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai options = []string{"ro"} } if sb.ResolvPath() != "" { - if err := securityLabel(sb.ResolvPath(), mountLabel, false); err != nil { + if err := securityLabel(sb.ResolvPath(), mountLabel, false, false); err != nil { return nil, err } ctr.SpecAddMount(rspec.Mount{ @@ -532,7 +560,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai } if sb.HostnamePath() != "" { - if err := securityLabel(sb.HostnamePath(), mountLabel, false); err != nil { + if err := securityLabel(sb.HostnamePath(), mountLabel, false, false); err != nil { return nil, err } ctr.SpecAddMount(rspec.Mount{ @@ -543,7 +571,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai }) } - if !isInCRIMounts("/etc/hosts", containerConfig.Mounts) && hostNetwork(containerConfig) { + if !isInCRIMounts("/etc/hosts", containerConfig.Mounts) && hostNet { // Only bind mount for host netns and when CRI does not give us any hosts file ctr.SpecAddMount(rspec.Mount{ Destination: "/etc/hosts", @@ -587,12 +615,6 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai } }() - // TODO: eventually, this should be in the container package, but it's going through a lot of churn - // and SpecAddAnnotations is already passed too many arguments - if err := s.Runtime().FilterDisallowedAnnotations(sb.RuntimeHandler(), ctr.Config().Annotations); err != nil { - return nil, err - } - // Get RDT class rdtClass, err := s.Config().Rdt().ContainerClassFromAnnotations(metadata.Name, containerConfig.Annotations, sb.Annotations()) if err != nil { @@ -791,7 +813,7 @@ func setupWorkingDirectory(rootfs, mountLabel, containerCwd string) error { return err } if mountLabel != "" { - if err1 := securityLabel(fp, mountLabel, false); err1 != nil { + if err1 := securityLabel(fp, mountLabel, false, false); err1 != nil { return err1 } } @@ -821,9 +843,11 @@ func clearReadOnly(m *rspec.Mount) { m.Options = append(m.Options, "rw") } -func addOCIBindMounts(ctx context.Context, mountLabel string, containerConfig *types.ContainerConfig, specgen *generate.Generator, bindMountPrefix string, absentMountSourcesToReject []string) ([]oci.ContainerVolume, []rspec.Mount, error) { +func addOCIBindMounts(ctx context.Context, ctr ctrIface.Container, mountLabel, bindMountPrefix string, absentMountSourcesToReject []string, maybeRelabel, skipRelabel bool) ([]oci.ContainerVolume, []rspec.Mount, error) { volumes := []oci.ContainerVolume{} ociMounts := []rspec.Mount{} + containerConfig := ctr.Config() + specgen := ctr.Spec() mounts := containerConfig.Mounts // Sort mounts in number of parts. This ensures that high level mounts don't @@ -929,7 +953,9 @@ func addOCIBindMounts(ctx context.Context, mountLabel string, containerConfig *t } if m.SelinuxRelabel { - if err := securityLabel(src, mountLabel, false); err != nil { + if skipRelabel { + logrus.Debugf("Skipping relabel for %s because of super privileged container (type: spc_t)", src) + } else if err := securityLabel(src, mountLabel, false, maybeRelabel); err != nil { return nil, nil, err } } diff --git a/server/container_create_linux_test.go b/server/container_create_linux_test.go index 27c0d2375fb..4653fcb1081 100644 --- a/server/container_create_linux_test.go +++ b/server/container_create_linux_test.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package server @@ -6,28 +7,38 @@ import ( "context" "testing" + "github.com/cri-o/cri-o/pkg/container" "github.com/cri-o/cri-o/server/cri/types" - "github.com/opencontainers/runtime-tools/generate" ) func TestAddOCIBindsForDev(t *testing.T) { - specgen, err := generate.New("linux") + ctr, err := container.New() if err != nil { t.Error(err) } - config := &types.ContainerConfig{ + if err := ctr.SetConfig(&types.ContainerConfig{ Mounts: []*types.Mount{ { ContainerPath: "/dev", HostPath: "/dev", }, }, + Metadata: &types.ContainerMetadata{ + Name: "testctr", + }, + }, &types.PodSandboxConfig{ + Metadata: &types.PodSandboxMetadata{ + Name: "testpod", + }, + }); err != nil { + t.Error(err) } - _, binds, err := addOCIBindMounts(context.Background(), "", config, &specgen, "", nil) + + _, binds, err := addOCIBindMounts(context.Background(), ctr, "", "", nil, false, false) if err != nil { t.Error(err) } - for _, m := range specgen.Mounts() { + for _, m := range ctr.Spec().Mounts() { if m.Destination == "/dev" { t.Error("/dev shouldn't be in the spec if it's bind mounted from kube") } @@ -45,19 +56,29 @@ func TestAddOCIBindsForDev(t *testing.T) { } func TestAddOCIBindsForSys(t *testing.T) { - specgen, err := generate.New("linux") + ctr, err := container.New() if err != nil { t.Error(err) } - config := &types.ContainerConfig{ + if err := ctr.SetConfig(&types.ContainerConfig{ Mounts: []*types.Mount{ { ContainerPath: "/sys", HostPath: "/sys", }, }, + Metadata: &types.ContainerMetadata{ + Name: "testctr", + }, + }, &types.PodSandboxConfig{ + Metadata: &types.PodSandboxMetadata{ + Name: "testpod", + }, + }); err != nil { + t.Error(err) } - _, binds, err := addOCIBindMounts(context.Background(), "", config, &specgen, "", nil) + + _, binds, err := addOCIBindMounts(context.Background(), ctr, "", "", nil, false, false) if err != nil { t.Error(err) } diff --git a/server/label_linux.go b/server/label_linux.go index 0d687800296..d53bb294f5e 100644 --- a/server/label_linux.go +++ b/server/label_linux.go @@ -5,10 +5,20 @@ import ( "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) -func securityLabel(path, secLabel string, shared bool) error { +func securityLabel(path, secLabel string, shared, maybeRelabel bool) error { + if maybeRelabel { + currentLabel, err := label.FileLabel(path) + if err == nil && currentLabel == secLabel { + logrus.Debugf( + "Skipping relabel for %s, as TrySkipVolumeSELinuxLabel is true and the label of the top level of the volume is already correct", + path) + return nil + } + } if err := label.Relabel(path, secLabel, shared); err != nil && !errors.Is(err, unix.ENOTSUP) { return fmt.Errorf("relabel failed %s: %v", path, err) } diff --git a/server/label_unsupported.go b/server/label_unsupported.go index 39ebd94a5f4..a3eb836df4f 100644 --- a/server/label_unsupported.go +++ b/server/label_unsupported.go @@ -1,7 +1,8 @@ +//go:build !linux // +build !linux package server -func securityLabel(path string, seclabel string, shared bool) error { +func securityLabel(path string, seclabel string, shared, maybeRelabel bool) error { return nil } diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go index 25e0676efe3..bd2d7330833 100644 --- a/server/metrics/metrics.go +++ b/server/metrics/metrics.go @@ -71,9 +71,10 @@ func New(config *libconfig.MetricsConfig) *Metrics { ), metricOperationsLatencyTotal: prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Subsystem: collectors.Subsystem, - Name: collectors.OperationsLatencyTotal.String(), - Help: "Latency in microseconds of CRI-O operations. Broken down by operation type.", + Subsystem: collectors.Subsystem, + Name: collectors.OperationsLatencyTotal.String(), + Help: "Latency in microseconds of CRI-O operations. Broken down by operation type.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, []string{"operation_type"}, ), diff --git a/server/sandbox_status.go b/server/sandbox_status.go index 5be37282151..0f54192b475 100644 --- a/server/sandbox_status.go +++ b/server/sandbox_status.go @@ -81,17 +81,23 @@ func toPodIPs(ips []string) (result []*types.PodIP) { } func createSandboxInfo(c *oci.Container) (map[string]string, error) { + var info interface{} if c.Spoofed() { - return map[string]string{"info": "{}"}, nil - } - info := struct { - Image string `json:"image"` - Pid int `json:"pid"` - RuntimeSpec spec.Spec `json:"runtimeSpec,omitempty"` - }{ - c.Image(), - c.State().Pid, - c.Spec(), + info = struct { + RuntimeSpec spec.Spec `json:"runtimeSpec,omitempty"` + }{ + c.Spec(), + } + } else { + info = struct { + Image string `json:"image"` + Pid int `json:"pid"` + RuntimeSpec spec.Spec `json:"runtimeSpec,omitempty"` + }{ + c.Image(), + c.State().Pid, + c.Spec(), + } } bytes, err := json.Marshal(info) if err != nil { diff --git a/server/sandbox_stop_linux.go b/server/sandbox_stop_linux.go index 908a5542edf..ef394fdbbb7 100644 --- a/server/sandbox_stop_linux.go +++ b/server/sandbox_stop_linux.go @@ -88,6 +88,10 @@ func (s *Server) stopPodSandbox(ctx context.Context, sb *sandbox.Sandbox) error } } + if err := sb.CloseManagedNamespaces(); err != nil { + return errors.Wrap(err, "unable to close managed namespaces") + } + if err := sb.UnmountShm(); err != nil { return err } diff --git a/test/devices.bats b/test/devices.bats index 369947e75ab..24dad7fee81 100644 --- a/test/devices.bats +++ b/test/devices.bats @@ -15,18 +15,6 @@ function teardown() { cleanup_test } -function create_device_runtime() { - cat << EOF > "$CRIO_CONFIG_DIR/01-device.conf" -[crio.runtime] -default_runtime = "device" -[crio.runtime.runtimes.device] -runtime_path = "$RUNTIME_BINARY_PATH" -runtime_root = "$RUNTIME_ROOT" -runtime_type = "$RUNTIME_TYPE" -allowed_annotations = ["io.kubernetes.cri-o.Devices"] -EOF -} - @test "additional devices support" { OVERRIDE_OPTIONS="--additional-devices /dev/null:/dev/qifoo:rwm" start_crio pod_id=$(crictl runp "$TESTDATA"/sandbox_config.json) @@ -80,7 +68,7 @@ EOF } @test "annotation devices support" { - create_device_runtime + create_runtime_with_allowed_annotation "device" "io.kubernetes.cri-o.Devices" start_crio jq ' .annotations."io.kubernetes.cri-o.Devices" = "/dev/null:/dev/qifoo:rwm"' \ @@ -110,7 +98,7 @@ EOF } @test "annotation should override configured additional_devices" { - create_device_runtime + create_runtime_with_allowed_annotation "device" "io.kubernetes.cri-o.Devices" OVERRIDE_OPTIONS="--additional-devices /dev/urandom:/dev/qifoo:rwm" start_crio @@ -128,7 +116,7 @@ EOF } @test "annotation should configure multiple devices" { - create_device_runtime + create_runtime_with_allowed_annotation "device" "io.kubernetes.cri-o.Devices" start_crio jq ' .annotations."io.kubernetes.cri-o.Devices" = "/dev/null:/dev/qifoo:rwm,/dev/urandom:/dev/peterfoo:rwm"' \ @@ -147,7 +135,7 @@ EOF } @test "annotation should fail if one device is invalid" { - create_device_runtime + create_runtime_with_allowed_annotation "device" "io.kubernetes.cri-o.Devices" start_crio jq ' .annotations."io.kubernetes.cri-o.Devices" = "/dev/null:/dev/qifoo:rwm,/dove/null"' \ diff --git a/test/drop_infra.bats b/test/drop_infra.bats index db01c950ed9..c1a663ff97f 100644 --- a/test/drop_infra.bats +++ b/test/drop_infra.bats @@ -33,3 +33,11 @@ function teardown() { output=$(runtime list) [[ "$output" = *"$pod_id"* ]] } + +@test "test infra ctr dropped status" { + jq '.linux.security_context.namespace_options.pid = 1' \ + "$TESTDATA"/sandbox_config.json > "$TESTDIR"/sandbox_no_infra.json + pod_id=$(crictl runp "$TESTDIR"/sandbox_no_infra.json) + output=$(crictl inspectp "$pod_id" | jq .info) + [[ "$output" != "{}" ]] +} diff --git a/test/helpers.bash b/test/helpers.bash index 1e3f9cecae1..ff4a41696f8 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -499,7 +499,7 @@ function ping_pod_from_pod() { # in such an environment without giving all containers NET_RAW capability # rather than reducing the security of the tests for all cases, skip this check # instead - if grep -i 'Red Hat\|CentOS' /etc/redhat-release | grep -q " 7"; then + if is_rhel_7; then return fi @@ -507,6 +507,10 @@ function ping_pod_from_pod() { crictl exec --sync "$2" ping6 -W 1 -c 2 "$ip" } +function is_rhel_7() { + grep -i 'Red Hat\|CentOS' /etc/redhat-release | grep -q " 7" +} + function cleanup_network_conf() { rm -rf "$CRIO_CNI_CONFIG" } @@ -548,3 +552,17 @@ function fail() { function is_cgroup_v2() { test "$(stat -f -c%T /sys/fs/cgroup)" = "cgroup2fs" } + +function create_runtime_with_allowed_annotation() { + local NAME="$1" + local ANNOTATION="$2" + cat <"$CRIO_CONFIG_DIR/01-$NAME.conf" +[crio.runtime] +default_runtime = "$NAME" +[crio.runtime.runtimes.$NAME] +runtime_path = "$RUNTIME_BINARY_PATH" +runtime_root = "$RUNTIME_ROOT" +runtime_type = "$RUNTIME_TYPE" +allowed_annotations = ["$ANNOTATION"] +EOF +} diff --git a/test/metrics.bats b/test/metrics.bats index 4a9e2523b06..fdab166ba74 100644 --- a/test/metrics.bats +++ b/test/metrics.bats @@ -34,6 +34,21 @@ function teardown() { curl -sf "http://localhost:$PORT/metrics" | grep crio_operations } +@test "metrics with operations quantile" { + # start crio with custom port + PORT=$(free_port) + CONTAINER_ENABLE_METRICS=true CONTAINER_METRICS_PORT=$PORT start_crio + + for ((i = 0; i < 100; i++)); do + crictl version + done + + # get metrics + curl -sf "http://localhost:$PORT/metrics" | grep 'container_runtime_crio_operations_latency_microseconds_total{operation_type="Version",quantile="0.5"}' + curl -sf "http://localhost:$PORT/metrics" | grep 'container_runtime_crio_operations_latency_microseconds_total{operation_type="Version",quantile="0.9"}' + curl -sf "http://localhost:$PORT/metrics" | grep 'container_runtime_crio_operations_latency_microseconds_total{operation_type="Version",quantile="0.99"}' +} + @test "secure metrics with random port" { openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \ -subj "/C=US/ST=State/L=City/O=Org/CN=Name" \ diff --git a/test/mocks/cmdrunner/cmdrunner.go b/test/mocks/cmdrunner/cmdrunner.go index 4d62ceeb76a..48e22c50458 100644 --- a/test/mocks/cmdrunner/cmdrunner.go +++ b/test/mocks/cmdrunner/cmdrunner.go @@ -5,6 +5,7 @@ package cmdrunnermock import ( + exec "os/exec" reflect "reflect" gomock "github.com/golang/mock/gomock" @@ -52,3 +53,22 @@ func (mr *MockCommandRunnerMockRecorder) CombinedOutput(arg0 interface{}, arg1 . varargs := append([]interface{}{arg0}, arg1...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CombinedOutput", reflect.TypeOf((*MockCommandRunner)(nil).CombinedOutput), varargs...) } + +// Command mocks base method. +func (m *MockCommandRunner) Command(arg0 string, arg1 ...string) *exec.Cmd { + m.ctrl.T.Helper() + varargs := []interface{}{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Command", varargs...) + ret0, _ := ret[0].(*exec.Cmd) + return ret0 +} + +// Command indicates an expected call of Command. +func (mr *MockCommandRunnerMockRecorder) Command(arg0 interface{}, arg1 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Command", reflect.TypeOf((*MockCommandRunner)(nil).Command), varargs...) +} diff --git a/test/selinux.bats b/test/selinux.bats index 78b7303167d..48be809a84d 100644 --- a/test/selinux.bats +++ b/test/selinux.bats @@ -20,3 +20,90 @@ function teardown() { ctr_id=$(crictl create "$pod_id" "$TESTDATA"/container_redis.json "$TESTDIR"/sandbox.json) crictl start "$ctr_id" } + +@test "selinux skips relabeling if TrySkipVolumeSELinuxLabel annotation is present" { + if [[ $(getenforce) != "Enforcing" ]]; then + skip "not enforcing" + fi + + # RHEL/CentOS 7's container-selinux package replaces container_file_t with svirt_sandbox_file_t + # under the hood. This causes the annotation to not work correctly. + if is_rhel_7; then + skip "fails on RHEL 7 or earlier" + fi + + VOLUME="$TESTDIR"/dir + FILE="$VOLUME"/file + mkdir "$VOLUME" + touch "$FILE" + + create_runtime_with_allowed_annotation "selinux" "io.kubernetes.cri-o.TrySkipVolumeSELinuxLabel" + start_crio + + jq ' .linux.security_context.selinux_options = {"level": "s0:c100,c200"} + | .annotations["io.kubernetes.cri-o.TrySkipVolumeSELinuxLabel"] = "true"' \ + "$TESTDATA"/sandbox_config.json > "$TESTDIR"/sandbox.json + + jq --arg path "$VOLUME" \ + ' .mounts = [ { + host_path: $path, + container_path: "/tmp/path", + selinux_relabel: true + } ]' \ + "$TESTDATA"/container_redis.json > "$TESTDIR"/container.json + + pod_id=$(crictl runp "$TESTDIR"/sandbox.json) + ctr_id=$(crictl create "$pod_id" "$TESTDIR"/container.json "$TESTDIR"/sandbox.json) + + crictl rm "$ctr_id" + + # shellcheck disable=SC2010 + oldlabel=$(ls -Z "$FILE" | grep -o '[a-z,_]*_u:[a-z,_]*_r:[a-z,_]*_t:[c,s,0-9,:,\,]* ') + + # Label file, but not top dir. This will show us the directory was not relabeled (as expected) + chcon --reference "$TESTDIR"/container.json "$FILE" # || \ + + # shellcheck disable=SC2010 + label=$(ls -Z "$FILE" | grep -o '[a-z,_]*_u:[a-z,_]*_r:[a-z,_]*_t:[c,s,0-9,:,\,]* ') + [[ "$oldlabel" != "$label" ]] + + # Recreate. Since top level is already labeled right, there won't be a relabel. + ctr_id=$(crictl create "$pod_id" "$TESTDIR"/container.json "$TESTDIR"/sandbox.json) + # shellcheck disable=SC2010 + newlabel=$(ls -Z "$FILE" | grep -o '[a-z,_]*_u:[a-z,_]*_r:[a-z,_]*_t:[c,s,0-9,:,\,]* ') + [[ "$label" == "$newlabel" ]] +} + +@test "selinux skips relabeling for super priviliged container" { + if [[ $(getenforce) != "Enforcing" ]]; then + skip "not enforcing" + fi + VOLUME="$TESTDIR"/dir + mkdir -p "$VOLUME" + + # shellcheck disable=SC2012 + OLDLABEL=$(ls -dZ "$VOLUME" | awk '{ printf $1 }') + + start_crio + + jq '.linux.security_context.selinux_options = {"type": "spc_t"}' \ + "$TESTDATA"/sandbox_config.json > "$TESTDIR"/sandbox.json + + jq --arg path "$VOLUME" \ + '.mounts = [{ + host_path: $path, + container_path: "/tmp/path", + selinux_relabel: true + }]' \ + "$TESTDATA"/container_redis.json > "$TESTDIR"/container.json + + pod_id=$(crictl runp "$TESTDIR"/sandbox.json) + ctr_id=$(crictl create "$pod_id" "$TESTDIR"/container.json "$TESTDIR"/sandbox.json) + + crictl rm "$ctr_id" + + # shellcheck disable=SC2012 + NEWLABEL=$(ls -dZ "$VOLUME" | awk '{ printf $1 }') + + [[ "$OLDLABEL" == "$NEWLABEL" ]] +} diff --git a/test/shm_size.bats b/test/shm_size.bats index b6dc7c75438..41ca968e49f 100644 --- a/test/shm_size.bats +++ b/test/shm_size.bats @@ -23,7 +23,7 @@ EOF } @test "check /dev/shm is changed" { - create_shmsize_runtime + create_runtime_with_allowed_annotation "shmsize" "io.kubernetes.cri-o.ShmSize" start_crio # Run base container to ensure it creates at all pod_id=$(crictl runp <(jq '.annotations."io.kubernetes.cri-o.ShmSize" = "16Mi"' "$TESTDATA"/sandbox_config.json)) @@ -40,7 +40,7 @@ EOF } @test "check /dev/shm fails with incorrect values" { - create_shmsize_runtime + create_runtime_with_allowed_annotation "shmsize" "io.kubernetes.cri-o.ShmSize" start_crio # Ensure pod fails if /dev/shm size is negative ! crictl runp <(jq '.annotations."io.kubernetes.cri-o.ShmSize" = "-1"' "$TESTDATA"/sandbox_config.json) diff --git a/test/workloads.bats b/test/workloads.bats index d4d262b2f07..3b5ff4d4c90 100644 --- a/test/workloads.bats +++ b/test/workloads.bats @@ -269,3 +269,20 @@ function check_conmon_fields() { ctr_id=$(crictl run "$ctrconfig" "$sboxconfig") check_conmon_fields "$ctr_id" "$shares" "$set" } + +@test "should succeed to create workload with empty resources" { + cat << EOF > "$CRIO_CONFIG_DIR/01-workload.conf" +[crio.runtime.workloads.management] +activation_annotation = "$activation" +annotation_prefix = "$prefix" +EOF + start_crio + + jq --arg act "$activation" ' .annotations[$act] = "true"' \ + "$TESTDATA"/sandbox_config.json > "$sboxconfig" + + jq --arg act "$activation" ' .annotations[$act] = "true"' \ + "$TESTDATA"/container_sleep.json > "$ctrconfig" + + ctr_id=$(crictl run "$ctrconfig" "$sboxconfig") +} diff --git a/utils/cmdrunner/cmdrunner.go b/utils/cmdrunner/cmdrunner.go index eebb37c1eb3..6a1d1536d8b 100644 --- a/utils/cmdrunner/cmdrunner.go +++ b/utils/cmdrunner/cmdrunner.go @@ -4,12 +4,68 @@ import ( "os/exec" ) +// Use a singleton instance because there are many modules that may want access +// and having it all go through a shared object like the config or server would +// add a lot of complexity. +var commandRunner CommandRunner + +// CommandRunner is an interface for executing commands. +// It gives the option to change the way commands are run server-wide. type CommandRunner interface { + Command(string, ...string) *exec.Cmd CombinedOutput(string, ...string) ([]byte, error) } -type RealCommandRunner struct{} +// prependableCommandRunner is an implementation of CommandRunner. +// It gives the option for all commands that are run to be prepended by another command +// and arguments. +type prependableCommandRunner struct { + prependCmd string + prependArgs []string +} + +// PrependCommandsWith updates the commandRunner singleton to have the configured prepended args and command. +func PrependCommandsWith(prependCmd string, prependArgs ...string) { + commandRunner = &prependableCommandRunner{ + prependCmd: prependCmd, + prependArgs: prependArgs, + } +} + +// CombinedOutput calls CombinedOutput on the defined commandRunner, +// or the default implementation in the exec package if there's no commandRunner defined. +func CombinedOutput(command string, args ...string) ([]byte, error) { + if commandRunner == nil { + return exec.Command(command, args...).CombinedOutput() + } + return commandRunner.CombinedOutput(command, args...) +} + +// CombinedOutput returns the combined output of the command, given the prepended cmd/args that were defined. +func (c *prependableCommandRunner) CombinedOutput(command string, args ...string) ([]byte, error) { + return c.Command(command, args...).CombinedOutput() +} + +// Command calls Command on the defined commandRunner, +// or the default implementation in the exec package if there's no commandRunner defined. +func Command(cmd string, args ...string) *exec.Cmd { + if commandRunner == nil { + return exec.Command(cmd, args...) + } + return commandRunner.Command(cmd, args...) +} -func (c *RealCommandRunner) CombinedOutput(command string, args ...string) ([]byte, error) { - return exec.Command(command, args...).CombinedOutput() +// Command creates an exec.Cmd object. If prependCmd is defined, the command will be prependCmd +// and the args will be prependArgs + cmd + args. +// Otherwise, cmd and args will be as inputted. +func (c *prependableCommandRunner) Command(cmd string, args ...string) *exec.Cmd { + realCmd := cmd + realArgs := args + if c.prependCmd != "" { + realCmd = c.prependCmd + realArgs = c.prependArgs + realArgs = append(realArgs, cmd) + realArgs = append(realArgs, args...) + } + return exec.Command(realCmd, realArgs...) } diff --git a/utils/cmdrunner/cmdrunner_test.go b/utils/cmdrunner/cmdrunner_test.go new file mode 100644 index 00000000000..c83e9391cb2 --- /dev/null +++ b/utils/cmdrunner/cmdrunner_test.go @@ -0,0 +1,53 @@ +package cmdrunner_test + +import ( + "os/exec" + + "github.com/cri-o/cri-o/utils/cmdrunner" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = t.Describe("CommandRunner", func() { + It("command should not prepend if not configured", func() { + // Given + cmd := "ls" + baseline, err := exec.Command(cmd).CombinedOutput() + Expect(err).To(BeNil()) + + // When + output, err := cmdrunner.CombinedOutput(cmd) + Expect(err).To(BeNil()) + + // Then + Expect(output).To(Equal(baseline)) + }) + It("command should prepend if configured", func() { + // Given + cmd := "ls" + cmdrunner.PrependCommandsWith("which") + baseline, err := exec.Command(cmd).CombinedOutput() + Expect(err).To(BeNil()) + + // When + output, err := cmdrunner.CombinedOutput(cmd) + Expect(err).To(BeNil()) + + // Then + Expect(output).NotTo(Equal(baseline)) + }) + It("command should not prepend if only args are configured", func() { + // Given + cmd := "ls" + cmdrunner.PrependCommandsWith("", "-l") + baseline, err := exec.Command(cmd).CombinedOutput() + Expect(err).To(BeNil()) + + // When + output, err := cmdrunner.CombinedOutput(cmd) + Expect(err).To(BeNil()) + + // Then + Expect(output).To(Equal(baseline)) + }) +}) diff --git a/utils/cmdrunner/cmdrunner_test_inject.go b/utils/cmdrunner/cmdrunner_test_inject.go new file mode 100644 index 00000000000..f056d8d653e --- /dev/null +++ b/utils/cmdrunner/cmdrunner_test_inject.go @@ -0,0 +1,15 @@ +//go:build test +// +build test + +// All *_inject.go files are meant to be used by tests only. Purpose of this +// files is to provide a way to inject mocked data into the current setup. + +package cmdrunner + +import ( + runnerMock "github.com/cri-o/cri-o/test/mocks/cmdrunner" +) + +func SetMocked(runner *runnerMock.MockCommandRunner) { + commandRunner = runner +} diff --git a/utils/cmdrunner/suite_test.go b/utils/cmdrunner/suite_test.go new file mode 100644 index 00000000000..55d950677e9 --- /dev/null +++ b/utils/cmdrunner/suite_test.go @@ -0,0 +1,26 @@ +package cmdrunner_test + +import ( + "testing" + + . "github.com/cri-o/cri-o/test/framework" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +// TestCommandRunner runs the created specs +func TestCommandRunner(t *testing.T) { + RegisterFailHandler(Fail) + RunFrameworkSpecs(t, "CommandRunner") +} + +var t *TestFramework + +var _ = BeforeSuite(func() { + t = NewTestFramework(NilFunc, NilFunc) + t.Setup() +}) + +var _ = AfterSuite(func() { + t.Teardown() +}) diff --git a/utils/utils.go b/utils/utils.go index 14482f913ef..4e4608d38c9 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -8,7 +8,6 @@ import ( "io" "io/ioutil" "os" - "os/exec" "path/filepath" "runtime" "strconv" @@ -18,6 +17,7 @@ import ( "github.com/containers/podman/v3/pkg/lookup" "github.com/cri-o/cri-o/internal/dbusmgr" "github.com/cri-o/cri-o/server/cri/types" + "github.com/cri-o/cri-o/utils/cmdrunner" securejoin "github.com/cyphar/filepath-securejoin" "github.com/opencontainers/runc/libcontainer/user" "github.com/pkg/errors" @@ -31,7 +31,7 @@ import ( // ExecCmd executes a command with args and returns its output as a string along // with an error, if any func ExecCmd(name string, args ...string) (string, error) { - cmd := exec.Command(name, args...) + cmd := cmdrunner.Command(name, args...) var stdout bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &stdout diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/label/label_linux.go b/vendor/github.com/opencontainers/selinux/go-selinux/label/label_linux.go index 14e1e38c248..12de0ae5d65 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/label/label_linux.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/label/label_linux.go @@ -103,9 +103,11 @@ func SetFileCreateLabel(fileLabel string) error { return selinux.SetFSCreateLabel(fileLabel) } -// Relabel changes the label of path to the filelabel string. +// Relabel changes the label of path and all the entries beneath the path. // It changes the MCS label to s0 if shared is true. // This will allow all containers to share the content. +// +// The path itself is guaranteed to be relabeled last. func Relabel(path string, fileLabel string, shared bool) error { if !selinux.GetEnabled() || fileLabel == "" { return nil diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/selinux.go b/vendor/github.com/opencontainers/selinux/go-selinux/selinux.go index 9ffd77afabe..cad467507a5 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/selinux.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/selinux.go @@ -38,6 +38,8 @@ var ( // CategoryRange allows the upper bound on the category range to be adjusted CategoryRange = DefaultCategoryRange + + privContainerMountLabel string ) // Context is a representation of the SELinux label broken into 4 parts @@ -253,6 +255,8 @@ func CopyLevel(src, dest string) (string, error) { // Chcon changes the fpath file object to the SELinux label label. // If fpath is a directory and recurse is true, then Chcon walks the // directory tree setting the label. +// +// The fpath itself is guaranteed to be relabeled last. func Chcon(fpath string, label string, recurse bool) error { return chcon(fpath, label, recurse) } @@ -280,5 +284,7 @@ func GetDefaultContextWithLevel(user, level, scon string) (string, error) { // PrivContainerMountLabel returns mount label for privileged containers func PrivContainerMountLabel() string { + // Make sure label is initialized. + _ = label("") return privContainerMountLabel } diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go index a804473e465..b045843ad6e 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go @@ -9,15 +9,14 @@ import ( "fmt" "io" "io/ioutil" + "math/big" "os" "path" "path/filepath" - "regexp" "strconv" "strings" "sync" - "github.com/bits-and-blooms/bitset" "golang.org/x/sys/unix" ) @@ -34,8 +33,6 @@ const ( xattrNameSelinux = "security.selinux" ) -var policyRoot = filepath.Join(selinuxDir, readConfig(selinuxTypeTag)) - type selinuxState struct { enabledSet bool enabled bool @@ -47,7 +44,7 @@ type selinuxState struct { type level struct { sens uint - cats *bitset.BitSet + cats *big.Int } type mlsRange struct { @@ -70,7 +67,6 @@ const ( ) var ( - assignRegex = regexp.MustCompile(`^([^=]+)=(.*)$`) readOnlyFileLabel string state = selinuxState{ mcsList: make(map[string]bool), @@ -79,8 +75,24 @@ var ( // for attrPath() attrPathOnce sync.Once haveThreadSelf bool + + // for policyRoot() + policyRootOnce sync.Once + policyRootVal string + + // for label() + loadLabelsOnce sync.Once + labels map[string]string ) +func policyRoot() string { + policyRootOnce.Do(func() { + policyRootVal = filepath.Join(selinuxDir, readConfig(selinuxTypeTag)) + }) + + return policyRootVal +} + func (s *selinuxState) setEnable(enabled bool) bool { s.Lock() defer s.Unlock() @@ -222,7 +234,7 @@ func readConfig(target string) string { scanner := bufio.NewScanner(in) for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) + line := bytes.TrimSpace(scanner.Bytes()) if len(line) == 0 { // Skip blank lines continue @@ -231,11 +243,12 @@ func readConfig(target string) string { // Skip comments continue } - if groups := assignRegex.FindStringSubmatch(line); groups != nil { - key, val := strings.TrimSpace(groups[1]), strings.TrimSpace(groups[2]) - if key == target { - return strings.Trim(val, "\"") - } + fields := bytes.SplitN(line, []byte{'='}, 2) + if len(fields) != 2 { + continue + } + if bytes.Equal(fields[0], []byte(target)) { + return string(bytes.Trim(fields[1], `"`)) } } return "" @@ -274,12 +287,15 @@ func readCon(fpath string) (string, error) { if err := isProcHandle(in); err != nil { return "", err } + return readConFd(in) +} - var retval string - if _, err := fmt.Fscanf(in, "%s", &retval); err != nil { +func readConFd(in *os.File) (string, error) { + data, err := ioutil.ReadAll(in) + if err != nil { return "", err } - return strings.Trim(retval, "\x00"), nil + return string(bytes.TrimSuffix(data, []byte{0})), nil } // classIndex returns the int index for an object class in the loaded policy, @@ -389,7 +405,7 @@ func writeCon(fpath, val string) error { _, err = out.Write(nil) } if err != nil { - return &os.PathError{Op: "write", Path: fpath, Err: err} + return err } return nil } @@ -439,8 +455,8 @@ func computeCreateContext(source string, target string, class string) (string, e } // catsToBitset stores categories in a bitset. -func catsToBitset(cats string) (*bitset.BitSet, error) { - bitset := &bitset.BitSet{} +func catsToBitset(cats string) (*big.Int, error) { + bitset := new(big.Int) catlist := strings.Split(cats, ",") for _, r := range catlist { @@ -455,14 +471,14 @@ func catsToBitset(cats string) (*bitset.BitSet, error) { return nil, err } for i := catstart; i <= catend; i++ { - bitset.Set(i) + bitset.SetBit(bitset, int(i), 1) } } else { cat, err := parseLevelItem(ranges[0], category) if err != nil { return nil, err } - bitset.Set(cat) + bitset.SetBit(bitset, int(cat), 1) } } @@ -532,37 +548,30 @@ func rangeStrToMLSRange(rangeStr string) (*mlsRange, error) { // bitsetToStr takes a category bitset and returns it in the // canonical selinux syntax -func bitsetToStr(c *bitset.BitSet) string { +func bitsetToStr(c *big.Int) string { var str string - i, e := c.NextSet(0) - len := 0 - for e { - if len == 0 { + + length := 0 + for i := int(c.TrailingZeroBits()); i < c.BitLen(); i++ { + if c.Bit(i) == 0 { + continue + } + if length == 0 { if str != "" { str += "," } - str += "c" + strconv.Itoa(int(i)) + str += "c" + strconv.Itoa(i) } - - next, e := c.NextSet(i + 1) - if e { - // consecutive cats - if next == i+1 { - len++ - i = next - continue - } - } - if len == 1 { - str += ",c" + strconv.Itoa(int(i)) - } else if len > 1 { - str += ".c" + strconv.Itoa(int(i)) + if c.Bit(i+1) == 1 { + length++ + continue } - if !e { - break + if length == 1 { + str += ",c" + strconv.Itoa(i) + } else if length > 1 { + str += ".c" + strconv.Itoa(i) } - len = 0 - i = next + length = 0 } return str @@ -575,13 +584,16 @@ func (l1 *level) equal(l2 *level) bool { if l1.sens != l2.sens { return false } - return l1.cats.Equal(l2.cats) + if l2.cats == nil || l1.cats == nil { + return l2.cats == l1.cats + } + return l1.cats.Cmp(l2.cats) == 0 } // String returns an mlsRange as a string. func (m mlsRange) String() string { low := "s" + strconv.Itoa(int(m.low.sens)) - if m.low.cats != nil && m.low.cats.Count() > 0 { + if m.low.cats != nil && m.low.cats.BitLen() > 0 { low += ":" + bitsetToStr(m.low.cats) } @@ -590,7 +602,7 @@ func (m mlsRange) String() string { } high := "s" + strconv.Itoa(int(m.high.sens)) - if m.high.cats != nil && m.high.cats.Count() > 0 { + if m.high.cats != nil && m.high.cats.BitLen() > 0 { high += ":" + bitsetToStr(m.high.cats) } @@ -640,10 +652,12 @@ func calculateGlbLub(sourceRange, targetRange string) (string, error) { /* find the intersecting categories */ if s.low.cats != nil && t.low.cats != nil { - outrange.low.cats = s.low.cats.Intersection(t.low.cats) + outrange.low.cats = new(big.Int) + outrange.low.cats.And(s.low.cats, t.low.cats) } if s.high.cats != nil && t.high.cats != nil { - outrange.high.cats = s.high.cats.Intersection(t.high.cats) + outrange.high.cats = new(big.Int) + outrange.high.cats.And(s.high.cats, t.high.cats) } return outrange.String(), nil @@ -664,11 +678,7 @@ func readWriteCon(fpath string, val string) (string, error) { return "", err } - var retval string - if _, err := fmt.Fscanf(f, "%s", &retval); err != nil { - return "", err - } - return strings.Trim(retval, "\x00"), nil + return readConFd(f) } // setExecLabel sets the SELinux label that the kernel will use for any programs @@ -723,10 +733,10 @@ func keyLabel() (string, error) { // get returns the Context as a string func (c Context) get() string { - if c["level"] != "" { - return fmt.Sprintf("%s:%s:%s:%s", c["user"], c["role"], c["type"], c["level"]) + if level := c["level"]; level != "" { + return c["user"] + ":" + c["role"] + ":" + c["type"] + ":" + level } - return fmt.Sprintf("%s:%s:%s", c["user"], c["role"], c["type"]) + return c["user"] + ":" + c["role"] + ":" + c["type"] } // newContext creates a new Context struct from the specified label @@ -891,24 +901,21 @@ func openContextFile() (*os.File, error) { if f, err := os.Open(contextFile); err == nil { return f, nil } - lxcPath := filepath.Join(policyRoot, "/contexts/lxc_contexts") - return os.Open(lxcPath) + return os.Open(filepath.Join(policyRoot(), "/contexts/lxc_contexts")) } -var labels, privContainerMountLabel = loadLabels() - -func loadLabels() (map[string]string, string) { - labels := make(map[string]string) +func loadLabels() { + labels = make(map[string]string) in, err := openContextFile() if err != nil { - return labels, "" + return } defer in.Close() scanner := bufio.NewScanner(in) for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) + line := bytes.TrimSpace(scanner.Bytes()) if len(line) == 0 { // Skip blank lines continue @@ -917,38 +924,47 @@ func loadLabels() (map[string]string, string) { // Skip comments continue } - if groups := assignRegex.FindStringSubmatch(line); groups != nil { - key, val := strings.TrimSpace(groups[1]), strings.TrimSpace(groups[2]) - labels[key] = strings.Trim(val, "\"") + fields := bytes.SplitN(line, []byte{'='}, 2) + if len(fields) != 2 { + continue } + key, val := bytes.TrimSpace(fields[0]), bytes.TrimSpace(fields[1]) + labels[string(key)] = string(bytes.Trim(val, `"`)) } con, _ := NewContext(labels["file"]) con["level"] = fmt.Sprintf("s0:c%d,c%d", maxCategory-2, maxCategory-1) - reserveLabel(con.get()) - return labels, con.get() + privContainerMountLabel = con.get() + reserveLabel(privContainerMountLabel) +} + +func label(key string) string { + loadLabelsOnce.Do(func() { + loadLabels() + }) + return labels[key] } // kvmContainerLabels returns the default processLabel and mountLabel to be used // for kvm containers by the calling process. func kvmContainerLabels() (string, string) { - processLabel := labels["kvm_process"] + processLabel := label("kvm_process") if processLabel == "" { - processLabel = labels["process"] + processLabel = label("process") } - return addMcs(processLabel, labels["file"]) + return addMcs(processLabel, label("file")) } // initContainerLabels returns the default processLabel and file labels to be // used for containers running an init system like systemd by the calling process. func initContainerLabels() (string, string) { - processLabel := labels["init_process"] + processLabel := label("init_process") if processLabel == "" { - processLabel = labels["process"] + processLabel = label("process") } - return addMcs(processLabel, labels["file"]) + return addMcs(processLabel, label("file")) } // containerLabels returns an allocated processLabel and fileLabel to be used for @@ -958,9 +974,9 @@ func containerLabels() (processLabel string, fileLabel string) { return "", "" } - processLabel = labels["process"] - fileLabel = labels["file"] - readOnlyFileLabel = labels["ro_file"] + processLabel = label("process") + fileLabel = label("file") + readOnlyFileLabel = label("ro_file") if processLabel == "" || fileLabel == "" { return "", fileLabel @@ -1180,15 +1196,14 @@ func getDefaultContextFromReaders(c *defaultSECtx) (string, error) { } func getDefaultContextWithLevel(user, level, scon string) (string, error) { - userPath := filepath.Join(policyRoot, selinuxUsersDir, user) - defaultPath := filepath.Join(policyRoot, defaultContexts) - + userPath := filepath.Join(policyRoot(), selinuxUsersDir, user) fu, err := os.Open(userPath) if err != nil { return "", err } defer fu.Close() + defaultPath := filepath.Join(policyRoot(), defaultContexts) fd, err := os.Open(defaultPath) if err != nil { return "", err diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_stub.go b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_stub.go index b7218a0b6a8..42657759c38 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_stub.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_stub.go @@ -2,8 +2,6 @@ package selinux -const privContainerMountLabel = "" - func setDisabled() { } @@ -152,3 +150,7 @@ func disableSecOpt() []string { func getDefaultContextWithLevel(user, level, scon string) (string, error) { return "", nil } + +func label(_ string) string { + return "" +} diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go b/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go index 011fe862aad..202c80da59c 100644 --- a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go @@ -51,6 +51,9 @@ func WalkN(root string, walkFn WalkFunc, num int) error { var ( err error wg sync.WaitGroup + + rootLen = len(root) + rootEntry *walkArgs ) wg.Add(1) go func() { @@ -59,6 +62,11 @@ func WalkN(root string, walkFn WalkFunc, num int) error { close(files) return err } + if len(p) == rootLen { + // Root entry is processed separately below. + rootEntry = &walkArgs{path: p, info: &info} + return nil + } // add a file to the queue unless a callback sent an error select { case e := <-errCh: @@ -92,6 +100,10 @@ func WalkN(root string, walkFn WalkFunc, num int) error { wg.Wait() + if err == nil { + err = walkFn(rootEntry.path, *rootEntry.info, nil) + } + return err } diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go index 222820750c3..a5796b2c4f1 100644 --- a/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go @@ -1,3 +1,4 @@ +//go:build go1.16 // +build go1.16 package pwalkdir @@ -51,6 +52,9 @@ func WalkN(root string, walkFn fs.WalkDirFunc, num int) error { var ( err error wg sync.WaitGroup + + rootLen = len(root) + rootEntry *walkArgs ) wg.Add(1) go func() { @@ -59,6 +63,11 @@ func WalkN(root string, walkFn fs.WalkDirFunc, num int) error { close(files) return err } + if len(p) == rootLen { + // Root entry is processed separately below. + rootEntry = &walkArgs{path: p, entry: entry} + return nil + } // Add a file to the queue unless a callback sent an error. select { case e := <-errCh: @@ -92,6 +101,10 @@ func WalkN(root string, walkFn fs.WalkDirFunc, num int) error { wg.Wait() + if err == nil { + err = walkFn(rootEntry.path, rootEntry.entry, nil) + } + return err } diff --git a/vendor/modules.txt b/vendor/modules.txt index deff855d919..6f8a08b0870 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -813,7 +813,7 @@ github.com/opencontainers/runtime-tools/generate github.com/opencontainers/runtime-tools/generate/seccomp github.com/opencontainers/runtime-tools/specerror github.com/opencontainers/runtime-tools/validate -# github.com/opencontainers/selinux v1.8.4 +# github.com/opencontainers/selinux v1.9.1 ## explicit github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label