diff --git a/pkg/container/container.go b/pkg/container/container.go index 6f4b1ab3ac6..507fed226a5 100644 --- a/pkg/container/container.go +++ b/pkg/container/container.go @@ -2,12 +2,22 @@ package container import ( "context" + "encoding/json" "fmt" "path/filepath" + "strconv" "strings" + "time" + "github.com/containers/libpod/v2/pkg/annotations" "github.com/containers/storage/pkg/stringid" + "github.com/cri-o/cri-o/internal/lib" + "github.com/cri-o/cri-o/internal/lib/sandbox" + oci "github.com/cri-o/cri-o/internal/oci" + "github.com/cri-o/cri-o/internal/storage" "github.com/cri-o/cri-o/utils" + rspec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -62,6 +72,19 @@ type Container interface { // SelinuxLabel returns the container's SelinuxLabel // it takes the sandbox's label, which it falls back upon SelinuxLabel(string) ([]string, error) + + // spec functions + + // returns the spec + Spec() *generate.Generator + + // SpecAddMount adds a mount to the container's spec + // it takes the rspec mount object + // if there is already a mount at the path specified, it removes it. + SpecAddMount(rspec.Mount) + + // SpecAddAnnotations adds annotations to the spec. + SpecAddAnnotations(sandbox *sandbox.Sandbox, containerVolume []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool) error } // container is the hidden default type behind the Container interface @@ -72,13 +95,117 @@ type container struct { id string name string privileged bool + spec generate.Generator } // New creates a new, empty Sandbox instance -func New(ctx context.Context) Container { +func New(ctx context.Context) (Container, error) { + spec, err := generate.New("linux") + if err != nil { + return nil, err + } return &container{ - ctx: ctx, + ctx: ctx, + spec: spec, + }, nil +} + +// SpecAddMount adds a specified mount to the spec +func (c *container) SpecAddMount(r rspec.Mount) { + c.spec.RemoveMount(r.Destination) + c.spec.AddMount(r) +} + +// SpecAddAnnotation adds all annotations to the spec +func (c *container) SpecAddAnnotations(sb *sandbox.Sandbox, containerVolumes []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool) (err error) { + // Copied from k8s.io/kubernetes/pkg/kubelet/kuberuntime/labels.go + const podTerminationGracePeriodLabel = "io.kubernetes.pod.terminationGracePeriod" + + kubeAnnotations := c.Config().GetAnnotations() + created := time.Now() + labels := c.Config().GetLabels() + + image, err := c.Image() + if err != nil { + return err + } + logPath, err := c.LogPath(sb.LogDir()) + if err != nil { + return err + } + c.spec.AddAnnotation(annotations.Image, image) + c.spec.AddAnnotation(annotations.ImageName, imageResult.Name) + c.spec.AddAnnotation(annotations.ImageRef, imageResult.ID) + c.spec.AddAnnotation(annotations.Name, c.Name()) + c.spec.AddAnnotation(annotations.ContainerID, c.ID()) + c.spec.AddAnnotation(annotations.SandboxID, sb.ID()) + c.spec.AddAnnotation(annotations.SandboxName, sb.Name()) + c.spec.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer) + c.spec.AddAnnotation(annotations.LogPath, logPath) + c.spec.AddAnnotation(annotations.TTY, strconv.FormatBool(c.Config().Tty)) + c.spec.AddAnnotation(annotations.Stdin, strconv.FormatBool(c.Config().Stdin)) + c.spec.AddAnnotation(annotations.StdinOnce, strconv.FormatBool(c.Config().StdinOnce)) + c.spec.AddAnnotation(annotations.ResolvPath, sb.ResolvPath()) + c.spec.AddAnnotation(annotations.ContainerManager, lib.ContainerManagerCRIO) + c.spec.AddAnnotation(annotations.MountPoint, mountPoint) + c.spec.AddAnnotation(annotations.SeccompProfilePath, c.Config().GetLinux().GetSecurityContext().GetSeccompProfilePath()) + c.spec.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano)) + + metadataJSON, err := json.Marshal(c.Config().GetMetadata()) + if err != nil { + return err + } + c.spec.AddAnnotation(annotations.Metadata, string(metadataJSON)) + + labelsJSON, err := json.Marshal(labels) + if err != nil { + return err } + c.spec.AddAnnotation(annotations.Labels, string(labelsJSON)) + + volumesJSON, err := json.Marshal(containerVolumes) + if err != nil { + return err + } + c.spec.AddAnnotation(annotations.Volumes, string(volumesJSON)) + + kubeAnnotationsJSON, err := json.Marshal(kubeAnnotations) + if err != nil { + return err + } + c.spec.AddAnnotation(annotations.Annotations, string(kubeAnnotationsJSON)) + + for k, v := range kubeAnnotations { + c.spec.AddAnnotation(k, v) + } + for k, v := range labels { + c.spec.AddAnnotation(k, v) + } + for idx, ip := range sb.IPs() { + c.spec.AddAnnotation(fmt.Sprintf("%s.%d", annotations.IP, idx), ip) + } + + if isSystemd { + if t, ok := kubeAnnotations[podTerminationGracePeriodLabel]; ok { + // currently only supported by systemd, see + // https://github.com/opencontainers/runc/pull/2224 + c.spec.AddAnnotation("org.systemd.property.TimeoutStopUSec", "uint64 "+t+"000000") // sec to usec + } + if systemdHasCollectMode { + c.spec.AddAnnotation("org.systemd.property.CollectMode", "'inactive-or-failed'") + } + } + + if configStopSignal != "" { + // this key is defined in image-spec conversion document at https://github.com/opencontainers/image-spec/pull/492/files#diff-8aafbe2c3690162540381b8cdb157112R57 + c.spec.AddAnnotation("org.opencontainers.image.stopSignal", configStopSignal) + } + + return nil +} + +func (c *container) Spec() *generate.Generator { + return &c.spec } // SetConfig sets the configuration to the container and validates it diff --git a/pkg/container/container_setnameandid_test.go b/pkg/container/container_setnameandid_test.go index 2e4f8d910bb..b2c1ba9e439 100644 --- a/pkg/container/container_setnameandid_test.go +++ b/pkg/container/container_setnameandid_test.go @@ -54,9 +54,13 @@ var _ = t.Describe("Container:SetNameAndID", func() { It("should fail with config nil", func() { // Given // When - err := container.New(context.Background()).SetNameAndID() + container, err := container.New(context.Background()) + Expect(err).To(BeNil()) + + err = container.SetNameAndID() // Then + Expect(container).ToNot(BeNil()) Expect(err).NotTo(BeNil()) }) }) diff --git a/pkg/container/container_test.go b/pkg/container/container_test.go index a1869e1c02c..6b0896a2ff0 100644 --- a/pkg/container/container_test.go +++ b/pkg/container/container_test.go @@ -1,20 +1,141 @@ package container_test import ( + "encoding/json" + "strconv" + "time" + + "github.com/containers/libpod/v2/pkg/annotations" + "github.com/cri-o/cri-o/internal/lib" + "github.com/cri-o/cri-o/internal/lib/sandbox" + oci "github.com/cri-o/cri-o/internal/oci" + "github.com/cri-o/cri-o/internal/storage" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + rspec "github.com/opencontainers/runtime-spec/specs-go" pb "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + "k8s.io/kubernetes/pkg/kubelet/dockershim/network/hostport" ) var _ = t.Describe("Container", func() { var config *pb.ContainerConfig var sboxConfig *pb.PodSandboxConfig + const defaultMounts = 6 BeforeEach(func() { config = &pb.ContainerConfig{ Metadata: &pb.ContainerMetadata{Name: "name"}, } sboxConfig = &pb.PodSandboxConfig{} }) + t.Describe("SpecAddMount", func() { + It("should add the mount to the spec", func() { + sut.SpecAddMount(rspec.Mount{ + Destination: "test", + Type: "test", + Source: "test", + Options: []string{"test"}, + }) + Expect(len(sut.Spec().Mounts())).To(Equal(defaultMounts + 1)) + }) + It("should add only one copy to the spec", func() { + sut.SpecAddMount(rspec.Mount{ + Destination: "test", + Type: "test", + Source: "test", + Options: []string{"test"}, + }) + sut.SpecAddMount(rspec.Mount{ + Destination: "test", + Type: "test", + Source: "test", + Options: []string{"test"}, + }) + Expect(len(sut.Spec().Mounts())).To(Equal(defaultMounts + 1)) + }) + }) + t.Describe("Spec", func() { + It("should return the spec", func() { + Expect(sut.Spec()).ToNot(Equal(nil)) + }) + }) + t.Describe("SpecAddAnnotations", func() { + It("should set the spec annotations", func() { + // Given + sandboxConfig := &pb.PodSandboxConfig{ + Metadata: &pb.PodSandboxMetadata{Name: "name"}, + } + containerConfig := &pb.ContainerConfig{ + Metadata: &pb.ContainerMetadata{Name: "name"}, + Linux: &pb.LinuxContainerConfig{ + SecurityContext: &pb.LinuxContainerSecurityContext{ + Privileged: true, + }, + }, + Image: &pb.ImageSpec{ + Image: "img", + }, + } + err := sut.SetConfig(containerConfig, sandboxConfig) + Expect(err).To(BeNil()) + currentTime := time.Now() + volumes := []oci.ContainerVolume{} + imageResult := storage.ImageResult{} + mountPoint := "test" + configStopSignal := "test" + + sb, err := sandbox.New("sandboxID", "", "", "", "test", + make(map[string]string), make(map[string]string), "", "", + &pb.PodSandboxMetadata{}, "", "", false, "", "", "", + []*hostport.PortMapping{}, false, currentTime, "") + Expect(err).To(BeNil()) + + image, err := sut.Image() + Expect(err).To(BeNil()) + + logpath, err := sut.LogPath(sb.LogDir()) + Expect(err).To(BeNil()) + + metadataJSON, err := json.Marshal(sut.Config().GetMetadata()) + Expect(err).To(BeNil()) + + labelsJSON, err := json.Marshal(sut.Config().GetLabels()) + Expect(err).To(BeNil()) + + volumesJSON, err := json.Marshal(volumes) + Expect(err).To(BeNil()) + + kubeAnnotationsJSON, err := json.Marshal(sut.Config().GetAnnotations()) + Expect(err).To(BeNil()) + + Expect(currentTime).ToNot(BeNil()) + Expect(sb).ToNot(BeNil()) + + err = sut.SpecAddAnnotations(sb, volumes, mountPoint, configStopSignal, &imageResult, false, false) + Expect(err).To(BeNil()) + + Expect(sut.Spec().Config.Annotations[annotations.Image]).To(Equal(image)) + Expect(sut.Spec().Config.Annotations[annotations.ImageName]).To(Equal(imageResult.Name)) + Expect(sut.Spec().Config.Annotations[annotations.ImageRef]).To(Equal(imageResult.ID)) + Expect(sut.Spec().Config.Annotations[annotations.Name]).To(Equal(sut.Name())) + Expect(sut.Spec().Config.Annotations[annotations.ContainerID]).To(Equal(sut.ID())) + Expect(sut.Spec().Config.Annotations[annotations.SandboxID]).To(Equal(sb.ID())) + Expect(sut.Spec().Config.Annotations[annotations.SandboxName]).To(Equal(sb.Name())) + Expect(sut.Spec().Config.Annotations[annotations.ContainerType]).To(Equal(annotations.ContainerTypeContainer)) + Expect(sut.Spec().Config.Annotations[annotations.LogPath]).To(Equal(logpath)) + Expect(sut.Spec().Config.Annotations[annotations.TTY]).To(Equal(strconv.FormatBool(sut.Config().Tty))) + Expect(sut.Spec().Config.Annotations[annotations.Stdin]).To(Equal(strconv.FormatBool(sut.Config().Stdin))) + Expect(sut.Spec().Config.Annotations[annotations.StdinOnce]).To(Equal(strconv.FormatBool(sut.Config().StdinOnce))) + Expect(sut.Spec().Config.Annotations[annotations.ResolvPath]).To(Equal(sb.ResolvPath())) + Expect(sut.Spec().Config.Annotations[annotations.ContainerManager]).To(Equal(lib.ContainerManagerCRIO)) + Expect(sut.Spec().Config.Annotations[annotations.MountPoint]).To(Equal(mountPoint)) + Expect(sut.Spec().Config.Annotations[annotations.SeccompProfilePath]).To(Equal(sut.Config().GetLinux().GetSecurityContext().GetSeccompProfilePath())) + Expect(sut.Spec().Config.Annotations[annotations.Created]).ToNot(BeNil()) + Expect(sut.Spec().Config.Annotations[annotations.Metadata]).To(Equal(string(metadataJSON))) + Expect(sut.Spec().Config.Annotations[annotations.Labels]).To(Equal(string(labelsJSON))) + Expect(sut.Spec().Config.Annotations[annotations.Volumes]).To(Equal(string(volumesJSON))) + Expect(sut.Spec().Config.Annotations[annotations.Annotations]).To(Equal(string(kubeAnnotationsJSON))) + }) + }) t.Describe("FipsDisable", func() { It("should be true when set to true", func() { // Given diff --git a/pkg/container/suite_test.go b/pkg/container/suite_test.go index 753886e103d..b5eba861166 100644 --- a/pkg/container/suite_test.go +++ b/pkg/container/suite_test.go @@ -31,5 +31,7 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - sut = container.New(context.Background()) + var err error + sut, err = container.New(context.Background()) + Expect(err).To(BeNil()) }) diff --git a/server/container_create.go b/server/container_create.go index 08fd7a9e9e0..febd1b85dcc 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -482,7 +482,11 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return nil, fmt.Errorf("CreateContainer failed as the sandbox was stopped: %v", sbID) } - ctr := container.New(ctx) + ctr, err := container.New(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to create container") + } + if err := ctr.SetConfig(req.GetConfig(), req.GetSandboxConfig()); err != nil { return nil, errors.Wrap(err, "setting container config") } diff --git a/server/container_create_linux.go b/server/container_create_linux.go index 1eb7a3d2623..67df3e4d737 100644 --- a/server/container_create_linux.go +++ b/server/container_create_linux.go @@ -13,7 +13,6 @@ import ( "github.com/containers/buildah/pkg/secrets" "github.com/containers/buildah/util" - "github.com/containers/libpod/v2/pkg/annotations" "github.com/containers/libpod/v2/pkg/rootless" selinux "github.com/containers/libpod/v2/pkg/selinux" createconfig "github.com/containers/libpod/v2/pkg/spec" @@ -22,7 +21,6 @@ import ( "github.com/containers/storage/pkg/mount" "github.com/cri-o/cri-o/internal/config/cgmgr" "github.com/cri-o/cri-o/internal/config/node" - "github.com/cri-o/cri-o/internal/lib" "github.com/cri-o/cri-o/internal/lib/sandbox" "github.com/cri-o/cri-o/internal/log" oci "github.com/cri-o/cri-o/internal/oci" @@ -31,7 +29,6 @@ import ( ctrIface "github.com/cri-o/cri-o/pkg/container" "github.com/cri-o/cri-o/utils" securejoin "github.com/cyphar/filepath-securejoin" - json "github.com/json-iterator/go" "github.com/opencontainers/runc/libcontainer/devices" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/runtime-tools/generate" @@ -40,9 +37,6 @@ import ( pb "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) -// Copied from k8s.io/kubernetes/pkg/kubelet/kuberuntime/labels.go -const podTerminationGracePeriodLabel = "io.kubernetes.pod.terminationGracePeriod" - type configDevice struct { Device rspec.LinuxDevice Resource rspec.LinuxDeviceCgroup @@ -281,10 +275,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai securityContext := containerConfig.GetLinux().GetSecurityContext() // creates a spec Generator with the default spec. - specgen, err := generate.New("linux") - if err != nil { - return nil, err - } + specgen := ctr.Spec() specgen.HostSpecific = true specgen.ClearProcessRlimits() @@ -307,13 +298,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai } for target, mode := range mounts { if !isInCRIMounts(target, containerConfig.GetMounts()) { - mnt := rspec.Mount{ + ctr.SpecAddMount(rspec.Mount{ Destination: target, Type: "tmpfs", Source: "tmpfs", Options: append(options, mode), - } - specgen.AddMount(mnt) + }) } } } @@ -352,10 +342,6 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai imageRef = imgResult.RepoDigests[0] } - specgen.AddAnnotation(annotations.Image, image) - specgen.AddAnnotation(annotations.ImageName, imageName) - specgen.AddAnnotation(annotations.ImageRef, imageRef) - labelOptions, err := ctr.SelinuxLabel(sb.ProcessLabel()) if err != nil { return nil, err @@ -415,17 +401,11 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai } }() - containerVolumes, ociMounts, err := addOCIBindMounts(ctx, mountLabel, containerConfig, &specgen, s.config.RuntimeConfig.BindMountPrefix) + containerVolumes, ociMounts, err := addOCIBindMounts(ctx, mountLabel, containerConfig, specgen, s.config.RuntimeConfig.BindMountPrefix) if err != nil { return nil, err } - volumesJSON, err := json.Marshal(containerVolumes) - if err != nil { - return nil, err - } - specgen.AddAnnotation(annotations.Volumes, string(volumesJSON)) - configuredDevices, err := getDevicesFromConfig(ctx, &s.config) if err != nil { return nil, err @@ -443,7 +423,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai return nil, err } - if err := addDevices(ctx, sb, containerConfig, privilegedWithoutHostDevices, &specgen); err != nil { + if err := addDevices(ctx, sb, containerConfig, privilegedWithoutHostDevices, specgen); err != nil { return nil, err } @@ -453,14 +433,6 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai return nil, err } - kubeAnnotations := containerConfig.GetAnnotations() - for k, v := range kubeAnnotations { - specgen.AddAnnotation(k, v) - } - for k, v := range labels { - specgen.AddAnnotation(k, v) - } - // set this container's apparmor profile if it is set by sandbox if s.Config().AppArmor().IsEnabled() && !ctr.Privileged() { profile, err := s.Config().AppArmor().Apply( @@ -518,18 +490,6 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai specgen.SetLinuxCgroupsPath(s.config.CgroupManager().ContainerCgroupPath(sb.CgroupParent(), containerID)) - if s.config.CgroupManager().IsSystemd() { - if t, ok := kubeAnnotations[podTerminationGracePeriodLabel]; ok { - // currently only supported by systemd, see - // https://github.com/opencontainers/runc/pull/2224 - specgen.AddAnnotation("org.systemd.property.TimeoutStopUSec", - "uint64 "+t+"000000") // sec to usec - } - if node.SystemdHasCollectMode() { - specgen.AddAnnotation("org.systemd.property.CollectMode", "'inactive-or-failed'") - } - } - if ctr.Privileged() { specgen.SetupPrivileged(true) } else { @@ -542,7 +502,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai // Clear default capabilities from spec specgen.ClearProcessCapabilities() capabilities.AddCapabilities = append(capabilities.AddCapabilities, s.config.DefaultCapabilities...) - err = setupCapabilities(&specgen, capabilities) + err = setupCapabilities(specgen, capabilities) if err != nil { return nil, err } @@ -591,7 +551,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai } // Join the namespace paths for the pod sandbox container. - if err := configureGeneratorGivenNamespacePaths(sb.NamespacePaths(), specgen); err != nil { + if err := configureGeneratorGivenNamespacePaths(sb.NamespacePaths(), *specgen); err != nil { return nil, errors.Wrap(err, "failed to configure namespaces in container create") } @@ -619,36 +579,28 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai if !isInCRIMounts("/sys", containerConfig.GetMounts()) { specgen.RemoveMount("/sys") - specgen.RemoveMount("/sys/fs/cgroup") - sysMnt := rspec.Mount{ + ctr.SpecAddMount(rspec.Mount{ Destination: "/sys", Type: "sysfs", Source: "sysfs", Options: []string{"nosuid", "noexec", "nodev", "ro"}, - } - specgen.AddMount(sysMnt) + }) } } if ctr.Privileged() { - specgen.RemoveMount("/sys") - specgen.RemoveMount("/sys/fs/cgroup") - - sysMnt := rspec.Mount{ + ctr.SpecAddMount(rspec.Mount{ Destination: "/sys", Type: "sysfs", Source: "sysfs", Options: []string{"nosuid", "noexec", "nodev", "rw"}, - } - specgen.AddMount(sysMnt) - - cgroupMnt := rspec.Mount{ + }) + ctr.SpecAddMount(rspec.Mount{ Destination: "/sys/fs/cgroup", Type: "cgroup", Source: "cgroup", Options: []string{"nosuid", "noexec", "nodev", "rw", "relatime"}, - } - specgen.AddMount(cgroupMnt) + }) } containerImageConfig := containerInfo.Config @@ -668,7 +620,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai if err != nil { return nil, err } - setupSystemd(specgen.Mounts(), specgen) + setupSystemd(specgen.Mounts(), *specgen) } // When running on cgroupv2, automatically add a cgroup namespace for not privileged containers. @@ -678,21 +630,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai } } - for idx, ip := range sb.IPs() { - specgen.AddAnnotation(fmt.Sprintf("%s.%d", annotations.IP, idx), ip) - } - - // Remove the default /dev/shm mount to ensure we overwrite it - specgen.RemoveMount("/dev/shm") - - mnt := rspec.Mount{ + ctr.SpecAddMount(rspec.Mount{ + Destination: "/dev/shm", Type: "bind", Source: sb.ShmPath(), - Destination: "/dev/shm", Options: []string{"rw", "bind"}, - } - // bind mount the pod shm - specgen.AddMount(mnt) + }) options := []string{"rw"} if readOnlyRootfs { @@ -702,95 +645,57 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai if err := securityLabel(sb.ResolvPath(), mountLabel, false); err != nil { return nil, err } - - mnt = rspec.Mount{ + ctr.SpecAddMount(rspec.Mount{ + Destination: "/etc/resolv.conf", Type: "bind", Source: sb.ResolvPath(), - Destination: "/etc/resolv.conf", Options: []string{"bind", "nodev", "nosuid", "noexec"}, - } - // bind mount the pod resolver file - specgen.AddMount(mnt) + }) } if sb.HostnamePath() != "" { if err := securityLabel(sb.HostnamePath(), mountLabel, false); err != nil { return nil, err } - - mnt = rspec.Mount{ + ctr.SpecAddMount(rspec.Mount{ + Destination: "/etc/hostname", Type: "bind", Source: sb.HostnamePath(), - Destination: "/etc/hostname", Options: append(options, "bind"), - } - specgen.AddMount(mnt) + }) } if !isInCRIMounts("/etc/hosts", containerConfig.GetMounts()) && hostNetwork(containerConfig) { // Only bind mount for host netns and when CRI does not give us any hosts file - mnt = rspec.Mount{ + ctr.SpecAddMount(rspec.Mount{ + Destination: "/etc/hosts", Type: "bind", Source: "/etc/hosts", - Destination: "/etc/hosts", Options: append(options, "bind"), - } - specgen.AddMount(mnt) + }) } if ctr.Privileged() { - setOCIBindMountsPrivileged(&specgen) + setOCIBindMountsPrivileged(specgen) } // Set hostname and add env for hostname specgen.SetHostname(sb.Hostname()) specgen.AddProcessEnv("HOSTNAME", sb.Hostname()) - specgen.AddAnnotation(annotations.Name, containerName) - specgen.AddAnnotation(annotations.ContainerID, containerID) - specgen.AddAnnotation(annotations.SandboxID, sb.ID()) - specgen.AddAnnotation(annotations.SandboxName, sb.Name()) - specgen.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer) - specgen.AddAnnotation(annotations.LogPath, logPath) - specgen.AddAnnotation(annotations.TTY, fmt.Sprintf("%v", containerConfig.Tty)) - specgen.AddAnnotation(annotations.Stdin, fmt.Sprintf("%v", containerConfig.Stdin)) - specgen.AddAnnotation(annotations.StdinOnce, fmt.Sprintf("%v", containerConfig.StdinOnce)) - specgen.AddAnnotation(annotations.ResolvPath, sb.ResolvPath()) - specgen.AddAnnotation(annotations.ContainerManager, lib.ContainerManagerCRIO) - created := time.Now() - specgen.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano)) - - metadataJSON, err := json.Marshal(metadata) - if err != nil { - return nil, err - } - specgen.AddAnnotation(annotations.Metadata, string(metadataJSON)) - - labelsJSON, err := json.Marshal(labels) - if err != nil { - return nil, err - } - specgen.AddAnnotation(annotations.Labels, string(labelsJSON)) - - kubeAnnotationsJSON, err := json.Marshal(kubeAnnotations) - if err != nil { - return nil, err - } - specgen.AddAnnotation(annotations.Annotations, string(kubeAnnotationsJSON)) - - spp := securityContext.GetSeccompProfilePath() + spp := containerConfig.GetLinux().GetSecurityContext().GetSeccompProfilePath() if !ctr.Privileged() { - if err := s.setupSeccomp(ctx, &specgen, spp); err != nil { + if err := s.setupSeccomp(ctx, specgen, spp); err != nil { return nil, err } } - specgen.AddAnnotation(annotations.SeccompProfilePath, spp) mountPoint, err := s.StorageRuntimeServer().StartContainer(containerID) if err != nil { return nil, fmt.Errorf("failed to mount container %s(%s): %v", containerName, containerID, err) } + defer func() { if retErr != nil { log.Infof(ctx, "createCtrLinux: stopping storage container %s", containerID) @@ -799,11 +704,9 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai } } }() - specgen.AddAnnotation(annotations.MountPoint, mountPoint) - - if containerImageConfig.Config.StopSignal != "" { - // this key is defined in image-spec conversion document at https://github.com/opencontainers/image-spec/pull/492/files#diff-8aafbe2c3690162540381b8cdb157112R57 - specgen.AddAnnotation("org.opencontainers.image.stopSignal", containerImageConfig.Config.StopSignal) + err = ctr.SpecAddAnnotations(sb, containerVolumes, mountPoint, containerImageConfig.Config.StopSignal, imgResult, s.config.CgroupManager().IsSystemd(), node.SystemdHasCollectMode()) + if err != nil { + return nil, err } // First add any configured environment variables from crio config. @@ -819,13 +722,13 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai // Setup user and groups if linux != nil { - if err := setupContainerUser(ctx, &specgen, mountPoint, mountLabel, containerInfo.RunDir, securityContext, containerImageConfig); err != nil { + if err := setupContainerUser(ctx, specgen, mountPoint, mountLabel, containerInfo.RunDir, securityContext, containerImageConfig); err != nil { return nil, err } } // Add image volumes - volumeMounts, err := addImageVolumes(ctx, mountPoint, s, &containerInfo, mountLabel, &specgen) + volumeMounts, err := addImageVolumes(ctx, mountPoint, s, &containerInfo, mountLabel, specgen) if err != nil { return nil, err } @@ -851,7 +754,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai // This option has been deprecated, once it is removed in the later versions, delete the server/secrets.go file as well log.Warnf(ctx, "--default-mounts has been deprecated and will be removed in future versions. Add mounts to either %q or %q", secrets.DefaultMountsFile, secrets.OverrideMountsFile) var err error - secretMounts, err = addSecretsBindMounts(ctx, mountLabel, containerInfo.RunDir, s.config.DefaultMounts, specgen) + secretMounts, err = addSecretsBindMounts(ctx, mountLabel, containerInfo.RunDir, s.config.DefaultMounts, *specgen) if err != nil { return nil, fmt.Errorf("failed to mount secrets: %v", err) } @@ -867,13 +770,13 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai sort.Sort(orderedMounts(mounts)) for _, m := range mounts { - mnt = rspec.Mount{ + rspecMount := rspec.Mount{ Type: "bind", - Source: m.Source, - Destination: m.Destination, Options: append(m.Options, "bind"), + Destination: m.Destination, + Source: m.Source, } - specgen.AddMount(mnt) + ctr.SpecAddMount(rspecMount) } if s.ContainerServer.Hooks != nil { @@ -900,7 +803,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai crioAnnotations := specgen.Config.Annotations - ociContainer, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.RuntimeHandler(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal) + ociContainer, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, labels, crioAnnotations, ctr.Config().GetAnnotations(), image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.RuntimeHandler(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal) if err != nil { return nil, err } @@ -910,7 +813,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai ociContainer.SetIDMappings(containerIDMappings) if containerIDMappings != nil { - if err := s.finalizeUserMapping(&specgen, containerIDMappings); err != nil { + if err := s.finalizeUserMapping(specgen, containerIDMappings); err != nil { return nil, err } @@ -937,7 +840,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai } if os.Getenv(rootlessEnvName) != "" { - makeOCIConfigurationRootless(&specgen) + makeOCIConfigurationRootless(specgen) } saveOptions := generate.ExportOptions{}