diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index 465ef43c600..fd257df0434 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -328,6 +328,9 @@ The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. Th "io.kubernetes.cri-o.UnifiedCgroup.$CTR_NAME" for configuring the cgroup v2 unified block for a container. "io.containers.trace-syscall" for tracing syscalls via the OCI seccomp BPF hook. +**platform_runtime_paths**={} + A mapping of platforms to the corresponding runtime executable paths for the runtime handler. + ### CRIO.RUNTIME.WORKLOADS TABLE The "crio.runtime.workloads" table defines a list of workloads - a way to customize the behavior of a pod and container. A workload is chosen for a pod based on whether the workload's **activation_annotation** is an annotation on the pod. diff --git a/internal/factory/container/container.go b/internal/factory/container/container.go index d19bb488257..81eee914644 100644 --- a/internal/factory/container/container.go +++ b/internal/factory/container/container.go @@ -104,7 +104,7 @@ type Container interface { SpecAddMount(rspec.Mount) // SpecAddAnnotations adds annotations to the spec. - SpecAddAnnotations(ctx context.Context, sandbox *sandbox.Sandbox, containerVolume []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool) error + SpecAddAnnotations(ctx context.Context, sandbox *sandbox.Sandbox, containerVolume []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool, seccompRef, platformRuntimePath string) error // SpecAddDevices adds devices from the server config, and container CRI config SpecAddDevices([]device.Device, []device.Device, bool, bool) error @@ -162,7 +162,7 @@ func (c *container) SpecAddMount(r rspec.Mount) { } // SpecAddAnnotation adds all annotations to the spec -func (c *container) SpecAddAnnotations(ctx context.Context, sb *sandbox.Sandbox, containerVolumes []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool) (err error) { +func (c *container) SpecAddAnnotations(ctx context.Context, sb *sandbox.Sandbox, containerVolumes []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool, seccompRef, platformRuntimePath string) (err error) { ctx, span := log.StartSpan(ctx) defer span.End() // Copied from k8s.io/kubernetes/pkg/kubelet/kuberuntime/labels.go @@ -228,6 +228,8 @@ func (c *container) SpecAddAnnotations(ctx context.Context, sb *sandbox.Sandbox, c.spec.AddAnnotation(annotations.MountPoint, mountPoint) c.spec.AddAnnotation(annotations.SeccompProfilePath, c.Config().Linux.SecurityContext.SeccompProfilePath) c.spec.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano)) + // for retrieving the runtime path for a given platform. + c.spec.AddAnnotation(crioann.PlatformRuntimePath, platformRuntimePath) metadataJSON, err := json.Marshal(c.Config().Metadata) if err != nil { diff --git a/internal/factory/container/container_test.go b/internal/factory/container/container_test.go index 0e4fbc31891..dc22c69c4bf 100644 --- a/internal/factory/container/container_test.go +++ b/internal/factory/container/container_test.go @@ -117,7 +117,7 @@ var _ = t.Describe("Container", func() { Expect(currentTime).ToNot(BeNil()) Expect(sb).ToNot(BeNil()) - err = sut.SpecAddAnnotations(context.Background(), sb, volumes, mountPoint, configStopSignal, &imageResult, false, false) + err = sut.SpecAddAnnotations(context.Background(), sb, volumes, mountPoint, configStopSignal, &imageResult, false, false, "foo", "") Expect(err).To(BeNil()) Expect(sut.Spec().Config.Annotations[annotations.Image]).To(Equal(image)) diff --git a/internal/lib/container_server.go b/internal/lib/container_server.go index e91c17c5931..4feaac8899a 100644 --- a/internal/lib/container_server.go +++ b/internal/lib/container_server.go @@ -425,6 +425,11 @@ func (c *ContainerServer) LoadContainer(ctx context.Context, id string) (retErr imgRef = "" } + platformRuntimePath, ok := m.Annotations[crioann.PlatformRuntimePath] + if !ok { + platformRuntimePath = "" + } + kubeAnnotations := make(map[string]string) if err := json.Unmarshal([]byte(m.Annotations[annotations.Annotations]), &kubeAnnotations); err != nil { return err @@ -455,6 +460,8 @@ func (c *ContainerServer) LoadContainer(ctx context.Context, id string) (retErr } ctr.SetCreated() + ctr.SetRuntimePathForPlatform(platformRuntimePath) + c.AddContainer(ctx, ctr) return c.ctrIDIndex.Add(id) diff --git a/internal/oci/container.go b/internal/oci/container.go index 87273a6e736..455454ecc25 100644 --- a/internal/oci/container.go +++ b/internal/oci/container.go @@ -79,6 +79,7 @@ type Container struct { restoreArchive string restoreIsOCIImage bool resources *types.ContainerResources + runtimePath string // runtime path for a given platform } func (c *Container) CRIAttributes() *types.ContainerAttributes { @@ -755,3 +756,16 @@ func (c *Container) SetResources(s *specs.Spec) { func (c *Container) GetResources() *types.ContainerResources { return c.resources } + +// SetRuntimePathForPlatform sets the runtime path for a given platform. +func (c *Container) SetRuntimePathForPlatform(runtimePath string) { + c.runtimePath = runtimePath +} + +// RuntimePathForPlatform returns the runtime path for a given platform. +func (c *Container) RuntimePathForPlatform(r *runtimeOCI) string { + if c.runtimePath == "" { + return r.handler.RuntimePath + } + return c.runtimePath +} diff --git a/internal/oci/oci.go b/internal/oci/oci.go index fe2d5ffdefb..6da8c0c893d 100644 --- a/internal/oci/oci.go +++ b/internal/oci/oci.go @@ -146,6 +146,19 @@ func (r *Runtime) PrivilegedWithoutHostDevices(handler string) (bool, error) { return rh.PrivilegedWithoutHostDevices, nil } +// PlatformRuntimePath returns the runtime path for a given platform. +func (r *Runtime) PlatformRuntimePath(handler, platform string) (string, error) { + rh, err := r.getRuntimeHandler(handler) + if err != nil { + return "", err + } + if runtimePath, ok := rh.PlatformRuntimePaths[platform]; ok { + return runtimePath, nil + } + + return "", nil +} + // AllowedAnnotations returns the allowed annotations for this runtime. func (r *Runtime) AllowedAnnotations(handler string) ([]string, error) { rh, err := r.getRuntimeHandler(handler) diff --git a/internal/oci/runtime_oci.go b/internal/oci/runtime_oci.go index 6b227f44ced..5771dfe7b67 100644 --- a/internal/oci/runtime_oci.go +++ b/internal/oci/runtime_oci.go @@ -112,7 +112,7 @@ func (r *runtimeOCI) CreateContainer(ctx context.Context, c *Container, cgroupPa "-P", c.conmonPidFilePath(), "-p", filepath.Join(c.bundlePath, "pidfile"), "--persist-dir", c.dir, - "-r", r.handler.RuntimePath, + "-r", c.RuntimePathForPlatform(r), "--runtime-arg", fmt.Sprintf("%s=%s", rootFlag, r.root), "--socket-dir-path", r.config.ContainerAttachSocketDir, "--syslog", @@ -409,7 +409,7 @@ func (r *runtimeOCI) ExecContainer(ctx context.Context, c *Container, cmd []stri args := r.defaultRuntimeArgs() args = append(args, "exec", "--process", processFile, c.ID()) - execCmd := cmdrunner.Command(r.handler.RuntimePath, args...) // nolint: gosec + execCmd := cmdrunner.Command(c.RuntimePathForPlatform(r), args...) // nolint: gosec if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found { execCmd.Env = append(execCmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v)) } @@ -525,7 +525,7 @@ func (r *runtimeOCI) ExecSyncContainer(ctx context.Context, c *Container, comman args := []string{ "-c", c.ID(), "-n", c.name, - "-r", r.handler.RuntimePath, + "-r", c.RuntimePathForPlatform(r), "-p", pidFile, "-e", "-l", logPath, @@ -764,7 +764,7 @@ func (r *runtimeOCI) UpdateContainer(ctx context.Context, c *Container, res *rsp return nil } - cmd := cmdrunner.Command(r.handler.RuntimePath, rootFlag, r.root, "update", "--resources", "-", c.ID()) // nolint: gosec + cmd := cmdrunner.Command(c.RuntimePathForPlatform(r), rootFlag, r.root, "update", "--resources", "-", c.ID()) // nolint: gosec var stdout bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &stdout @@ -1439,8 +1439,8 @@ func (r *runtimeOCI) defaultRuntimeArgs() []string { func (r *runtimeOCI) CheckpointContainer(ctx context.Context, c *Container, specgen *rspec.Spec, leaveRunning bool) error { c.opLock.Lock() defer c.opLock.Unlock() - - if err := r.checkpointRestoreSupported(); err != nil { + runtimePath := c.RuntimePathForPlatform(r) + if err := r.checkpointRestoreSupported(runtimePath); err != nil { return err } @@ -1481,7 +1481,7 @@ func (r *runtimeOCI) CheckpointContainer(ctx context.Context, c *Container, spec _, err := r.runtimeCmd(args...) if err != nil { - return fmt.Errorf("running %q %q failed: %w", r.handler.RuntimePath, args, err) + return fmt.Errorf("running %q %q failed: %w", runtimePath, args, err) } c.SetCheckpointedAt(time.Now()) @@ -1496,7 +1496,7 @@ func (r *runtimeOCI) CheckpointContainer(ctx context.Context, c *Container, spec // RestoreContainer restores a container. func (r *runtimeOCI) RestoreContainer(ctx context.Context, c *Container, cgroupParent, mountLabel string) error { - if err := r.checkpointRestoreSupported(); err != nil { + if err := r.checkpointRestoreSupported(c.RuntimePathForPlatform(r)); err != nil { return err } @@ -1560,11 +1560,11 @@ func (r *runtimeOCI) RestoreContainer(ctx context.Context, c *Container, cgroupP return nil } -func (r *runtimeOCI) checkpointRestoreSupported() error { +func (r *runtimeOCI) checkpointRestoreSupported(runtimePath string) error { if !criu.CheckForCriu(criu.PodCriuVersion) { return fmt.Errorf("checkpoint/restore requires at least CRIU %d", criu.PodCriuVersion) } - if !crutils.CRRuntimeSupportsCheckpointRestore(r.handler.RuntimePath) { + if !crutils.CRRuntimeSupportsCheckpointRestore(runtimePath) { return fmt.Errorf("configured runtime does not support checkpoint/restore") } return nil diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index e31bbc1f03a..c6c5c80ada3 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -57,6 +57,9 @@ const ( // LinkLogsAnnotations indicates that CRI-O should link the pod containers logs into the specified // emptyDir volume LinkLogsAnnotation = "io.kubernetes.cri-o.LinkLogs" + + // PlatformRuntimePath indicates the runtime path that CRI-O should use for a specific platform. + PlatformRuntimePath = "io.kubernetes.cri-o.PlatformRuntimePath" ) var AllAllowedAnnotations = []string{ diff --git a/pkg/config/config.go b/pkg/config/config.go index e85b412b5e7..14a4220477b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -214,6 +214,10 @@ type RuntimeHandler struct { // MonitorExecCgroup indicates whether to move exec probes to the container's cgroup. MonitorExecCgroup string `toml:"monitor_exec_cgroup,omitempty"` + + // PlatformRuntimePaths defines a configuration option that specifies + // the runtime paths for different platforms. + PlatformRuntimePaths map[string]string `toml:"platform_runtime_paths,omitempty"` } // Multiple runtime Handlers in a map diff --git a/pkg/config/template.go b/pkg/config/template.go index 61979da657b..3e47aad667b 100644 --- a/pkg/config/template.go +++ b/pkg/config/template.go @@ -1168,6 +1168,7 @@ const templateStringCrioRuntimeRuntimesRuntimeHandler = `# The "crio.runtime.run # monitor_env = [] # privileged_without_host_devices = false # allowed_annotations = [] +# platform_runtime_paths = { "os/arch" = "/path/to/binary" } # Where: # - runtime-handler: Name used to identify the runtime. # - runtime_path (optional, string): Absolute path to the runtime executable in @@ -1200,6 +1201,8 @@ const templateStringCrioRuntimeRuntimesRuntimeHandler = `# The "crio.runtime.run # should be moved to the container's cgroup # - monitor_env (optional, array of strings): Environment variables to pass to the montior. # Replaces deprecated option "conmon_env". +# - platform_runtime_paths (optional, map): A mapping of platforms to the corresponding +# runtime executable paths for the runtime handler. # # Using the seccomp notifier feature: # @@ -1242,6 +1245,10 @@ const templateStringCrioRuntimeRuntimesRuntimeHandler = `# The "crio.runtime.run {{ if $runtime_handler.AllowedAnnotations }}{{ $.Comment }}allowed_annotations = [ {{ range $opt := $runtime_handler.AllowedAnnotations }}{{ $.Comment }}{{ printf "\t%q,\n" $opt }}{{ end }}{{ $.Comment }}]{{ end }} {{ $.Comment }}privileged_without_host_devices = {{ $runtime_handler.PrivilegedWithoutHostDevices }} +{{ if $runtime_handler.PlatformRuntimePaths }}platform_runtime_paths = { +{{- $first := true }}{{- range $key, $value := $runtime_handler.PlatformRuntimePaths }} +{{- if not $first }},{{ end }}{{- printf "%q = %q" $key $value }}{{- $first = false }}{{- end }}} +{{ end }} {{ end }} ` diff --git a/server/container_create_linux.go b/server/container_create_linux.go index aa563b7b28d..dfdcb4e545f 100644 --- a/server/container_create_linux.go +++ b/server/container_create_linux.go @@ -623,6 +623,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrfactory.Cont specgen.AddProcessEnv("HOSTNAME", sb.Hostname()) created := time.Now() + seccompRef := types.SecurityProfile_Unconfined.String() if !ctr.Privileged() { notifier, err := s.config.Seccomp().Setup( ctx, @@ -652,7 +653,13 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrfactory.Cont specgen.Config.Linux.IntelRdt = &rspec.LinuxIntelRdt{ClosID: rdt.ResctrlPrefix + rdtClass} } - err = ctr.SpecAddAnnotations(ctx, sb, containerVolumes, mountPoint, containerImageConfig.Config.StopSignal, imgResult, s.config.CgroupManager().IsSystemd(), node.SystemdHasCollectMode()) + // compute the runtime path for a given container + platform := containerInfo.Config.OS + "/" + containerInfo.Config.Architecture + runtimePath, err := s.Runtime().PlatformRuntimePath(sb.RuntimeHandler(), platform) + if err != nil { + return nil, err + } + err = ctr.SpecAddAnnotations(ctx, sb, containerVolumes, mountPoint, containerImageConfig.Config.StopSignal, imgResult, s.config.CgroupManager().IsSystemd(), node.SystemdHasCollectMode(), seccompRef, runtimePath) if err != nil { return nil, err } @@ -865,7 +872,10 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrfactory.Cont ociContainer.SetSpec(specgen.Config) ociContainer.SetMountPoint(mountPoint) - ociContainer.SetSeccompProfilePath(containerConfig.Linux.SecurityContext.SeccompProfilePath) + ociContainer.SetSeccompProfilePath(seccompRef) + if runtimePath != "" { + ociContainer.SetRuntimePathForPlatform(runtimePath) + } for _, cv := range containerVolumes { ociContainer.AddVolume(cv) diff --git a/test/helpers.bash b/test/helpers.bash index ba7d27ce9de..de4034b90ae 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -105,6 +105,7 @@ IMAGES=( quay.io/crio/redis:alpine quay.io/crio/stderr-test:latest quay.io/crio/etc-permission:latest + quay.io/crio/hello-wasm:latest ) function img2dir() { diff --git a/test/image.bats b/test/image.bats index b2e10e7164e..733bc71cc98 100644 --- a/test/image.bats +++ b/test/image.bats @@ -1,6 +1,7 @@ #!/usr/bin/env bats load helpers +CRUN_WASM_BINARY=${CRUN_WASM_BINARY:-$(command -v crun-wasm || true)} IMAGE=quay.io/crio/pause SIGNED_IMAGE=registry.access.redhat.com/rhel7-atomic:latest @@ -302,3 +303,28 @@ function teardown() { cleanup_images } + +@test "run container in pod with crun-wasm enabled" { + if [ -z "$CRUN_WASM_BINARY" ] || [[ "$RUNTIME_TYPE" == "vm" ]]; then + skip "crun-wasm not installed or runtime type is VM" + fi + cat << EOF > "$CRIO_CONFIG_DIR/99-crun-wasm.conf" +[crio.runtime] +default_runtime = "crun-wasm" + +[crio.runtime.runtimes.crun-wasm] +runtime_path = "/usr/bin/crun" + +platform_runtime_paths = {"wasi/wasm32" = "/usr/bin/crun-wasm", "abc/def" = "/usr/bin/acme"} +EOF + start_crio + + jq '.metadata.name = "podsandbox-wasm" + |.image.image = "quay.io/crio/hello-wasm:latest" + | del(.command, .args, .linux.resources)' \ + "$TESTDATA"/container_config.json > "$TESTDIR/wasm.json" + + ctr_id=$(crictl run "$TESTDIR/wasm.json" "$TESTDATA/sandbox_config.json") + output=$(crictl logs "$ctr_id") + [[ "$output" == *"Hello, world!"* ]] +}