cri-o · openshift-merge-robot · Aug 9, 2023 · Jul 27, 2023 · Jul 31, 2023
@@ -328,6 +328,9 @@ The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.  Th
   "io.kubernetes.cri-o.UnifiedCgroup.$CTR_NAME" for configuring the cgroup v2 unified block for a container.
   "io.containers.trace-syscall" for tracing syscalls via the OCI seccomp BPF hook.
 
+**platform_runtime_paths**={}
+  A mapping of platforms to the corresponding runtime executable paths for the runtime handler.
+
 ### CRIO.RUNTIME.WORKLOADS TABLE
 The "crio.runtime.workloads" table defines a list of workloads - a way to customize the behavior of a pod and container.
 A workload is chosen for a pod based on whether the workload's **activation_annotation** is an annotation on the pod.

@@ -104,7 +104,7 @@ type Container interface {
 	SpecAddMount(rspec.Mount)
 
 	// SpecAddAnnotations adds annotations to the spec.
-	SpecAddAnnotations(ctx context.Context, sandbox *sandbox.Sandbox, containerVolume []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool) error
+	SpecAddAnnotations(ctx context.Context, sandbox *sandbox.Sandbox, containerVolume []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool, seccompRef, platformRuntimePath string) error
 
 	// SpecAddDevices adds devices from the server config, and container CRI config
 	SpecAddDevices([]device.Device, []device.Device, bool, bool) error
@@ -162,7 +162,7 @@ func (c *container) SpecAddMount(r rspec.Mount) {
 }
 
 // SpecAddAnnotation adds all annotations to the spec
-func (c *container) SpecAddAnnotations(ctx context.Context, sb *sandbox.Sandbox, containerVolumes []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool) (err error) {
+func (c *container) SpecAddAnnotations(ctx context.Context, sb *sandbox.Sandbox, containerVolumes []oci.ContainerVolume, mountPoint, configStopSignal string, imageResult *storage.ImageResult, isSystemd, systemdHasCollectMode bool, seccompRef, platformRuntimePath string) (err error) {
 	ctx, span := log.StartSpan(ctx)
 	defer span.End()
 	// Copied from k8s.io/kubernetes/pkg/kubelet/kuberuntime/labels.go
@@ -228,6 +228,8 @@ func (c *container) SpecAddAnnotations(ctx context.Context, sb *sandbox.Sandbox,
 	c.spec.AddAnnotation(annotations.MountPoint, mountPoint)
 	c.spec.AddAnnotation(annotations.SeccompProfilePath, c.Config().Linux.SecurityContext.SeccompProfilePath)
 	c.spec.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano))
+	// for retrieving the runtime path for a given platform.
+	c.spec.AddAnnotation(crioann.PlatformRuntimePath, platformRuntimePath)
 
 	metadataJSON, err := json.Marshal(c.Config().Metadata)
 	if err != nil {

@@ -117,7 +117,7 @@ var _ = t.Describe("Container", func() {
 			Expect(currentTime).ToNot(BeNil())
 			Expect(sb).ToNot(BeNil())
 
-			err = sut.SpecAddAnnotations(context.Background(), sb, volumes, mountPoint, configStopSignal, &imageResult, false, false)
+			err = sut.SpecAddAnnotations(context.Background(), sb, volumes, mountPoint, configStopSignal, &imageResult, false, false, "foo", "")
 			Expect(err).To(BeNil())
 
 			Expect(sut.Spec().Config.Annotations[annotations.Image]).To(Equal(image))

@@ -425,6 +425,11 @@ func (c *ContainerServer) LoadContainer(ctx context.Context, id string) (retErr
 		imgRef = ""
 	}
 
+	platformRuntimePath, ok := m.Annotations[crioann.PlatformRuntimePath]
+	if !ok {
+		platformRuntimePath = ""
+	}
+
 	kubeAnnotations := make(map[string]string)
 	if err := json.Unmarshal([]byte(m.Annotations[annotations.Annotations]), &kubeAnnotations); err != nil {
 		return err
@@ -455,6 +460,8 @@ func (c *ContainerServer) LoadContainer(ctx context.Context, id string) (retErr
 	}
 	ctr.SetCreated()
 
+	ctr.SetRuntimePathForPlatform(platformRuntimePath)
+
 	c.AddContainer(ctx, ctr)
 
 	return c.ctrIDIndex.Add(id)

@@ -79,6 +79,7 @@ type Container struct {
 	restoreArchive     string
 	restoreIsOCIImage  bool
 	resources          *types.ContainerResources
+	runtimePath        string // runtime path for a given platform
 }
 
 func (c *Container) CRIAttributes() *types.ContainerAttributes {
@@ -755,3 +756,16 @@ func (c *Container) SetResources(s *specs.Spec) {
 func (c *Container) GetResources() *types.ContainerResources {
 	return c.resources
 }
+
+// SetRuntimePathForPlatform sets the runtime path for a given platform.
+func (c *Container) SetRuntimePathForPlatform(runtimePath string) {
+	c.runtimePath = runtimePath
+}
+
+// RuntimePathForPlatform returns the runtime path for a given platform.
+func (c *Container) RuntimePathForPlatform(r *runtimeOCI) string {
+	if c.runtimePath == "" {
+		return r.handler.RuntimePath
+	}
+	return c.runtimePath
+}
@@ -146,6 +146,19 @@ func (r *Runtime) PrivilegedWithoutHostDevices(handler string) (bool, error) {
 	return rh.PrivilegedWithoutHostDevices, nil
 }
 
+// PlatformRuntimePath returns the runtime path for a given platform.
+func (r *Runtime) PlatformRuntimePath(handler, platform string) (string, error) {
+	rh, err := r.getRuntimeHandler(handler)
+	if err != nil {
+		return "", err
+	}
+	if runtimePath, ok := rh.PlatformRuntimePaths[platform]; ok {
+		return runtimePath, nil
+	}
+
+	return "", nil
+}
+
 // AllowedAnnotations returns the allowed annotations for this runtime.
 func (r *Runtime) AllowedAnnotations(handler string) ([]string, error) {
 	rh, err := r.getRuntimeHandler(handler)

@@ -112,7 +112,7 @@ func (r *runtimeOCI) CreateContainer(ctx context.Context, c *Container, cgroupPa
 		"-P", c.conmonPidFilePath(),
 		"-p", filepath.Join(c.bundlePath, "pidfile"),
 		"--persist-dir", c.dir,
-		"-r", r.handler.RuntimePath,
+		"-r", c.RuntimePathForPlatform(r),
 		"--runtime-arg", fmt.Sprintf("%s=%s", rootFlag, r.root),
 		"--socket-dir-path", r.config.ContainerAttachSocketDir,
 		"--syslog",
@@ -409,7 +409,7 @@ func (r *runtimeOCI) ExecContainer(ctx context.Context, c *Container, cmd []stri
 
 	args := r.defaultRuntimeArgs()
 	args = append(args, "exec", "--process", processFile, c.ID())
-	execCmd := cmdrunner.Command(r.handler.RuntimePath, args...) // nolint: gosec
+	execCmd := cmdrunner.Command(c.RuntimePathForPlatform(r), args...) // nolint: gosec
 	if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found {
 		execCmd.Env = append(execCmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v))
 	}
@@ -525,7 +525,7 @@ func (r *runtimeOCI) ExecSyncContainer(ctx context.Context, c *Container, comman
 	args := []string{
 		"-c", c.ID(),
 		"-n", c.name,
-		"-r", r.handler.RuntimePath,
+		"-r", c.RuntimePathForPlatform(r),
 		"-p", pidFile,
 		"-e",
 		"-l", logPath,
@@ -764,7 +764,7 @@ func (r *runtimeOCI) UpdateContainer(ctx context.Context, c *Container, res *rsp
 		return nil
 	}
 
-	cmd := cmdrunner.Command(r.handler.RuntimePath, rootFlag, r.root, "update", "--resources", "-", c.ID()) // nolint: gosec
+	cmd := cmdrunner.Command(c.RuntimePathForPlatform(r), rootFlag, r.root, "update", "--resources", "-", c.ID()) // nolint: gosec
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
 	cmd.Stdout = &stdout
@@ -1439,8 +1439,8 @@ func (r *runtimeOCI) defaultRuntimeArgs() []string {
 func (r *runtimeOCI) CheckpointContainer(ctx context.Context, c *Container, specgen *rspec.Spec, leaveRunning bool) error {
 	c.opLock.Lock()
 	defer c.opLock.Unlock()
-
-	if err := r.checkpointRestoreSupported(); err != nil {
+	runtimePath := c.RuntimePathForPlatform(r)
+	if err := r.checkpointRestoreSupported(runtimePath); err != nil {
 		return err
 	}
 
@@ -1481,7 +1481,7 @@ func (r *runtimeOCI) CheckpointContainer(ctx context.Context, c *Container, spec
 
 	_, err := r.runtimeCmd(args...)
 	if err != nil {
-		return fmt.Errorf("running %q %q failed: %w", r.handler.RuntimePath, args, err)
+		return fmt.Errorf("running %q %q failed: %w", runtimePath, args, err)
 	}
 
 	c.SetCheckpointedAt(time.Now())
@@ -1496,7 +1496,7 @@ func (r *runtimeOCI) CheckpointContainer(ctx context.Context, c *Container, spec
 
 // RestoreContainer restores a container.
 func (r *runtimeOCI) RestoreContainer(ctx context.Context, c *Container, cgroupParent, mountLabel string) error {
-	if err := r.checkpointRestoreSupported(); err != nil {
+	if err := r.checkpointRestoreSupported(c.RuntimePathForPlatform(r)); err != nil {
 		return err
 	}
 
@@ -1560,11 +1560,11 @@ func (r *runtimeOCI) RestoreContainer(ctx context.Context, c *Container, cgroupP
 	return nil
 }
 
-func (r *runtimeOCI) checkpointRestoreSupported() error {
+func (r *runtimeOCI) checkpointRestoreSupported(runtimePath string) error {
 	if !criu.CheckForCriu(criu.PodCriuVersion) {
 		return fmt.Errorf("checkpoint/restore requires at least CRIU %d", criu.PodCriuVersion)
 	}
-	if !crutils.CRRuntimeSupportsCheckpointRestore(r.handler.RuntimePath) {
+	if !crutils.CRRuntimeSupportsCheckpointRestore(runtimePath) {
 		return fmt.Errorf("configured runtime does not support checkpoint/restore")
 	}
 	return nil

@@ -57,6 +57,9 @@ const (
 	// LinkLogsAnnotations indicates that CRI-O should link the pod containers logs into the specified
 	// emptyDir volume
 	LinkLogsAnnotation = "io.kubernetes.cri-o.LinkLogs"
+
+	// PlatformRuntimePath indicates the runtime path that CRI-O should use for a specific platform.
+	PlatformRuntimePath = "io.kubernetes.cri-o.PlatformRuntimePath"
 )
 
 var AllAllowedAnnotations = []string{

@@ -214,6 +214,10 @@ type RuntimeHandler struct {
 
 	// MonitorExecCgroup indicates whether to move exec probes to the container's cgroup.
 	MonitorExecCgroup string `toml:"monitor_exec_cgroup,omitempty"`
+
+	// PlatformRuntimePaths defines a configuration option that specifies
+	// the runtime paths for different platforms.
+	PlatformRuntimePaths map[string]string `toml:"platform_runtime_paths,omitempty"`
 }
 
 // Multiple runtime Handlers in a map

@@ -1168,6 +1168,7 @@ const templateStringCrioRuntimeRuntimesRuntimeHandler = `# The "crio.runtime.run
 # monitor_env = []
 # privileged_without_host_devices = false
 # allowed_annotations = []
+# platform_runtime_paths = { "os/arch" = "/path/to/binary" }
 # Where:
 # - runtime-handler: Name used to identify the runtime.
 # - runtime_path (optional, string): Absolute path to the runtime executable in
@@ -1200,6 +1201,8 @@ const templateStringCrioRuntimeRuntimesRuntimeHandler = `# The "crio.runtime.run
 #   should be moved to the container's cgroup
 # - monitor_env (optional, array of strings): Environment variables to pass to the montior.
 #   Replaces deprecated option "conmon_env".
+# - platform_runtime_paths (optional, map): A mapping of platforms to the corresponding
+#   runtime executable paths for the runtime handler.
 #
 # Using the seccomp notifier feature:
 #
@@ -1242,6 +1245,10 @@ const templateStringCrioRuntimeRuntimesRuntimeHandler = `# The "crio.runtime.run
 {{ if $runtime_handler.AllowedAnnotations }}{{ $.Comment }}allowed_annotations = [
 {{ range $opt := $runtime_handler.AllowedAnnotations }}{{ $.Comment }}{{ printf "\t%q,\n" $opt }}{{ end }}{{ $.Comment }}]{{ end }}
 {{ $.Comment }}privileged_without_host_devices = {{ $runtime_handler.PrivilegedWithoutHostDevices }}
+{{ if $runtime_handler.PlatformRuntimePaths }}platform_runtime_paths = {
+{{- $first := true }}{{- range $key, $value := $runtime_handler.PlatformRuntimePaths }}
+{{- if not $first }},{{ end }}{{- printf "%q = %q" $key $value }}{{- $first = false }}{{- end }}}
+{{ end }}
 {{ end }}
 `
 

@@ -623,6 +623,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrfactory.Cont
 	specgen.AddProcessEnv("HOSTNAME", sb.Hostname())
 
 	created := time.Now()
+	seccompRef := types.SecurityProfile_Unconfined.String()
 	if !ctr.Privileged() {
 		notifier, err := s.config.Seccomp().Setup(
 			ctx,
@@ -652,7 +653,13 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrfactory.Cont
 		specgen.Config.Linux.IntelRdt = &rspec.LinuxIntelRdt{ClosID: rdt.ResctrlPrefix + rdtClass}
 	}
 
-	err = ctr.SpecAddAnnotations(ctx, sb, containerVolumes, mountPoint, containerImageConfig.Config.StopSignal, imgResult, s.config.CgroupManager().IsSystemd(), node.SystemdHasCollectMode())
+	// compute the runtime path for a given container
+	platform := containerInfo.Config.OS + "/" + containerInfo.Config.Architecture
+	runtimePath, err := s.Runtime().PlatformRuntimePath(sb.RuntimeHandler(), platform)
+	if err != nil {
+		return nil, err
+	}
+	err = ctr.SpecAddAnnotations(ctx, sb, containerVolumes, mountPoint, containerImageConfig.Config.StopSignal, imgResult, s.config.CgroupManager().IsSystemd(), node.SystemdHasCollectMode(), seccompRef, runtimePath)
 	if err != nil {
 		return nil, err
 	}
@@ -865,7 +872,10 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrfactory.Cont
 
 	ociContainer.SetSpec(specgen.Config)
 	ociContainer.SetMountPoint(mountPoint)
-	ociContainer.SetSeccompProfilePath(containerConfig.Linux.SecurityContext.SeccompProfilePath)
+	ociContainer.SetSeccompProfilePath(seccompRef)
+	if runtimePath != "" {
+		ociContainer.SetRuntimePathForPlatform(runtimePath)
+	}
 
 	for _, cv := range containerVolumes {
 		ociContainer.AddVolume(cv)

@@ -105,6 +105,7 @@ IMAGES=(
     quay.io/crio/redis:alpine
     quay.io/crio/stderr-test:latest
     quay.io/crio/etc-permission:latest
+    quay.io/crio/hello-wasm:latest
 )
 
 function img2dir() {

@@ -1,6 +1,7 @@
 #!/usr/bin/env bats
 
 load helpers
+CRUN_WASM_BINARY=${CRUN_WASM_BINARY:-$(command -v crun-wasm || true)}
 
 IMAGE=quay.io/crio/pause
 SIGNED_IMAGE=registry.access.redhat.com/rhel7-atomic:latest
@@ -302,3 +303,28 @@ function teardown() {
 
 	cleanup_images
 }
+
+@test "run container in pod with crun-wasm enabled" {
+	if [ -z "$CRUN_WASM_BINARY" ] || [[ "$RUNTIME_TYPE" == "vm" ]]; then
+		skip "crun-wasm not installed or runtime type is VM"
+	fi
+	cat << EOF > "$CRIO_CONFIG_DIR/99-crun-wasm.conf"
+[crio.runtime]
+default_runtime = "crun-wasm"
+
+[crio.runtime.runtimes.crun-wasm]
+runtime_path = "/usr/bin/crun"
+
+platform_runtime_paths = {"wasi/wasm32" = "/usr/bin/crun-wasm", "abc/def" = "/usr/bin/acme"}
+EOF
+	start_crio
+
+	jq '.metadata.name = "podsandbox-wasm"
+		|.image.image = "quay.io/crio/hello-wasm:latest"
+		| del(.command, .args, .linux.resources)' \
+		"$TESTDATA"/container_config.json > "$TESTDIR/wasm.json"
+
+	ctr_id=$(crictl run "$TESTDIR/wasm.json" "$TESTDATA/sandbox_config.json")
+	output=$(crictl logs "$ctr_id")
+	[[ "$output" == *"Hello, world!"* ]]
+}