cri-o · openshift-merge-robot · Mar 7, 2022 · Jan 26, 2022 · Jan 25, 2022 · Jan 25, 2022
@@ -20,6 +20,7 @@ h
 --big-files-temporary-dir
 --bind-mount-prefix
 --blockio-config-file
+--cdi-spec-dirs
 --cgroup-manager
 --clean-shutdown-file
 --cni-config-dir

@@ -16,6 +16,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l apparmor-profile -r -d 'Na
 complete -c crio -n '__fish_crio_no_subcommand' -f -l big-files-temporary-dir -r -d 'Path to the temporary directory to use for storing big files, used to store image blobs and data streams related to containers image management.'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l bind-mount-prefix -r -d 'A prefix to use for the source of the bind mounts. This option would be useful if you were running CRI-O in a container. And had `/` mounted on `/host` in your container. Then if you ran CRI-O with the `--bind-mount-prefix=/host` option, CRI-O would add /host to any bind mounts it is handed over CRI. If Kubernetes asked to have `/var/lib/foobar` bind mounted into the container, then CRI-O would bind mount `/host/var/lib/foobar`. Since CRI-O itself is running in a container with `/` or the host mounted on `/host`, the container would end up with `/var/lib/foobar` from the host mounted in the container rather then `/var/lib/foobar` from the CRI-O container. (default: "")'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l blockio-config-file -r -d 'Path to the blockio class configuration file for configuring the cgroup blockio controller.'
+complete -c crio -n '__fish_crio_no_subcommand' -f -l cdi-spec-dirs -r -d 'Directories to scan for CDI Spec files'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l cgroup-manager -r -d 'cgroup manager (cgroupfs or systemd)'
 complete -c crio -n '__fish_crio_no_subcommand' -l clean-shutdown-file -r -d 'Location for CRI-O to lay down the clean shutdown file. It indicates whether we\'ve had time to sync changes to disk before shutting down. If not found, crio wipe will clear the storage directory'
 complete -c crio -n '__fish_crio_no_subcommand' -l cni-config-dir -r -d 'CNI configuration files directory'

@@ -27,6 +27,7 @@ it later with **--config**. Global options will modify the output.'
         '--big-files-temporary-dir'
         '--bind-mount-prefix'
         '--blockio-config-file'
+        '--cdi-spec-dirs'
         '--cgroup-manager'
         '--clean-shutdown-file'
         '--cni-config-dir'

@@ -18,6 +18,7 @@ crio
 [--big-files-temporary-dir]=[value]
 [--bind-mount-prefix]=[value]
 [--blockio-config-file]=[value]
+[--cdi-spec-dirs]=[value]
 [--cgroup-manager]=[value]
 [--clean-shutdown-file]=[value]
 [--cni-config-dir]=[value]
@@ -148,6 +149,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
 
 **--blockio-config-file**="": Path to the blockio class configuration file for configuring the cgroup blockio controller.
 
+**--cdi-spec-dirs**="": Directories to scan for CDI Spec files (default: [/etc/cdi /var/run/cdi])
+
 **--cgroup-manager**="": cgroup manager (cgroupfs or systemd) (default: systemd)
 
 **--clean-shutdown-file**="": Location for CRI-O to lay down the clean shutdown file. It indicates whether we've had time to sync changes to disk before shutting down. If not found, crio wipe will clear the storage directory (default: /var/lib/crio/clean.shutdown)

@@ -141,6 +141,17 @@ the container runtime configuration.
 **blockio_config_file**=""
   Path to the blockio class configuration file for configuring the cgroup blockio controller.
 
+**cdi_spec_dirs**=[]
+  Directories to scan for Container Device Interface Specifications to enable CDI device injection. For more details about CDI and the syntax of CDI Spec files please refer to https://github.com/container-orchestrated-devices/container-device-interface.
+
+  Directories later in the list have precedence over earlier ones. The default directory list is:
+```
+  cdi_spec_dirs = [
+	  "/etc/cdi",
+	  "/var/run/cdi",
+  ]
+```
+
 **irqbalance_config_file**="/etc/sysconfig/irqbalance"
   Used to change irqbalance service config file which is used by CRI-O.
   For CentOS/SUSE, this file is located at /etc/sysconfig/irqbalance. For Ubuntu, this file is located at /etc/default/irqbalance.

@@ -6,6 +6,7 @@ require (
 	github.com/BurntSushi/toml v0.4.1
 	github.com/Microsoft/go-winio v0.5.1
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220224133719-e5457123010b
 	github.com/containerd/cgroups v1.0.2
 	github.com/containerd/containerd v1.5.8
 	github.com/containerd/cri-containerd v1.19.0
@@ -62,7 +63,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.2.0
 	golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef
+	golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c
 	google.golang.org/grpc v1.43.0
 	k8s.io/api v0.22.2
 	k8s.io/apimachinery v0.22.2
@@ -94,7 +95,6 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.5 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/cilium/ebpf v0.6.2 // indirect
-	github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 // indirect
 	github.com/containerd/console v1.0.2 // indirect
 	github.com/containerd/fifo v1.0.0 // indirect
 	github.com/containerd/go-runc v1.0.0 // indirect

@@ -239,8 +239,9 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
-github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 h1:Kn0s9/APRtr5dk/83aXj97WX0+PYnJK9BO8g0Xclm0I=
 github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9/go.mod h1:eQt66kIaJpUhCrjCtBFQGQxGLbAUl0OuuwjTH16ON4s=
+github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220224133719-e5457123010b h1:kfww7SrumR0HoZTwEXRDy2CzwhhmigCWK3mvvhYDaUQ=
+github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220224133719-e5457123010b/go.mod h1:E1zcucIkq9P3eyNmY+68dBQsTcsXJh9cgRo2IVNScKQ=
 github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
@@ -1074,6 +1075,7 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m
 github.com/opencontainers/runtime-spec v1.0.3-0.20210709190330-896175883324 h1:EPCbQw9GYqjcp8Y5Oz/RzEO3hlxA1MiQJ3DiYgzzUqg=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210709190330-896175883324/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
+github.com/opencontainers/runtime-tools v0.0.0-20190417131837-cd1349b7c47e/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/runtime-tools v0.9.1-0.20210326182921-59cdde06764b h1:BDT83NWjBPS3Esj/BS1Cf9F4NYzYmiitJZ0TlXx5M4w=
 github.com/opencontainers/runtime-tools v0.9.1-0.20210326182921-59cdde06764b/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
@@ -1700,8 +1702,9 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef h1:fPxZ3Umkct3LZ8gK9nbk+DWDJ9fstZa2grBn+lWVKPs=
 golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c h1:DHcbWVXeY+0Y8HHKR+rbLwnoh2F4tNCY7rTiHJ30RmA=
+golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

@@ -261,6 +261,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
 	if ctx.IsSet("allowed-devices") {
 		config.AllowedDevices = StringSliceTrySplit(ctx, "allowed-devices")
 	}
+	if ctx.IsSet("cdi-spec-dirs") {
+		config.CDISpecDirs = StringSliceTrySplit(ctx, "cdi-spec-dirs")
+	}
 	if ctx.IsSet("device-ownership-from-security-context") {
 		config.DeviceOwnershipFromSecurityContext = ctx.Bool("device-ownership-from-security-context")
 	}
@@ -842,6 +845,12 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
 			Value:   cli.NewStringSlice(defConf.AdditionalDevices...),
 			EnvVars: []string{"CONTAINER_ADDITIONAL_DEVICES"},
 		},
+		&cli.StringSliceFlag{
+			Name:    "cdi-spec-dirs",
+			Usage:   "Directories to scan for CDI Spec files",
+			Value:   cli.NewStringSlice(defConf.CDISpecDirs...),
+			EnvVars: []string{"CONTAINER_CDI_SPEC_DIRS"},
+		},
 		&cli.BoolFlag{
 			Name:  "device-ownership-from-security-context",
 			Usage: "Set devices' uid/gid ownership from runAsUser/runAsGroup",

@@ -1,11 +1,15 @@
 package container
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	devicecfg "github.com/cri-o/cri-o/internal/config/device"
+
+	"github.com/cri-o/cri-o/internal/log"
 	"github.com/cri-o/cri-o/utils"
 	securejoin "github.com/cyphar/filepath-securejoin"
 	"github.com/opencontainers/runc/libcontainer/devices"
@@ -39,8 +43,13 @@ func (c *container) SpecAddDevices(configuredDevices, annotationDevices []device
 		return err
 	}
 
-	// Finally, add container config devices
-	return c.specAddContainerConfigDevices(enableDeviceOwnershipFromSecurityContext)
+	// Then, add container config devices
+	if err := c.specAddContainerConfigDevices(enableDeviceOwnershipFromSecurityContext); err != nil {
+		return err
+	}
+
+	// Finally, inject CDI devices
+	return c.specInjectCDIDevices()
 }
 
 func (c *container) specAddHostDevicesIfPrivileged(privilegedWithoutHostDevices bool) error {
@@ -165,6 +174,38 @@ func (c *container) specAddContainerConfigDevices(enableDeviceOwnershipFromSecur
 	return nil
 }
 
+func (c *container) specInjectCDIDevices() error {
+	// TODO: Once CRI is extended with native CDI support this will need to be updated...
+	_, names, err := cdi.ParseAnnotations(c.Config().GetAnnotations())
+	if err != nil {
+		return errors.Wrap(err, "failed to parse CDI device annotations")
+	}
+	if names == nil {
+		return nil
+	}
+
+	registry := cdi.GetRegistry()
+	if err := registry.Refresh(); err != nil {
+		// We don't consider registry refresh failure a fatal error.
+		// For instance, a dynamically generated invalid CDI Spec file for
+		// any particular vendor shouldn't prevent injection of devices of
+		// different vendors. CDI itself knows better and it will fail the
+		// injection if necessary.
+
+		log.Warnf(context.TODO(), "CDI registry has errors: %v", err)
+	}
+
+	if _, err := registry.InjectDevices(c.Spec().Config, names...); err != nil {
+		return errors.Wrap(err, "CDI device injection failed")
+	}
+
+	// One crucial thing to keep in mind is that CDI device injection
+	// might add OCI Spec environment variables, hooks, and mounts as
+	// well. Therefore it is important that none of the corresponding
+	// OCI Spec fields are reset up in the call stack once we return.
+	return nil
+}
+
 // getDeviceUserGroupID() is used to find the right uid/gid
 // value for the device node created in the container namespace.
 // The runtime executes mknod() and chmod()s the created

@@ -1,8 +1,16 @@
 package container_test
 
 import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/opencontainers/runc/libcontainer/devices"
 	types "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
@@ -180,4 +188,152 @@ var _ = t.Describe("Container", func() {
 			})
 		}
 	})
+
+	t.Describe("SpecAdd(CDI)Devices", func() {
+		writeCDISpecFiles := func(content []string) error {
+			if len(content) == 0 {
+				return nil
+			}
+
+			dir := t.MustTempDir("cdi")
+			for idx, data := range content {
+				file := filepath.Join(dir, fmt.Sprintf("spec-%d.yaml", idx))
+				err := ioutil.WriteFile(file, []byte(data), 0o644)
+				if err != nil {
+					return err
+				}
+			}
+
+			return cdi.GetRegistry(cdi.WithSpecDirs(dir)).Refresh()
+		}
+
+		type testdata struct {
+			testDescription string
+			cdiSpecFiles    []string
+			annotations     map[string]string
+			expectError     bool
+			expectDevices   []rspec.LinuxDevice
+			expectEnv       []string
+		}
+
+		tests := []testdata{
+			{
+				testDescription: "Expect no CDI error for nil annotations",
+			},
+			{
+				testDescription: "Expect no CDI error for empty annotations",
+				annotations:     map[string]string{},
+			},
+			{
+				testDescription: "Expect CDI error for invalid CDI device reference in annotations",
+				annotations: map[string]string{
+					cdi.AnnotationPrefix + "devices": "foobar",
+				},
+				expectError: true,
+			},
+			{
+				testDescription: "Expect CDI error for unresolvable devices",
+				annotations: map[string]string{
+					cdi.AnnotationPrefix + "vendor1_devices": "vendor1.com/device=no-such-dev",
+				},
+				expectError: true,
+			},
+			{
+				testDescription: "Expect properly injected resolvable CDI devices",
+				cdiSpecFiles: []string{
+					`
+cdiVersion: "0.2.0"
+kind: "vendor1.com/device"
+devices:
+  - name: foo
+    containerEdits:
+      deviceNodes:
+        - path: /dev/loop8
+          type: b
+          major: 7
+          minor: 8
+      env:
+        - FOO=injected
+containerEdits:
+  env:
+    - "VENDOR1=present"
+`,
+					`
+cdiVersion: "0.2.0"
+kind: "vendor2.com/device"
+devices:
+  - name: bar
+    containerEdits:
+      deviceNodes:
+        - path: /dev/loop9
+          type: b
+          major: 7
+          minor: 9
+      env:
+        - BAR=injected
+containerEdits:
+  env:
+    - "VENDOR2=present"
+`,
+				},
+				annotations: map[string]string{
+					cdi.AnnotationPrefix + "vendor1_devices": "vendor1.com/device=foo",
+					cdi.AnnotationPrefix + "vendor2_devices": "vendor2.com/device=bar",
+				},
+				expectDevices: []rspec.LinuxDevice{
+					{
+						Path:  "/dev/loop8",
+						Type:  "b",
+						Major: 7,
+						Minor: 8,
+					},
+					{
+						Path:  "/dev/loop9",
+						Type:  "b",
+						Major: 7,
+						Minor: 9,
+					},
+				},
+				expectEnv: []string{
+					"FOO=injected",
+					"VENDOR1=present",
+					"BAR=injected",
+					"VENDOR2=present",
+				},
+			},
+		}
+
+		for _, test := range tests {
+			test := test
+			It(test.testDescription, func() {
+				// Given
+				config := &types.ContainerConfig{
+					Metadata:    &types.ContainerMetadata{Name: "name"},
+					Annotations: test.annotations,
+					Linux: &types.LinuxContainerConfig{
+						SecurityContext: &types.LinuxContainerSecurityContext{},
+					},
+					Devices: []*types.Device{},
+				}
+				sboxConfig := &types.PodSandboxConfig{
+					Linux: &types.LinuxPodSandboxConfig{
+						SecurityContext: &types.LinuxSandboxSecurityContext{},
+					},
+				}
+				Expect(sut.SetConfig(config, sboxConfig)).To(BeNil())
+				Expect(sut.SetPrivileged()).To(BeNil())
+				Expect(writeCDISpecFiles(test.cdiSpecFiles)).To(BeNil())
+
+				// When
+				err := sut.SpecAddDevices(nil, nil, false, false)
+
+				// Then
+				Expect(err != nil).To(Equal(test.expectError))
+				if err == nil {
+					Expect(sut.Spec().Config.Process.Env).Should(ContainElements(test.expectEnv))
+					Expect(sut.Spec().Config.Linux.Devices).Should(ContainElements(test.expectDevices))
+				}
+			})
+		}
+	})
 })