cri-o · alicefr · Feb 11, 2022 · Feb 11, 2022 · Jun 28, 2022 · Feb 11, 2022
@@ -35,6 +35,33 @@ jobs:
           JOBS: '1'
           RUN_CRITEST: '1'
 
+  multi-storage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+          key: go-integration-${{ hashFiles('**/go.sum') }}
+          restore-keys: go-integration-
+      - run: scripts/github-actions-packages
+      - run: scripts/github-actions-setup
+      - run: make all test-binaries
+      - run: scripts/setup_storage
+      - run: sudo -E test/test_runner.sh multi_storage.bats critest.bats
+        env:
+          LVM_DEVICE: '/dev/crio-test'
+          JOBS: '1'
+          RUN_CRITEST: '1'
+          CONTAINER_RUNTIMES: 'runctest:/usr/bin/runc:/run/runctest:oci:false'
+          CONTAINER_RUNTIME_STORAGE: 'runctest:devicemapper:dm.directlvm_device=/dev/crio-test:dm.fs=ext4:dm.directlvm_device_force=true'
+          RUNTIME_HANDLER: "runctest"
+
   critest-conmonrs:
     runs-on: ubuntu-latest
     steps:

@@ -352,7 +352,7 @@ mock-criostorage: ${MOCKGEN}
 	${MOCKGEN} \
 		-package criostoragemock \
 		-destination ${MOCK_PATH}/criostorage/criostorage.go \
-		github.com/cri-o/cri-o/internal/storage ImageServer,RuntimeServer
+		github.com/cri-o/cri-o/internal/storage ImageServer,RuntimeServer,MultiStoreServer,MultiStore
 
 mock-lib-config: ${MOCKGEN}
 	${MOCKGEN} \

@@ -65,7 +65,18 @@ func crioWipe(c *cli.Context) error {
 	// Note: this is only needed if the node rebooted.
 	// If there wasn't time to sync, we should clear the storage directory
 	if shouldWipeContainers && shutdownWasUnclean(config) {
-		return handleCleanShutdown(config, store)
+		var lastError error
+		for _, v := range store.GetStore() {
+			err := handleCleanShutdown(config, v)
+			if err != nil {
+				if lastError != nil {
+					lastError = fmt.Errorf("%w, %s", lastError, err.Error())
+				} else {
+					lastError = err
+				}
+			}
+		}
+		return lastError
 	}
 
 	// If crio is configured to wipe internally (and `--force` wasn't set)
@@ -133,7 +144,7 @@ func handleCleanShutdown(config *crioconf.Config, store cstorage.Store) error {
 }
 
 type ContainerStore struct {
-	store cstorage.Store
+	store storage.MultiStore
 }
 
 func (c ContainerStore) wipeCrio(shouldWipeImages bool) error {

@@ -91,6 +91,7 @@ h
 --registry
 --root
 --runroot
+--runtime-storage
 --runtimes
 --seccomp-profile
 --seccomp-use-default-when-empty

@@ -126,6 +126,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l read-only -d 'Setup all un
 complete -c crio -n '__fish_crio_no_subcommand' -f -l registry -r -d 'Registry to be prepended when pulling unqualified images, can be specified multiple times'
 complete -c crio -n '__fish_crio_no_subcommand' -l root -s r -r -d 'The CRI-O root directory'
 complete -c crio -n '__fish_crio_no_subcommand' -l runroot -r -d 'The CRI-O state directory'
+complete -c crio -n '__fish_crio_no_subcommand' -f -l runtime-storage -r -d 'Storage for the runtime, format is runtime_name:storage_driver:opt1:opt2...:optN. The runtime_name needs to be listed with the  runtimes, otherwise this option doesn\'t have any effect.'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l runtimes -r -d 'OCI runtimes, format is runtime_name:runtime_path:runtime_root:runtime_type:privileged_without_host_devices:runtime_config_path'
 complete -c crio -n '__fish_crio_no_subcommand' -l seccomp-profile -r -d 'Path to the seccomp.json profile to be used as the runtime\'s default. If not specified, then the internal default seccomp profile will be used. (default: "")'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l seccomp-use-default-when-empty -d 'Use the default seccomp profile when an empty one is specified'

@@ -98,6 +98,7 @@ it later with **--config**. Global options will modify the output.'
         '--registry'
         '--root'
         '--runroot'
+        '--runtime-storage'
         '--runtimes'
         '--seccomp-profile'
         '--seccomp-use-default-when-empty'

@@ -88,6 +88,7 @@ crio
 [--registry]=[value]
 [--root|-r]=[value]
 [--runroot]=[value]
+[--runtime-storage]=[value]
 [--runtimes]=[value]
 [--seccomp-profile]=[value]
 [--seccomp-use-default-when-empty]
@@ -331,6 +332,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
 
 **--runroot**="": The CRI-O state directory (default: /run/containers/storage)
 
+**--runtime-storage**="": Storage for the runtime, format is runtime_name:storage_driver:opt1:opt2...:optN. The runtime_name needs to be listed with the  runtimes, otherwise this option doesn't have any effect.
+
 **--runtimes**="": OCI runtimes, format is runtime_name:runtime_path:runtime_root:runtime_type:privileged_without_host_devices:runtime_config_path
 
 **--seccomp-profile**="": Path to the seccomp.json profile to be used as the runtime's default. If not specified, then the internal default seccomp profile will be used. (default: "")

@@ -25,7 +25,7 @@ require (
 	github.com/containers/kubensmnt v1.2.0
 	github.com/containers/ocicrypt v1.1.5
 	github.com/containers/podman/v4 v4.2.0
-	github.com/containers/storage v1.42.0
+	github.com/containers/storage v1.42.1-0.20220725075157-a010eb437cdc
 	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/cpuguy83/go-md2man v1.0.10
 	github.com/creack/pty v1.1.18
@@ -53,6 +53,7 @@ require (
 	github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab
 	github.com/opencontainers/runtime-tools v0.9.1-0.20220714195903-17b3287fafb7
 	github.com/opencontainers/selinux v1.10.1
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.13.0
 	github.com/psampaz/go-mod-outdated v0.8.0
 	github.com/sirupsen/logrus v1.9.0
@@ -284,7 +285,6 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/proglottis/gpgme v0.1.3 // indirect

@@ -647,8 +647,9 @@ github.com/containers/psgo v1.7.2 h1:WbCvsY9w+nCv3j4der0mbD3PSRUv/W8l+G0YrZrdSDc
 github.com/containers/psgo v1.7.2/go.mod h1:SLpqxsPOHtTqRygjutCPXmeU2PoEFzV3gzJplN4BMx0=
 github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4=
 github.com/containers/storage v1.38.0/go.mod h1:lBzt28gAk5ADZuRtwdndRJyqX22vnRaXmlF+7ktfMYc=
-github.com/containers/storage v1.42.0 h1:zm2AQD4NDeTB3JQ8X+Wo5+VRqNB+b4ocEd7Qj6ylPJA=
 github.com/containers/storage v1.42.0/go.mod h1:JiUJwOgOo1dr2DdOUc1MRe2GCAXABYoYmOdPF8yvH78=
+github.com/containers/storage v1.42.1-0.20220725075157-a010eb437cdc h1:qge3uikiNA7oJ3CC9IEZWA63FGTLv5GXIXYMgRJ0nVQ=
+github.com/containers/storage v1.42.1-0.20220725075157-a010eb437cdc/go.mod h1:JiUJwOgOo1dr2DdOUc1MRe2GCAXABYoYmOdPF8yvH78=
 github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
 github.com/coredns/corefile-migration v1.0.17/go.mod h1:XnhgULOEouimnzgn0t4WPuFDN2/PJQcTxdWKC5eXNGE=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=

@@ -129,15 +129,42 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
 	if ctx.IsSet("decryption-keys-path") {
 		config.DecryptionKeysPath = ctx.String("decryption-keys-path")
 	}
-
 	if ctx.IsSet("runtimes") {
+		type storageOpts struct {
+			storage string
+			opts    []string
+		}
 		runtimes := StringSliceTrySplit(ctx, "runtimes")
+		storageRuntimeMap := make(map[string]storageOpts)
+		if ctx.IsSet("runtime-storage") {
+			for _, s := range StringSliceTrySplit(ctx, "runtime-storage") {
+				fields := strings.Split(s, ":")
+				if len(fields) < 2 {
+					return fmt.Errorf("invalid format runtime-storage: %s", s)
+				}
+				var opts []string
+				if len(fields) > 2 {
+					opts = fields[2:]
+				}
+				storageRuntimeMap[fields[0]] = storageOpts{
+					storage: fields[1],
+					opts:    opts,
+				}
+			}
+		}
+
 		for _, r := range runtimes {
 			fields := strings.Split(r, ":")
 
 			runtimeType := libconfig.DefaultRuntimeType
 			privilegedWithoutHostDevices := false
 			runtimeConfigPath := ""
+			storage := ""
+			var opts []string
+			if v, ok := storageRuntimeMap[fields[0]]; ok {
+				storage = v.storage
+				opts = v.opts
+			}
 
 			switch len(fields) {
 			case 6:
@@ -158,6 +185,8 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
 					RuntimeType:                  runtimeType,
 					PrivilegedWithoutHostDevices: privilegedWithoutHostDevices,
 					RuntimeConfigPath:            runtimeConfigPath,
+					Storage:                      storage,
+					StorageOptions:               opts,
 				}
 			default:
 				return fmt.Errorf("wrong format for --runtimes: %q", r)
@@ -584,6 +613,11 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
 			Usage:   "OCI runtimes, format is runtime_name:runtime_path:runtime_root:runtime_type:privileged_without_host_devices:runtime_config_path",
 			EnvVars: []string{"CONTAINER_RUNTIMES"},
 		},
+		&cli.StringSliceFlag{
+			Name:    "runtime-storage",
+			Usage:   "Storage for the runtime, format is runtime_name:storage_driver:opt1:opt2...:optN. The runtime_name needs to be listed with the  runtimes, otherwise this option doesn't have any effect.",
+			EnvVars: []string{"CONTAINER_RUNTIME_STORAGE"},
+		},
 		&cli.StringFlag{
 			Name:      "seccomp-profile",
 			Usage:     fmt.Sprintf("Path to the seccomp.json profile to be used as the runtime's default. If not specified, then the internal default seccomp profile will be used. (default: %q)", defConf.SeccompProfile),

@@ -94,6 +94,17 @@ func (s *sandbox) SetConfig(config *types.PodSandboxConfig) error {
 	return nil
 }
 
+// ConstructSandboxNameFromConfig returns the pod sandbox name from the configuration
+func ConstructSandboxNameFromConfig(config *types.PodSandboxConfig) string {
+	return strings.Join([]string{
+		"k8s",
+		config.Metadata.Name,
+		config.Metadata.Namespace,
+		config.Metadata.Uid,
+		fmt.Sprintf("%d", config.Metadata.Attempt),
+	}, "_")
+}
+
 // SetNameAndID sets the sandbox name and ID
 func (s *sandbox) SetNameAndID() error {
 	if s.config == nil {
@@ -109,13 +120,7 @@ func (s *sandbox) SetNameAndID() error {
 	}
 
 	s.id = stringid.GenerateNonCryptoID()
-	s.name = strings.Join([]string{
-		"k8s",
-		s.config.Metadata.Name,
-		s.config.Metadata.Namespace,
-		s.config.Metadata.Uid,
-		fmt.Sprintf("%d", s.config.Metadata.Attempt),
-	}, "_")
+	s.name = ConstructSandboxNameFromConfig(s.config)
 
 	return nil
 }

@@ -103,7 +103,11 @@ func (c *ContainerServer) getDiff(id string) (rchanges []archive.Change, err err
 	if err != nil {
 		return nil, err
 	}
-	changes, err := c.store.Changes("", layerID)
+	store, err := c.Store().GetStoreForContainer(id)
+	if err != nil {
+		return nil, err
+	}
+	changes, err := store.Changes("", layerID)
 	if err == nil {
 		for _, c := range changes {
 			if containerMounts[c.Path] {
@@ -166,7 +170,11 @@ func (c *ContainerServer) exportCheckpoint(ctr *oci.Container, specgen *rspec.Sp
 	if err != nil {
 		return fmt.Errorf("error exporting root file-system diff for %q: %w", id, err)
 	}
-	mountPoint, err := c.StorageImageServer().GetStore().Mount(id, specgen.Linux.MountLabel)
+	is, err := c.StorageImageServerPerContainer(id)
+	if err != nil {
+		return err
+	}
+	mountPoint, err := is.GetStore().Mount(id, specgen.Linux.MountLabel)
 	if err != nil {
 		return fmt.Errorf("not able to get mountpoint for container %q: %w", id, err)
 	}

@@ -15,7 +15,12 @@ func (c *ContainerServer) GetStorageContainer(container string) (*cstorage.Conta
 	if err != nil {
 		return nil, err
 	}
-	return c.store.Container(ociCtr.ID())
+	store, err := c.Store().GetStoreForContainer(ociCtr.ID())
+	if err != nil {
+		return nil, err
+	}
+
+	return store.Container(ociCtr.ID())
 }
 
 // GetContainerTopLayerID gets the ID of the top layer of the given container