From 83518f0981759138ec6fcde414def7f2c751d641 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Mon, 27 Sep 2021 16:02:46 -0400 Subject: [PATCH 1/3] add allowed annotations to workloads Signed-off-by: Peter Hunt --- internal/oci/oci.go | 18 ++------ internal/oci/oci_test.go | 23 ++++------- pkg/config/config.go | 65 ++++++++++++++++++++++++----- pkg/config/workloads.go | 70 +++++++++++++++++++++++++++++++- server/container_create_linux.go | 2 +- server/sandbox_run_linux.go | 5 ++- server/utils.go | 23 +++++++++++ test/shm_size.bats | 12 ------ test/workloads.bats | 57 ++++++++++++++++++++++++++ 9 files changed, 217 insertions(+), 58 deletions(-) diff --git a/internal/oci/oci.go b/internal/oci/oci.go index b59811eed1b..bf98cb06e36 100644 --- a/internal/oci/oci.go +++ b/internal/oci/oci.go @@ -200,23 +200,13 @@ func (r *Runtime) PrivilegedWithoutHostDevices(handler string) (bool, error) { return rh.PrivilegedWithoutHostDevices, nil } -// FilterDisallowedAnnotations filters annotations that are not specified in the allowed_annotations map -// for a given handler. -// This function returns an error if the runtime handler can't be found. -// The annotations map is mutated in-place. -func (r *Runtime) FilterDisallowedAnnotations(handler string, annotations map[string]string) error { +func (r *Runtime) AllowedAnnotations(handler string) ([]string, error) { rh, err := r.getRuntimeHandler(handler) if err != nil { - return err - } - for ann := range annotations { - for _, disallowed := range rh.DisallowedAnnotations { - if strings.HasPrefix(ann, disallowed) { - delete(annotations, disallowed) - } - } + return []string{}, err } - return nil + + return rh.AllowedAnnotations, nil } // RuntimeType returns the type of runtimeHandler diff --git a/internal/oci/oci_test.go b/internal/oci/oci_test.go index 5336e1d73e2..953e69a7bde 100644 --- a/internal/oci/oci_test.go +++ b/internal/oci/oci_test.go @@ -121,32 +121,23 @@ var _ = t.Describe("Oci", func() { Expect(err).To(BeNil()) Expect(runtimeType).To(Equal(config.RuntimeTypeVM)) }) - Context("FilterDisallowedAnnotations", func() { - It("should succeed to filter disallowed annotation", func() { + Context("AllowedAnnotations", func() { + It("should succeed to return allowed annotation", func() { // Given - testAnn := map[string]string{ - annotations.DevicesAnnotation: "/dev", - annotations.IRQLoadBalancingAnnotation: "true", - } Expect(runtimes[performanceRuntime].ValidateRuntimeAllowedAnnotations()).To(BeNil()) // When - err := sut.FilterDisallowedAnnotations(performanceRuntime, testAnn) + foundAnn, err := sut.AllowedAnnotations(performanceRuntime) // Then Expect(err).To(BeNil()) - _, ok := testAnn[annotations.DevicesAnnotation] - Expect(ok).To(Equal(false)) - - _, ok = testAnn[annotations.IRQLoadBalancingAnnotation] - Expect(ok).To(Equal(true)) + Expect(foundAnn).NotTo(ContainElement(annotations.DevicesAnnotation)) + Expect(foundAnn).To(ContainElement(annotations.IRQLoadBalancingAnnotation)) }) - It("should fail to filter disallowed annotation of unknown runtime", func() { + It("should fail to return allowed annotation of unknown runtime", func() { // Given - testAnn := map[string]string{} - // When - err := sut.FilterDisallowedAnnotations("invalid", testAnn) + _, err := sut.AllowedAnnotations("invalid") // Then Expect(err).NotTo(BeNil()) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7f7568d4f61..c5b8db0c27a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -913,6 +913,10 @@ func (c *RuntimeConfig) Validate(systemContext *types.SystemContext, onExecution return errors.Wrap(err, "runtime validation") } + if err := c.ValidateAllowedAnnotations(); err != nil { + return errors.Wrap(err, "allowed annotations validation") + } + // Validate the system registries configuration if _, err := sysregistriesv2.GetRegistries(systemContext); err != nil { return errors.Wrap(err, "invalid registries") @@ -1224,22 +1228,61 @@ func (r *RuntimeHandler) ValidateRuntimeConfigPath(name string) error { } func (r *RuntimeHandler) ValidateRuntimeAllowedAnnotations() error { - disallowedAnnotations := make(map[string]struct{}) + disallowed, err := validateAllowedAndGenerateDisallowedAnnotations(r.AllowedAnnotations) + if err != nil { + return err + } + logrus.Debugf( + "Allowed annotations for runtime: %v", r.AllowedAnnotations, + ) + r.DisallowedAnnotations = disallowed + return nil +} + +func validateAllowedAndGenerateDisallowedAnnotations(allowed []string) (disallowed []string, _ error) { + disallowedMap := make(map[string]struct{}) for _, ann := range annotations.AllAllowedAnnotations { - disallowedAnnotations[ann] = struct{}{} + disallowedMap[ann] = struct{}{} } - for _, allowed := range r.AllowedAnnotations { - if _, ok := disallowedAnnotations[allowed]; !ok { - return errors.Errorf("invalid allowed_annotation: %s", allowed) + for _, ann := range allowed { + if _, ok := disallowedMap[ann]; !ok { + return nil, errors.Errorf("invalid allowed_annotation: %s", ann) } - delete(disallowedAnnotations, allowed) + delete(disallowedMap, ann) } - for ann := range disallowedAnnotations { - r.DisallowedAnnotations = append(r.DisallowedAnnotations, ann) + disallowed = make([]string, 0, len(disallowedMap)) + for ann := range disallowedMap { + disallowed = append(disallowed, ann) + } + return disallowed, nil +} + +// In the interim between adding workload level allowed annotations +// and disabling runtime level allowed annotations, we need to do a separate +// validation step to ensure neither list are stepping on the other's toes. +// Instead of complicated logic, declare workload level allowed annotations to +// always overwrite runtime level ones. +func (c *RuntimeConfig) ValidateAllowedAnnotations() error { + var workloadHasAnnotation bool + for _, wl := range c.Workloads { + if len(wl.AllowedAnnotations) != 0 { + workloadHasAnnotation = true + } + } + if !workloadHasAnnotation { + for _, wl := range c.Workloads { + wl.AllowedAnnotations = []string{} + wl.DisallowedAnnotations = []string{} + } + logrus.Infof("Workload does not have an allowed annotation configured. Clearing allowed annotations from runtimes") + return nil + } + logrus.Infof("Workload has an allowed annotation configured. Clearing allowed annotations from runtimes") + for name, rh := range c.Runtimes { + logrus.Infof("Clearing allowed annotations from %s", name) + rh.AllowedAnnotations = []string{} + rh.DisallowedAnnotations = []string{} } - logrus.Debugf( - "Allowed annotations for runtime: %v", r.AllowedAnnotations, - ) return nil } diff --git a/pkg/config/workloads.go b/pkg/config/workloads.go index 5ecfa6453bb..376476772b1 100644 --- a/pkg/config/workloads.go +++ b/pkg/config/workloads.go @@ -2,9 +2,11 @@ package config import ( "encoding/json" + "strings" "github.com/opencontainers/runtime-tools/generate" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) @@ -30,6 +32,17 @@ type WorkloadConfig struct { // the annotation with the resource and value, the default value will apply. // Default values do not need to be specified. Resources *Resources `toml:"resources"` + // AllowedAnnotations is a slice of experimental annotations that this workload is allowed to process. + // The currently recognized values are: + // "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod. + // "io.kubernetes.cri-o.Devices" for configuring devices for the pod. + // "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm. + // "io.kubernetes.cri-o.UnifiedCgroup.$CTR_NAME" for configuring the cgroup v2 unified block for a container. + // "io.containers.trace-syscall" for tracing syscalls via the OCI seccomp BPF hook. + AllowedAnnotations []string `toml:"allowed_annotations,omitempty"` + + // DisallowedAnnotations is the slice of experimental annotations that are not allowed for this workload. + DisallowedAnnotations []string } func (w Workloads) Validate() error { @@ -45,9 +58,53 @@ func (w *WorkloadConfig) Validate(workloadName string) error { if w.ActivationAnnotation == "" { return errors.Errorf("annotation shouldn't be empty for workload %q", workloadName) } + if err := w.ValidateWorkloadAllowedAnnotations(); err != nil { + return err + } return w.Resources.ValidateDefaults() } +func (w *WorkloadConfig) ValidateWorkloadAllowedAnnotations() error { + disallowed, err := validateAllowedAndGenerateDisallowedAnnotations(w.AllowedAnnotations) + if err != nil { + return err + } + logrus.Debugf( + "Allowed annotations for workload: %v", w.AllowedAnnotations, + ) + w.DisallowedAnnotations = disallowed + return nil +} + +func (w Workloads) AllowedAnnotations(toFind map[string]string) []string { + workload := w.workloadGivenActivationAnnotation(toFind) + if workload == nil { + return []string{} + } + return workload.AllowedAnnotations +} + +// FilterDisallowedAnnotations filters annotations that are not specified in the allowed_annotations map +// for a given handler. +// This function returns an error if the runtime handler can't be found. +// The annotations map is mutated in-place. +func (w Workloads) FilterDisallowedAnnotations(allowed []string, toFilter map[string]string) error { + disallowed, err := validateAllowedAndGenerateDisallowedAnnotations(allowed) + if err != nil { + return err + } + logrus.Warnf("Allowed annotations are specified for workload %v", allowed) + + for ann := range toFilter { + for _, d := range disallowed { + if strings.HasPrefix(ann, d) { + delete(toFilter, d) + } + } + } + return nil +} + func (w Workloads) MutateSpecGivenAnnotations(ctrName string, specgen *generate.Generator, sboxAnnotations map[string]string) error { workload := w.workloadGivenActivationAnnotation(sboxAnnotations) if workload == nil { @@ -73,9 +130,9 @@ func (w Workloads) workloadGivenActivationAnnotation(sboxAnnotations map[string] return nil } -func resourcesFromAnnotation(prefix, ctrName string, annotations map[string]string, defaultResources *Resources) (*Resources, error) { +func resourcesFromAnnotation(prefix, ctrName string, allAnnotations map[string]string, defaultResources *Resources) (*Resources, error) { annotationKey := prefix + "/" + ctrName - value, ok := annotations[annotationKey] + value, ok := allAnnotations[annotationKey] if !ok { return defaultResources, nil } @@ -84,6 +141,9 @@ func resourcesFromAnnotation(prefix, ctrName string, annotations map[string]stri if err := json.Unmarshal([]byte(value), &resources); err != nil { return nil, err } + if resources == nil { + return nil, nil + } if resources.CPUSet == "" { resources.CPUSet = defaultResources.CPUSet @@ -96,6 +156,9 @@ func resourcesFromAnnotation(prefix, ctrName string, annotations map[string]stri } func (r *Resources) ValidateDefaults() error { + if r == nil { + return nil + } if r.CPUSet == "" { return nil } @@ -104,6 +167,9 @@ func (r *Resources) ValidateDefaults() error { } func (r *Resources) MutateSpec(specgen *generate.Generator) { + if r == nil { + return + } if r.CPUSet != "" { specgen.SetLinuxResourcesCPUCpus(r.CPUSet) } diff --git a/server/container_create_linux.go b/server/container_create_linux.go index cb2a97ca907..4df71492861 100644 --- a/server/container_create_linux.go +++ b/server/container_create_linux.go @@ -141,7 +141,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, ctr ctrIface.Contai // TODO: eventually, this should be in the container package, but it's going through a lot of churn // and SpecAddAnnotations is already being passed too many arguments // Filter early so any use of the annotations don't use the wrong values - if err := s.Runtime().FilterDisallowedAnnotations(sb.RuntimeHandler(), ctr.Config().Annotations); err != nil { + if err := s.FilterDisallowedAnnotations(sb.Annotations(), ctr.Config().Annotations, sb.RuntimeHandler()); err != nil { return nil, err } diff --git a/server/sandbox_run_linux.go b/server/sandbox_run_linux.go index 929a559a333..3ac7a88306e 100644 --- a/server/sandbox_run_linux.go +++ b/server/sandbox_run_linux.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package server @@ -338,8 +339,8 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ return nil, err } - if err := s.Runtime().FilterDisallowedAnnotations(runtimeHandler, sbox.Config().Annotations); err != nil { - return nil, errors.Wrap(err, "filter disallowed annotations") + if err := s.FilterDisallowedAnnotations(sbox.Config().Annotations, sbox.Config().Annotations, runtimeHandler); err != nil { + return nil, err } kubeAnnotations := sbox.Config().Annotations diff --git a/server/utils.go b/server/utils.go index 6e77a1ef983..d5cc40170c5 100644 --- a/server/utils.go +++ b/server/utils.go @@ -209,3 +209,26 @@ func (s *Server) getResourceOrWait(ctx context.Context, name, resourceType strin return "", errors.Wrap(err, "Kubelet may be retrying requests that are timing out in CRI-O due to system load") } + +// FilterDisallowedAnnotations is a common place to have a map of annotations filtered for both runtimes and workloads. +// This function exists until the support for runtime level allowed annotations is dropped. +// toFind is used to find the workload for the specific pod or container, toFilter are the annotations +// for which disallowed annotations will be filtered. They may be the same. +// After this function, toFilter will no longer container disallowed annotations. +func (s *Server) FilterDisallowedAnnotations(toFind, toFilter map[string]string, runtimeHandler string) error { + // Only one of these Filter* will actually do any filtering, as the runtime DisallowedAnnotations + // were scrubbed at the config validation step if there were workload AllowedAnnotations configured. + // When runtime level allowed annotations are deprecated, this will be dropped. + // TODO: eventually, this should be in the container package, but it's going through a lot of churn + // and SpecAddAnnotations is already passed too many arguments + rtAllowed, err := s.Runtime().AllowedAnnotations(runtimeHandler) + if err != nil { + return err + } + allowed := s.config.Workloads.AllowedAnnotations(toFind) + if len(allowed) == 0 { + allowed = rtAllowed + } + + return s.config.Workloads.FilterDisallowedAnnotations(allowed, toFilter) +} diff --git a/test/shm_size.bats b/test/shm_size.bats index 41ca968e49f..8fe7002f059 100644 --- a/test/shm_size.bats +++ b/test/shm_size.bats @@ -10,18 +10,6 @@ function teardown() { cleanup_test } -function create_shmsize_runtime() { - cat << EOF > "$CRIO_CONFIG_DIR/01-shmsize.conf" -[crio.runtime] -default_runtime = "shmsize" -[crio.runtime.runtimes.shmsize] -runtime_path = "$RUNTIME_BINARY_PATH" -runtime_root = "$RUNTIME_ROOT" -runtime_type = "$RUNTIME_TYPE" -allowed_annotations = ["io.kubernetes.cri-o.ShmSize"] -EOF -} - @test "check /dev/shm is changed" { create_runtime_with_allowed_annotation "shmsize" "io.kubernetes.cri-o.ShmSize" start_crio diff --git a/test/workloads.bats b/test/workloads.bats index d4d262b2f07..8ed489fc5c8 100644 --- a/test/workloads.bats +++ b/test/workloads.bats @@ -29,6 +29,14 @@ cpuset = "$cpuset" EOF } +function create_workload_with_allowed_annotation() { + cat << EOF > "$CRIO_CONFIG_DIR/01-workload.conf" +[crio.runtime.workloads.management] +activation_annotation = "$activation" +allowed_annotations = ["$1"] +EOF +} + function check_cpu_fields() { local ctr_id="$1" local cpushares="$2" @@ -269,3 +277,52 @@ function check_conmon_fields() { ctr_id=$(crictl run "$ctrconfig" "$sboxconfig") check_conmon_fields "$ctr_id" "$shares" "$set" } + +@test "test workload allowed annotation should not work if not configured" { + create_workload_with_allowed_annotation "io.kubernetes.cri-o.ShmSize" + + start_crio + + jq '.annotations."io.kubernetes.cri-o.ShmSize" = "16Mi"' \ + "$TESTDATA"/sandbox_config.json > "$sboxconfig" + + ctrconfig="$TESTDATA"/container_sleep.json + ctr_id=$(crictl run "$ctrconfig" "$sboxconfig") + + df=$(crictl exec --sync "$ctr_id" df | grep /dev/shm) + [[ "$df" != *'16384'* ]] +} + +@test "test workload allowed annotation overrides runtime" { + create_workload_with_allowed_annotation "io.kubernetes.cri-o.userns-mode" + create_runtime_with_allowed_annotation "shmsize" "io.kubernetes.cri-o.ShmSize" + + start_crio + + jq '.annotations."io.kubernetes.cri-o.ShmSize" = "16Mi"' \ + "$TESTDATA"/sandbox_config.json > "$sboxconfig" + + ctrconfig="$TESTDATA"/container_sleep.json + ctr_id=$(crictl run "$ctrconfig" "$sboxconfig") + + df=$(crictl exec --sync "$ctr_id" df | grep /dev/shm) + [[ "$df" != *'16384'* ]] +} + +@test "test workload allowed annotation works for pod" { + create_workload_with_allowed_annotation "io.kubernetes.cri-o.ShmSize" + + name=POD + start_crio + + jq --arg act "$activation" \ + ' .annotations[$act] = "true" + | .annotations."io.kubernetes.cri-o.ShmSize" = "16Mi"' \ + "$TESTDATA"/sandbox_config.json > "$sboxconfig" + + ctrconfig="$TESTDATA"/container_sleep.json + ctr_id=$(crictl run "$ctrconfig" "$sboxconfig") + + df=$(crictl exec --sync "$ctr_id" df | grep /dev/shm) + [[ "$df" == *'16384'* ]] +} From b7a80f137a1d03798885f4e1c64f5c9b71317fe3 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Tue, 28 Sep 2021 13:12:01 -0400 Subject: [PATCH 2/3] update documentation for workloads as well as mark runtime class version of allowed_annotations as deprecated Signed-off-by: Peter Hunt --- docs/crio.conf.5.md | 32 ++++++++++++++++++++++++++++++++ internal/oci/oci.go | 1 + pkg/config/workloads.go | 34 +++++++++++++++++++--------------- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index f0f8757a226..57048a72e72 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -278,11 +278,43 @@ The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. Th **allowed_annotations**=[] A list of experimental annotations this runtime handler is allowed to process. + This field is currently DEPRECATED. If you'd like to use allowed_annotations, please use a workload. The currently recognized values are: "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod. "io.kubernetes.cri-o.Devices" for configuring devices for the pod. "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm. "io.kubernetes.cri-o.UnifiedCgroup.$CTR_NAME" for configuring the cgroup v2 unified block for a container. + "io.containers.trace-syscall" for tracing syscalls via the OCI seccomp BPF hook. + +### CRIO.RUNTIME.WORKLOADS TABLE +The "crio.runtime.workloads" table defines a list of workloads - a way to customize the behavior of a pod and container. +A workload is chosen for a pod based on whether the workload's **activation_annotation** is an annotation on the pod. + +**activation_annotation**="" + activation_annotation is the pod annotation that activates these workload settings. + +**annotation_prefix**="" + annotation_prefix is the way a pod can override a specific resource for a container. + The full annotation must be of the form `$annotation_prefix.$resource/$ctrname = $value`. + +**allowed_annotations**=[] + allowed_annotations is a slice of experimental annotations that this workload is allowed to process. + The currently recognized values are: + "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod. + "io.kubernetes.cri-o.Devices" for configuring devices for the pod. + "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm. + "io.kubernetes.cri-o.UnifiedCgroup.$CTR_NAME" for configuring the cgroup v2 unified block for a container. + "io.containers.trace-syscall" for tracing syscalls via the OCI seccomp BPF hook. + +### CRIO.RUNTIME.WORKLOAD.RESOURCES TABLE +The resources table is a structure for overriding certain resources for pods using this workload. +This structure provides a default value, and can be overridden by using the AnnotationPrefix. + +**cpushares**="" +Specifies the number of CPU shares this pod has access to. + +**cpuset**="" +Specifies the cpuset this pod has access to. ## CRIO.IMAGE TABLE The `crio.image` table contains settings pertaining to the management of OCI images. diff --git a/internal/oci/oci.go b/internal/oci/oci.go index bf98cb06e36..638640c8eb1 100644 --- a/internal/oci/oci.go +++ b/internal/oci/oci.go @@ -200,6 +200,7 @@ func (r *Runtime) PrivilegedWithoutHostDevices(handler string) (bool, error) { return rh.PrivilegedWithoutHostDevices, nil } +// AllowedAnnotations returns the allowed annotations for this runtime. func (r *Runtime) AllowedAnnotations(handler string) ([]string, error) { rh, err := r.getRuntimeHandler(handler) if err != nil { diff --git a/pkg/config/workloads.go b/pkg/config/workloads.go index 376476772b1..d5eeaf1ef98 100644 --- a/pkg/config/workloads.go +++ b/pkg/config/workloads.go @@ -10,11 +10,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) -type Resources struct { - CPUShares uint64 `json:"cpushares,omitempty"` - CPUSet string `json:"cpuset,omitempty"` -} - type Workloads map[string]*WorkloadConfig type WorkloadConfig struct { @@ -23,15 +18,6 @@ type WorkloadConfig struct { // AnnotationPrefix is the way a pod can override a specific resource for a container. // The full annotation must be of the form $annotation_prefix.$resource/$ctrname = $value AnnotationPrefix string `toml:"annotation_prefix"` - // Resources are the names of the resources that can be overridden by annotation. - // The key of the map is the resource name. The following resources are supported: - // `cpushares`: configure cpu shares for a given container - // `cpuset`: configure cpuset for a given container - // The value of the map is the default value for that resource. - // If a container is configured to use this workload, and does not specify - // the annotation with the resource and value, the default value will apply. - // Default values do not need to be specified. - Resources *Resources `toml:"resources"` // AllowedAnnotations is a slice of experimental annotations that this workload is allowed to process. // The currently recognized values are: // "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod. @@ -40,9 +26,27 @@ type WorkloadConfig struct { // "io.kubernetes.cri-o.UnifiedCgroup.$CTR_NAME" for configuring the cgroup v2 unified block for a container. // "io.containers.trace-syscall" for tracing syscalls via the OCI seccomp BPF hook. AllowedAnnotations []string `toml:"allowed_annotations,omitempty"` - // DisallowedAnnotations is the slice of experimental annotations that are not allowed for this workload. DisallowedAnnotations []string + // Resources are the names of the resources that can be overridden by annotation. + // The key of the map is the resource name. The following resources are supported: + // `cpushares`: configure cpu shares for a given container + // `cpuset`: configure cpuset for a given container + // The value of the map is the default value for that resource. + // If a container is configured to use this workload, and does not specify + // the annotation with the resource and value, the default value will apply. + // Default values do not need to be specified. + Resources *Resources `toml:"resources"` +} + +// Resources is a structure for overriding certain resources for the pod. +// This resources structure provides a default value, and can be overridden +// by using the AnnotationPrefix. +type Resources struct { + // Specifies the number of CPU shares this pod has access to. + CPUShares uint64 `json:"cpushares,omitempty"` + // Specifies the cpuset this pod has access to. + CPUSet string `json:"cpuset,omitempty"` } func (w Workloads) Validate() error { From 01ee37390be8135d3a92e1376ca75b24a710cf48 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Thu, 30 Sep 2021 12:30:07 -0400 Subject: [PATCH 3/3] docs: emphasize deprecation notice Signed-off-by: Peter Hunt --- docs/crio.conf.5.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index 57048a72e72..ae4f1dbc95e 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -55,9 +55,9 @@ CRI-O reads its storage defaults from the containers-storage.conf(5) file locate only happen when CRI-O has been upgraded **internal_wipe**=true + **This option is currently DEPRECATED, and will be removed in the future.** Whether CRI-O should wipe containers after a reboot and images after an upgrade when the server starts. If set to false, one must run `crio wipe` to wipe the containers and images in these situations. - This option is deprecated, and will be removed in the future. **clean_shutdown_file**="/var/lib/crio/clean.shutdown" Location for CRI-O to lay down the clean shutdown file. @@ -277,8 +277,8 @@ The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. Th Whether this runtime handler prevents host devices from being passed to privileged containers. **allowed_annotations**=[] + **This field is currently DEPRECATED. If you'd like to use allowed_annotations, please use a workload.** A list of experimental annotations this runtime handler is allowed to process. - This field is currently DEPRECATED. If you'd like to use allowed_annotations, please use a workload. The currently recognized values are: "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod. "io.kubernetes.cri-o.Devices" for configuring devices for the pod.