From f75a76d5a05b5978a1657e51023b537de750bfb7 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Wed, 21 Apr 2021 12:50:00 -0400 Subject: [PATCH 1/2] workloads: update how overrides are specified as well as mark the feature as experimental Signed-off-by: Peter Hunt --- pkg/config/template.go | 3 ++- pkg/config/workloads.go | 29 ++++++++++++++++++++++------- test/workloads.bats | 12 ++++++------ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/pkg/config/template.go b/pkg/config/template.go index 502ddfaa134..0af00a2d466 100644 --- a/pkg/config/template.go +++ b/pkg/config/template.go @@ -960,6 +960,7 @@ allowed_annotations = [ const templateStringCrioRuntimeWorkloads = `# The workloads table defines ways to customize containers with different resources # that work based on annotations, rather than the CRI. +# Note, the behavior of this table is EXPERIMENTAL and may change at any time. # Each workload, has a name, activation_annotation, annotation_prefix and set of resources it supports mutating. # The currently supported resources are "cpu" (to configure the cpu shares) and "cpuset" to configure the cpuset. # Each resource can have a default value specified, or be empty. @@ -978,7 +979,7 @@ const templateStringCrioRuntimeWorkloads = `# The workloads table defines ways t # This workload supports setting cpuset and cpu resources. # annotation_prefix is used to customize the different resources. # To configure the cpu shares a container gets in the example above, the pod would have to have the following annotation: -# "io.crio.workload-type.cpu/{container_name} = {shares}" +# "io.crio.workload-type/$container_name = {"cpushares": "value"}" {{ range $workload_type, $workload_config := .Workloads }} [crio.runtime.workloads.{{ $workload_type }}] activation_annotation = "{{ $workload_config.ActivationAnnotation }}" diff --git a/pkg/config/workloads.go b/pkg/config/workloads.go index c52a60c6f4b..336bd4b2ea6 100644 --- a/pkg/config/workloads.go +++ b/pkg/config/workloads.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "strconv" "github.com/opencontainers/runtime-tools/generate" @@ -63,8 +64,11 @@ func (w Workloads) MutateSpecGivenAnnotations(ctrName string, specgen *generate. if workload == nil { return nil } - for resource, defaultValue := range workload.Resources { - value := valueFromAnnotation(resource, defaultValue, workload.AnnotationPrefix, ctrName, sboxAnnotations) + resources, err := resourcesFromAnnotation(workload.AnnotationPrefix, ctrName, sboxAnnotations, workload.Resources) + if err != nil { + return err + } + for resource, value := range resources { if value == "" { continue } @@ -72,13 +76,14 @@ func (w Workloads) MutateSpecGivenAnnotations(ctrName string, specgen *generate. m, ok := mutators[resource] if !ok { // CRI-O bug - panic(errors.Errorf("resource %s is not defined", resource)) + return errors.Errorf("resource %s is not defined", resource) } if err := m.MutateSpec(specgen, value); err != nil { return errors.Wrapf(err, "mutating spec given workload %s", workload.ActivationAnnotation) } } + return nil } @@ -93,13 +98,23 @@ func (w Workloads) workloadGivenActivationAnnotation(sboxAnnotations map[string] return nil } -func valueFromAnnotation(resource, defaultValue, prefix, ctrName string, annotations map[string]string) string { - annotationKey := prefix + "." + resource + "/" + ctrName +func resourcesFromAnnotation(prefix, ctrName string, annotations, defaultResources map[string]string) (map[string]string, error) { + annotationKey := prefix + "/" + ctrName value, ok := annotations[annotationKey] if !ok { - return defaultValue + return defaultResources, nil + } + + var resources map[string]string + if err := json.Unmarshal([]byte(value), &resources); err != nil { + return nil, err + } + for defaultKey, defaultVal := range defaultResources { + if value, ok := resources[defaultKey]; !ok || value == "" { + resources[defaultKey] = defaultVal + } } - return value + return resources, nil } var mutators = map[string]Mutator{ diff --git a/test/workloads.bats b/test/workloads.bats index 5d1a35312be..764ac912b96 100644 --- a/test/workloads.bats +++ b/test/workloads.bats @@ -71,12 +71,12 @@ function check_cpu_fields() { start_crio - jq --arg act "$activation" --arg set "$set" --arg setkey "$prefix.cpuset/$name" \ + jq --arg act "$activation" --arg set "{\"cpuset\": \"$set\"}" --arg setkey "$prefix/$name" \ ' .annotations[$act] = "true" | .annotations[$setkey] = $set' \ "$TESTDATA"/sandbox_config.json > "$sboxconfig" - jq --arg act "$activation" --arg name "$name" --arg set "$set" --arg setkey "$prefix.cpuset/$name" \ + jq --arg act "$activation" --arg name "$name" --arg set "{\"cpuset\": \"$set\"}" --arg setkey "$prefix/$name" \ ' .annotations[$act] = "true" | .annotations[$setkey] = $set | .metadata.name = $name' \ @@ -94,12 +94,12 @@ function check_cpu_fields() { start_crio - jq --arg act "$activation" --arg set "$set" --arg setkey "$prefix.cpuset/$name" \ + jq --arg act "$activation" --arg set "{\"cpuset\": \"$set\"}" --arg setkey "$prefix/$name" \ ' .annotations[$act] = "true" | .annotations[$setkey] = $set' \ "$TESTDATA"/sandbox_config.json > "$sboxconfig" - jq --arg act "$activation" --arg name "$name" --arg set "$set" --arg setkey "$prefix.cpuset/$name" \ + jq --arg act "$activation" --arg name "$name" --arg set "{\"cpuset\": \"$set\"}" --arg setkey "$prefix/$name" \ ' .annotations[$act] = "true" | .annotations[$setkey] = $set | .metadata.name = $name' \ @@ -117,11 +117,11 @@ function check_cpu_fields() { start_crio - jq --arg act "$activation" --arg set "$set" --arg setkey "$prefix.cpuset/$name" \ + jq --arg act "$activation" --arg set "{\"cpuset\": \"$set\"}" --arg setkey "$prefix/$name" \ ' .annotations[$setkey] = $set' \ "$TESTDATA"/sandbox_config.json > "$sboxconfig" - jq --arg act "$activation" --arg name "$name" --arg set "$set" --arg setkey "$prefix.cpuset/$name" \ + jq --arg act "$activation" --arg name "$name" --arg set "{\"cpuset\": \"$set\"}" --arg setkey "$prefix/$name" \ ' .annotations[$setkey] = $set | .metadata.name = $name | del(.linux.resources.cpu_shares)' \ From 671a5c74700689a7c317fbec665aec252c90f82c Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Thu, 22 Apr 2021 16:15:51 -0400 Subject: [PATCH 2/2] workloads: move to more concrete type Signed-off-by: Peter Hunt --- pkg/config/template.go | 10 +++-- pkg/config/workloads.go | 99 ++++++++++------------------------------- test/workloads.bats | 4 +- 3 files changed, 33 insertions(+), 80 deletions(-) diff --git a/pkg/config/template.go b/pkg/config/template.go index 0af00a2d466..47c9760789b 100644 --- a/pkg/config/template.go +++ b/pkg/config/template.go @@ -972,7 +972,9 @@ const templateStringCrioRuntimeWorkloads = `# The workloads table defines ways t # [crio.runtime.workloads.workload-type] # activation_annotation = "io.crio/workload" # annotation_prefix = "io.crio.workload-type" -# resources = { "cpu" = "", "cpuset" = "0-1", } +# [crio.runtime.workloads.workload-type.resources] +# cpuset = 0 +# cpushares = "0-1" # Where: # The workload name is workload-type. # To specify, the pod must have the "io.crio.workload" annotation (this is a precise string match). @@ -984,9 +986,9 @@ const templateStringCrioRuntimeWorkloads = `# The workloads table defines ways t [crio.runtime.workloads.{{ $workload_type }}] activation_annotation = "{{ $workload_config.ActivationAnnotation }}" annotation_prefix = "{{ $workload_config.AnnotationPrefix }}" -{{ if $workload_config.Resources }} -resources = { {{ range $resource, $default := $workload_config.Resources }}{{ printf "%q = %q, " $resource $default }}{{ end }}} -{{ end }} +[crio.runtime.workloads.{{ $workload_type }}.resources] +cpuset = "{{ $workload_config.Resources.CPUSet }}" +cpushares = {{ $workload_config.Resources.CPUShares }} {{ end }} ` diff --git a/pkg/config/workloads.go b/pkg/config/workloads.go index 336bd4b2ea6..5ecfa6453bb 100644 --- a/pkg/config/workloads.go +++ b/pkg/config/workloads.go @@ -2,18 +2,16 @@ package config import ( "encoding/json" - "strconv" "github.com/opencontainers/runtime-tools/generate" "github.com/pkg/errors" - libresource "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) -const ( - CPUShareResource = "cpushares" - CPUSetResource = "cpuset" -) +type Resources struct { + CPUShares uint64 `json:"cpushares,omitempty"` + CPUSet string `json:"cpuset,omitempty"` +} type Workloads map[string]*WorkloadConfig @@ -31,7 +29,7 @@ type WorkloadConfig struct { // If a container is configured to use this workload, and does not specify // the annotation with the resource and value, the default value will apply. // Default values do not need to be specified. - Resources map[string]string `toml:"resources"` + Resources *Resources `toml:"resources"` } func (w Workloads) Validate() error { @@ -47,16 +45,7 @@ func (w *WorkloadConfig) Validate(workloadName string) error { if w.ActivationAnnotation == "" { return errors.Errorf("annotation shouldn't be empty for workload %q", workloadName) } - for resource, defaultValue := range w.Resources { - m, ok := mutators[resource] - if !ok { - return errors.Errorf("process resource %s for workload %s: resource not supported", resource, workloadName) - } - if err := m.ValidateDefault(defaultValue); err != nil { - return errors.Wrapf(err, "process resource %s for workload %s: default value %s invalid", resource, workloadName, defaultValue) - } - } - return nil + return w.Resources.ValidateDefaults() } func (w Workloads) MutateSpecGivenAnnotations(ctrName string, specgen *generate.Generator, sboxAnnotations map[string]string) error { @@ -68,21 +57,7 @@ func (w Workloads) MutateSpecGivenAnnotations(ctrName string, specgen *generate. if err != nil { return err } - for resource, value := range resources { - if value == "" { - continue - } - - m, ok := mutators[resource] - if !ok { - // CRI-O bug - return errors.Errorf("resource %s is not defined", resource) - } - - if err := m.MutateSpec(specgen, value); err != nil { - return errors.Wrapf(err, "mutating spec given workload %s", workload.ActivationAnnotation) - } - } + resources.MutateSpec(specgen) return nil } @@ -98,67 +73,41 @@ func (w Workloads) workloadGivenActivationAnnotation(sboxAnnotations map[string] return nil } -func resourcesFromAnnotation(prefix, ctrName string, annotations, defaultResources map[string]string) (map[string]string, error) { +func resourcesFromAnnotation(prefix, ctrName string, annotations map[string]string, defaultResources *Resources) (*Resources, error) { annotationKey := prefix + "/" + ctrName value, ok := annotations[annotationKey] if !ok { return defaultResources, nil } - var resources map[string]string + var resources *Resources if err := json.Unmarshal([]byte(value), &resources); err != nil { return nil, err } - for defaultKey, defaultVal := range defaultResources { - if value, ok := resources[defaultKey]; !ok || value == "" { - resources[defaultKey] = defaultVal - } - } - return resources, nil -} -var mutators = map[string]Mutator{ - CPUShareResource: new(cpuShareMutator), - CPUSetResource: new(cpusetMutator), -} + if resources.CPUSet == "" { + resources.CPUSet = defaultResources.CPUSet + } + if resources.CPUShares == 0 { + resources.CPUShares = defaultResources.CPUShares + } -type Mutator interface { - ValidateDefault(string) error - MutateSpec(*generate.Generator, string) error + return resources, nil } -type cpusetMutator struct{} - -func (m *cpusetMutator) ValidateDefault(set string) error { - if set == "" { +func (r *Resources) ValidateDefaults() error { + if r.CPUSet == "" { return nil } - _, err := cpuset.Parse(set) + _, err := cpuset.Parse(r.CPUSet) return err } -func (*cpusetMutator) MutateSpec(specgen *generate.Generator, configuredValue string) error { - specgen.SetLinuxResourcesCPUCpus(configuredValue) - return nil -} - -type cpuShareMutator struct{} - -func (*cpuShareMutator) ValidateDefault(cpuShare string) error { - if cpuShare == "" { - return nil +func (r *Resources) MutateSpec(specgen *generate.Generator) { + if r.CPUSet != "" { + specgen.SetLinuxResourcesCPUCpus(r.CPUSet) } - if _, err := libresource.ParseQuantity(cpuShare); err != nil { - return err - } - return nil -} - -func (*cpuShareMutator) MutateSpec(specgen *generate.Generator, configuredValue string) error { - u, err := strconv.ParseUint(configuredValue, 0, 64) - if err != nil { - return err + if r.CPUShares != 0 { + specgen.SetLinuxResourcesCPUShares(r.CPUShares) } - specgen.SetLinuxResourcesCPUShares(u) - return nil } diff --git a/test/workloads.bats b/test/workloads.bats index 764ac912b96..49848bfd7bf 100644 --- a/test/workloads.bats +++ b/test/workloads.bats @@ -21,7 +21,9 @@ function create_workload() { [crio.runtime.workloads.management] activation_annotation = "$activation" annotation_prefix = "$prefix" -resources = { "cpushares" = "$cpushares", "cpuset" = "$cpuset" } +[crio.runtime.workloads.management.resources] +cpushares = $cpushares +cpuset = "$cpuset" EOF }