From bbe9a7a2c5300a4548476204f86b1275f0c6cdeb Mon Sep 17 00:00:00 2001 From: Sohan Kunkerkar Date: Fri, 28 Apr 2023 15:18:25 -0400 Subject: [PATCH 1/2] *: add support for pinned_images in crio configuration Fixes: https://github.com/cri-o/cri-o/issues/6835 Signed-off-by: Sohan Kunkerkar --- completions/bash/crio | 1 + completions/fish/crio.fish | 1 + completions/zsh/_crio | 1 + docs/crio.8.md | 3 + docs/crio.conf.5.md | 3 + internal/criocli/criocli.go | 9 +++ internal/lib/container_server.go | 2 +- internal/storage/image.go | 104 +++++++++++++++++++++++-------- internal/storage/image_test.go | 71 ++++++++++++++++++--- pkg/config/config.go | 5 ++ pkg/config/template.go | 11 ++++ server/image_list.go | 1 + 12 files changed, 174 insertions(+), 38 deletions(-) diff --git a/completions/bash/crio b/completions/bash/crio index a92e7ed8102..fb93350d0d6 100755 --- a/completions/bash/crio +++ b/completions/bash/crio @@ -90,6 +90,7 @@ h --pause-image --pause-image-auth-file --pids-limit +--pinned-images --pinns-path --profile --profile-cpu diff --git a/completions/fish/crio.fish b/completions/fish/crio.fish index 85832729fc4..985b454642b 100644 --- a/completions/fish/crio.fish +++ b/completions/fish/crio.fish @@ -127,6 +127,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l pause-command -r -d 'Path complete -c crio -n '__fish_crio_no_subcommand' -f -l pause-image -r -d 'Image which contains the pause executable.' complete -c crio -n '__fish_crio_no_subcommand' -l pause-image-auth-file -r -d 'Path to a config file containing credentials for --pause-image.' complete -c crio -n '__fish_crio_no_subcommand' -f -l pids-limit -r -d 'Maximum number of processes allowed in a container. This option is deprecated. The Kubelet flag \'--pod-pids-limit\' should be used instead.' +complete -c crio -n '__fish_crio_no_subcommand' -f -l pinned-images -r -d 'A list of images that will be excluded from the kubelet\'s garbage collection.' complete -c crio -n '__fish_crio_no_subcommand' -l pinns-path -r -d 'The path to find the pinns binary, which is needed to manage namespace lifecycle. Will be searched for in $PATH if empty.' complete -c crio -n '__fish_crio_no_subcommand' -f -l profile -d 'Enable pprof remote profiler on localhost:6060.' complete -c crio -n '__fish_crio_no_subcommand' -f -l profile-cpu -r -d 'Write a pprof CPU profile to the provided path.' diff --git a/completions/zsh/_crio b/completions/zsh/_crio index ff3ce85e6d8..6c71a3a7792 100644 --- a/completions/zsh/_crio +++ b/completions/zsh/_crio @@ -97,6 +97,7 @@ it later with **--config**. Global options will modify the output.' '--pause-image' '--pause-image-auth-file' '--pids-limit' + '--pinned-images' '--pinns-path' '--profile' '--profile-cpu' diff --git a/docs/crio.8.md b/docs/crio.8.md index 049c7a27812..c084ba4a952 100644 --- a/docs/crio.8.md +++ b/docs/crio.8.md @@ -89,6 +89,7 @@ crio [--pause-image-auth-file]=[value] [--pause-image]=[value] [--pids-limit]=[value] +[--pinned-images]=[value] [--pinns-path]=[value] [--profile-cpu]=[value] [--profile-mem]=[value] @@ -344,6 +345,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...] **--pids-limit**="": Maximum number of processes allowed in a container. This option is deprecated. The Kubelet flag '--pod-pids-limit' should be used instead. (default: 0) +**--pinned-images**="": A list of images that will be excluded from the kubelet's garbage collection. + **--pinns-path**="": The path to find the pinns binary, which is needed to manage namespace lifecycle. Will be searched for in $PATH if empty. **--profile**: Enable pprof remote profiler on localhost:6060. diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index ea73cf498bf..3096d42f368 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -407,6 +407,9 @@ CRI-O reads its configured registries defaults from the system wide containers-r **pause_command**="/pause" The command to run to have a container stay in the paused state. This option supports live configuration reload. +**pinned_images**=[] + A list of images to be excluded from the kubelet's garbage collection. It allows specifying image names using either exact, glob, or keyword patterns. Exact matches must match the entire name, glob matches can have a wildcard * at the end, and keyword matches can have wildcards on both ends. + **signature_policy**="" Path to the file which decides what sort of policy we use when deciding whether or not to trust an image that we've pulled. It is not recommended that this option be used, as the default behavior of using the system-wide default policy (i.e., /etc/containers/policy.json) is most often preferred. Please refer to containers-policy.json(5) for more details. diff --git a/internal/criocli/criocli.go b/internal/criocli/criocli.go index 7edb9a35615..1df8fdce083 100644 --- a/internal/criocli/criocli.go +++ b/internal/criocli/criocli.go @@ -404,6 +404,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error { if ctx.IsSet("hostnetwork-disable-selinux") { config.HostNetworkDisableSELinux = ctx.Bool("hostnetwork-disable-selinux") } + if ctx.IsSet("pinned-images") { + config.PinnedImages = StringSliceTrySplit(ctx, "pinned-images") + } return nil } @@ -1123,6 +1126,12 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag { EnvVars: []string{"CONTAINER_HOSTNETWORK_DISABLE_SELINUX"}, Value: defConf.HostNetworkDisableSELinux, }, + &cli.StringSliceFlag{ + Name: "pinned-images", + Usage: "A list of images that will be excluded from the kubelet's garbage collection.", + EnvVars: []string{"CONTAINER_PINNED_IMAGES"}, + Value: cli.NewStringSlice(defConf.PinnedImages...), + }, } } diff --git a/internal/lib/container_server.go b/internal/lib/container_server.go index e91c17c5931..bb0b88b1525 100644 --- a/internal/lib/container_server.go +++ b/internal/lib/container_server.go @@ -102,7 +102,7 @@ func New(ctx context.Context, configIface libconfig.Iface) (*ContainerServer, er return nil, fmt.Errorf("cannot create container server: interface is nil") } - imageService, err := storage.GetImageService(ctx, config.SystemContext, store, config.DefaultTransport, config.InsecureRegistries) + imageService, err := storage.GetImageService(ctx, store, config) if err != nil { return nil, err } diff --git a/internal/storage/image.go b/internal/storage/image.go index 3821e948d39..212660901a4 100644 --- a/internal/storage/image.go +++ b/internal/storage/image.go @@ -9,6 +9,7 @@ import ( "net" "os" "path/filepath" + "regexp" "sort" "strings" "sync" @@ -28,6 +29,7 @@ import ( systemdDbus "github.com/coreos/go-systemd/v22/dbus" "github.com/cri-o/cri-o/internal/config/node" "github.com/cri-o/cri-o/internal/dbusmgr" + "github.com/cri-o/cri-o/pkg/config" "github.com/cri-o/cri-o/utils" "github.com/godbus/dbus/v5" json "github.com/json-iterator/go" @@ -62,6 +64,7 @@ type ImageResult struct { Labels map[string]string OCIConfig *specs.Image Annotations map[string]string + Pinned bool // pinned image to prevent it from garbage collection } type indexInfo struct { @@ -75,11 +78,12 @@ type indexInfo struct { // Every field in imageCacheItem are fixed properties of an "image", which in this // context is the image.ID stored in c/storage, and thus don't need to be recomputed. type imageCacheItem struct { - config *specs.Image - size *uint64 - configDigest digest.Digest - info *types.ImageInspectInfo - annotations map[string]string + config *specs.Image + size *uint64 + configDigest digest.Digest + info *types.ImageInspectInfo + annotations map[string]string + isImagePinned bool } type imageCache map[string]imageCacheItem @@ -91,11 +95,13 @@ type imageLookupService struct { } type imageService struct { - lookup *imageLookupService - store storage.Store - imageCache imageCache - imageCacheLock sync.Mutex - ctx context.Context + lookup *imageLookupService + store storage.Store + imageCache imageCache + imageCacheLock sync.Mutex + ctx context.Context + config *config.Config + regexForPinnedImages []*regexp.Regexp } // ImageBeingPulled map[string]bool to keep track of the images haven't done pulling. @@ -215,7 +221,7 @@ func (svc *imageService) makeRepoDigests(knownRepoDigests, tags []string, img *s return imageDigest, repoDigests } -func (svc *imageService) buildImageCacheItem(systemContext *types.SystemContext, ref types.ImageReference) (imageCacheItem, error) { +func (svc *imageService) buildImageCacheItem(systemContext *types.SystemContext, ref types.ImageReference, image *storage.Image) (imageCacheItem, error) { imageFull, err := ref.NewImage(svc.ctx, systemContext) if err != nil { return imageCacheItem{}, err @@ -249,13 +255,14 @@ func (svc *imageService) buildImageCacheItem(systemContext *types.SystemContext, return imageCacheItem{}, err } } - + name, _, _ := sortNamesByType(image.Names) return imageCacheItem{ - config: imageConfig, - size: size, - configDigest: configDigest, - info: info, - annotations: ociManifest.Annotations, + config: imageConfig, + size: size, + configDigest: configDigest, + info: info, + annotations: ociManifest.Annotations, + isImagePinned: FilterPinnedImage(name, svc.regexForPinnedImages), }, nil } @@ -286,6 +293,7 @@ func (svc *imageService) buildImageResult(image *storage.Image, cacheItem imageC Labels: cacheItem.info.Labels, OCIConfig: cacheItem.config, Annotations: cacheItem.annotations, + Pinned: cacheItem.isImagePinned, } } @@ -295,7 +303,7 @@ func (svc *imageService) appendCachedResult(systemContext *types.SystemContext, cacheItem, ok := svc.imageCache[image.ID] svc.imageCacheLock.Unlock() if !ok { - cacheItem, err = svc.buildImageCacheItem(systemContext, ref) + cacheItem, err = svc.buildImageCacheItem(systemContext, ref, image) if err != nil { return results, err } @@ -380,7 +388,7 @@ func (svc *imageService) ImageStatus(systemContext *types.SystemContext, nameOrI svc.imageCacheLock.Unlock() if !ok { - cacheItem, err = svc.buildImageCacheItem(systemContext, ref) // Single-use-only, not actually cached + cacheItem, err = svc.buildImageCacheItem(systemContext, ref, image) // Single-use-only, not actually cached if err != nil { return nil, err } @@ -830,7 +838,7 @@ func (svc *imageService) ResolveNames(systemContext *types.SystemContext, imageN // which will prepend the passed-in DefaultTransport value to an image name if // a name that's passed to its PullImage() method can't be resolved to an image // in the store and can't be resolved to a source on its own. -func GetImageService(ctx context.Context, sc *types.SystemContext, store storage.Store, defaultTransport string, insecureRegistries []string) (ImageServer, error) { +func GetImageService(ctx context.Context, store storage.Store, serverConfig *config.Config) (ImageServer, error) { if store == nil { var err error storeOpts, err := storage.DefaultStoreOptions(rootless.IsRootless(), rootless.GetRootlessUID()) @@ -843,20 +851,22 @@ func GetImageService(ctx context.Context, sc *types.SystemContext, store storage } } ils := &imageLookupService{ - DefaultTransport: defaultTransport, + DefaultTransport: serverConfig.DefaultTransport, IndexConfigs: make(map[string]*indexInfo), InsecureRegistryCIDRs: make([]*net.IPNet, 0), } is := &imageService{ - lookup: ils, - store: store, - imageCache: make(map[string]imageCacheItem), - ctx: ctx, + lookup: ils, + store: store, + imageCache: make(map[string]imageCacheItem), + ctx: ctx, + config: serverConfig, + regexForPinnedImages: CompileRegexpsForPinnedImages(serverConfig.PinnedImages), } - insecureRegistries = append(insecureRegistries, "127.0.0.0/8") + serverConfig.InsecureRegistries = append(serverConfig.InsecureRegistries, "127.0.0.0/8") // Split --insecure-registry into CIDR and registry-specific settings. - for _, r := range insecureRegistries { + for _, r := range serverConfig.InsecureRegistries { // Check if CIDR was passed to --insecure-registry _, ipnet, err := net.ParseCIDR(r) if err == nil { @@ -873,3 +883,43 @@ func GetImageService(ctx context.Context, sc *types.SystemContext, store storage return is, nil } + +// FilterPinnedImage checks if the give image needs to be pinned +// to exclude from kubelet's image GC. +func FilterPinnedImage(image string, pinnedImages []*regexp.Regexp) bool { + if len(pinnedImages) == 0 { + return false + } + + for _, pinnedImage := range pinnedImages { + if pinnedImage.MatchString(image) { + return true + } + } + return false +} + +// CompileRegexpsForPinnedImages compiles regular expressions for the given +// list of pinned images. +func CompileRegexpsForPinnedImages(patterns []string) []*regexp.Regexp { + regexps := make([]*regexp.Regexp, 0, len(patterns)) + for _, pattern := range patterns { + var re *regexp.Regexp + switch { + case strings.HasPrefix(pattern, "*") && strings.HasSuffix(pattern, "*"): + // keyword pattern + keyword := regexp.QuoteMeta(pattern[1 : len(pattern)-1]) + re = regexp.MustCompile("(?i)" + keyword) + case strings.HasSuffix(pattern, "*"): + // glob pattern + pattern = regexp.QuoteMeta(pattern[:len(pattern)-1]) + ".*" + re = regexp.MustCompile("(?i)" + pattern) + default: + // exact pattern + re = regexp.MustCompile("(?i)^" + regexp.QuoteMeta(pattern) + "$") + } + regexps = append(regexps, re) + } + + return regexps +} diff --git a/internal/storage/image_test.go b/internal/storage/image_test.go index f2ec42d53dd..25edaa63d43 100644 --- a/internal/storage/image_test.go +++ b/internal/storage/image_test.go @@ -8,6 +8,7 @@ import ( "github.com/containers/image/v5/types" cs "github.com/containers/storage" "github.com/cri-o/cri-o/internal/storage" + "github.com/cri-o/cri-o/pkg/config" containerstoragemock "github.com/cri-o/cri-o/test/mocks/containerstorage" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" @@ -54,9 +55,18 @@ var _ = t.Describe("Image", func() { ctx = &types.SystemContext{ SystemRegistriesConfPath: t.MustTempFile("registries"), } + config := &config.Config{ + SystemContext: &types.SystemContext{ + SystemRegistriesConfPath: t.MustTempFile("registries"), + }, + ImageConfig: config.ImageConfig{ + DefaultTransport: "docker://", + InsecureRegistries: []string{}, + }, + } sut, err = storage.GetImageService( - context.Background(), ctx, storeMock, "docker://", []string{}, + context.Background(), storeMock, config, ) Expect(err).To(BeNil()) Expect(sut).NotTo(BeNil()) @@ -81,7 +91,7 @@ var _ = t.Describe("Image", func() { // Given // When imageService, err := storage.GetImageService( - context.Background(), nil, storeMock, "", []string{}, + context.Background(), storeMock, &config.Config{}, ) // Then @@ -92,12 +102,18 @@ var _ = t.Describe("Image", func() { It("should succeed with custom registries.conf", func() { // Given // When - imageService, err := storage.GetImageService( - context.Background(), - &types.SystemContext{ + config := &config.Config{ + SystemContext: &types.SystemContext{ SystemRegistriesConfPath: "../../test/registries.conf", }, - storeMock, "", []string{}, + ImageConfig: config.ImageConfig{ + DefaultTransport: "", + InsecureRegistries: []string{}, + }, + } + imageService, err := storage.GetImageService( + context.Background(), + storeMock, config, ) // Then @@ -280,11 +296,15 @@ var _ = t.Describe("Image", func() { storeMock.EXPECT().Image(gomock.Any()). Return(&cs.Image{ID: "id"}, nil), ) - + config := &config.Config{ + SystemContext: ctx, + ImageConfig: config.ImageConfig{ + DefaultTransport: "", + InsecureRegistries: []string{}, + }, + } // Create an empty file for the registries config path - sut, err := storage.GetImageService(context.Background(), - ctx, storeMock, "", []string{}, - ) + sut, err := storage.GetImageService(context.Background(), storeMock, config) Expect(err).To(BeNil()) Expect(sut).NotTo(BeNil()) @@ -746,4 +766,35 @@ var _ = t.Describe("Image", func() { Expect(res).To(BeNil()) }) }) + + t.Describe("CompileRegexpsForPinnedImages", func() { + It("should return regexps for exact patterns", func() { + patterns := []string{"quay.io/crio/pause:latest", "docker.io/crio/sandbox:latest"} + regexps := storage.CompileRegexpsForPinnedImages(patterns) + Expect(len(regexps)).To(Equal(len(patterns))) + Expect(regexps[0].MatchString("quay.io/crio/pause:latest")).To(BeTrue()) + Expect(regexps[1].MatchString("docker.io/crio/sandbox:latest")).To(BeTrue()) + }) + + It("should return regexps for keyword patterns", func() { + patterns := []string{"*Fedora*"} + regexps := storage.CompileRegexpsForPinnedImages(patterns) + Expect(len(regexps)).To(Equal(len(patterns))) + Expect(regexps[0].MatchString("quay.io/crio/Fedora34:latest")).To(BeTrue()) + }) + + It("should return regexps for glob patterns", func() { + patterns := []string{"quay.io/*", "*Fedora*", "docker.io/*"} + regexps := storage.CompileRegexpsForPinnedImages(patterns) + Expect(len(regexps)).To(Equal(len(patterns))) + Expect(regexps[0].MatchString("quay.io/test/image")).To(BeTrue()) + Expect(regexps[1].MatchString("gcr.io/CRIO-Fedora34")).To(BeTrue()) + Expect(regexps[2].MatchString("docker.io/test/image")).To(BeTrue()) + }) + + It("should panic for invalid pattern", func() { + patterns := []string{"*"} + Expect(func() { storage.CompileRegexpsForPinnedImages(patterns) }).To(Panic()) + }) + }) }) diff --git a/pkg/config/config.go b/pkg/config/config.go index 127f45ea536..1fc2e65fe75 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -469,6 +469,11 @@ type ImageConfig struct { // PauseCommand is the path of the binary we run in an infra // container that's been instantiated using PauseImage. PauseCommand string `toml:"pause_command"` + // PinnedImages is a list of container images that should be pinned + // and not subject to garbage collection by kubelet. + // Pinned images will remain in the container runtime's storage until + // they are manually removed. Default value: empty list (no images pinned) + PinnedImages []string `toml:"pinned_images"` // SignaturePolicyPath is the name of the file which decides what sort // of policy we use when deciding whether or not to trust an image that // we've pulled. Outside of testing situations, it is strongly advised diff --git a/pkg/config/template.go b/pkg/config/template.go index 4b0ab9cc2a2..01fa796c149 100644 --- a/pkg/config/template.go +++ b/pkg/config/template.go @@ -480,6 +480,11 @@ func initCrioTemplateConfig(c *Config) ([]*templateConfigValue, error) { group: crioImageConfig, isDefaultValue: simpleEqual(dc.PauseCommand, c.PauseCommand), }, + { + templateString: templateStringCrioImagePinnedImages, + group: crioImageConfig, + isDefaultValue: stringSliceEqual(dc.PinnedImages, c.PinnedImages), + }, { templateString: templateStringCrioImageSignaturePolicy, group: crioImageConfig, @@ -1324,6 +1329,12 @@ const templateStringCrioImagePauseCommand = `# The command to run to have a cont ` +const templateStringCrioImagePinnedImages = `# List of container images to be skipped from the kubelet's garbage collection. +{{ $.Comment }}pinned_images = [ +{{ range $opt := .PinnedImages }}{{ $.Comment }}{{ printf "\t%q,\n" $opt }}{{ end }}{{ $.Comment }}] + +` + const templateStringCrioImageSignaturePolicy = `# Path to the file which decides what sort of policy we use when deciding # whether or not to trust an image that we've pulled. It is not recommended that # this option be used, as the default behavior of using the system-wide default diff --git a/server/image_list.go b/server/image_list.go index 0e1c736a907..34cf1a2b3af 100644 --- a/server/image_list.go +++ b/server/image_list.go @@ -56,6 +56,7 @@ func ConvertImage(from *storage.ImageResult) *types.Image { Id: from.ID, RepoTags: repoTags, RepoDigests: repoDigests, + Pinned: from.Pinned, } uid, username := getUserFromImage(from.User) From f9abf50c995e8d79815bc09051c4e2e184517a7c Mon Sep 17 00:00:00 2001 From: Sohan Kunkerkar Date: Fri, 5 May 2023 11:10:34 -0400 Subject: [PATCH 2/2] pkg/config: reload pinned_images when the new config is provided Signed-off-by: Sohan Kunkerkar --- pkg/config/reload.go | 15 +++++++++++++++ pkg/config/reload_test.go | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/pkg/config/reload.go b/pkg/config/reload.go index 7520641871f..7be989d70e1 100644 --- a/pkg/config/reload.go +++ b/pkg/config/reload.go @@ -72,6 +72,7 @@ func (c *Config) Reload() error { if err := c.ReloadPauseImage(newConfig); err != nil { return err } + c.ReloadPinnedImages(newConfig) if err := c.ReloadRegistries(); err != nil { return err } @@ -159,6 +160,20 @@ func (c *Config) ReloadPauseImage(newConfig *Config) error { return nil } +// ReloadPinnedImages updates the PinnedImages with the provided `newConfig`. +func (c *Config) ReloadPinnedImages(newConfig *Config) { + updatedPinnedImages := make([]string, len(newConfig.PinnedImages)) + for i, image := range newConfig.PinnedImages { + if i < len(c.PinnedImages) && image == c.PinnedImages[i] { + updatedPinnedImages[i] = c.PinnedImages[i] + } else { + updatedPinnedImages[i] = image + } + } + logrus.Infof("Updated new pinned images: %+v", updatedPinnedImages) + c.PinnedImages = updatedPinnedImages +} + // ReloadRegistries reloads the registry configuration from the Configs // `SystemContext`. The method errors in case of any update failure. func (c *Config) ReloadRegistries() error { diff --git a/pkg/config/reload_test.go b/pkg/config/reload_test.go index c383472703a..1dd3bdb028d 100644 --- a/pkg/config/reload_test.go +++ b/pkg/config/reload_test.go @@ -408,4 +408,22 @@ var _ = t.Describe("Config", func() { Expect(sut.Runtimes["existing"].PrivilegedWithoutHostDevices).To(BeTrue()) }) }) + + t.Describe("ReloadPinnedImages", func() { + It("should update PinnedImages with newConfig's PinnedImages if they are different", func() { + sut.PinnedImages = []string{"image1", "image4", "image3"} + newConfig := &config.Config{} + newConfig.PinnedImages = []string{"image5"} + sut.ReloadPinnedImages(newConfig) + Expect(sut.PinnedImages).To(Equal([]string{"image5"})) + }) + + It("should not update PinnedImages if they are the same as newConfig's PinnedImages", func() { + sut.PinnedImages = []string{"image1", "image2", "image3"} + newConfig := &config.Config{} + newConfig.PinnedImages = []string{"image1", "image2", "image3"} + sut.ReloadPinnedImages(newConfig) + Expect(sut.PinnedImages).To(Equal([]string{"image1", "image2", "image3"})) + }) + }) })