cri-o · openshift-merge-robot · May 9, 2023 · Apr 28, 2023 · May 5, 2023 · mtrmac
@@ -90,6 +90,7 @@ h
 --pause-image
 --pause-image-auth-file
 --pids-limit
+--pinned-images
 --pinns-path
 --profile
 --profile-cpu

@@ -127,6 +127,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l pause-command -r -d 'Path
 complete -c crio -n '__fish_crio_no_subcommand' -f -l pause-image -r -d 'Image which contains the pause executable.'
 complete -c crio -n '__fish_crio_no_subcommand' -l pause-image-auth-file -r -d 'Path to a config file containing credentials for --pause-image.'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l pids-limit -r -d 'Maximum number of processes allowed in a container. This option is deprecated. The Kubelet flag \'--pod-pids-limit\' should be used instead.'
+complete -c crio -n '__fish_crio_no_subcommand' -f -l pinned-images -r -d 'A list of images that will be excluded from the kubelet\'s garbage collection.'
 complete -c crio -n '__fish_crio_no_subcommand' -l pinns-path -r -d 'The path to find the pinns binary, which is needed to manage namespace lifecycle. Will be searched for in $PATH if empty.'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l profile -d 'Enable pprof remote profiler on localhost:6060.'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l profile-cpu -r -d 'Write a pprof CPU profile to the provided path.'

@@ -97,6 +97,7 @@ it later with **--config**. Global options will modify the output.'
         '--pause-image'
         '--pause-image-auth-file'
         '--pids-limit'
+        '--pinned-images'
         '--pinns-path'
         '--profile'
         '--profile-cpu'

@@ -89,6 +89,7 @@ crio
 [--pause-image-auth-file]=[value]
 [--pause-image]=[value]
 [--pids-limit]=[value]
+[--pinned-images]=[value]
 [--pinns-path]=[value]
 [--profile-cpu]=[value]
 [--profile-mem]=[value]
@@ -344,6 +345,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
 
 **--pids-limit**="": Maximum number of processes allowed in a container. This option is deprecated. The Kubelet flag '--pod-pids-limit' should be used instead. (default: 0)
 
+**--pinned-images**="": A list of images that will be excluded from the kubelet's garbage collection.
+
 **--pinns-path**="": The path to find the pinns binary, which is needed to manage namespace lifecycle. Will be searched for in $PATH if empty.
 
 **--profile**: Enable pprof remote profiler on localhost:6060.

@@ -407,6 +407,9 @@ CRI-O reads its configured registries defaults from the system wide containers-r
 **pause_command**="/pause"
   The command to run to have a container stay in the paused state. This option supports live configuration reload.
 
+**pinned_images**=[]
+  A list of images to be excluded from the kubelet's garbage collection. It allows specifying image names using either exact, glob, or keyword patterns. Exact matches must match the entire name, glob matches can have a wildcard * at the end, and keyword matches can have wildcards on both ends.
+
 **signature_policy**=""
   Path to the file which decides what sort of policy we use when deciding whether or not to trust an image that we've pulled. It is not recommended that this option be used, as the default behavior of using the system-wide default policy (i.e., /etc/containers/policy.json) is most often preferred. Please refer to containers-policy.json(5) for more details.
 

@@ -404,6 +404,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
 	if ctx.IsSet("hostnetwork-disable-selinux") {
 		config.HostNetworkDisableSELinux = ctx.Bool("hostnetwork-disable-selinux")
 	}
+	if ctx.IsSet("pinned-images") {
+		config.PinnedImages = StringSliceTrySplit(ctx, "pinned-images")
+	}
 	return nil
 }
 
@@ -1123,6 +1126,12 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
 			EnvVars: []string{"CONTAINER_HOSTNETWORK_DISABLE_SELINUX"},
 			Value:   defConf.HostNetworkDisableSELinux,
 		},
+		&cli.StringSliceFlag{
+			Name:    "pinned-images",
+			Usage:   "A list of images that will be excluded from the kubelet's garbage collection.",
+			EnvVars: []string{"CONTAINER_PINNED_IMAGES"},
+			Value:   cli.NewStringSlice(defConf.PinnedImages...),
+		},
 	}
 }
 

@@ -102,7 +102,7 @@ func New(ctx context.Context, configIface libconfig.Iface) (*ContainerServer, er
 		return nil, fmt.Errorf("cannot create container server: interface is nil")
 	}
 
-	imageService, err := storage.GetImageService(ctx, config.SystemContext, store, config.DefaultTransport, config.InsecureRegistries)
+	imageService, err := storage.GetImageService(ctx, store, config)
 	if err != nil {
 		return nil, err
 	}

@@ -9,6 +9,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
@@ -28,6 +29,7 @@ import (
 	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
 	"github.com/cri-o/cri-o/internal/config/node"
 	"github.com/cri-o/cri-o/internal/dbusmgr"
+	"github.com/cri-o/cri-o/pkg/config"
 	"github.com/cri-o/cri-o/utils"
 	"github.com/godbus/dbus/v5"
 	json "github.com/json-iterator/go"
@@ -62,6 +64,7 @@ type ImageResult struct {
 	Labels       map[string]string
 	OCIConfig    *specs.Image
 	Annotations  map[string]string
+	Pinned       bool // pinned image to prevent it from garbage collection
 }
 
 type indexInfo struct {
@@ -75,11 +78,12 @@ type indexInfo struct {
 // Every field in imageCacheItem are fixed properties of an "image", which in this
 // context is the image.ID stored in c/storage, and thus don't need to be recomputed.
 type imageCacheItem struct {
-	config       *specs.Image
-	size         *uint64
-	configDigest digest.Digest
-	info         *types.ImageInspectInfo
-	annotations  map[string]string
+	config        *specs.Image
+	size          *uint64
+	configDigest  digest.Digest
+	info          *types.ImageInspectInfo
+	annotations   map[string]string
+	isImagePinned bool
 }
 
 type imageCache map[string]imageCacheItem
@@ -91,11 +95,13 @@ type imageLookupService struct {
 }
 
 type imageService struct {
-	lookup         *imageLookupService
-	store          storage.Store
-	imageCache     imageCache
-	imageCacheLock sync.Mutex
-	ctx            context.Context
+	lookup               *imageLookupService
+	store                storage.Store
+	imageCache           imageCache
+	imageCacheLock       sync.Mutex
+	ctx                  context.Context
+	config               *config.Config
+	regexForPinnedImages []*regexp.Regexp
 }
 
 // ImageBeingPulled map[string]bool to keep track of the images haven't done pulling.
@@ -215,7 +221,7 @@ func (svc *imageService) makeRepoDigests(knownRepoDigests, tags []string, img *s
 	return imageDigest, repoDigests
 }
 
-func (svc *imageService) buildImageCacheItem(systemContext *types.SystemContext, ref types.ImageReference) (imageCacheItem, error) {
+func (svc *imageService) buildImageCacheItem(systemContext *types.SystemContext, ref types.ImageReference, image *storage.Image) (imageCacheItem, error) {
 	imageFull, err := ref.NewImage(svc.ctx, systemContext)
 	if err != nil {
 		return imageCacheItem{}, err
@@ -249,13 +255,14 @@ func (svc *imageService) buildImageCacheItem(systemContext *types.SystemContext,
 			return imageCacheItem{}, err
 		}
 	}
-
+	name, _, _ := sortNamesByType(image.Names)
 	return imageCacheItem{
-		config:       imageConfig,
-		size:         size,
-		configDigest: configDigest,
-		info:         info,
-		annotations:  ociManifest.Annotations,
+		config:        imageConfig,
+		size:          size,
+		configDigest:  configDigest,
+		info:          info,
+		annotations:   ociManifest.Annotations,
+		isImagePinned: FilterPinnedImage(name, svc.regexForPinnedImages),
 	}, nil
 }
 
@@ -286,6 +293,7 @@ func (svc *imageService) buildImageResult(image *storage.Image, cacheItem imageC
 		Labels:       cacheItem.info.Labels,
 		OCIConfig:    cacheItem.config,
 		Annotations:  cacheItem.annotations,
+		Pinned:       cacheItem.isImagePinned,
 	}
 }
 
@@ -295,7 +303,7 @@ func (svc *imageService) appendCachedResult(systemContext *types.SystemContext,
 	cacheItem, ok := svc.imageCache[image.ID]
 	svc.imageCacheLock.Unlock()
 	if !ok {
-		cacheItem, err = svc.buildImageCacheItem(systemContext, ref)
+		cacheItem, err = svc.buildImageCacheItem(systemContext, ref, image)
 		if err != nil {
 			return results, err
 		}
@@ -380,7 +388,7 @@ func (svc *imageService) ImageStatus(systemContext *types.SystemContext, nameOrI
 	svc.imageCacheLock.Unlock()
 
 	if !ok {
-		cacheItem, err = svc.buildImageCacheItem(systemContext, ref) // Single-use-only, not actually cached
+		cacheItem, err = svc.buildImageCacheItem(systemContext, ref, image) // Single-use-only, not actually cached
 		if err != nil {
 			return nil, err
 		}
@@ -830,7 +838,7 @@ func (svc *imageService) ResolveNames(systemContext *types.SystemContext, imageN
 // which will prepend the passed-in DefaultTransport value to an image name if
 // a name that's passed to its PullImage() method can't be resolved to an image
 // in the store and can't be resolved to a source on its own.
-func GetImageService(ctx context.Context, sc *types.SystemContext, store storage.Store, defaultTransport string, insecureRegistries []string) (ImageServer, error) {
+func GetImageService(ctx context.Context, store storage.Store, serverConfig *config.Config) (ImageServer, error) {
 	if store == nil {
 		var err error
 		storeOpts, err := storage.DefaultStoreOptions(rootless.IsRootless(), rootless.GetRootlessUID())
@@ -843,20 +851,22 @@ func GetImageService(ctx context.Context, sc *types.SystemContext, store storage
 		}
 	}
 	ils := &imageLookupService{
-		DefaultTransport:      defaultTransport,
+		DefaultTransport:      serverConfig.DefaultTransport,
 		IndexConfigs:          make(map[string]*indexInfo),
 		InsecureRegistryCIDRs: make([]*net.IPNet, 0),
 	}
 	is := &imageService{
-		lookup:     ils,
-		store:      store,
-		imageCache: make(map[string]imageCacheItem),
-		ctx:        ctx,
+		lookup:               ils,
+		store:                store,
+		imageCache:           make(map[string]imageCacheItem),
+		ctx:                  ctx,
+		config:               serverConfig,
+		regexForPinnedImages: CompileRegexpsForPinnedImages(serverConfig.PinnedImages),
 	}
 
-	insecureRegistries = append(insecureRegistries, "127.0.0.0/8")
+	serverConfig.InsecureRegistries = append(serverConfig.InsecureRegistries, "127.0.0.0/8")
 	// Split --insecure-registry into CIDR and registry-specific settings.
-	for _, r := range insecureRegistries {
+	for _, r := range serverConfig.InsecureRegistries {
 		// Check if CIDR was passed to --insecure-registry
 		_, ipnet, err := net.ParseCIDR(r)
 		if err == nil {
@@ -873,3 +883,43 @@ func GetImageService(ctx context.Context, sc *types.SystemContext, store storage
 
 	return is, nil
 }
+
+// FilterPinnedImage checks if the give image needs to be pinned
+// to exclude from kubelet's image GC.
+func FilterPinnedImage(image string, pinnedImages []*regexp.Regexp) bool {
+	if len(pinnedImages) == 0 {
+		return false
+	}
+
+	for _, pinnedImage := range pinnedImages {
+		if pinnedImage.MatchString(image) {
+			return true
+		}
+	}
+	return false
+}
+
+// CompileRegexpsForPinnedImages compiles regular expressions for the given
+// list of pinned images.
+func CompileRegexpsForPinnedImages(patterns []string) []*regexp.Regexp {
+	regexps := make([]*regexp.Regexp, 0, len(patterns))
+	for _, pattern := range patterns {
+		var re *regexp.Regexp
+		switch {
+		case strings.HasPrefix(pattern, "*") && strings.HasSuffix(pattern, "*"):
+			// keyword pattern
+			keyword := regexp.QuoteMeta(pattern[1 : len(pattern)-1])
+			re = regexp.MustCompile("(?i)" + keyword)
+		case strings.HasSuffix(pattern, "*"):
+			// glob pattern
+			pattern = regexp.QuoteMeta(pattern[:len(pattern)-1]) + ".*"
+			re = regexp.MustCompile("(?i)" + pattern)
+		default:
+			// exact pattern
+			re = regexp.MustCompile("(?i)^" + regexp.QuoteMeta(pattern) + "$")
+		}
+		regexps = append(regexps, re)
+	}
+
+	return regexps
+}
@@ -8,6 +8,7 @@ import (
 	"github.com/containers/image/v5/types"
 	cs "github.com/containers/storage"
 	"github.com/cri-o/cri-o/internal/storage"
+	"github.com/cri-o/cri-o/pkg/config"
 	containerstoragemock "github.com/cri-o/cri-o/test/mocks/containerstorage"
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo/v2"
@@ -54,9 +55,18 @@ var _ = t.Describe("Image", func() {
 		ctx = &types.SystemContext{
 			SystemRegistriesConfPath: t.MustTempFile("registries"),
 		}
+		config := &config.Config{
+			SystemContext: &types.SystemContext{
+				SystemRegistriesConfPath: t.MustTempFile("registries"),
+			},
+			ImageConfig: config.ImageConfig{
+				DefaultTransport:   "docker://",
+				InsecureRegistries: []string{},
+			},
+		}
 
 		sut, err = storage.GetImageService(
-			context.Background(), ctx, storeMock, "docker://", []string{},
+			context.Background(), storeMock, config,
 		)
 		Expect(err).To(BeNil())
 		Expect(sut).NotTo(BeNil())
@@ -81,7 +91,7 @@ var _ = t.Describe("Image", func() {
 			// Given
 			// When
 			imageService, err := storage.GetImageService(
-				context.Background(), nil, storeMock, "", []string{},
+				context.Background(), storeMock, &config.Config{},
 			)
 
 			// Then
@@ -92,12 +102,18 @@ var _ = t.Describe("Image", func() {
 		It("should succeed with custom registries.conf", func() {
 			// Given
 			// When
-			imageService, err := storage.GetImageService(
-				context.Background(),
-				&types.SystemContext{
+			config := &config.Config{
+				SystemContext: &types.SystemContext{
 					SystemRegistriesConfPath: "../../test/registries.conf",
 				},
-				storeMock, "", []string{},
+				ImageConfig: config.ImageConfig{
+					DefaultTransport:   "",
+					InsecureRegistries: []string{},
+				},
+			}
+			imageService, err := storage.GetImageService(
+				context.Background(),
+				storeMock, config,
 			)
 
 			// Then
@@ -280,11 +296,15 @@ var _ = t.Describe("Image", func() {
 				storeMock.EXPECT().Image(gomock.Any()).
 					Return(&cs.Image{ID: "id"}, nil),
 			)
-
+			config := &config.Config{
+				SystemContext: ctx,
+				ImageConfig: config.ImageConfig{
+					DefaultTransport:   "",
+					InsecureRegistries: []string{},
+				},
+			}
 			// Create an empty file for the registries config path
-			sut, err := storage.GetImageService(context.Background(),
-				ctx, storeMock, "", []string{},
-			)
+			sut, err := storage.GetImageService(context.Background(), storeMock, config)
 			Expect(err).To(BeNil())
 			Expect(sut).NotTo(BeNil())
 
@@ -746,4 +766,35 @@ var _ = t.Describe("Image", func() {
 			Expect(res).To(BeNil())
 		})
 	})
+
+	t.Describe("CompileRegexpsForPinnedImages", func() {
+		It("should return regexps for exact patterns", func() {
+			patterns := []string{"quay.io/crio/pause:latest", "docker.io/crio/sandbox:latest"}
+			regexps := storage.CompileRegexpsForPinnedImages(patterns)
+			Expect(len(regexps)).To(Equal(len(patterns)))
+			Expect(regexps[0].MatchString("quay.io/crio/pause:latest")).To(BeTrue())
+			Expect(regexps[1].MatchString("docker.io/crio/sandbox:latest")).To(BeTrue())
+		})
+
+		It("should return regexps for keyword patterns", func() {
+			patterns := []string{"*Fedora*"}
+			regexps := storage.CompileRegexpsForPinnedImages(patterns)
+			Expect(len(regexps)).To(Equal(len(patterns)))
+			Expect(regexps[0].MatchString("quay.io/crio/Fedora34:latest")).To(BeTrue())
+		})
+
+		It("should return regexps for glob patterns", func() {
+			patterns := []string{"quay.io/*", "*Fedora*", "docker.io/*"}
+			regexps := storage.CompileRegexpsForPinnedImages(patterns)
+			Expect(len(regexps)).To(Equal(len(patterns)))
+			Expect(regexps[0].MatchString("quay.io/test/image")).To(BeTrue())
+			Expect(regexps[1].MatchString("gcr.io/CRIO-Fedora34")).To(BeTrue())
+			Expect(regexps[2].MatchString("docker.io/test/image")).To(BeTrue())
+		})
+
+		It("should panic for invalid pattern", func() {
+			patterns := []string{"*"}
+			Expect(func() { storage.CompileRegexpsForPinnedImages(patterns) }).To(Panic())
+		})
+	})
 })