cri-o · haircommander · Dec 7, 2020 · Dec 7, 2020 · Dec 8, 2020 · Dec 8, 2020
diff --git a/completions/bash/crio b/completions/bash/crio
@@ -37,7 +37,6 @@ h
 --default-transport
 --default-ulimits
 --drop-infra-ctr
---enable-custom-shm-size
 --enable-metrics
 --gid-mappings
 --global-auth-file

diff --git a/completions/fish/crio.fish b/completions/fish/crio.fish
@@ -43,7 +43,6 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l default-sysctls -r -d 'Sys
 complete -c crio -n '__fish_crio_no_subcommand' -f -l default-transport -r -d 'A prefix to prepend to image names that cannot be pulled as-is'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l default-ulimits -r -d 'Ulimits to apply to containers by default (name=soft:hard) (default: [])'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l drop-infra-ctr -d 'Determines whether pods are created without an infra container (when the pod is not using a pod level PID namespace). Requires ManageNSLifecycle to be true (default: false)'
-complete -c crio -n '__fish_crio_no_subcommand' -f -l enable-custom-shm-size -d 'Enable users to set a custom shm size instead of using the default value of 64M'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l enable-metrics -d 'Enable metrics endpoint for the server on localhost:9090'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l gid-mappings -r -d 'Specify the GID mappings to use for the user namespace (default: "")'
 complete -c crio -n '__fish_crio_no_subcommand' -l global-auth-file -r -d 'Path to a file like /var/lib/kubelet/config.json holding credentials necessary for pulling images from secure registries (default: "")'

diff --git a/completions/zsh/_crio b/completions/zsh/_crio
@@ -7,7 +7,7 @@ it later with **--config**. Global options will modify the output.' 'version:dis
   _describe 'commands' cmds
 
   local -a opts
-  opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-custom-shm-size' '--enable-metrics' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--insecure-registry' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--manage-ns-lifecycle' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
+  opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-metrics' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--insecure-registry' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--manage-ns-lifecycle' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
   _describe 'global options' opts
 
   return

diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
@@ -25,7 +25,7 @@
     - name: clone build and install kubernetes
       include: "build/kubernetes.yml"
       vars:
-        k8s_git_version: "release-1.19"
+        k8s_git_version: "release-1.20"
         k8s_github_fork: "kubernetes"
         crio_socket: "/var/run/crio/crio.sock"
       when: "(cgroupv2 is undefined) or (cgroupv2 == False) | bool"
@@ -148,7 +148,7 @@
       include: "build/kubernetes.yml"
       vars:
         force_clone: true
-        k8s_git_version: "master"
+        k8s_git_version: "release-1.20"
         k8s_github_fork: "kubernetes"
         crio_socket: "/var/run/crio/crio.sock"
 
@@ -166,7 +166,7 @@
       include: "build/kubernetes.yml"
       vars:
         force_clone: true
-        k8s_git_version: "master"
+        k8s_git_version: "release-1.20"
         k8s_github_fork: "kubernetes"
         crio_socket: "/var/run/crio/crio.sock"
 
@@ -189,15 +189,15 @@
       include: "build/kubernetes.yml"
       vars:
         force_clone: true
-        k8s_git_version: "master"
+        k8s_git_version: "release-1.20"
         k8s_github_fork: "kubernetes"
         crio_socket: "/var/run/crio/crio.sock"
       when: "(cgroupv2 is undefined) or (cgroupv2 == False) | bool"
     - name: clone build and install kubernetes for cgroup v2
       include: "build/kubernetes.yml"
       vars:
         force_clone: true
-        k8s_git_version: "master"
+        k8s_git_version: "release-1.20"
         k8s_github_fork: "kubernetes"
         crio_socket: "/var/run/crio/crio.sock"
       when: "cgroupv2 | bool"

diff --git a/docs/crio.8.md b/docs/crio.8.md
@@ -36,7 +36,6 @@ crio
 [--default-transport]=[value]
 [--default-ulimits]=[value]
 [--drop-infra-ctr]
-[--enable-custom-shm-size]
 [--enable-metrics]
 [--gid-mappings]=[value]
 [--global-auth-file]=[value]
@@ -173,8 +172,6 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
 
 **--drop-infra-ctr**: Determines whether pods are created without an infra container (when the pod is not using a pod level PID namespace). Requires ManageNSLifecycle to be true (default: false)
 
-**--enable-custom-shm-size**: Enable users to set a custom shm size instead of using the default value of 64M
-
 **--enable-metrics**: Enable metrics endpoint for the server on localhost:9090
 
 **--gid-mappings**="": Specify the GID mappings to use for the user namespace (default: "")
@@ -279,15 +276,15 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
 
 **--seccomp-use-default-when-empty**="": Use the default seccomp profile when an empty one is specified (default: false)
 
-**--selinux**: Enable selinux support (default: false)
+**--selinux**: Enable selinux support (default: true)
 
 **--separate-pull-cgroup**="": [EXPERIMENTAL] Pull in new cgroup (default: "")
 
 **--signature-policy**="": Path to signature policy JSON file. (default: "", to use the system-wide default)
 
-**--storage-driver, -s**="": OCI storage driver (default: "")
+**--storage-driver, -s**="": OCI storage driver (default: "overlay")
 
-**--storage-opt**="": OCI storage driver option (default: [])
+**--storage-opt**="": OCI storage driver option (default: [overlay.mountopt=nodev])
 
 **--stream-address**="": Bind address for streaming socket (default: 127.0.0.1)
 

diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md
@@ -252,6 +252,7 @@ The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.  Th
   The currently recognized values are:
   "io.kubernetes.cri-o.userns-mode" for configuring a user namespace for the pod.
   "io.kubernetes.cri-o.Devices" for configuring devices for the pod.
+  "io.kubernetes.cri-o.ShmSize" for configuring the size of /dev/shm.
 
 ## CRIO.IMAGE TABLE
 The `crio.image` table contains settings pertaining to the management of OCI images.

diff --git a/internal/criocli/criocli.go b/internal/criocli/criocli.go
@@ -281,9 +281,6 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
 	if ctx.IsSet("enable-metrics") {
 		config.EnableMetrics = ctx.Bool("enable-metrics")
 	}
-	if ctx.IsSet("enable-custom-shm-size") {
-		config.EnableCustomShmSize = ctx.Bool("enable-custom-shm-size")
-	}
 	if ctx.IsSet("metrics-port") {
 		config.MetricsPort = ctx.Int("metrics-port")
 	}
@@ -647,11 +644,6 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
 			Usage:   "Enable metrics endpoint for the server on localhost:9090",
 			EnvVars: []string{"CONTAINER_ENABLE_METRICS"},
 		},
-		&cli.BoolFlag{
-			Name:    "enable-custom-shm-size",
-			Usage:   "Enable users to set a custom shm size instead of using the default value of 64M",
-			EnvVars: []string{"CONTAINER_ENABLE_CUSTOM_SHM_SIZE"},
-		},
 		&cli.IntFlag{
 			Name:    "metrics-port",
 			Value:   9090,

diff --git a/internal/oci/oci.go b/internal/oci/oci.go
@@ -220,6 +220,10 @@ func (r *Runtime) AllowIRQLoadBalancingAnnotation(handler string) (bool, error)
 	return r.allowAnnotation(handler, annotations.IRQLoadBalancingAnnotation)
 }
 
+func (r *Runtime) AllowShmSizeAnnotation(handler string) (bool, error) {
+	return r.allowAnnotation(handler, annotations.ShmSizeAnnotation)
+}
+
 func (r *Runtime) allowAnnotation(handler, annotation string) (bool, error) {
 	rh, err := r.getRuntimeHandler(handler)
 	if err != nil {

diff --git a/internal/resourcestore/resourcestore.go b/internal/resourcestore/resourcestore.go
@@ -0,0 +1,157 @@
+package resourcestore
+
+import (
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const sleepTimeBeforeCleanup = 1 * time.Minute
+
+// ResourceStore is a structure that saves information about a recently created resource.
+// Resources can be added and retrieved from the store. A retrieval (Get) also removes the Resource from the store.
+// The ResourceStore comes with a cleanup routine that loops through the resources and marks them as stale, or removes
+// them if they're already stale, then sleeps for `timeout`.
+// Thus, it takes between `timeout` and `2*timeout` for unrequested resources to be cleaned up.
+// Another routine can request a watcher for a resource by calling WatcherForResource.
+// All watchers will be notified when the resource has successfully been created.
+type ResourceStore struct {
+	resources map[string]*Resource
+	timeout   time.Duration
+	sync.Mutex
+}
+
+// Resource contains the actual resource itself (which must implement the IdentifiableCreatable interface),
+// as well as stores function pointers that pertain to how that resource should be cleaned up,
+// and keeps track of other requests that are watching for the successful creation of this resource.
+type Resource struct {
+	resource     IdentifiableCreatable
+	cleanupFuncs []func()
+	watchers     []chan struct{}
+	stale        bool
+	name         string
+}
+
+// IdentifiableCreatable are the qualities needed by the caller of the resource.
+// Once a resource is retrieved, SetCreated() will be called, indicating to the server
+// that resource is ready to be listed and operated upon, and ID() will be used to identify the
+// newly created resource to the server.
+type IdentifiableCreatable interface {
+	ID() string
+	SetCreated()
+}
+
+// New creates a new ResourceStore, with a default timeout, and starts the cleanup function
+func New() *ResourceStore {
+	return NewWithTimeout(sleepTimeBeforeCleanup)
+}
+
+// NewWithTimeout is used for testing purposes. It allows the caller to set the timeout, allowing for faster tests.
+// Most callers should use New instead.
+func NewWithTimeout(timeout time.Duration) *ResourceStore {
+	rc := &ResourceStore{
+		resources: make(map[string]*Resource),
+		timeout:   timeout,
+	}
+	go rc.cleanupStaleResources()
+	return rc
+}
+
+// cleanupStaleResources is responsible for cleaning up resources that haven't been gotten
+// from the store.
+// It runs on a loop, sleeping `sleepTimeBeforeCleanup` between each loop.
+// A resource will first be marked as stale before being cleaned up.
+// This means a resource will stay in the store between `sleepTimeBeforeCleanup` and `2*sleepTimeBeforeCleanup`.
+// When a resource is cleaned up, it's removed from the store and its cleanupFuncs are called.
+func (rc *ResourceStore) cleanupStaleResources() {
+	for {
+		time.Sleep(rc.timeout)
+		resourcesToReap := []*Resource{}
+		rc.Lock()
+		for name, r := range rc.resources {
+			if r.stale {
+				resourcesToReap = append(resourcesToReap, r)
+				delete(rc.resources, name)
+			}
+			r.stale = true
+		}
+		// no need to hold the lock when running the cleanup functions
+		rc.Unlock()
+
+		for _, r := range resourcesToReap {
+			logrus.Infof("cleaning up stale resource %s", r.name)
+			for _, f := range r.cleanupFuncs {
+				f()
+			}
+		}
+	}
+}
+
+// Get attempts to look up a resource by its name.
+// If it's found, it's removed from the store, and it is set as created.
+// Get returns an empty ID if the resource is not found,
+// and returns the value of the Resource's ID() method if it is.
+func (rc *ResourceStore) Get(name string) string {
+	rc.Lock()
+	defer rc.Unlock()
+
+	r, ok := rc.resources[name]
+	if !ok {
+		return ""
+	}
+	delete(rc.resources, name)
+	r.resource.SetCreated()
+	return r.resource.ID()
+}
+
+// Put takes a unique resource name (retrieved from the client request, not generated by the server),
+// a newly created resource, and functions to clean up that newly created resource.
+// It adds the Resource to the ResourceStore. It expects name to be unique, and
+// returns an error if a duplicate name is detected.
+func (rc *ResourceStore) Put(name string, resource IdentifiableCreatable, cleanupFuncs []func()) error {
+	rc.Lock()
+	defer rc.Unlock()
+
+	r, ok := rc.resources[name]
+	// if we don't already have a resource, create it
+	if !ok {
+		r = &Resource{}
+		rc.resources[name] = r
+	}
+	// make sure the resource hasn't already been added to the store
+	if r.resource != nil || r.cleanupFuncs != nil {
+		return errors.Errorf("failed to add entry %s to ResourceStore; entry already exists", name)
+	}
+
+	r.resource = resource
+	r.cleanupFuncs = cleanupFuncs
+	r.name = name
+
+	// now the resource is created, notify the watchers
+	for _, w := range r.watchers {
+		w <- struct{}{}
+	}
+	return nil
+}
+
+// WatcherForResource looks up a Resource by name, and gives it a watcher if it's found.
+// A watcher can be used for concurrent processes to wait for the resource to be created.
+// This is useful for situations where clients retry requests quickly after they "fail" because
+// they've taken too long. Adding a watcher allows the server to slow down the client, but still
+// return the resource in a timely manner once it's actually created.
+func (rc *ResourceStore) WatcherForResource(name string) chan struct{} {
+	rc.Lock()
+	defer rc.Unlock()
+	watcher := make(chan struct{}, 1)
+	r, ok := rc.resources[name]
+	if !ok {
+		rc.resources[name] = &Resource{
+			watchers: []chan struct{}{watcher},
+		}
+		return watcher
+	}
+	r.watchers = append(r.watchers, watcher)
+	return watcher
+}