cri-o · haircommander · Feb 22, 2021 · Apr 5, 2021 · Apr 26, 2021 · Apr 15, 2021
@@ -51,6 +51,15 @@ func crioWipe(c *cli.Context) error {
 		}
 	}
 
+	// If crio is configured to wipe internally (and `--force` wasn't set)
+	// the `crio wipe` command has nothing left to do,
+	// as the remaining work will be done on server startup.
+	if config.InternalWipe && !c.IsSet("force") {
+		return nil
+	}
+
+	logrus.Infof("Internal wipe not set, meaning crio wipe will wipe. In the future, all wipes after reboot will happen when starting the crio server.")
+
 	// if we should not wipe, exit with no error
 	if !shouldWipeContainers {
 		// we should not wipe images without wiping containers

@@ -45,6 +45,7 @@ h
 --hooks-dir
 --image-volumes
 --insecure-registry
+--internal-wipe
 --listen
 --log
 --log-dir

@@ -83,6 +83,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l insecure-registry -r -d 'E
        be enabled for testing purposes**. For increased security, users should add
        their CA to their system\'s list of trusted CAs instead of using
        \'--insecure-registry\'.'
+complete -c crio -n '__fish_crio_no_subcommand' -f -l internal-wipe -d 'Whether CRI-O should wipe containers after a reboot and images after an upgrade when the server starts. If set to false, one must run `crio wipe` to wipe the containers and images in these situations.'
 complete -c crio -n '__fish_crio_no_subcommand' -l listen -r -d 'Path to the CRI-O socket'
 complete -c crio -n '__fish_crio_no_subcommand' -l log -r -d 'Set the log file path where internal debug information is written'
 complete -c crio -n '__fish_crio_no_subcommand' -l log-dir -r -d 'Default log directory where all logs will go unless directly specified by the kubelet'

@@ -7,7 +7,7 @@ it later with **--config**. Global options will modify the output.' 'version:dis
   _describe 'commands' cmds
 
   local -a opts
-  opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-metrics' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--insecure-registry' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--manage-ns-lifecycle' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-idle-timeout' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
+  opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-metrics' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--insecure-registry' '--internal-wipe' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--manage-ns-lifecycle' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-idle-timeout' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
   _describe 'global options' opts
 
   return

@@ -45,6 +45,7 @@ crio
 [--hooks-dir]=[value]
 [--image-volumes]=[value]
 [--insecure-registry]=[value]
+[--internal-wipe]
 [--listen]=[value]
 [--log-dir]=[value]
 [--log-filter]=[value]
@@ -222,6 +223,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
        their CA to their system's list of trusted CAs instead of using
        '--insecure-registry'. (default: [])
 
+**--internal-wipe**: Whether CRI-O should wipe containers after a reboot and images after an upgrade when the server starts. If set to false, one must run `crio wipe` to wipe the containers and images in these situations.
+
 **--listen**="": Path to the CRI-O socket (default: /var/run/crio/crio.sock)
 
 **--log**="": Set the log file path where internal debug information is written
@@ -268,7 +271,7 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
 
 **--root, -r**="": The CRI-O root directory (default: /var/lib/containers/storage)
 
-**--runroot**="": The CRI-O state directory (default: /var/run/containers/storage)
+**--runroot**="": The CRI-O state directory (default: /run/containers/storage)
 
 **--runtimes**="": OCI runtimes, format is runtime_name:runtime_path:runtime_root:runtime_type (default: [])
 

@@ -54,6 +54,10 @@ CRI-O reads its storage defaults from the containers-storage.conf(5) file locate
   It is used to check if crio wipe should wipe images, which should
   only happen when CRI-O has been upgraded
 
+**internal_wipe**=false
+  Whether CRI-O should wipe containers after a reboot and images after an upgrade when the server starts.
+  If set to false, one must run `crio wipe` to wipe the containers and images in these situations.
+
 ## CRIO.API TABLE
 The `crio.api` table contains settings for the kubelet/gRPC interface.
 

@@ -281,6 +281,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
 	if ctx.IsSet("version-file-persist") {
 		config.VersionFilePersist = ctx.String("version-file-persist")
 	}
+	if ctx.IsSet("internal-wipe") {
+		config.InternalWipe = ctx.Bool("internal-wipe")
+	}
 	if ctx.IsSet("enable-metrics") {
 		config.EnableMetrics = ctx.Bool("enable-metrics")
 	}
@@ -802,6 +805,12 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
 			EnvVars:   []string{"CONTAINER_VERSION_FILE_PERSIST"},
 			TakesFile: true,
 		},
+		&cli.BoolFlag{
+			Name:    "internal-wipe",
+			Usage:   "Whether CRI-O should wipe containers after a reboot and images after an upgrade when the server starts. If set to false, one must run `crio wipe` to wipe the containers and images in these situations.",
+			Value:   defConf.InternalWipe,
+			EnvVars: []string{"CONTAINER_INTERNAL_WIPE"},
+		},
 	}
 }
 

@@ -145,23 +145,23 @@ func New(ctx context.Context, configIface libconfig.Iface) (*ContainerServer, er
 }
 
 // LoadSandbox loads a sandbox from the disk into the sandbox store
-func (c *ContainerServer) LoadSandbox(id string) (retErr error) {
+func (c *ContainerServer) LoadSandbox(id string) (sb *sandbox.Sandbox, retErr error) {
 	config, err := c.store.FromContainerDirectory(id, "config.json")
 	if err != nil {
-		return err
+		return nil, err
 	}
 	var m rspec.Spec
 	if err := json.Unmarshal(config, &m); err != nil {
-		return errors.Wrap(err, "error unmarshalling sandbox spec")
+		return nil, errors.Wrap(err, "error unmarshalling sandbox spec")
 	}
 	labels := make(map[string]string)
 	if err := json.Unmarshal([]byte(m.Annotations[annotations.Labels]), &labels); err != nil {
-		return errors.Wrapf(err, "error unmarshalling %s annotation", annotations.Labels)
+		return nil, errors.Wrapf(err, "error unmarshalling %s annotation", annotations.Labels)
 	}
 	name := m.Annotations[annotations.Name]
 	name, err = c.ReservePodName(id, name)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer func() {
 		if retErr != nil {
@@ -170,7 +170,7 @@ func (c *ContainerServer) LoadSandbox(id string) (retErr error) {
 	}()
 	var metadata pb.PodSandboxMetadata
 	if err := json.Unmarshal([]byte(m.Annotations[annotations.Metadata]), &metadata); err != nil {
-		return errors.Wrapf(err, "error unmarshalling %s annotation", annotations.Metadata)
+		return nil, errors.Wrapf(err, "error unmarshalling %s annotation", annotations.Metadata)
 	}
 
 	processLabel := m.Process.SelinuxLabel
@@ -180,65 +180,56 @@ func (c *ContainerServer) LoadSandbox(id string) (retErr error) {
 
 	kubeAnnotations := make(map[string]string)
 	if err := json.Unmarshal([]byte(m.Annotations[annotations.Annotations]), &kubeAnnotations); err != nil {
-		return errors.Wrapf(err, "error unmarshalling %s annotation", annotations.Annotations)
+		return nil, errors.Wrapf(err, "error unmarshalling %s annotation", annotations.Annotations)
 	}
 
 	portMappings := []*hostport.PortMapping{}
 	if err := json.Unmarshal([]byte(m.Annotations[annotations.PortMappings]), &portMappings); err != nil {
-		return errors.Wrapf(err, "error unmarshalling %s annotation", annotations.PortMappings)
+		return nil, errors.Wrapf(err, "error unmarshalling %s annotation", annotations.PortMappings)
 	}
 
 	privileged := isTrue(m.Annotations[annotations.PrivilegedRuntime])
 	hostNetwork := isTrue(m.Annotations[annotations.HostNetwork])
 	nsOpts := pb.NamespaceOption{}
 	if err := json.Unmarshal([]byte(m.Annotations[annotations.NamespaceOptions]), &nsOpts); err != nil {
-		return errors.Wrapf(err, "error unmarshalling %s annotation", annotations.NamespaceOptions)
+		return nil, errors.Wrapf(err, "error unmarshalling %s annotation", annotations.NamespaceOptions)
 	}
 
 	created, err := time.Parse(time.RFC3339Nano, m.Annotations[annotations.Created])
 	if err != nil {
-		return errors.Wrap(err, "parsing created timestamp annotation")
+		return nil, errors.Wrap(err, "parsing created timestamp annotation")
 	}
 
-	sb, err := sandbox.New(id, m.Annotations[annotations.Namespace], name, m.Annotations[annotations.KubeName], filepath.Dir(m.Annotations[annotations.LogPath]), labels, kubeAnnotations, processLabel, mountLabel, &metadata, m.Annotations[annotations.ShmPath], m.Annotations[annotations.CgroupParent], privileged, m.Annotations[annotations.RuntimeHandler], m.Annotations[annotations.ResolvPath], m.Annotations[annotations.HostName], portMappings, hostNetwork, created)
+	sb, err = sandbox.New(id, m.Annotations[annotations.Namespace], name, m.Annotations[annotations.KubeName], filepath.Dir(m.Annotations[annotations.LogPath]), labels, kubeAnnotations, processLabel, mountLabel, &metadata, m.Annotations[annotations.ShmPath], m.Annotations[annotations.CgroupParent], privileged, m.Annotations[annotations.RuntimeHandler], m.Annotations[annotations.ResolvPath], m.Annotations[annotations.HostName], portMappings, hostNetwork, created)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	sb.AddHostnamePath(m.Annotations[annotations.HostnamePath])
 	sb.SetSeccompProfilePath(spp)
 	sb.SetNamespaceOptions(&nsOpts)
 
 	// We add an NS only if we can load a permanent one.
 	// Otherwise, the sandbox will live in the host namespace.
-	if c.config.ManageNSLifecycle {
-		netNsPath, err := configNsPath(&m, rspec.NetworkNamespace)
-		if err == nil {
-			if nsErr := sb.NetNsJoin(netNsPath); nsErr != nil {
-				return nsErr
-			}
-		}
-		ipcNsPath, err := configNsPath(&m, rspec.IPCNamespace)
-		if err == nil {
-			if nsErr := sb.IpcNsJoin(ipcNsPath); nsErr != nil {
-				return nsErr
-			}
-		}
-		utsNsPath, err := configNsPath(&m, rspec.UTSNamespace)
+	namespacesToJoin := []struct {
+		rspecNS  rspec.LinuxNamespaceType
+		joinFunc func(string) error
+	}{
+		{rspecNS: rspec.NetworkNamespace, joinFunc: sb.NetNsJoin},
+		{rspecNS: rspec.IPCNamespace, joinFunc: sb.IpcNsJoin},
+		{rspecNS: rspec.UTSNamespace, joinFunc: sb.UtsNsJoin},
+		{rspecNS: rspec.UserNamespace, joinFunc: sb.UserNsJoin},
+	}
+	for _, namespaceToJoin := range namespacesToJoin {
+		path, err := configNsPath(&m, namespaceToJoin.rspecNS)
 		if err == nil {
-			if nsErr := sb.UtsNsJoin(utsNsPath); nsErr != nil {
-				return nsErr
-			}
-		}
-		userNsPath, err := configNsPath(&m, rspec.UserNamespace)
-		if err == nil {
-			if nsErr := sb.UserNsJoin(userNsPath); nsErr != nil {
-				return nsErr
+			if nsErr := namespaceToJoin.joinFunc(path); nsErr != nil {
+				return sb, nsErr
 			}
 		}
 	}
 
 	if err := c.AddSandbox(sb); err != nil {
-		return err
+		return sb, err
 	}
 
 	defer func() {
@@ -251,19 +242,19 @@ func (c *ContainerServer) LoadSandbox(id string) (retErr error) {
 
 	sandboxPath, err := c.store.ContainerRunDirectory(id)
 	if err != nil {
-		return err
+		return sb, err
 	}
 
 	sandboxDir, err := c.store.ContainerDirectory(id)
 	if err != nil {
-		return err
+		return sb, err
 	}
 
 	cID := m.Annotations[annotations.ContainerID]
 
 	cname, err := c.ReserveContainerName(cID, m.Annotations[annotations.ContainerName])
 	if err != nil {
-		return err
+		return sb, err
 	}
 	defer func() {
 		if retErr != nil {
@@ -283,15 +274,15 @@ func (c *ContainerServer) LoadSandbox(id string) (retErr error) {
 	if !wasSpoofed {
 		scontainer, err = oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], labels, m.Annotations, kubeAnnotations, m.Annotations[annotations.Image], "", "", nil, id, false, false, false, sb.RuntimeHandler(), sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
 		if err != nil {
-			return err
+			return sb, err
 		}
 		scontainer.SetSpec(&m)
 		scontainer.SetMountPoint(m.Annotations[annotations.MountPoint])
 
 		if m.Annotations[annotations.Volumes] != "" {
 			containerVolumes := []oci.ContainerVolume{}
 			if err = json.Unmarshal([]byte(m.Annotations[annotations.Volumes]), &containerVolumes); err != nil {
-				return fmt.Errorf("failed to unmarshal container volumes: %v", err)
+				return sb, fmt.Errorf("failed to unmarshal container volumes: %v", err)
 			}
 			for _, cv := range containerVolumes {
 				scontainer.AddVolume(cv)
@@ -302,50 +293,28 @@ func (c *ContainerServer) LoadSandbox(id string) (retErr error) {
 	}
 
 	if err := c.ContainerStateFromDisk(scontainer); err != nil {
-		return fmt.Errorf("error reading sandbox state from disk %q: %v", scontainer.ID(), err)
+		return sb, fmt.Errorf("error reading sandbox state from disk %q: %v", scontainer.ID(), err)
 	}
 
 	// We write back the state because it is possible that crio did not have a chance to
 	// read the exit file and persist exit code into the state on reboot.
 	if err := c.ContainerStateToDisk(scontainer); err != nil {
-		return fmt.Errorf("failed to write container %q state to disk: %v", scontainer.ID(), err)
+		return sb, fmt.Errorf("failed to write container %q state to disk: %v", scontainer.ID(), err)
 	}
 
 	if err := sb.SetInfraContainer(scontainer); err != nil {
-		return err
-	}
-
-	// We add an NS only if we can load a permanent one.
-	// Otherwise, the sandbox will live in the host namespace.
-	if c.config.ManageNSLifecycle || wasSpoofed {
-		namespacesToJoin := []struct {
-			rspecNS  rspec.LinuxNamespaceType
-			joinFunc func(string) error
-		}{
-			{rspecNS: rspec.NetworkNamespace, joinFunc: sb.NetNsJoin},
-			{rspecNS: rspec.IPCNamespace, joinFunc: sb.IpcNsJoin},
-			{rspecNS: rspec.UTSNamespace, joinFunc: sb.UtsNsJoin},
-			{rspecNS: rspec.UserNamespace, joinFunc: sb.UserNsJoin},
-		}
-		for _, namespaceToJoin := range namespacesToJoin {
-			path, err := configNsPath(&m, namespaceToJoin.rspecNS)
-			if err == nil {
-				if nsErr := namespaceToJoin.joinFunc(path); err != nil {
-					return nsErr
-				}
-			}
-		}
+		return sb, err
 	}
 
 	sb.SetCreated()
 	if err := label.ReserveLabel(processLabel); err != nil {
-		return err
+		return sb, err
 	}
 
 	sb.RestoreStopped()
 
 	if err := c.ctrIDIndex.Add(scontainer.ID()); err != nil {
-		return err
+		return sb, err
 	}
 	defer func() {
 		if retErr != nil {
@@ -355,9 +324,9 @@ func (c *ContainerServer) LoadSandbox(id string) (retErr error) {
 		}
 	}()
 	if err := c.podIDIndex.Add(id); err != nil {
-		return err
+		return sb, err
 	}
-	return nil
+	return sb, nil
 }
 
 func configNsPath(spec *rspec.Spec, nsType rspec.LinuxNamespaceType) (string, error) {