cri-o · openshift-merge-robot · Aug 23, 2021 · Feb 4, 2021 · Mar 25, 2021 · Mar 29, 2021
@@ -241,6 +241,28 @@ func main() {
 			logrus.Fatal(err)
 		}
 
+		if config.CleanShutdownFile != "" {
+			// clear out the shutdown file
+			if err := os.Remove(config.CleanShutdownFile); err != nil && !os.IsNotExist(err) {
+				logrus.Error(err)
+			}
+
+			// Write "$CleanShutdownFile".supported to show crio-wipe that
+			// we should be wiping if the CleanShutdownFile wasn't found.
+			// This protects us from wiping after an upgrade from a version that don't support
+			// CleanShutdownFile.
+			f, err := os.Create(config.CleanShutdownSupportedFileName())
+			if err != nil {
+				logrus.Errorf("Writing clean shutdown supported file: %v", err)
+			}
+			f.Close()
+
+			// and sync the changes to disk
+			if err := utils.SyncParent(config.CleanShutdownFile); err != nil {
+				logrus.Errorf("failed to sync parent directory of clean shutdown file: %v", err)
+			}
+		}
+
 		runtime.RegisterRuntimeServiceServer(grpcServer, service)
 		runtime.RegisterImageServiceServer(grpcServer, service)
 

@@ -7,6 +7,7 @@ import (
 	"github.com/cri-o/cri-o/internal/criocli"
 	"github.com/cri-o/cri-o/internal/storage"
 	"github.com/cri-o/cri-o/internal/version"
+	crioconf "github.com/cri-o/cri-o/pkg/config"
 	json "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -32,17 +33,25 @@ func crioWipe(c *cli.Context) error {
 		return err
 	}
 
+	store, err := config.GetStore()
+	if err != nil {
+		return err
+	}
 	shouldWipeImages := true
 	shouldWipeContainers := true
-	// First, check if we need to upgrade at all
+
 	if !c.IsSet("force") {
-		// there are two locations we check before wiping:
-		// one in a temporary directory. This is to check whether the node has rebooted.
-		// if so, we should remove containers
+		// First, check if the node was rebooted.
+		// We know this happened because the VersionFile (which lives in a tmpfs)
+		// will not be there.
 		shouldWipeContainers, err = version.ShouldCrioWipe(config.VersionFile)
 		if err != nil {
-			logrus.Infof("%v: triggering wipe of containers", err.Error())
+			logrus.Infof("checking whether cri-o should wipe containers: %v", err)
 		}
+
+		// there are two locations we check before wiping:
+		// one in a temporary directory. This is to check whether the node has rebooted.
+		// if so, we should remove containers
 		// another is needed in a persistent directory. This is to check whether we've upgraded
 		// if we've upgraded, we should wipe images
 		shouldWipeImages, err = version.ShouldCrioWipe(config.VersionFilePersist)
@@ -51,6 +60,13 @@ func crioWipe(c *cli.Context) error {
 		}
 	}
 
+	// Then, check whether crio has shutdown with time to sync.
+	// Note: this is only needed if the node rebooted.
+	// If there wasn't time to sync, we should clear the storage directory
+	if shouldWipeContainers && shutdownWasUnclean(config) {
+		return handleCleanShutdown(config, store)
+	}
+
 	// If crio is configured to wipe internally (and `--force` wasn't set)
 	// the `crio wipe` command has nothing left to do,
 	// as the remaining work will be done on server startup.
@@ -73,11 +89,6 @@ func crioWipe(c *cli.Context) error {
 		return nil
 	}
 
-	store, err := config.GetStore()
-	if err != nil {
-		return err
-	}
-
 	cstore := ContainerStore{store}
 	if err := cstore.wipeCrio(shouldWipeImages); err != nil {
 		return err
@@ -86,6 +97,40 @@ func crioWipe(c *cli.Context) error {
 	return nil
 }
 
+func shutdownWasUnclean(config *crioconf.Config) bool {
+	// CleanShutdownFile not configured, skip
+	if config.CleanShutdownFile == "" {
+		return false
+	}
+	// CleanShutdownFile isn't supported, skip
+	if _, err := os.Stat(config.CleanShutdownSupportedFileName()); err != nil {
+		return false
+	}
+	// CleanShutdownFile is present, indicating clean shutdown
+	if _, err := os.Stat(config.CleanShutdownFile); err == nil {
+		return false
+	}
+	return true
+}
+
+func handleCleanShutdown(config *crioconf.Config, store cstorage.Store) error {
+	logrus.Infof("file %s not found. Wiping storage directory %s because of suspected dirty shutdown", config.CleanShutdownFile, store.GraphRoot())
+	// If we do not do this, we may leak other resources that are not directly in the graphroot.
+	// Erroring here should not be fatal though, it's a best effort cleanup
+	if err := store.Wipe(); err != nil {
+		logrus.Infof("failed to wipe storage cleanly: %v", err)
+	}
+	// unmount storage or else we will fail with EBUSY
+	if _, err := store.Shutdown(false); err != nil {
+		return errors.Errorf("failed to shutdown storage before wiping: %v", err)
+	}
+	// totally remove storage, whatever is left (possibly orphaned layers)
+	if err := os.RemoveAll(store.GraphRoot()); err != nil {
+		return errors.Errorf("failed to remove storage directory: %v", err)
+	}
+	return nil
+}
+
 type ContainerStore struct {
 	store cstorage.Store
 }
@@ -96,14 +141,14 @@ func (c ContainerStore) wipeCrio(shouldWipeImages bool) error {
 		return err
 	}
 	if len(crioContainers) != 0 {
-		logrus.Infof("wiping containers")
+		logrus.Infof("Wiping containers")
 	}
 	for _, id := range crioContainers {
 		c.deleteContainer(id)
 	}
 	if shouldWipeImages {
 		if len(crioImages) != 0 {
-			logrus.Infof("wiping images")
+			logrus.Infof("Wiping images")
 		}
 		for _, id := range crioImages {
 			c.deleteImage(id)

@@ -17,6 +17,7 @@ h
 --big-files-temporary-dir
 --bind-mount-prefix
 --cgroup-manager
+--clean-shutdown-file
 --cni-config-dir
 --cni-default-network
 --cni-plugin-dir

@@ -14,6 +14,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l apparmor-profile -r -d 'Na
 complete -c crio -n '__fish_crio_no_subcommand' -f -l big-files-temporary-dir -r -d 'Path to the temporary directory to use for storing big files, used to store image blobs and data streams related to containers image management.'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l bind-mount-prefix -r -d 'A prefix to use for the source of the bind mounts. This option would be useful if you were running CRI-O in a container. And had `/` mounted on `/host` in your container. Then if you ran CRI-O with the `--bind-mount-prefix=/host` option, CRI-O would add /host to any bind mounts it is handed over CRI. If Kubernetes asked to have `/var/lib/foobar` bind mounted into the container, then CRI-O would bind mount `/host/var/lib/foobar`. Since CRI-O itself is running in a container with `/` or the host mounted on `/host`, the container would end up with `/var/lib/foobar` from the host mounted in the container rather then `/var/lib/foobar` from the CRI-O container. (default: "")'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l cgroup-manager -r -d 'cgroup manager (cgroupfs or systemd)'
+complete -c crio -n '__fish_crio_no_subcommand' -l clean-shutdown-file -r -d 'Location for CRI-O to lay down the clean shutdown file. It indicates whether we\'ve had time to sync changes to disk before shutting down. If not found, crio wipe will clear the storage directory'
 complete -c crio -n '__fish_crio_no_subcommand' -l cni-config-dir -r -d 'CNI configuration files directory'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l cni-default-network -r -d 'Name of the default CNI network to select. If not set or "", then CRI-O will pick-up the first one found in --cni-config-dir.'
 complete -c crio -n '__fish_crio_no_subcommand' -f -l cni-plugin-dir -r -d 'CNI plugin binaries directory'

@@ -7,7 +7,7 @@ it later with **--config**. Global options will modify the output.' 'version:dis
   _describe 'commands' cmds
 
   local -a opts
-  opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-metrics' '--enable-profile-unix-socket' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--infra-ctr-cpuset' '--insecure-registry' '--internal-wipe' '--irqbalance-config-file' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--manage-ns-lifecycle' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-idle-timeout' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
+  opts=('--additional-devices' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--cgroup-manager' '--clean-shutdown-file' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--drop-infra-ctr' '--enable-metrics' '--enable-profile-unix-socket' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--infra-ctr-cpuset' '--insecure-registry' '--internal-wipe' '--irqbalance-config-file' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--manage-ns-lifecycle' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-port' '--read-only' '--registries-conf' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-idle-timeout' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
   _describe 'global options' opts
 
   return

@@ -16,6 +16,7 @@ crio
 [--big-files-temporary-dir]=[value]
 [--bind-mount-prefix]=[value]
 [--cgroup-manager]=[value]
+[--clean-shutdown-file]=[value]
 [--cni-config-dir]=[value]
 [--cni-default-network]=[value]
 [--cni-plugin-dir]=[value]
@@ -128,6 +129,8 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]
 
 **--cgroup-manager**="": cgroup manager (cgroupfs or systemd) (default: systemd)
 
+**--clean-shutdown-file**="": Location for CRI-O to lay down the clean shutdown file. It indicates whether we've had time to sync changes to disk before shutting down. If not found, crio wipe will clear the storage directory (default: /var/lib/crio/clean.shutdown)
+
 **--cni-config-dir**="": CNI configuration files directory (default: /etc/cni/net.d/)
 
 **--cni-default-network**="": Name of the default CNI network to select. If not set or "", then CRI-O will pick-up the first one found in --cni-config-dir.

@@ -54,6 +54,11 @@ CRI-O reads its storage defaults from the containers-storage.conf(5) file locate
   It is used to check if crio wipe should wipe images, which should
   only happen when CRI-O has been upgraded
 
+**clean_shutdown_file**="/var/lib/crio/clean.shutdown"
+  Location for CRI-O to lay down the clean shutdown file.
+  It is used to check whether crio had time to sync before shutting down.
+  If not found, crio wipe will clear the storage directory.
+
 **internal_wipe**=false
   Whether CRI-O should wipe containers after a reboot and images after an upgrade when the server starts.
   If set to false, one must run `crio wipe` to wipe the containers and images in these situations.

@@ -284,6 +284,9 @@ func mergeConfig(config *libconfig.Config, ctx *cli.Context) error {
 	if ctx.IsSet("version-file-persist") {
 		config.VersionFilePersist = ctx.String("version-file-persist")
 	}
+	if ctx.IsSet("clean-shutdown-file") {
+		config.CleanShutdownFile = ctx.String("clean-shutdown-file")
+	}
 	if ctx.IsSet("internal-wipe") {
 		config.InternalWipe = ctx.Bool("internal-wipe")
 	}
@@ -841,6 +844,13 @@ func getCrioFlags(defConf *libconfig.Config) []cli.Flag {
 			Usage:   "CPU set to run infra containers, if not specified CRI-O will use all online CPUs to run infra containers (default: '').",
 			EnvVars: []string{"CONTAINER_INFRA_CTR_CPUSET"},
 		},
+		&cli.StringFlag{
+			Name:      "clean-shutdown-file",
+			Usage:     "Location for CRI-O to lay down the clean shutdown file. It indicates whether we've had time to sync changes to disk before shutting down. If not found, crio wipe will clear the storage directory",
+			Value:     defConf.CleanShutdownFile,
+			EnvVars:   []string{"CONTAINER_CLEAN_SHUTDOWN_FILE"},
+			TakesFile: true,
+		},
 	}
 }
 

@@ -148,6 +148,10 @@ type RootConfig struct {
 	// that checks whether we've upgraded
 	VersionFilePersist string `toml:"version_file_persist"`
 
+	// CleanShutdownFile is the location CRI-O will lay down the clean shutdown file
+	// that checks whether we've had time to sync before shutting down
+	CleanShutdownFile string `toml:"clean_shutdown_file"`
+
 	// InternalWipe is whether CRI-O should wipe containers and images after a reboot when the server starts.
 	// If set to false, one must use the external command `crio wipe` to wipe the containers and images in these situations.
 	InternalWipe bool `toml:"internal_wipe"`
@@ -586,6 +590,7 @@ func DefaultConfig() (*Config, error) {
 			LogDir:             "/var/log/crio/pods",
 			VersionFile:        CrioVersionPathTmp,
 			VersionFilePersist: CrioVersionPathPersist,
+			CleanShutdownFile:  CrioCleanShutdownFile,
 		},
 		APIConfig: APIConfig{
 			Listen:             CrioSocketPath,
@@ -754,6 +759,10 @@ func (c *RootConfig) Validate(onExecution bool) error {
 	return nil
 }
 
+func (c *RootConfig) CleanShutdownSupportedFileName() string {
+	return c.CleanShutdownFile + ".supported"
+}
+
 // Validate is the main entry point for runtime configuration validation
 // The parameter `onExecution` specifies if the validation should include
 // execution checks. It returns an `error` on validation failure, otherwise

@@ -25,4 +25,9 @@ const (
 	// CrioVersionPathPersist is where the CRI-O version file is located
 	// used to check whether we've upgraded, and thus need to remove images
 	CrioVersionPathPersist = "/var/lib/crio/version"
+
+	// CrioCleanShutdownFile is the location CRI-O will lay down the clean shutdown file
+	// that checks whether we've had time to sync before shutting down.
+	// If not, crio wipe will clear the storage directory.
+	CrioCleanShutdownFile = "/var/lib/crio/clean.shutdown"
 )
@@ -62,6 +62,11 @@ version_file = "{{ .VersionFile }}"
 # only happen when CRI-O has been upgraded
 version_file_persist = "{{ .VersionFilePersist }}"
 
+# Location for CRI-O to lay down the clean shutdown file.
+# It is used to check whether crio had time to sync before shutting down.
+# If not found, crio wipe will clear the storage directory.
+clean_shutdown_file = "{{ .CleanShutdownFile }}"
+
 # InternalWipe is whether CRI-O should wipe containers and images after a reboot when the server starts.
 # If set to false, one must use the external command 'crio wipe' to wipe the containers and images in these situations.
 internal_wipe = {{ .InternalWipe }}

@@ -29,6 +29,7 @@ import (
 	"github.com/cri-o/cri-o/internal/version"
 	libconfig "github.com/cri-o/cri-o/pkg/config"
 	"github.com/cri-o/cri-o/server/metrics"
+	"github.com/cri-o/cri-o/utils"
 	"github.com/fsnotify/fsnotify"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -312,7 +313,38 @@ func (s *Server) Shutdown(ctx context.Context) error {
 	// crio.service restart!!!
 	s.cleanupSandboxesOnShutdown(ctx)
 
-	return s.ContainerServer.Shutdown()
+	if err := s.ContainerServer.Shutdown(); err != nil {
+		return err
+	}
+
+	// first, make sure we sync all storage changes
+	if err := utils.Sync(s.Store().GraphRoot()); err != nil {
+		return errors.Wrapf(err, "failed to sync graph root after shutting down")
+	}
+
+	if s.config.CleanShutdownFile != "" {
+		// then, we write the CleanShutdownFile
+		// we do this after the sync, to ensure ordering.
+		// Otherwise, we may run into situations where the CleanShutdownFile
+		// is written before storage, causing us to think a corrupted storage
+		// is not so.
+		f, err := os.Create(s.config.CleanShutdownFile)
+		if err != nil {
+			return errors.Wrapf(err, "failed to write file to indicate a clean shutdown")
+		}
+		f.Close()
+
+		// finally, attempt to sync the newly created file to disk.
+		// It's still possible we crash after Create but before this Sync,
+		// which will lead us to think storage wasn't synced.
+		// However, that's much less likely than if we don't have a second Sync,
+		// and less risky than if we don't Sync after the Create
+		if err := utils.SyncParent(s.config.CleanShutdownFile); err != nil {
+			return errors.Wrapf(err, "failed to sync clean shutdown file")
+		}
+	}
+
+	return nil
 }
 
 // configureMaxThreads sets the Go runtime max threads threshold

@@ -2,6 +2,7 @@ package server_test
 
 import (
 	"context"
+	"os"
 
 	cstorage "github.com/containers/storage"
 	"github.com/cri-o/cri-o/server"
@@ -267,13 +268,18 @@ var _ = t.Describe("Server", func() {
 			// Given
 			gomock.InOrder(
 				storeMock.EXPECT().Shutdown(gomock.Any()).Return(nil, nil),
+				storeMock.EXPECT().GraphRoot().Return(emptyDir),
 			)
 
 			// When
 			err := sut.Shutdown(context.Background())
 
 			// Then
 			Expect(err).To(BeNil())
+
+			// expect cri-o to have created the clean shutdown file
+			_, err = os.Stat(sut.Config().CleanShutdownFile)
+			Expect(err).To(BeNil())
 		})
 	})
 

@@ -142,6 +142,7 @@ var beforeEach = func() {
 	serverConfig.ContainerAttachSocketDir = testPath
 	serverConfig.ContainerExitsDir = path.Join(testPath, "exits")
 	serverConfig.LogDir = path.Join(testPath, "log")
+	serverConfig.CleanShutdownFile = path.Join(testPath, "clean.shutdown")
 
 	// We want a directory that is guaranteed to exist, but it must
 	// be empty so we don't erroneously load anything and make tests