diff --git a/internal/resourcestore/resourcecleaner.go b/internal/resourcestore/resourcecleaner.go new file mode 100644 index 00000000000..addcb7c7f8b --- /dev/null +++ b/internal/resourcestore/resourcecleaner.go @@ -0,0 +1,33 @@ +package resourcestore + +// A CleanupFunc is a function that cleans up one piece of +// the associated resource. +type CleanupFunc func() + +// ResourceCleaner is a structure that tracks +// how to cleanup a resource. +// CleanupFuncs can be added to it, and it can be told to +// Cleanup the resource +type ResourceCleaner struct { + funcs []CleanupFunc +} + +// NewResourceCleaner creates a new ResourceCleaner +func NewResourceCleaner() *ResourceCleaner { + return &ResourceCleaner{ + funcs: make([]CleanupFunc, 0), + } +} + +// Add adds a new CleanupFunc to the ResourceCleaner +func (r *ResourceCleaner) Add(f func()) { + r.funcs = append(r.funcs, CleanupFunc(f)) +} + +// Cleanup cleans up the resource, running +// the cleanup funcs in opposite chronological order +func (r *ResourceCleaner) Cleanup() { + for i := len(r.funcs) - 1; i >= 0; i-- { + r.funcs[i]() + } +} diff --git a/internal/resourcestore/resourcestore.go b/internal/resourcestore/resourcestore.go index e4c716484ab..cf4ceec8f56 100644 --- a/internal/resourcestore/resourcestore.go +++ b/internal/resourcestore/resourcestore.go @@ -29,11 +29,11 @@ type ResourceStore struct { // as well as stores function pointers that pertain to how that resource should be cleaned up, // and keeps track of other requests that are watching for the successful creation of this resource. type Resource struct { - resource IdentifiableCreatable - cleanupFuncs []func() - watchers []chan struct{} - stale bool - name string + resource IdentifiableCreatable + cleaner *ResourceCleaner + watchers []chan struct{} + stale bool + name string } // wasPut checks that a resource has been fully defined yet. @@ -83,7 +83,7 @@ func (rc *ResourceStore) Close() { // It runs on a loop, sleeping `sleepTimeBeforeCleanup` between each loop. // A resource will first be marked as stale before being cleaned up. // This means a resource will stay in the store between `sleepTimeBeforeCleanup` and `2*sleepTimeBeforeCleanup`. -// When a resource is cleaned up, it's removed from the store and its cleanupFuncs are called. +// When a resource is cleaned up, it's removed from the store and the cleanup funcs in its cleaner are called. func (rc *ResourceStore) cleanupStaleResources() { for { select { @@ -105,9 +105,7 @@ func (rc *ResourceStore) cleanupStaleResources() { for _, r := range resourcesToReap { logrus.Infof("cleaning up stale resource %s", r.name) - for _, f := range r.cleanupFuncs { - f() - } + r.cleaner.Cleanup() } } } @@ -138,7 +136,7 @@ func (rc *ResourceStore) Get(name string) string { // a newly created resource, and functions to clean up that newly created resource. // It adds the Resource to the ResourceStore. It expects name to be unique, and // returns an error if a duplicate name is detected. -func (rc *ResourceStore) Put(name string, resource IdentifiableCreatable, cleanupFuncs []func()) error { +func (rc *ResourceStore) Put(name string, resource IdentifiableCreatable, cleaner *ResourceCleaner) error { rc.Lock() defer rc.Unlock() @@ -154,7 +152,7 @@ func (rc *ResourceStore) Put(name string, resource IdentifiableCreatable, cleanu } r.resource = resource - r.cleanupFuncs = cleanupFuncs + r.cleaner = cleaner r.name = name // now the resource is created, notify the watchers diff --git a/internal/resourcestore/resourcestore_test.go b/internal/resourcestore/resourcestore_test.go index 0b600dbf046..2e1c0c8298a 100644 --- a/internal/resourcestore/resourcestore_test.go +++ b/internal/resourcestore/resourcestore_test.go @@ -30,14 +30,14 @@ func (e *entry) SetCreated() { var _ = t.Describe("ResourceStore", func() { // Setup the test var ( - sut *resourcestore.ResourceStore - cleanupFuncs []func() - e *entry + sut *resourcestore.ResourceStore + cleaner *resourcestore.ResourceCleaner + e *entry ) Context("no timeout", func() { BeforeEach(func() { sut = resourcestore.New() - cleanupFuncs = make([]func(), 0) + cleaner = resourcestore.NewResourceCleaner() e = &entry{ id: testID, } @@ -49,7 +49,7 @@ var _ = t.Describe("ResourceStore", func() { // Given // When - Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil()) + Expect(sut.Put(testName, e, cleaner)).To(BeNil()) // Then id := sut.Get(testName) @@ -62,14 +62,14 @@ var _ = t.Describe("ResourceStore", func() { // Given // When - Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil()) + Expect(sut.Put(testName, e, cleaner)).To(BeNil()) // Then - Expect(sut.Put(testName, e, cleanupFuncs)).NotTo(BeNil()) + Expect(sut.Put(testName, e, cleaner)).NotTo(BeNil()) }) It("Get should call SetCreated", func() { // When - Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil()) + Expect(sut.Put(testName, e, cleaner)).To(BeNil()) // Then id := sut.Get(testName) @@ -95,7 +95,7 @@ var _ = t.Describe("ResourceStore", func() { } // When - Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil()) + Expect(sut.Put(testName, e, cleaner)).To(BeNil()) // Then Expect(waitWatcherSet(watcher1)).To(BeTrue()) Expect(waitWatcherSet(watcher2)).To(BeTrue()) @@ -103,7 +103,7 @@ var _ = t.Describe("ResourceStore", func() { }) Context("with timeout", func() { BeforeEach(func() { - cleanupFuncs = make([]func(), 0) + cleaner = resourcestore.NewResourceCleaner() e = &entry{ id: testID, } @@ -117,7 +117,7 @@ var _ = t.Describe("ResourceStore", func() { sut = resourcestore.NewWithTimeout(timeout) timedOutChan := make(chan bool) - cleanupFuncs = append(cleanupFuncs, func() { + cleaner.Add(func() { timedOutChan <- true }) go func() { @@ -126,7 +126,7 @@ var _ = t.Describe("ResourceStore", func() { }() // When - Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil()) + Expect(sut.Put(testName, e, cleaner)).To(BeNil()) // Then didStoreCallTimeoutFunc := <-timedOutChan diff --git a/server/container_create.go b/server/container_create.go index 9ed863b4b05..13e5b727475 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -13,6 +13,7 @@ import ( "github.com/containers/storage/pkg/stringid" "github.com/cri-o/cri-o/internal/lib/sandbox" "github.com/cri-o/cri-o/internal/log" + "github.com/cri-o/cri-o/internal/resourcestore" "github.com/cri-o/cri-o/internal/storage" "github.com/cri-o/cri-o/pkg/config" "github.com/cri-o/cri-o/pkg/container" @@ -425,15 +426,13 @@ func (s *Server) CreateContainer(ctx context.Context, req *types.CreateContainer return nil, errors.Wrap(err, "setting container name and ID") } - cleanupFuncs := make([]func(), 0) + resourceCleaner := resourcestore.NewResourceCleaner() defer func() { // no error, no need to cleanup if retErr == nil || isContextError(retErr) { return } - for i := len(cleanupFuncs) - 1; i >= 0; i-- { - cleanupFuncs[i]() - } + resourceCleaner.Cleanup() }() if _, err = s.ReserveContainerName(ctr.ID(), ctr.Name()); err != nil { @@ -444,7 +443,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *types.CreateContainer return nil, errors.Wrapf(err, resourceErr.Error()) } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "createCtr: releasing container name %s", ctr.Name()) s.ReleaseContainerName(ctr.Name()) }) @@ -453,7 +452,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *types.CreateContainer if err != nil { return nil, err } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "createCtr: deleting container %s from storage", ctr.ID()) err2 := s.StorageRuntimeServer().DeleteContainer(ctr.ID()) if err2 != nil { @@ -462,7 +461,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *types.CreateContainer }) s.addContainer(newContainer) - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "createCtr: removing container %s", newContainer.ID()) s.removeContainer(newContainer) }) @@ -470,7 +469,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *types.CreateContainer if err := s.CtrIDIndex().Add(ctr.ID()); err != nil { return nil, err } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "createCtr: deleting container ID %s from idIndex", ctr.ID()) if err := s.CtrIDIndex().Delete(ctr.ID()); err != nil { log.Warnf(ctx, "couldn't delete ctr id %s from idIndex", ctr.ID()) @@ -485,7 +484,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *types.CreateContainer if err := s.createContainerPlatform(ctx, newContainer, sb.CgroupParent(), mappings); err != nil { return nil, err } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { if retErr != nil { log.Infof(ctx, "createCtr: removing container ID %s from runtime", ctr.ID()) if err := s.Runtime().DeleteContainer(ctx, newContainer); err != nil { @@ -499,7 +498,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *types.CreateContainer } if isContextError(ctx.Err()) { - if err := s.resourceStore.Put(ctr.Name(), newContainer, cleanupFuncs); err != nil { + if err := s.resourceStore.Put(ctr.Name(), newContainer, resourceCleaner); err != nil { log.Errorf(ctx, "createCtr: failed to save progress of container %s: %v", newContainer.ID(), err) } log.Infof(ctx, "createCtr: context was either canceled or the deadline was exceeded: %v", ctx.Err()) diff --git a/server/sandbox_run_linux.go b/server/sandbox_run_linux.go index d464c76385a..38523c4ae0c 100644 --- a/server/sandbox_run_linux.go +++ b/server/sandbox_run_linux.go @@ -24,6 +24,7 @@ import ( libsandbox "github.com/cri-o/cri-o/internal/lib/sandbox" "github.com/cri-o/cri-o/internal/log" oci "github.com/cri-o/cri-o/internal/oci" + "github.com/cri-o/cri-o/internal/resourcestore" ann "github.com/cri-o/cri-o/pkg/annotations" libconfig "github.com/cri-o/cri-o/pkg/config" "github.com/cri-o/cri-o/pkg/sandbox" @@ -307,15 +308,13 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ return nil, errors.Wrap(err, "setting pod sandbox name and id") } - cleanupFuncs := make([]func(), 0) + resourceCleaner := resourcestore.NewResourceCleaner() defer func() { // no error, no need to cleanup if retErr == nil || isContextError(retErr) { return } - for i := len(cleanupFuncs) - 1; i >= 0; i-- { - cleanupFuncs[i]() - } + resourceCleaner.Cleanup() }() if _, err := s.ReservePodName(sbox.ID(), sbox.Name()); err != nil { @@ -326,7 +325,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ return nil, errors.Wrapf(err, resourceErr.Error()) } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: releasing pod sandbox name: %s", sbox.Name()) s.ReleasePodName(sbox.Name()) }) @@ -359,7 +358,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ if err != nil { return nil, err } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: releasing container name: %s", containerName) s.ReleaseContainerName(containerName) }) @@ -397,7 +396,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ if err != nil { return nil, fmt.Errorf("error creating pod sandbox with name %q: %v", sbox.Name(), err) } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: removing pod sandbox from storage: %s", sbox.ID()) if err2 := s.StorageRuntimeServer().RemovePodSandbox(sbox.ID()); err2 != nil { log.Warnf(ctx, "could not cleanup pod sandbox %q: %v", sbox.ID(), err2) @@ -540,7 +539,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ return nil, err } pathsToChown = append(pathsToChown, shmPath) - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: unmounting shmPath for sandbox %s", sbox.ID()) if err2 := unix.Unmount(shmPath, unix.MNT_DETACH); err2 != nil { log.Warnf(ctx, "failed to unmount shm for sandbox: %v", err2) @@ -566,7 +565,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ return nil, err } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: deleting container ID from idIndex for sandbox %s", sbox.ID()) if err2 := s.CtrIDIndex().Delete(sbox.ID()); err2 != nil { log.Warnf(ctx, "could not delete ctr id %s from idIndex", sbox.ID()) @@ -663,7 +662,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ if err := s.addSandbox(sb); err != nil { return nil, err } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: removing pod sandbox %s", sbox.ID()) if err := s.removeSandbox(sbox.ID()); err != nil { log.Warnf(ctx, "could not remove pod sandbox: %v", err) @@ -674,7 +673,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ return nil, err } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: deleting pod ID %s from idIndex", sbox.ID()) if err := s.PodIDIndex().Delete(sbox.ID()); err != nil { log.Warnf(ctx, "could not delete pod id %s from idIndex", sbox.ID()) @@ -694,7 +693,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ // set up namespaces nsCleanupFuncs, err := s.configureGeneratorForSandboxNamespaces(hostNetwork, hostIPC, hostPID, sandboxIDMappings, sysctls, sb, g) // We want to cleanup after ourselves if we are managing any namespaces and fail in this function. - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: cleaning up namespaces after failing to run sandbox %s", sbox.ID()) for idx := range nsCleanupFuncs { if err2 := nsCleanupFuncs[idx](); err2 != nil { @@ -714,7 +713,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ if err != nil { return nil, err } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: stopping network for sandbox %s", sb.ID()) if err2 := s.networkStop(ctx, sb); err2 != nil { log.Errorf(ctx, "error stopping network on cleanup: %v", err2) @@ -749,7 +748,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ if err != nil { return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, sb.Name(), sbox.ID(), err) } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: stopping storage container for sandbox %s", sbox.ID()) if err2 := s.StorageRuntimeServer().StopContainer(sbox.ID()); err2 != nil { log.Warnf(ctx, "could not stop storage container: %v: %v", sbox.ID(), err2) @@ -887,7 +886,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ } s.addInfraContainer(container) - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { log.Infof(ctx, "runSandbox: removing infra container %s", container.ID()) s.removeInfraContainer(container) }) @@ -912,7 +911,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ return nil, err } - cleanupFuncs = append(cleanupFuncs, func() { + resourceCleaner.Add(func() { // Clean-up steps from RemovePodSanbox log.Infof(ctx, "runSandbox: stopping container %s", container.ID()) if err2 := s.Runtime().StopContainer(ctx, container, int64(10)); err2 != nil { @@ -941,7 +940,7 @@ func (s *Server) runPodSandbox(ctx context.Context, req *types.RunPodSandboxRequ sb.AddIPs(ips) if isContextError(ctx.Err()) { - if err := s.resourceStore.Put(sbox.Name(), sb, cleanupFuncs); err != nil { + if err := s.resourceStore.Put(sbox.Name(), sb, resourceCleaner); err != nil { log.Errorf(ctx, "runSandbox: failed to save progress of sandbox %s: %v", sbox.ID(), err) } log.Infof(ctx, "runSandbox: context was either canceled or the deadline was exceeded: %v", ctx.Err())