cri-o · openshift-merge-robot · Feb 15, 2021 · Feb 2, 2021 · Feb 2, 2021 · Feb 2, 2021
diff --git a/internal/resourcestore/resourcestore.go b/internal/resourcestore/resourcestore.go
@@ -20,6 +20,8 @@ const sleepTimeBeforeCleanup = 1 * time.Minute
 type ResourceStore struct {
 	resources map[string]*Resource
 	timeout   time.Duration
+	closeChan chan struct{}
+	closed    bool
 	sync.Mutex
 }
 
@@ -34,6 +36,12 @@ type Resource struct {
 	name         string
 }
 
+// wasPut checks that a resource has been fully defined yet.
+// This is defined as a resource that only has watchers, but no associated resource.
+func (r *Resource) wasPut() bool {
+	return r != nil && r.resource != nil
+}
+
 // IdentifiableCreatable are the qualities needed by the caller of the resource.
 // Once a resource is retrieved, SetCreated() will be called, indicating to the server
 // that resource is ready to be listed and operated upon, and ID() will be used to identify the
@@ -53,12 +61,23 @@ func New() *ResourceStore {
 func NewWithTimeout(timeout time.Duration) *ResourceStore {
 	rc := &ResourceStore{
 		resources: make(map[string]*Resource),
+		closeChan: make(chan struct{}, 1),
 		timeout:   timeout,
 	}
 	go rc.cleanupStaleResources()
 	return rc
 }
 
+func (rc *ResourceStore) Close() {
+	rc.Lock()
+	defer rc.Unlock()
+	if rc.closed {
+		return
+	}
+	close(rc.closeChan)
+	rc.closed = true
+}
+
 // cleanupStaleResources is responsible for cleaning up resources that haven't been gotten
 // from the store.
 // It runs on a loop, sleeping `sleepTimeBeforeCleanup` between each loop.
@@ -67,7 +86,11 @@ func NewWithTimeout(timeout time.Duration) *ResourceStore {
 // When a resource is cleaned up, it's removed from the store and its cleanupFuncs are called.
 func (rc *ResourceStore) cleanupStaleResources() {
 	for {
-		time.Sleep(rc.timeout)
+		select {
+		case <-rc.closeChan:
+			return
+		case <-time.After(rc.timeout):
+		}
 		resourcesToReap := []*Resource{}
 		rc.Lock()
 		for name, r := range rc.resources {
@@ -101,6 +124,11 @@ func (rc *ResourceStore) Get(name string) string {
 	if !ok {
 		return ""
 	}
+	// It is possible there are existing watchers,
+	// but no resource created yet
+	if !r.wasPut() {
+		return ""
+	}
 	delete(rc.resources, name)
 	r.resource.SetCreated()
 	return r.resource.ID()
@@ -121,7 +149,7 @@ func (rc *ResourceStore) Put(name string, resource IdentifiableCreatable, cleanu
 		rc.resources[name] = r
 	}
 	// make sure the resource hasn't already been added to the store
-	if r.resource != nil || r.cleanupFuncs != nil {
+	if ok && r.wasPut() {
 		return errors.Errorf("failed to add entry %s to ResourceStore; entry already exists", name)
 	}
 
@@ -136,7 +164,9 @@ func (rc *ResourceStore) Put(name string, resource IdentifiableCreatable, cleanu
 	return nil
 }
 
-// WatcherForResource looks up a Resource by name, and gives it a watcher if it's found.
+// WatcherForResource looks up a Resource by name, and gives it a watcher.
+// If no entry exists for that resource, a placeholder is created and a watcher is given to that
+// placeholder resource.
 // A watcher can be used for concurrent processes to wait for the resource to be created.
 // This is useful for situations where clients retry requests quickly after they "fail" because
 // they've taken too long. Adding a watcher allows the server to slow down the client, but still

diff --git a/internal/resourcestore/resourcestore_test.go b/internal/resourcestore/resourcestore_test.go
@@ -34,81 +34,106 @@ var _ = t.Describe("ResourceStore", func() {
 		cleanupFuncs []func()
 		e            *entry
 	)
-	BeforeEach(func() {
-		sut = resourcestore.New()
-		cleanupFuncs = make([]func(), 0)
-		e = &entry{
-			id: testID,
-		}
-	})
-	It("Put should be able to get resource after adding", func() {
-		// Given
+	Context("no timeout", func() {
+		BeforeEach(func() {
+			sut = resourcestore.New()
+			cleanupFuncs = make([]func(), 0)
+			e = &entry{
+				id: testID,
+			}
+		})
+		AfterEach(func() {
+			sut.Close()
+		})
+		It("Put should be able to get resource after adding", func() {
+			// Given
 
-		// When
-		Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil())
+			// When
+			Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil())
 
-		// Then
-		id := sut.Get(testName)
-		Expect(id).To(Equal(e.id))
+			// Then
+			id := sut.Get(testName)
+			Expect(id).To(Equal(e.id))
 
-		id = sut.Get(testName)
-		Expect(id).To(BeEmpty())
-	})
-	It("Put should fail to readd resource", func() {
-		// Given
+			id = sut.Get(testName)
+			Expect(id).To(BeEmpty())
+		})
+		It("Put should fail to readd resource", func() {
+			// Given
 
-		// When
-		Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil())
+			// When
+			Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil())
 
-		// Then
-		Expect(sut.Put(testName, e, cleanupFuncs)).NotTo(BeNil())
-	})
-	It("Get should call SetCreated", func() {
-		// When
-		Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil())
-
-		// Then
-		id := sut.Get(testName)
-		Expect(id).To(Equal(e.id))
-		Expect(e.created).To(BeTrue())
-	})
-})
+			// Then
+			Expect(sut.Put(testName, e, cleanupFuncs)).NotTo(BeNil())
+		})
+		It("Get should call SetCreated", func() {
+			// When
+			Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil())
+
+			// Then
+			id := sut.Get(testName)
+			Expect(id).To(Equal(e.id))
+			Expect(e.created).To(BeTrue())
+		})
+		It("Should not fail to Get after retrieving Watcher", func() {
+			// When
+			_ = sut.WatcherForResource(testName)
 
-var _ = t.Describe("ResourceStore and timeout", func() {
-	// Setup the test
-	var (
-		sut          *resourcestore.ResourceStore
-		cleanupFuncs []func()
-		e            *entry
-	)
-	BeforeEach(func() {
-		cleanupFuncs = make([]func(), 0)
-		e = &entry{
-			id: testID,
-		}
+			// Then
+			id := sut.Get(testName)
+			Expect(id).To(BeEmpty())
+		})
+		It("Should be able to get multiple Watchers", func() {
+			// Given
+			watcher1 := sut.WatcherForResource(testName)
+			watcher2 := sut.WatcherForResource(testName)
+
+			waitWatcherSet := func(watcher chan struct{}) bool {
+				<-watcher
+				return true
+			}
+
+			// When
+			Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil())
+			// Then
+			Expect(waitWatcherSet(watcher1)).To(BeTrue())
+			Expect(waitWatcherSet(watcher2)).To(BeTrue())
+		})
 	})
-	It("Put should call cleanup funcs after timeout", func() {
-		// Given
-		timeout := 2 * time.Second
-		sut = resourcestore.NewWithTimeout(timeout)
-
-		timedOutChan := make(chan bool)
-		cleanupFuncs = append(cleanupFuncs, func() {
-			timedOutChan <- true
+	Context("with timeout", func() {
+		BeforeEach(func() {
+			cleanupFuncs = make([]func(), 0)
+			e = &entry{
+				id: testID,
+			}
+		})
+		AfterEach(func() {
+			sut.Close()
+		})
+		It("Put should call cleanup funcs after timeout", func() {
+			// Given
+			timeout := 2 * time.Second
+			sut = resourcestore.NewWithTimeout(timeout)
+
+			timedOutChan := make(chan bool)
+			cleanupFuncs = append(cleanupFuncs, func() {
+				timedOutChan <- true
+			})
+			go func() {
+				time.Sleep(timeout * 3)
+				timedOutChan <- false
+			}()
+
+			// When
+			Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil())
+
+			// Then
+			didStoreCallTimeoutFunc := <-timedOutChan
+			Expect(didStoreCallTimeoutFunc).To(Equal(true))
+
+			id := sut.Get(testName)
+			Expect(id).To(BeEmpty())
 		})
-		go func() {
-			time.Sleep(timeout * 3)
-			timedOutChan <- false
-		}()
-
-		// When
-		Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil())
-
-		// Then
-		didStoreCallTimeoutFunc := <-timedOutChan
-		Expect(didStoreCallTimeoutFunc).To(Equal(true))
-
-		id := sut.Get(testName)
-		Expect(id).To(BeEmpty())
 	})
 })
diff --git a/server/server.go b/server/server.go
@@ -276,6 +276,7 @@ func (s *Server) Shutdown(ctx context.Context) error {
 	// notice this won't trigger just on system halt but also on normal
 	// crio.service restart!!!
 	s.cleanupSandboxesOnShutdown(ctx)
+	s.resourceStore.Close()
 
 	return s.ContainerServer.Shutdown()
 }