Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Fix goroutine leak in federation service controller #33359

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
k8s-github-robot merged 2 commits into from
Sep 24, 2016
Merged

Fix goroutine leak in federation service controller #33359

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d8ff487
Fix goroutine leak in federation service controller
Sep 21, 2016
690a06b
Handle review comments for Fix goroutine leak in federation service c…
Sep 23, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion federation/pkg/federation-controller/service/endpoint_helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,34 @@ import (
// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (sc *ServiceController) clusterEndpointWorker() {
fedClient := sc.federationClient
// process all pending events in endpointWorkerDoneChan
ForLoop:
for {
select {
case clusterName := <-sc.endpointWorkerDoneChan:
Copy link
Copy Markdown
Contributor

@madhusudancs madhusudancs Sep 23, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function is important here.

Notice the defer statement inside this block. This func() exists for that sole reason. At the end of each loop, we need to say we are done for that key, so we defer cache.endpointQueue.Done(key). If you remove the func() {} wrapping, defer will be run at the end of the enclosing goroutine and the keys won't be removed from the queue until then. So I don't think you should be removing the func() {} wrapping.

Copy link
Copy Markdown
Author

@shashidharatd shashidharatd Sep 23, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agreed, missed that, will change

sc.endpointWorkerMap[clusterName] = false
default:
// non-blocking, comes here if all existing events are processed
break ForLoop
}
}

for clusterName, cache := range sc.clusterCache.clientMap {
workerExist, found := sc.endpointWorkerMap[clusterName]
if found && workerExist {
continue
}

// create a worker only if the previous worker has finished and gone out of scope
go func(cache *clusterCache, clusterName string) {
fedClient := sc.federationClient
for {
func() {
key, quit := cache.endpointQueue.Get()
// update endpoint cache
if quit {
// send signal that current worker has finished tasks and is going out of scope
sc.endpointWorkerDoneChan <- clusterName
return
}
defer cache.endpointQueue.Done(key)
Expand All @@ -49,6 +69,7 @@ func (sc *ServiceController) clusterEndpointWorker() {
}()
}
}(cache, clusterName)
sc.endpointWorkerMap[clusterName] = true
}
}

Expand Down
25 changes: 23 additions & 2 deletions federation/pkg/federation-controller/service/service_helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,23 +35,44 @@ import (
// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (sc *ServiceController) clusterServiceWorker() {
fedClient := sc.federationClient
// process all pending events in serviceWorkerDoneChan
ForLoop:
for {
select {
case clusterName := <-sc.serviceWorkerDoneChan:
sc.serviceWorkerMap[clusterName] = false
default:
// non-blocking, comes here if all existing events are processed
break ForLoop
}
}

for clusterName, cache := range sc.clusterCache.clientMap {
workerExist, found := sc.serviceWorkerMap[clusterName]
if found && workerExist {
Copy link
Copy Markdown
Contributor

@madhusudancs madhusudancs Sep 23, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comment as above.

continue
}

// create a worker only if the previous worker has finished and gone out of scope
go func(cache *clusterCache, clusterName string) {
fedClient := sc.federationClient
for {
func() {
key, quit := cache.serviceQueue.Get()
defer cache.serviceQueue.Done(key)
if quit {
// send signal that current worker has finished tasks and is going out of scope
sc.serviceWorkerDoneChan <- clusterName
return
}
defer cache.serviceQueue.Done(key)
err := sc.clusterCache.syncService(key.(string), clusterName, cache, sc.serviceCache, fedClient, sc)
if err != nil {
glog.Errorf("Failed to sync service: %+v", err)
}
}()
}
}(cache, clusterName)
sc.serviceWorkerMap[clusterName] = true
}
}

Expand Down
17 changes: 17 additions & 0 deletions federation/pkg/federation-controller/service/servicecontroller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ const (
UserAgentName = "federation-service-controller"
KubeAPIQPS = 20.0
KubeAPIBurst = 30

maxNoOfClusters = 100
)

type cachedService struct {
Expand Down Expand Up @@ -118,6 +120,16 @@ type ServiceController struct {
// services that need to be synced
queue *workqueue.Type
knownClusterSet sets.String
// endpoint worker map contains all the clusters registered with an indication that worker exist
// key clusterName
endpointWorkerMap map[string]bool
// channel for worker to signal that it is going out of existence
endpointWorkerDoneChan chan string
// service worker map contains all the clusters registered with an indication that worker exist
// key clusterName
serviceWorkerMap map[string]bool
// channel for worker to signal that it is going out of existence
serviceWorkerDoneChan chan string
}

// New returns a new service controller to keep DNS provider service resources
Expand Down Expand Up @@ -204,6 +216,11 @@ func New(federationClient federation_release_1_4.Interface, dns dnsprovider.Inte
},
},
)

s.endpointWorkerMap = make(map[string]bool)
s.serviceWorkerMap = make(map[string]bool)
s.endpointWorkerDoneChan = make(chan string, maxNoOfClusters)
s.serviceWorkerDoneChan = make(chan string, maxNoOfClusters)
return s
}

Expand Down