Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1184800

Browse files
author
Kubernetes Submit Queue
authored
Merge pull request #30730 from janetkuo/prevent-overlapping-deployment
Automatic merge from submit-queue Handle overlapping deployments gracefully Fixes #30028
2 parents feb4d20 + c5cef18 commit 1184800

10 files changed

Lines changed: 357 additions & 41 deletions

File tree

‎pkg/client/cache/listers.go‎

Lines changed: 82 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -285,12 +285,12 @@ func (s *StoreToReplicationControllerLister) GetPodControllers(pod *api.Pod) (co
285285

286286
// StoreToDeploymentLister gives a store List and Exists methods. The store must contain only Deployments.
287287
type StoreToDeploymentLister struct {
288-
Store
288+
Indexer
289289
}
290290

291291
// Exists checks if the given deployment exists in the store.
292292
func (s *StoreToDeploymentLister) Exists(deployment *extensions.Deployment) (bool, error) {
293-
_, exists, err := s.Store.Get(deployment)
293+
_, exists, err := s.Indexer.Get(deployment)
294294
if err != nil {
295295
return false, err
296296
}
@@ -300,40 +300,112 @@ func (s *StoreToDeploymentLister) Exists(deployment *extensions.Deployment) (boo
300300
// StoreToDeploymentLister lists all deployments in the store.
301301
// TODO: converge on the interface in pkg/client
302302
func (s *StoreToDeploymentLister) List() (deployments []extensions.Deployment, err error) {
303-
for _, c := range s.Store.List() {
303+
for _, c := range s.Indexer.List() {
304304
deployments = append(deployments, *(c.(*extensions.Deployment)))
305305
}
306306
return deployments, nil
307307
}
308308

309309
// GetDeploymentsForReplicaSet returns a list of deployments managing a replica set. Returns an error only if no matching deployments are found.
310310
func (s *StoreToDeploymentLister) GetDeploymentsForReplicaSet(rs *extensions.ReplicaSet) (deployments []extensions.Deployment, err error) {
311-
var d extensions.Deployment
312-
313311
if len(rs.Labels) == 0 {
314312
err = fmt.Errorf("no deployments found for ReplicaSet %v because it has no labels", rs.Name)
315313
return
316314
}
317315

318316
// TODO: MODIFY THIS METHOD so that it checks for the podTemplateSpecHash label
319-
for _, m := range s.Store.List() {
320-
d = *m.(*extensions.Deployment)
321-
if d.Namespace != rs.Namespace {
317+
dList, err := s.Deployments(rs.Namespace).List(labels.Everything())
318+
if err != nil {
319+
return
320+
}
321+
for _, d := range dList {
322+
selector, err := unversioned.LabelSelectorAsSelector(d.Spec.Selector)
323+
if err != nil {
324+
return nil, fmt.Errorf("invalid label selector: %v", err)
325+
}
326+
// If a deployment with a nil or empty selector creeps in, it should match nothing, not everything.
327+
if selector.Empty() || !selector.Matches(labels.Set(rs.Labels)) {
322328
continue
323329
}
330+
deployments = append(deployments, d)
331+
}
332+
if len(deployments) == 0 {
333+
err = fmt.Errorf("could not find deployments set for ReplicaSet %s in namespace %s with labels: %v", rs.Name, rs.Namespace, rs.Labels)
334+
}
335+
return
336+
}
337+
338+
type storeToDeploymentNamespacer struct {
339+
indexer Indexer
340+
namespace string
341+
}
342+
343+
// storeToDeploymentNamespacer lists deployments under its namespace in the store.
344+
func (s storeToDeploymentNamespacer) List(selector labels.Selector) (deployments []extensions.Deployment, err error) {
345+
if s.namespace == api.NamespaceAll {
346+
for _, m := range s.indexer.List() {
347+
d := *(m.(*extensions.Deployment))
348+
if selector.Matches(labels.Set(d.Labels)) {
349+
deployments = append(deployments, d)
350+
}
351+
}
352+
return
353+
}
354+
355+
key := &extensions.Deployment{ObjectMeta: api.ObjectMeta{Namespace: s.namespace}}
356+
items, err := s.indexer.Index(NamespaceIndex, key)
357+
if err != nil {
358+
// Ignore error; do slow search without index.
359+
glog.Warningf("can not retrieve list of objects using index : %v", err)
360+
for _, m := range s.indexer.List() {
361+
d := *(m.(*extensions.Deployment))
362+
if s.namespace == d.Namespace && selector.Matches(labels.Set(d.Labels)) {
363+
deployments = append(deployments, d)
364+
}
365+
}
366+
return deployments, nil
367+
}
368+
for _, m := range items {
369+
d := *(m.(*extensions.Deployment))
370+
if selector.Matches(labels.Set(d.Labels)) {
371+
deployments = append(deployments, d)
372+
}
373+
}
374+
return
375+
}
324376

377+
func (s *StoreToDeploymentLister) Deployments(namespace string) storeToDeploymentNamespacer {
378+
return storeToDeploymentNamespacer{s.Indexer, namespace}
379+
}
380+
381+
// GetDeploymentsForPods returns a list of deployments managing a pod. Returns an error only if no matching deployments are found.
382+
func (s *StoreToDeploymentLister) GetDeploymentsForPod(pod *api.Pod) (deployments []extensions.Deployment, err error) {
383+
if len(pod.Labels) == 0 {
384+
err = fmt.Errorf("no deployments found for Pod %v because it has no labels", pod.Name)
385+
return
386+
}
387+
388+
if len(pod.Labels[extensions.DefaultDeploymentUniqueLabelKey]) == 0 {
389+
return
390+
}
391+
392+
dList, err := s.Deployments(pod.Namespace).List(labels.Everything())
393+
if err != nil {
394+
return
395+
}
396+
for _, d := range dList {
325397
selector, err := unversioned.LabelSelectorAsSelector(d.Spec.Selector)
326398
if err != nil {
327399
return nil, fmt.Errorf("invalid label selector: %v", err)
328400
}
329401
// If a deployment with a nil or empty selector creeps in, it should match nothing, not everything.
330-
if selector.Empty() || !selector.Matches(labels.Set(rs.Labels)) {
402+
if selector.Empty() || !selector.Matches(labels.Set(pod.Labels)) {
331403
continue
332404
}
333405
deployments = append(deployments, d)
334406
}
335407
if len(deployments) == 0 {
336-
err = fmt.Errorf("could not find deployments set for ReplicaSet %s in namespace %s with labels: %v", rs.Name, rs.Namespace, rs.Labels)
408+
err = fmt.Errorf("could not find deployments set for Pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels)
337409
}
338410
return
339411
}

‎pkg/controller/deployment/deployment_controller.go‎

Lines changed: 81 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ package deployment
2323
import (
2424
"fmt"
2525
"reflect"
26+
"sort"
2627
"time"
2728

2829
"github.com/golang/glog"
@@ -36,6 +37,7 @@ import (
3637
"k8s.io/kubernetes/pkg/controller"
3738
"k8s.io/kubernetes/pkg/controller/deployment/util"
3839
"k8s.io/kubernetes/pkg/controller/framework"
40+
"k8s.io/kubernetes/pkg/labels"
3941
"k8s.io/kubernetes/pkg/runtime"
4042
"k8s.io/kubernetes/pkg/util/metrics"
4143
utilruntime "k8s.io/kubernetes/pkg/util/runtime"
@@ -108,7 +110,7 @@ func NewDeploymentController(client clientset.Interface, resyncPeriod controller
108110
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
109111
}
110112

111-
dc.dStore.Store, dc.dController = framework.NewInformer(
113+
dc.dStore.Indexer, dc.dController = framework.NewIndexerInformer(
112114
&cache.ListWatch{
113115
ListFunc: func(options api.ListOptions) (runtime.Object, error) {
114116
return dc.client.Extensions().Deployments(api.NamespaceAll).List(options)
@@ -125,6 +127,7 @@ func NewDeploymentController(client clientset.Interface, resyncPeriod controller
125127
// This will enter the sync loop and no-op, because the deployment has been deleted from the store.
126128
DeleteFunc: dc.deleteDeploymentNotification,
127129
},
130+
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
128131
)
129132

130133
dc.rsStore.Store, dc.rsController = framework.NewInformer(
@@ -252,7 +255,6 @@ func (dc *DeploymentController) addReplicaSet(obj interface{}) {
252255
}
253256

254257
// getDeploymentForReplicaSet returns the deployment managing the given ReplicaSet.
255-
// TODO: Surface that we are ignoring multiple deployments for a given ReplicaSet.
256258
func (dc *DeploymentController) getDeploymentForReplicaSet(rs *extensions.ReplicaSet) *extensions.Deployment {
257259
deployments, err := dc.dStore.GetDeploymentsForReplicaSet(rs)
258260
if err != nil || len(deployments) == 0 {
@@ -262,8 +264,11 @@ func (dc *DeploymentController) getDeploymentForReplicaSet(rs *extensions.Replic
262264
// Because all ReplicaSet's belonging to a deployment should have a unique label key,
263265
// there should never be more than one deployment returned by the above method.
264266
// If that happens we should probably dynamically repair the situation by ultimately
265-
// trying to clean up one of the controllers, for now we just return one of the two,
266-
// likely randomly.
267+
// trying to clean up one of the controllers, for now we just return the older one
268+
if len(deployments) > 1 {
269+
sort.Sort(util.BySelectorLastUpdateTime(deployments))
270+
glog.Errorf("user error! more than one deployment is selecting replica set %s/%s with labels: %#v, returning %s/%s", rs.Namespace, rs.Name, rs.Labels, deployments[0].Namespace, deployments[0].Name)
271+
}
267272
return &deployments[0]
268273
}
269274

@@ -321,22 +326,20 @@ func (dc *DeploymentController) deleteReplicaSet(obj interface{}) {
321326
}
322327
}
323328

324-
// getDeploymentForPod returns the deployment managing the ReplicaSet that manages the given Pod.
325-
// TODO: Surface that we are ignoring multiple deployments for a given Pod.
329+
// getDeploymentForPod returns the deployment that manages the given Pod.
330+
// If there are multiple deployments for a given Pod, only return the oldest one.
326331
func (dc *DeploymentController) getDeploymentForPod(pod *api.Pod) *extensions.Deployment {
327-
rss, err := dc.rsStore.GetPodReplicaSets(pod)
328-
if err != nil {
329-
glog.V(4).Infof("Error: %v. No ReplicaSets found for pod %v, deployment controller will avoid syncing.", err, pod.Name)
332+
deployments, err := dc.dStore.GetDeploymentsForPod(pod)
333+
if err != nil || len(deployments) == 0 {
334+
glog.V(4).Infof("Error: %v. No deployment found for Pod %v, deployment controller will avoid syncing.", err, pod.Name)
330335
return nil
331336
}
332-
for _, rs := range rss {
333-
deployments, err := dc.dStore.GetDeploymentsForReplicaSet(&rs)
334-
if err == nil && len(deployments) > 0 {
335-
return &deployments[0]
336-
}
337+
338+
if len(deployments) > 1 {
339+
sort.Sort(util.BySelectorLastUpdateTime(deployments))
340+
glog.Errorf("user error! more than one deployment is selecting pod %s/%s with labels: %#v, returning %s/%s", pod.Namespace, pod.Name, pod.Labels, deployments[0].Namespace, deployments[0].Name)
337341
}
338-
glog.V(4).Infof("No deployments found for pod %v, deployment controller will avoid syncing.", pod.Name)
339-
return nil
342+
return &deployments[0]
340343
}
341344

342345
// When a pod is created, ensure its controller syncs
@@ -407,15 +410,28 @@ func (dc *DeploymentController) enqueueDeployment(deployment *extensions.Deploym
407410
return
408411
}
409412

410-
// TODO: Handle overlapping deployments better. Either disallow them at admission time or
411-
// deterministically avoid syncing deployments that fight over ReplicaSet's. Currently, we
412-
// only ensure that the same deployment is synced for a given ReplicaSet. When we
413-
// periodically relist all deployments there will still be some ReplicaSet instability. One
414-
// way to handle this is by querying the store for all deployments that this deployment
415-
// overlaps, as well as all deployments that overlap this deployments, and sorting them.
416413
dc.queue.Add(key)
417414
}
418415

416+
func (dc *DeploymentController) markDeploymentOverlap(deployment *extensions.Deployment, withDeployment string) (*extensions.Deployment, error) {
417+
if deployment.Annotations[util.OverlapAnnotation] == withDeployment {
418+
return deployment, nil
419+
}
420+
if deployment.Annotations == nil {
421+
deployment.Annotations = make(map[string]string)
422+
}
423+
deployment.Annotations[util.OverlapAnnotation] = withDeployment
424+
return dc.client.Extensions().Deployments(deployment.Namespace).Update(deployment)
425+
}
426+
427+
func (dc *DeploymentController) clearDeploymentOverlap(deployment *extensions.Deployment) (*extensions.Deployment, error) {
428+
if len(deployment.Annotations[util.OverlapAnnotation]) == 0 {
429+
return deployment, nil
430+
}
431+
delete(deployment.Annotations, util.OverlapAnnotation)
432+
return dc.client.Extensions().Deployments(deployment.Namespace).Update(deployment)
433+
}
434+
419435
// worker runs a worker thread that just dequeues items, processes them, and marks them done.
420436
// It enforces that the syncHandler is never invoked concurrently with the same key.
421437
func (dc *DeploymentController) worker() {
@@ -463,7 +479,7 @@ func (dc *DeploymentController) syncDeployment(key string) error {
463479
glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Now().Sub(startTime))
464480
}()
465481

466-
obj, exists, err := dc.dStore.Store.GetByKey(key)
482+
obj, exists, err := dc.dStore.Indexer.GetByKey(key)
467483
if err != nil {
468484
glog.Infof("Unable to retrieve deployment %v from store: %v", key, err)
469485
return err
@@ -491,6 +507,11 @@ func (dc *DeploymentController) syncDeployment(key string) error {
491507
return dc.syncStatusOnly(d)
492508
}
493509

510+
// Handle overlapping deployments by deterministically avoid syncing deployments that fight over ReplicaSets.
511+
if err = dc.handleOverlap(d); err != nil {
512+
return err
513+
}
514+
494515
if d.Spec.Paused {
495516
return dc.sync(d)
496517
}
@@ -518,3 +539,40 @@ func (dc *DeploymentController) syncDeployment(key string) error {
518539
}
519540
return fmt.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type)
520541
}
542+
543+
// handleOverlap relists all deployment in the same namespace for overlaps, and avoid syncing
544+
// the newer overlapping ones (only sync the oldest one). New/old is determined by when the
545+
// deployment's selector is last updated.
546+
func (dc *DeploymentController) handleOverlap(d *extensions.Deployment) error {
547+
selector, err := unversioned.LabelSelectorAsSelector(d.Spec.Selector)
548+
if err != nil {
549+
return fmt.Errorf("deployment %s/%s has invalid label selector: %v", d.Namespace, d.Name, err)
550+
}
551+
deployments, err := dc.dStore.Deployments(d.Namespace).List(labels.Everything())
552+
if err != nil {
553+
return fmt.Errorf("error listing deployments in namespace %s: %v", d.Namespace, err)
554+
}
555+
overlapping := false
556+
for i := range deployments {
557+
other := &deployments[i]
558+
if !selector.Empty() && selector.Matches(labels.Set(other.Spec.Template.Labels)) && d.UID != other.UID {
559+
overlapping = true
560+
// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
561+
d, _ = dc.markDeploymentOverlap(d, other.Name)
562+
other, _ = dc.markDeploymentOverlap(other, d.Name)
563+
// Skip syncing this one if older overlapping one is found
564+
// TODO: figure out a better way to determine which deployment to skip,
565+
// either with controller reference, or with validation.
566+
// Using oldest active replica set to determine which deployment to skip wouldn't make much difference,
567+
// since new replica set hasn't been created after selector update
568+
if util.SelectorUpdatedBefore(other, d) {
569+
return fmt.Errorf("found deployment %s/%s has overlapping selector with an older deployment %s/%s, skip syncing it", d.Namespace, d.Name, other.Namespace, other.Name)
570+
}
571+
}
572+
}
573+
if !overlapping {
574+
// We don't care if the overlapping annotation update failed or not (we don't make decision on it)
575+
d, _ = dc.clearDeploymentOverlap(d)
576+
}
577+
return nil
578+
}

‎pkg/controller/deployment/deployment_controller_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ func (f *fixture) run(deploymentName string) {
203203
c.rsStoreSynced = alwaysReady
204204
c.podStoreSynced = alwaysReady
205205
for _, d := range f.dStore {
206-
c.dStore.Store.Add(d)
206+
c.dStore.Indexer.Add(d)
207207
}
208208
for _, rs := range f.rsStore {
209209
c.rsStore.Store.Add(rs)
@@ -275,7 +275,7 @@ func TestDeploymentController_dontSyncDeploymentsWithEmptyPodSelector(t *testing
275275
d := newDeployment(1, nil)
276276
empty := unversioned.LabelSelector{}
277277
d.Spec.Selector = &empty
278-
controller.dStore.Store.Add(d)
278+
controller.dStore.Indexer.Add(d)
279279
// We expect the deployment controller to not take action here since it's configuration
280280
// is invalid, even though no replicasets exist that match it's selector.
281281
controller.syncDeployment(fmt.Sprintf("%s/%s", d.ObjectMeta.Namespace, d.ObjectMeta.Name))

‎pkg/controller/deployment/util/deployment_util.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@ const (
5858
RollbackTemplateUnchanged = "DeploymentRollbackTemplateUnchanged"
5959
// RollbackDone is the done rollback event reason
6060
RollbackDone = "DeploymentRollback"
61+
// OverlapAnnotation marks deployments with overlapping selector with other deployments
62+
// TODO: Delete this annotation when we gracefully handle overlapping selectors. See https://github.com/kubernetes/kubernetes/issues/2210
63+
OverlapAnnotation = "deployment.kubernetes.io/error-selector-overlapping-with"
64+
// SelectorUpdateAnnotation marks the last time deployment selector update
65+
// TODO: Delete this annotation when we gracefully handle overlapping selectors. See https://github.com/kubernetes/kubernetes/issues/2210
66+
SelectorUpdateAnnotation = "deployment.kubernetes.io/selector-updated-at"
6167
)
6268

6369
// MaxRevision finds the highest revision in the replica sets
@@ -791,3 +797,42 @@ func DeploymentDeepCopy(deployment *extensions.Deployment) (*extensions.Deployme
791797
}
792798
return copied, nil
793799
}
800+
801+
// SelectorUpdatedBefore returns true if the former deployment's selector
802+
// is updated before the latter, false otherwise
803+
func SelectorUpdatedBefore(d1, d2 *extensions.Deployment) bool {
804+
t1, t2 := LastSelectorUpdate(d1), LastSelectorUpdate(d2)
805+
return t1.Before(t2)
806+
}
807+
808+
// LastSelectorUpdate returns the last time given deployment's selector is updated
809+
func LastSelectorUpdate(d *extensions.Deployment) unversioned.Time {
810+
t := d.Annotations[SelectorUpdateAnnotation]
811+
if len(t) > 0 {
812+
parsedTime, err := time.Parse(t, time.RFC3339)
813+
// If failed to parse the time, use creation timestamp instead
814+
if err != nil {
815+
return d.CreationTimestamp
816+
}
817+
return unversioned.Time{Time: parsedTime}
818+
}
819+
// If it's never updated, use creation timestamp instead
820+
return d.CreationTimestamp
821+
}
822+
823+
// BySelectorLastUpdateTime sorts a list of deployments by the last update time of their selector,
824+
// first using their creation timestamp and then their names as a tie breaker.
825+
type BySelectorLastUpdateTime []extensions.Deployment
826+
827+
func (o BySelectorLastUpdateTime) Len() int { return len(o) }
828+
func (o BySelectorLastUpdateTime) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
829+
func (o BySelectorLastUpdateTime) Less(i, j int) bool {
830+
ti, tj := LastSelectorUpdate(&o[i]), LastSelectorUpdate(&o[j])
831+
if ti.Equal(tj) {
832+
if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) {
833+
return o[i].Name < o[j].Name
834+
}
835+
return o[i].CreationTimestamp.Before(o[j].CreationTimestamp)
836+
}
837+
return ti.Before(tj)
838+
}

0 commit comments

Comments
 (0)