Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 6e8d6ac

Browse files
author
Kubernetes Submit Queue
authored
Merge pull request #47078 from DirectXMan12/bug/only-update-when-needed
Automatic merge from submit-queue (batch tested with PRs 46979, 47078, 47138, 46916) HPA: only send updates when the status has changed This commit only sends updates if the status has actually changed. Since the HPA runs at a regular interval, this should reduce the volume of writes, especially on short HPA intervals with relatively constant metrics. Fixes #47077 **Release note**: ```release-note The HorizontalPodAutoscaler controller will now only send updates when it has new status information, reducing the number of writes caused by the controller. ```
2 parents 6bc4006 + 00a19f1 commit 6e8d6ac

3 files changed

Lines changed: 114 additions & 13 deletions

File tree

‎pkg/controller/podautoscaler/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ go_library(
3232
"//pkg/controller:go_default_library",
3333
"//pkg/controller/podautoscaler/metrics:go_default_library",
3434
"//vendor/github.com/golang/glog:go_default_library",
35+
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
3536
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
3637
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
3738
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

‎pkg/controller/podautoscaler/horizontal.go‎

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"time"
2323

2424
"github.com/golang/glog"
25+
apiequality "k8s.io/apimachinery/pkg/api/equality"
2526
"k8s.io/apimachinery/pkg/api/errors"
2627
"k8s.io/apimachinery/pkg/api/resource"
2728
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -371,14 +372,20 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
371372
return fmt.Errorf("failed to convert the given HPA to %s: %v", autoscalingv2.SchemeGroupVersion.String(), err)
372373
}
373374
hpa := hpaRaw.(*autoscalingv2.HorizontalPodAutoscaler)
375+
hpaStatusOriginalRaw, err := api.Scheme.DeepCopy(&hpa.Status)
376+
if err != nil {
377+
a.eventRecorder.Event(hpav1Shared, v1.EventTypeWarning, "FailedConvertHPA", err.Error())
378+
return fmt.Errorf("failed to deep-copy the HPA status: %v", err)
379+
}
380+
hpaStatusOriginal := hpaStatusOriginalRaw.(*autoscalingv2.HorizontalPodAutoscalerStatus)
374381

375382
reference := fmt.Sprintf("%s/%s/%s", hpa.Spec.ScaleTargetRef.Kind, hpa.Namespace, hpa.Spec.ScaleTargetRef.Name)
376383

377384
scale, err := a.scaleNamespacer.Scales(hpa.Namespace).Get(hpa.Spec.ScaleTargetRef.Kind, hpa.Spec.ScaleTargetRef.Name)
378385
if err != nil {
379386
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetScale", err.Error())
380387
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionFalse, "FailedGetScale", "the HPA controller was unable to get the target's current scale: %v", err)
381-
a.update(hpa)
388+
a.updateStatusIfNeeded(hpaStatusOriginal, hpa)
382389
return fmt.Errorf("failed to query scale subresource for %s: %v", reference, err)
383390
}
384391
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "SucceededGetScale", "the HPA controller was able to get the target's current scale")
@@ -412,7 +419,10 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
412419
} else {
413420
metricDesiredReplicas, metricName, metricStatuses, metricTimestamp, err = a.computeReplicasForMetrics(hpa, scale, hpa.Spec.Metrics)
414421
if err != nil {
415-
a.updateCurrentReplicasInStatus(hpa, currentReplicas)
422+
a.setCurrentReplicasInStatus(hpa, currentReplicas)
423+
if err := a.updateStatusIfNeeded(hpaStatusOriginal, hpa); err != nil {
424+
utilruntime.HandleError(err)
425+
}
416426
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedComputeMetricsReplicas", err.Error())
417427
return fmt.Errorf("failed to compute desired number of replicas based on listed metrics for %s: %v", reference, err)
418428
}
@@ -489,7 +499,10 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
489499
if err != nil {
490500
a.eventRecorder.Eventf(hpa, v1.EventTypeWarning, "FailedRescale", "New size: %d; reason: %s; error: %v", desiredReplicas, rescaleReason, err.Error())
491501
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionFalse, "FailedUpdateScale", "the HPA controller was unable to update the target scale: %v", err)
492-
a.updateCurrentReplicasInStatus(hpa, currentReplicas)
502+
a.setCurrentReplicasInStatus(hpa, currentReplicas)
503+
if err := a.updateStatusIfNeeded(hpaStatusOriginal, hpa); err != nil {
504+
utilruntime.HandleError(err)
505+
}
493506
return fmt.Errorf("failed to rescale %s: %v", reference, err)
494507
}
495508
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "SucceededRescale", "the HPA controller was able to update the target scale to %d", desiredReplicas)
@@ -501,7 +514,8 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
501514
desiredReplicas = currentReplicas
502515
}
503516

504-
return a.updateStatusWithReplicas(hpa, currentReplicas, desiredReplicas, metricStatuses, rescale)
517+
a.setStatus(hpa, currentReplicas, desiredReplicas, metricStatuses, rescale)
518+
return a.updateStatusIfNeeded(hpaStatusOriginal, hpa)
505519
}
506520

507521
func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
@@ -528,14 +542,14 @@ func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutos
528542
return false
529543
}
530544

531-
func (a *HorizontalController) updateCurrentReplicasInStatus(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas int32) {
532-
err := a.updateStatusWithReplicas(hpa, currentReplicas, hpa.Status.DesiredReplicas, hpa.Status.CurrentMetrics, false)
533-
if err != nil {
534-
utilruntime.HandleError(err)
535-
}
545+
// setCurrentReplicasInStatus sets the current replica count in the status of the HPA.
546+
func (a *HorizontalController) setCurrentReplicasInStatus(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas int32) {
547+
a.setStatus(hpa, currentReplicas, hpa.Status.DesiredReplicas, hpa.Status.CurrentMetrics, false)
536548
}
537549

538-
func (a *HorizontalController) updateStatusWithReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, metricStatuses []autoscalingv2.MetricStatus, rescale bool) error {
550+
// setStatus recreates the status of the given HPA, updating the current and
551+
// desired replicas, as well as the metric statuses
552+
func (a *HorizontalController) setStatus(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, metricStatuses []autoscalingv2.MetricStatus, rescale bool) {
539553
hpa.Status = autoscalingv2.HorizontalPodAutoscalerStatus{
540554
CurrentReplicas: currentReplicas,
541555
DesiredReplicas: desiredReplicas,
@@ -548,11 +562,19 @@ func (a *HorizontalController) updateStatusWithReplicas(hpa *autoscalingv2.Horiz
548562
now := metav1.NewTime(time.Now())
549563
hpa.Status.LastScaleTime = &now
550564
}
565+
}
551566

552-
return a.update(hpa)
567+
// updateStatusIfNeeded calls updateStatus only if the status of the new HPA is not the same as the old status
568+
func (a *HorizontalController) updateStatusIfNeeded(oldStatus *autoscalingv2.HorizontalPodAutoscalerStatus, newHPA *autoscalingv2.HorizontalPodAutoscaler) error {
569+
// skip a write if we wouldn't need to update
570+
if apiequality.Semantic.DeepEqual(oldStatus, &newHPA.Status) {
571+
return nil
572+
}
573+
return a.updateStatus(newHPA)
553574
}
554575

555-
func (a *HorizontalController) update(hpa *autoscalingv2.HorizontalPodAutoscaler) error {
576+
// updateStatus actually does the update request for the status of the given HPA
577+
func (a *HorizontalController) updateStatus(hpa *autoscalingv2.HorizontalPodAutoscaler) error {
556578
// convert back to autoscalingv1
557579
hpaRaw, err := UnsafeConvertToVersionVia(hpa, autoscalingv1.SchemeGroupVersion)
558580
if err != nil {

‎pkg/controller/podautoscaler/horizontal_test.go‎

Lines changed: 79 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -538,7 +538,7 @@ func (tc *testCase) verifyResults(t *testing.T) {
538538
}
539539
}
540540

541-
func (tc *testCase) runTest(t *testing.T) {
541+
func (tc *testCase) setupController(t *testing.T) (*HorizontalController, informers.SharedInformerFactory) {
542542
testClient, testMetricsClient, testCMClient := tc.prepareTestClient(t)
543543
if tc.testClient != nil {
544544
testClient = tc.testClient
@@ -598,6 +598,10 @@ func (tc *testCase) runTest(t *testing.T) {
598598
)
599599
hpaController.hpaListerSynced = alwaysReady
600600

601+
return hpaController, informerFactory
602+
}
603+
604+
func (tc *testCase) runTestWithController(t *testing.T, hpaController *HorizontalController, informerFactory informers.SharedInformerFactory) {
601605
stop := make(chan struct{})
602606
defer close(stop)
603607
informerFactory.Start(stop)
@@ -616,6 +620,11 @@ func (tc *testCase) runTest(t *testing.T) {
616620
tc.verifyResults(t)
617621
}
618622

623+
func (tc *testCase) runTest(t *testing.T) {
624+
hpaController, informerFactory := tc.setupController(t)
625+
tc.runTestWithController(t, hpaController, informerFactory)
626+
}
627+
619628
func TestScaleUp(t *testing.T) {
620629
tc := testCase{
621630
minReplicas: 2,
@@ -1594,4 +1603,73 @@ func TestScaleDownRCImmediately(t *testing.T) {
15941603
tc.runTest(t)
15951604
}
15961605

1606+
func TestAvoidUncessaryUpdates(t *testing.T) {
1607+
tc := testCase{
1608+
minReplicas: 2,
1609+
maxReplicas: 6,
1610+
initialReplicas: 3,
1611+
desiredReplicas: 3,
1612+
CPUTarget: 30,
1613+
CPUCurrent: 40,
1614+
verifyCPUCurrent: true,
1615+
reportedLevels: []uint64{400, 500, 700},
1616+
reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1617+
reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse},
1618+
useMetricsApi: true,
1619+
}
1620+
testClient, _, _ := tc.prepareTestClient(t)
1621+
tc.testClient = testClient
1622+
var savedHPA *autoscalingv1.HorizontalPodAutoscaler
1623+
testClient.PrependReactor("list", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
1624+
tc.Lock()
1625+
defer tc.Unlock()
1626+
1627+
if savedHPA != nil {
1628+
// fake out the verification logic and mark that we're done processing
1629+
go func() {
1630+
// wait a tick and then mark that we're finished (otherwise, we have no
1631+
// way to indicate that we're finished, because the function decides not to do anything)
1632+
time.Sleep(1 * time.Second)
1633+
tc.statusUpdated = true
1634+
tc.processed <- "test-hpa"
1635+
}()
1636+
return true, &autoscalingv1.HorizontalPodAutoscalerList{
1637+
Items: []autoscalingv1.HorizontalPodAutoscaler{*savedHPA},
1638+
}, nil
1639+
}
1640+
1641+
// fallthrough
1642+
return false, nil, nil
1643+
})
1644+
testClient.PrependReactor("update", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
1645+
tc.Lock()
1646+
defer tc.Unlock()
1647+
1648+
if savedHPA == nil {
1649+
// save the HPA and return it
1650+
savedHPA = action.(core.UpdateAction).GetObject().(*autoscalingv1.HorizontalPodAutoscaler)
1651+
return true, savedHPA, nil
1652+
}
1653+
1654+
assert.Fail(t, "should not have attempted to update the HPA when nothing changed")
1655+
// mark that we've processed this HPA
1656+
tc.processed <- ""
1657+
return true, nil, fmt.Errorf("unexpected call")
1658+
})
1659+
1660+
controller, informerFactory := tc.setupController(t)
1661+
1662+
// fake an initial processing loop to populate savedHPA
1663+
initialHPAs, err := testClient.Autoscaling().HorizontalPodAutoscalers("test-namespace").List(metav1.ListOptions{})
1664+
if err != nil {
1665+
t.Fatalf("unexpected error: %v", err)
1666+
}
1667+
if err := controller.reconcileAutoscaler(&initialHPAs.Items[0]); err != nil {
1668+
t.Fatalf("unexpected error: %v", err)
1669+
}
1670+
1671+
// actually run the test
1672+
tc.runTestWithController(t, controller, informerFactory)
1673+
}
1674+
15971675
// TODO: add more tests

0 commit comments

Comments
 (0)