Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 7e9a4be

Browse files
lingsamuelmborsz
lingsamuel
authored and
mborsz
committed
apiserver add metric etcd_lease_object_counts
Signed-off-by: Ling Samuel <[email protected]>
1 parent 67a75c2 commit 7e9a4be

File tree

15 files changed

+102
-51
lines changed

15 files changed

+102
-51
lines changed

‎cmd/kube-apiserver/app/options/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ go_test(
5858
"//pkg/kubelet/client:go_default_library",
5959
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
6060
"//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
61+
"//staging/src/k8s.io/apiserver/pkg/storage/etcd3:go_default_library",
6162
"//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
6263
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
6364
"//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library",

‎cmd/kube-apiserver/app/options/options_test.go‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,15 @@ import (
2525
"github.com/google/go-cmp/cmp"
2626
"github.com/google/go-cmp/cmp/cmpopts"
2727
"github.com/spf13/pflag"
28-
"k8s.io/component-base/logs"
29-
3028
"k8s.io/apiserver/pkg/admission"
3129
apiserveroptions "k8s.io/apiserver/pkg/server/options"
30+
"k8s.io/apiserver/pkg/storage/etcd3"
3231
"k8s.io/apiserver/pkg/storage/storagebackend"
3332
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
3433
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
3534
restclient "k8s.io/client-go/rest"
3635
cliflag "k8s.io/component-base/cli/flag"
36+
"k8s.io/component-base/logs"
3737
"k8s.io/component-base/metrics"
3838
kapi "k8s.io/kubernetes/pkg/apis/core"
3939
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
@@ -116,6 +116,7 @@ func TestAddFlags(t *testing.T) {
116116
"--request-timeout=2m",
117117
"--storage-backend=etcd3",
118118
"--service-cluster-ip-range=192.168.128.0/17",
119+
"--lease-reuse-duration-seconds=100",
119120
}
120121
fs.Parse(args)
121122

@@ -155,13 +156,15 @@ func TestAddFlags(t *testing.T) {
155156
TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
156157
CertFile: "/var/run/kubernetes/etcdce.crt",
157158
},
158-
Paging: true,
159-
Prefix: "/registry",
160-
CompactionInterval: storagebackend.DefaultCompactInterval,
161-
CountMetricPollPeriod: time.Minute,
162-
DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval,
163-
HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout,
164-
LeaseReuseDurationSeconds: storagebackend.DefaultLeaseReuseDurationSeconds,
159+
Paging: true,
160+
Prefix: "/registry",
161+
CompactionInterval: storagebackend.DefaultCompactInterval,
162+
CountMetricPollPeriod: time.Minute,
163+
DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval,
164+
HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout,
165+
LeaseManagerConfig: etcd3.LeaseManagerConfig{
166+
ReuseDurationSeconds: 100,
167+
},
165168
},
166169
DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
167170
DeleteCollectionWorkers: 1,

‎staging/src/k8s.io/apiserver/pkg/server/options/etcd.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
184184
fs.DurationVar(&s.StorageConfig.HealthcheckTimeout, "etcd-healthcheck-timeout", s.StorageConfig.HealthcheckTimeout,
185185
"The timeout to use when checking etcd health.")
186186

187-
fs.Int64Var(&s.StorageConfig.LeaseReuseDurationSeconds, "lease-reuse-duration-seconds", s.StorageConfig.LeaseReuseDurationSeconds,
187+
fs.Int64Var(&s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds, "lease-reuse-duration-seconds", s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds,
188188
"The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.")
189189
}
190190

‎staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ go_test(
3535
"//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library",
3636
"//staging/src/k8s.io/apiserver/pkg/features:go_default_library",
3737
"//staging/src/k8s.io/apiserver/pkg/storage:go_default_library",
38-
"//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
3938
"//staging/src/k8s.io/apiserver/pkg/storage/testing:go_default_library",
4039
"//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library",
4140
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",

‎staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager.go‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,28 @@ import (
2222
"time"
2323

2424
"go.etcd.io/etcd/clientv3"
25+
"k8s.io/apiserver/pkg/storage/etcd3/metrics"
26+
"k8s.io/klog/v2"
2527
)
2628

29+
const (
30+
defaultLeaseReuseDurationSeconds = 60
31+
largeLeaseThreshold = 5000
32+
)
33+
34+
// LeaseManagerConfig is configuration for creating a lease manager.
35+
type LeaseManagerConfig struct {
36+
// ReuseDurationSeconds specifies time in seconds that each lease is reused
37+
ReuseDurationSeconds int64
38+
}
39+
40+
// NewDefaultLeaseManagerConfig creates a LeaseManagerConfig with default values
41+
func NewDefaultLeaseManagerConfig() LeaseManagerConfig {
42+
return LeaseManagerConfig{
43+
ReuseDurationSeconds: defaultLeaseReuseDurationSeconds,
44+
}
45+
}
46+
2747
// leaseManager is used to manage leases requested from etcd. If a new write
2848
// needs a lease that has similar expiration time to the previous one, the old
2949
// lease will be reused to reduce the overhead of etcd, since lease operations
@@ -36,14 +56,15 @@ type leaseManager struct {
3656
prevLeaseExpirationTime time.Time
3757
// The period of time in seconds and percent of TTL that each lease is
3858
// reused. The minimum of them is used to avoid unreasonably large
39-
// numbers. We use var instead of const for testing purposes.
59+
// numbers.
4060
leaseReuseDurationSeconds int64
4161
leaseReuseDurationPercent float64
62+
leaseAttachedObjectCount int64
4263
}
4364

4465
// newDefaultLeaseManager creates a new lease manager using default setting.
45-
func newDefaultLeaseManager(client *clientv3.Client, leaseReuseDurationSeconds int64) *leaseManager {
46-
return newLeaseManager(client, leaseReuseDurationSeconds, 0.05)
66+
func newDefaultLeaseManager(client *clientv3.Client, config LeaseManagerConfig) *leaseManager {
67+
return newLeaseManager(client, config.ReuseDurationSeconds, 0.05)
4768
}
4869

4970
// newLeaseManager creates a new lease manager with the number of buffered
@@ -67,9 +88,15 @@ func (l *leaseManager) GetLease(ctx context.Context, ttl int64) (clientv3.LeaseI
6788
reuseDurationSeconds := l.getReuseDurationSecondsLocked(ttl)
6889
valid := now.Add(time.Duration(ttl) * time.Second).Before(l.prevLeaseExpirationTime)
6990
sufficient := now.Add(time.Duration(ttl+reuseDurationSeconds) * time.Second).After(l.prevLeaseExpirationTime)
91+
92+
// We count all operations that happened in the same lease, regardless of success or failure.
93+
// Currently each GetLease call only attach 1 object
94+
l.leaseAttachedObjectCount++
95+
7096
if valid && sufficient {
7197
return l.prevLeaseID, nil
7298
}
99+
73100
// request a lease with a little extra ttl from etcd
74101
ttl += reuseDurationSeconds
75102
lcr, err := l.client.Lease.Grant(ctx, ttl)
@@ -79,6 +106,12 @@ func (l *leaseManager) GetLease(ctx context.Context, ttl int64) (clientv3.LeaseI
79106
// cache the new lease id
80107
l.prevLeaseID = lcr.ID
81108
l.prevLeaseExpirationTime = now.Add(time.Duration(ttl) * time.Second)
109+
// refresh count
110+
if l.leaseAttachedObjectCount > largeLeaseThreshold {
111+
klog.Infof("The object count for lease %x is large: %v", l.prevLeaseID, l.leaseAttachedObjectCount)
112+
}
113+
metrics.UpdateLeaseObjectCount(l.leaseAttachedObjectCount)
114+
l.leaseAttachedObjectCount = 1
82115
return lcr.ID, nil
83116
}
84117

‎staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager_test.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ limitations under the License.
1717
package etcd3
1818

1919
import (
20-
"k8s.io/apiserver/pkg/storage/storagebackend"
2120
"testing"
2221
)
2322

@@ -35,7 +34,7 @@ func TestGetReuseDurationSeconds(t *testing.T) {
3534
duration: 50,
3635
},
3736
}
38-
lm := newDefaultLeaseManager(nil, storagebackend.DefaultLeaseReuseDurationSeconds)
37+
lm := newDefaultLeaseManager(nil, NewDefaultLeaseManagerConfig())
3938
for i := 0; i < len(testCases); i++ {
4039
dur := lm.getReuseDurationSecondsLocked(testCases[i].ttl)
4140
if dur != testCases[i].duration {

‎staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,15 @@ var (
6767
},
6868
[]string{"resource"},
6969
)
70+
etcdLeaseObjectCounts = compbasemetrics.NewHistogramVec(
71+
&compbasemetrics.HistogramOpts{
72+
Name: "etcd_lease_object_counts",
73+
Help: "Number of objects attached to a single etcd lease.",
74+
Buckets: []float64{10, 50, 100, 500, 1000, 2500, 5000},
75+
StabilityLevel: compbasemetrics.ALPHA,
76+
},
77+
[]string{},
78+
)
7079
)
7180

7281
var registerMetrics sync.Once
@@ -79,6 +88,7 @@ func Register() {
7988
legacyregistry.MustRegister(objectCounts)
8089
legacyregistry.MustRegister(dbTotalSize)
8190
legacyregistry.MustRegister(etcdBookmarkCounts)
91+
legacyregistry.MustRegister(etcdLeaseObjectCounts)
8292
})
8393
}
8494

@@ -111,3 +121,10 @@ func sinceInSeconds(start time.Time) float64 {
111121
func UpdateEtcdDbSize(ep string, size int64) {
112122
dbTotalSize.WithLabelValues(ep).Set(float64(size))
113123
}
124+
125+
// UpdateLeaseObjectCount sets the etcd_lease_object_counts metric.
126+
func UpdateLeaseObjectCount(count int64) {
127+
// Currently we only store one previous lease, since all the events have the same ttl.
128+
// See pkg/storage/etcd3/lease_manager.go
129+
etcdLeaseObjectCounts.WithLabelValues().Observe(float64(count))
130+
}

‎staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,11 @@ type objState struct {
8383
}
8484

8585
// New returns an etcd3 implementation of storage.Interface.
86-
func New(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, transformer value.Transformer, pagingEnabled bool, leaseReuseDurationSeconds int64) storage.Interface {
87-
return newStore(c, newFunc, pagingEnabled, leaseReuseDurationSeconds, codec, prefix, transformer)
86+
func New(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, transformer value.Transformer, pagingEnabled bool, leaseManagerConfig LeaseManagerConfig) storage.Interface {
87+
return newStore(c, codec, newFunc, prefix, transformer, pagingEnabled, leaseManagerConfig)
8888
}
8989

90-
func newStore(c *clientv3.Client, newFunc func() runtime.Object, pagingEnabled bool, leaseReuseDurationSeconds int64, codec runtime.Codec, prefix string, transformer value.Transformer) *store {
90+
func newStore(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, transformer value.Transformer, pagingEnabled bool, leaseManagerConfig LeaseManagerConfig) *store {
9191
versioner := APIObjectVersioner{}
9292
result := &store{
9393
client: c,
@@ -100,7 +100,7 @@ func newStore(c *clientv3.Client, newFunc func() runtime.Object, pagingEnabled b
100100
// keeps compatibility with etcd2 impl for custom prefixes that don't start with '/'
101101
pathPrefix: path.Join("/", prefix),
102102
watcher: newWatcher(c, codec, newFunc, versioner, transformer),
103-
leaseManager: newDefaultLeaseManager(c, leaseReuseDurationSeconds),
103+
leaseManager: newDefaultLeaseManager(c, leaseManagerConfig),
104104
}
105105
return result
106106
}

‎staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ import (
4747
examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
4848
"k8s.io/apiserver/pkg/features"
4949
"k8s.io/apiserver/pkg/storage"
50-
"k8s.io/apiserver/pkg/storage/storagebackend"
5150
storagetesting "k8s.io/apiserver/pkg/storage/testing"
5251
"k8s.io/apiserver/pkg/storage/value"
5352
utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -824,7 +823,7 @@ func TestTransformationFailure(t *testing.T) {
824823
codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
825824
cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
826825
defer cluster.Terminate(t)
827-
store := newStore(cluster.RandClient(), newPod, false, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)})
826+
store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, false, NewDefaultLeaseManagerConfig())
828827
ctx := context.Background()
829828

830829
preset := []struct {
@@ -901,8 +900,8 @@ func TestList(t *testing.T) {
901900
codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
902901
cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
903902
defer cluster.Terminate(t)
904-
store := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)})
905-
disablePagingStore := newStore(cluster.RandClient(), newPod, false, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)})
903+
store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, true, NewDefaultLeaseManagerConfig())
904+
disablePagingStore := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, false, NewDefaultLeaseManagerConfig())
906905
ctx := context.Background()
907906

908907
// Setup storage with the following structure:
@@ -1400,7 +1399,7 @@ func TestListContinuation(t *testing.T) {
14001399
etcdClient := cluster.RandClient()
14011400
recorder := &clientRecorder{KV: etcdClient.KV}
14021401
etcdClient.KV = recorder
1403-
store := newStore(etcdClient, newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", transformer)
1402+
store := newStore(etcdClient, codec, newPod, "", transformer, true, NewDefaultLeaseManagerConfig())
14041403
ctx := context.Background()
14051404

14061405
// Setup storage with the following structure:
@@ -1562,7 +1561,7 @@ func TestListContinuationWithFilter(t *testing.T) {
15621561
etcdClient := cluster.RandClient()
15631562
recorder := &clientRecorder{KV: etcdClient.KV}
15641563
etcdClient.KV = recorder
1565-
store := newStore(etcdClient, newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", transformer)
1564+
store := newStore(etcdClient, codec, newPod, "", transformer, true, NewDefaultLeaseManagerConfig())
15661565
ctx := context.Background()
15671566

15681567
preset := []struct {
@@ -1665,7 +1664,7 @@ func TestListInconsistentContinuation(t *testing.T) {
16651664
codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
16661665
cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
16671666
defer cluster.Terminate(t)
1668-
store := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)})
1667+
store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, true, NewDefaultLeaseManagerConfig())
16691668
ctx := context.Background()
16701669

16711670
// Setup storage with the following structure:
@@ -1813,7 +1812,9 @@ func testSetup(t *testing.T) (context.Context, *store, *integration.ClusterV3) {
18131812
// As 30s is the default timeout for testing in glboal configuration,
18141813
// we cannot wait longer than that in a single time: change it to 10
18151814
// for testing purposes. See apimachinery/pkg/util/wait/wait.go
1816-
store := newStore(cluster.RandClient(), newPod, true, 1, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)})
1815+
store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, true, LeaseManagerConfig{
1816+
ReuseDurationSeconds: 1,
1817+
})
18171818
ctx := context.Background()
18181819
return ctx, store, cluster
18191820
}
@@ -1855,7 +1856,7 @@ func TestPrefix(t *testing.T) {
18551856
"/registry": "/registry",
18561857
}
18571858
for configuredPrefix, effectivePrefix := range testcases {
1858-
store := newStore(cluster.RandClient(), nil, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, configuredPrefix, transformer)
1859+
store := newStore(cluster.RandClient(), codec, nil, configuredPrefix, transformer, true, NewDefaultLeaseManagerConfig())
18591860
if store.pathPrefix != effectivePrefix {
18601861
t.Errorf("configured prefix of %s, expected effective prefix of %s, got %s", configuredPrefix, effectivePrefix, store.pathPrefix)
18611862
}
@@ -2022,7 +2023,7 @@ func TestConsistentList(t *testing.T) {
20222023
transformer := &fancyTransformer{
20232024
transformer: &prefixTransformer{prefix: []byte(defaultTestPrefix)},
20242025
}
2025-
store := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", transformer)
2026+
store := newStore(cluster.RandClient(), codec, newPod, "", transformer, true, NewDefaultLeaseManagerConfig())
20262027
transformer.store = store
20272028

20282029
for i := 0; i < 5; i++ {

‎staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ import (
3838
"k8s.io/apiserver/pkg/apis/example"
3939
examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
4040
"k8s.io/apiserver/pkg/storage"
41-
"k8s.io/apiserver/pkg/storage/storagebackend"
4241
)
4342

4443
func TestWatch(t *testing.T) {
@@ -226,13 +225,13 @@ func TestWatchError(t *testing.T) {
226225
codec := &testCodec{apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)}
227226
cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
228227
defer cluster.Terminate(t)
229-
invalidStore := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte("test!")})
228+
invalidStore := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte("test!")}, true, NewDefaultLeaseManagerConfig())
230229
ctx := context.Background()
231230
w, err := invalidStore.Watch(ctx, "/abc", storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything})
232231
if err != nil {
233232
t.Fatalf("Watch failed: %v", err)
234233
}
235-
validStore := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte("test!")})
234+
validStore := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte("test!")}, true, NewDefaultLeaseManagerConfig())
236235
validStore.GuaranteedUpdate(ctx, "/abc", &example.Pod{}, true, nil, storage.SimpleUpdate(
237236
func(runtime.Object) (runtime.Object, error) {
238237
return &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, nil
@@ -323,7 +322,7 @@ func TestProgressNotify(t *testing.T) {
323322
}
324323
cluster := integration.NewClusterV3(t, clusterConfig)
325324
defer cluster.Terminate(t)
326-
store := newStore(cluster.RandClient(), newPod, false, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)})
325+
store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, false, NewDefaultLeaseManagerConfig())
327326
ctx := context.Background()
328327

329328
key := "/somekey"

0 commit comments

Comments
 (0)