Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit f5bb473

Browse files
openshift-merge-robotopenshift-merge-robot
authored
Merge pull request openshift#839 from danwinship/kube-proxy-rbac-proxy
Use kube-rbac-proxy for standalone kube-proxy metrics
2 parents d94a25c + 9c683fb commit f5bb473

File tree

6 files changed

+147
-33
lines changed

6 files changed

+147
-33
lines changed

‎bindata/kube-proxy/kube-proxy.yaml‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,6 @@ spec:
6666
ports:
6767
- name: healthz
6868
containerPort: {{.HealthzPort}}
69-
- name: metrics
70-
containerPort: {{.MetricsPort}}
7169
livenessProbe:
7270
httpGet:
7371
path: /healthz
@@ -82,6 +80,80 @@ spec:
8280
requests:
8381
cpu: 100m
8482
memory: 200Mi
83+
- name: kube-rbac-proxy
84+
image: {{.KubeRBACProxyImage}}
85+
command:
86+
- /bin/bash
87+
- -c
88+
- |
89+
#!/bin/bash
90+
set -euo pipefail
91+
TLS_PK=/etc/pki/tls/metrics-certs/tls.key
92+
TLS_CERT=/etc/pki/tls/metrics-certs/tls.crt
93+
94+
# As the secret mount is optional we must wait for the files to be present.
95+
# The service is created in monitor.yaml and this is created in kube-proxy.yaml.
96+
# If it isn't created there is probably an issue so we want to crashloop.
97+
retries=0
98+
while [[ "${retries}" -lt 100 ]]; do
99+
TS=$(
100+
curl \
101+
-s \
102+
--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt \
103+
-H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
104+
"https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/api/v1/namespaces/openshift-kube-proxy/services/openshift-kube-proxy" |
105+
python -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["creationTimestamp"])' 2>/dev/null || true
106+
)
107+
if [ -n "${TS}" ]; then
108+
break
109+
fi
110+
(( retries += 1 ))
111+
echo $(date -Iseconds) INFO: Failed to get openshift-kube-proxy service from API. Retry "${retries}"/100 1>&2
112+
sleep 20
113+
done
114+
if [ "${retries}" -ge 20 ]; then
115+
echo $(date -Iseconds) FATAL: Unable to get openshift-kube-proxy service from API.
116+
exit 1
117+
fi
118+
119+
TS=$(date -d "${TS}" +%s)
120+
WARN_TS=$(( ${TS} + $(( 20 * 60)) ))
121+
HAS_LOGGED_INFO=0
122+
123+
log_missing_certs(){
124+
CUR_TS=$(date +%s)
125+
if [[ "${CUR_TS}" -gt "WARN_TS" ]]; then
126+
echo $(date -Iseconds) WARN: kube-proxy-metrics-certs not mounted after 20 minutes.
127+
elif [[ "${HAS_LOGGED_INFO}" -eq 0 ]] ; then
128+
echo $(date -Iseconds) INFO: kube-proxy-metrics-certs not mounted. Waiting 20 minutes.
129+
HAS_LOGGED_INFO=1
130+
fi
131+
}
132+
133+
while [[ ! -f "${TLS_PK}" || ! -f "${TLS_CERT}" ]] ; do
134+
log_missing_certs
135+
sleep 5
136+
done
137+
138+
exec /usr/bin/kube-rbac-proxy \
139+
--logtostderr \
140+
--secure-listen-address=:{{.MetricsPort}} \
141+
--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \
142+
--upstream=http://127.0.0.1:29102/ \
143+
--tls-private-key-file=${TLS_PK} \
144+
--tls-cert-file=${TLS_CERT}
145+
ports:
146+
- containerPort: {{.MetricsPort}}
147+
name: https
148+
resources:
149+
requests:
150+
cpu: 10m
151+
memory: 20Mi
152+
terminationMessagePolicy: FallbackToLogsOnError
153+
volumeMounts:
154+
- name: kube-proxy-metrics-certs
155+
mountPath: /etc/pki/tls/metrics-certs
156+
readOnly: True
85157
restartPolicy: Always
86158
tolerations:
87159
- operator: Exists
@@ -94,3 +166,10 @@ spec:
94166
- name: config
95167
configMap:
96168
name: proxy-config
169+
# Must be optional because the sdn-metrics-certs is a service serving
170+
# certificate and those cannot be generated without the service proxy
171+
# running
172+
- name: kube-proxy-metrics-certs
173+
secret:
174+
secretName: kube-proxy-metrics-certs
175+
optional: true

‎bindata/kube-proxy/monitor.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
---
12
apiVersion: monitoring.coreos.com/v1
23
kind: ServiceMonitor
34
metadata:
@@ -11,6 +12,11 @@ spec:
1112
endpoints:
1213
- interval: 30s
1314
port: metrics
15+
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
16+
scheme: https
17+
tlsConfig:
18+
caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
19+
serverName: openshift-kube-proxy.openshift-kube-proxy.svc
1420
jobLabel: app
1521
namespaceSelector:
1622
matchNames:
@@ -22,6 +28,8 @@ spec:
2228
apiVersion: v1
2329
kind: Service
2430
metadata:
31+
annotations:
32+
service.beta.openshift.io/serving-cert-secret-name: kube-proxy-metrics-certs
2533
labels:
2634
app: kube-proxy
2735
name: openshift-kube-proxy
@@ -30,6 +38,7 @@ spec:
3038
selector:
3139
app: kube-proxy
3240
clusterIP: None
41+
publishNotReadyAddresses: true
3342
ports:
3443
- name: metrics
3544
port: {{.MetricsPort}}

‎pkg/network/kube_proxy.go‎

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import (
1919
// - pluginDefaults
2020
// - conf.KubeProxyConfig.ProxyArguments
2121
// - pluginOverrides
22-
func kubeProxyConfiguration(pluginDefaults map[string]operv1.ProxyArgumentList, conf *operv1.NetworkSpec, pluginOverrides map[string]operv1.ProxyArgumentList) (jsonConf, metricsPort, healthzPort string, err error) {
22+
func kubeProxyConfiguration(pluginDefaults map[string]operv1.ProxyArgumentList, conf *operv1.NetworkSpec, pluginOverrides map[string]operv1.ProxyArgumentList) (string, error) {
2323
p := conf.KubeProxyConfig
2424

2525
args := map[string]operv1.ProxyArgumentList{}
@@ -33,14 +33,7 @@ func kubeProxyConfiguration(pluginDefaults map[string]operv1.ProxyArgumentList,
3333
args = k8sutil.MergeKubeProxyArguments(args, p.ProxyArguments)
3434
args = k8sutil.MergeKubeProxyArguments(args, pluginOverrides)
3535

36-
if len(args["metrics-port"]) == 1 {
37-
metricsPort = args["metrics-port"][0]
38-
}
39-
if len(args["healthz-port"]) == 1 {
40-
healthzPort = args["healthz-port"][0]
41-
}
42-
jsonConf, err = k8sutil.GenerateKubeProxyConfiguration(args)
43-
return
36+
return k8sutil.GenerateKubeProxyConfiguration(args)
4437
}
4538

4639
// acceptsKubeProxyConfig determines if the desired network type allows
@@ -103,20 +96,18 @@ func validateKubeProxy(conf *operv1.NetworkSpec) []error {
10396
}
10497
}
10598

106-
// Don't allow ports to be overridden. Before 4.7, standalone kube-proxy used the
107-
// same ports as openshift-sdn (metrics 9101, healthz 10256). In 4.7 and later,
108-
// the defaults are 9102 and 10255 to allow openshift-sdn and kube-proxy to be run
109-
// together, but we still allow the old values to avoid breaking old clusters.
99+
// Don't allow ports to be overridden. For backward compatibility, we allow
100+
// explicitly specifying the (old) default values, though we prefer for them to be
101+
// left blank.
110102
if p.ProxyArguments != nil {
111103
if val, ok := p.ProxyArguments["metrics-port"]; ok {
112-
if len(val) != 1 || (val[0] != "9102" && val[0] != "9101") {
113-
out = append(out, errors.Errorf("kube-proxy --metrics-port must be 9102 or 9101"))
104+
if len(val) != 1 || val[0] != "9101" {
105+
out = append(out, errors.Errorf("kube-proxy --metrics-port cannot be overridden"))
114106
}
115107
}
116-
117108
if val, ok := p.ProxyArguments["healthz-port"]; ok {
118-
if len(val) != 1 || (val[0] != "10255" && val[0] != "10256") {
119-
out = append(out, errors.Errorf("kube-proxy --healthz-port must be 10255 or 10256"))
109+
if len(val) != 1 || val[0] != "10256" {
110+
out = append(out, errors.Errorf("kube-proxy --healthz-port cannot be overridden"))
120111
}
121112
}
122113
}
@@ -187,21 +178,34 @@ func renderStandaloneKubeProxy(conf *operv1.NetworkSpec, manifestDir string) ([]
187178
return nil, nil
188179
}
189180

181+
metricsPort := "9102"
182+
healthzPort := "10255"
183+
if val, ok := conf.KubeProxyConfig.ProxyArguments["metrics-port"]; ok {
184+
metricsPort = val[0]
185+
}
186+
if val, ok := conf.KubeProxyConfig.ProxyArguments["healthz-port"]; ok {
187+
healthzPort = val[0]
188+
}
189+
190190
kpcDefaults := map[string]operv1.ProxyArgumentList{
191191
"metrics-bind-address": {"0.0.0.0"},
192-
"metrics-port": {"9102"},
193192
"healthz-port": {"10255"},
194193
"proxy-mode": {"iptables"},
195194
}
196-
197-
kpc, metricsPort, healthzPort, err := kubeProxyConfiguration(kpcDefaults, conf, nil)
195+
// Regardless of the public metrics port, kube-proxy itself must publish metrics on
196+
// port 29102.
197+
kpcOverrides := map[string]operv1.ProxyArgumentList{
198+
"metrics-port": {"29102"},
199+
}
200+
kpc, err := kubeProxyConfiguration(kpcDefaults, conf, kpcOverrides)
198201
if err != nil {
199202
return nil, errors.Wrapf(err, "failed to generate kube-proxy configuration file")
200203
}
201204

202205
data := render.MakeRenderData()
203206
data.Data["ReleaseVersion"] = os.Getenv("RELEASE_VERSION")
204207
data.Data["KubeProxyImage"] = os.Getenv("KUBE_PROXY_IMAGE")
208+
data.Data["KubeRBACProxyImage"] = os.Getenv("KUBE_RBAC_PROXY_IMAGE")
205209
data.Data["KUBERNETES_SERVICE_HOST"] = os.Getenv("KUBERNETES_SERVICE_HOST")
206210
data.Data["KUBERNETES_SERVICE_PORT"] = os.Getenv("KUBERNETES_SERVICE_PORT")
207211
data.Data["KubeProxyConfig"] = kpc

‎pkg/network/kube_proxy_test.go‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func TestKubeProxyConfig(t *testing.T) {
7373
errs := validateKubeProxy(&config)
7474
g.Expect(errs).To(HaveLen(0))
7575

76-
cfg, metricsPort, _, err := kubeProxyConfiguration(map[string]operv1.ProxyArgumentList{
76+
cfg, err := kubeProxyConfiguration(map[string]operv1.ProxyArgumentList{
7777
// special address+port combo
7878
"metrics-bind-address": {"1.2.3.4"},
7979
"metrics-port": {"999"},
@@ -84,7 +84,6 @@ func TestKubeProxyConfig(t *testing.T) {
8484
"conntrack-max-per-core": {"15"},
8585
})
8686
g.Expect(err).NotTo(HaveOccurred())
87-
g.Expect(metricsPort).To(Equal("999"))
8887
g.Expect(cfg).To(MatchYAML(`apiVersion: kubeproxy.config.k8s.io/v1alpha1
8988
bindAddress: "0.0.0.0"
9089
bindAddressHardFail: false
@@ -140,7 +139,7 @@ func TestKubeProxyIPv6Config(t *testing.T) {
140139
errs := validateKubeProxy(&configIPv6)
141140
g.Expect(errs).To(HaveLen(0))
142141

143-
cfg, metricsPort, _, err := kubeProxyConfiguration(
142+
cfg, err := kubeProxyConfiguration(
144143
map[string]operv1.ProxyArgumentList{
145144
// special address+port combo
146145
"metrics-bind-address": {"fd00:1234::4"},
@@ -152,7 +151,6 @@ func TestKubeProxyIPv6Config(t *testing.T) {
152151
"conntrack-max-per-core": {"15"},
153152
})
154153
g.Expect(err).NotTo(HaveOccurred())
155-
g.Expect(metricsPort).To(Equal("51999"))
156154
g.Expect(cfg).To(MatchYAML(`apiVersion: kubeproxy.config.k8s.io/v1alpha1
157155
bindAddress: "::"
158156
bindAddressHardFail: false
@@ -414,7 +412,7 @@ ipvs:
414412
tcpTimeout: 0s
415413
udpTimeout: 0s
416414
kind: KubeProxyConfiguration
417-
metricsBindAddress: 0.0.0.0:9102
415+
metricsBindAddress: 0.0.0.0:29102
418416
mode: iptables
419417
nodePortAddresses: null
420418
oomScoreAdj: null

‎pkg/network/openshift_sdn.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,21 +49,25 @@ func renderOpenShiftSDN(conf *operv1.NetworkSpec, manifestDir string) ([]*uns.Un
4949

5050
kpcDefaults := map[string]operv1.ProxyArgumentList{
5151
"metrics-bind-address": {"0.0.0.0"},
52-
"metrics-port": {"29101"},
5352
"healthz-port": {"10256"},
5453
"proxy-mode": {"iptables"},
5554
"iptables-masquerade-bit": {"0"},
5655
}
57-
58-
kpcOverrides := map[string]operv1.ProxyArgumentList{}
56+
// For backward compatibility we allow conf to specify `metrics-port: 9101` but
57+
// the daemonset always configures 9101 as the secure metrics port and 29101 as
58+
// the insecure metrics port exposed by kube-proxy itself. So just override
59+
// the value from conf (which we know is either "9101" or unspecified).
60+
kpcOverrides := map[string]operv1.ProxyArgumentList{
61+
"metrics-port": {"29101"},
62+
}
5963
if *c.EnableUnidling {
6064
// We already validated that proxy-mode was either unset or iptables.
6165
kpcOverrides["proxy-mode"] = operv1.ProxyArgumentList{"unidling+iptables"}
6266
} else if *conf.DeployKubeProxy {
6367
kpcOverrides["proxy-mode"] = operv1.ProxyArgumentList{"disabled"}
6468
}
6569

66-
kpc, _, _, err := kubeProxyConfiguration(kpcDefaults, conf, kpcOverrides)
70+
kpc, err := kubeProxyConfiguration(kpcDefaults, conf, kpcOverrides)
6771
if err != nil {
6872
return nil, errors.Wrap(err, "failed to build kube-proxy config")
6973
}

‎pkg/network/openshift_sdn_test.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ func TestProxyArgs(t *testing.T) {
244244
val, _, _ = uns.NestedString(cfg.Object, "bindAddress")
245245
g.Expect(val).To(Equal("1.2.3.4"))
246246

247-
//set proxy args
247+
// set proxy args
248248
config.KubeProxyConfig.ProxyArguments = map[string]operv1.ProxyArgumentList{
249249
"cluster-cidr": {"1.2.3.4/5"},
250250
"config-sync-period": {"1s", "2s"},
@@ -280,6 +280,26 @@ func TestProxyArgs(t *testing.T) {
280280
arg, _, _ = uns.NestedString(cfg.Object, "mode")
281281
g.Expect(arg).To(Equal("iptables"))
282282

283+
// Explicitly setting the metrics port to "9101" is allowed but does not affect
284+
// the actual configuration, which uses "29101". Other port values are not
285+
// allowed. (Even 29101!)
286+
config.KubeProxyConfig.ProxyArguments = map[string]operv1.ProxyArgumentList{
287+
"metrics-port": {"29101"},
288+
}
289+
errs := validateKubeProxy(config)
290+
g.Expect(errs).To(HaveLen(1))
291+
config.KubeProxyConfig.ProxyArguments = map[string]operv1.ProxyArgumentList{
292+
"metrics-port": {"9101"},
293+
}
294+
errs = validateKubeProxy(config)
295+
g.Expect(errs).To(HaveLen(0))
296+
297+
objs, err = renderOpenShiftSDN(config, manifestDir)
298+
g.Expect(err).NotTo(HaveOccurred())
299+
cfg = getProxyConfigFile(objs)
300+
301+
arg, _, _ = uns.NestedString(cfg.Object, "metricsBindAddress")
302+
g.Expect(arg).To(Equal("0.0.0.0:29101"))
283303
}
284304

285305
func TestOpenShiftSDNIsSafe(t *testing.T) {

0 commit comments

Comments
 (0)