From 57c8d45bc2dec955ddd104941dffcf50bfabf540 Mon Sep 17 00:00:00 2001 From: "Patrick McKee Jr." Date: Tue, 31 Oct 2023 23:55:59 -0400 Subject: [PATCH 1/6] expose prometheus port --- helm/coder/templates/service.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/helm/coder/templates/service.yaml b/helm/coder/templates/service.yaml index 1881f992a695e..baad53b4864c7 100644 --- a/helm/coder/templates/service.yaml +++ b/helm/coder/templates/service.yaml @@ -28,6 +28,15 @@ spec: nodePort: {{ .Values.coder.service.httpsNodePort }} {{ end }} {{- end }} + {{- if eq (include "coder.coderPrometheusEnabled" .) "true" }} + - name: "prometheus-http" + port: 2112 + targetPort: "prometheus-http" + protocol: TCP + {{ if eq .Values.coder.service.type "NodePort" }} + nodePort: {{ .Values.coder.service.prometheusNodePort }} + {{ end }} + {{- end }} {{- if eq "LoadBalancer" .Values.coder.service.type }} {{- with .Values.coder.service.loadBalancerIP }} loadBalancerIP: {{ . | quote }} From f6a88d9b921e59f2739c27a8241b67340f5fe0d7 Mon Sep 17 00:00:00 2001 From: "Patrick McKee Jr." Date: Wed, 1 Nov 2023 10:03:37 -0400 Subject: [PATCH 2/6] values --- helm/coder/templates/service.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/helm/coder/templates/service.yaml b/helm/coder/templates/service.yaml index baad53b4864c7..76ebbce89c23f 100644 --- a/helm/coder/templates/service.yaml +++ b/helm/coder/templates/service.yaml @@ -28,7 +28,10 @@ spec: nodePort: {{ .Values.coder.service.httpsNodePort }} {{ end }} {{- end }} - {{- if eq (include "coder.coderPrometheusEnabled" .) "true" }} + {{- range .Values.coder.env }} + {{- if eq .name "CODER_PROMETHEUS_ENABLE" }} + {{}} + {{- if eq .value "true" }} - name: "prometheus-http" port: 2112 targetPort: "prometheus-http" @@ -37,6 +40,8 @@ spec: nodePort: {{ .Values.coder.service.prometheusNodePort }} {{ end }} {{- end }} + {{- end }} + {{- end }} {{- if eq "LoadBalancer" .Values.coder.service.type }} {{- with .Values.coder.service.loadBalancerIP }} loadBalancerIP: {{ . | quote }} From 4c7a0792b9a0095c4b3e41b7fd8ce47f33c152f4 Mon Sep 17 00:00:00 2001 From: "Patrick McKee Jr." Date: Wed, 1 Nov 2023 23:44:11 -0400 Subject: [PATCH 3/6] lint fix --- helm/coder/templates/service.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/coder/templates/service.yaml b/helm/coder/templates/service.yaml index 76ebbce89c23f..a65b50bb87544 100644 --- a/helm/coder/templates/service.yaml +++ b/helm/coder/templates/service.yaml @@ -30,7 +30,6 @@ spec: {{- end }} {{- range .Values.coder.env }} {{- if eq .name "CODER_PROMETHEUS_ENABLE" }} - {{}} {{- if eq .value "true" }} - name: "prometheus-http" port: 2112 From b71d6d4d83ac95f452901579c8df0cba9d588c62 Mon Sep 17 00:00:00 2001 From: "Patrick McKee Jr." Date: Thu, 2 Nov 2023 00:29:08 -0400 Subject: [PATCH 4/6] updated values.yaml for nodeport --- helm/coder/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm/coder/values.yaml b/helm/coder/values.yaml index 2f064836d5321..0c4a3924ab07f 100644 --- a/helm/coder/values.yaml +++ b/helm/coder/values.yaml @@ -265,6 +265,9 @@ coder: # coder.service.httpsNodePort -- Enabled if coder.service.type is set to NodePort. # If not set, Kubernetes will allocate a port from the default range, 30000-32767. httpsNodePort: "" + # coder.service.prometheusNodePort -- Enabled if coder.service.type is set to NodePort. + # If not set, Kubernetes will allocate a port from the default range, 30000-32767. + prometheusNodePort: "" # coder.ingress -- The Ingress object to expose for Coder. ingress: From 505f31b2c91ba912d2051567d7794306b3259632 Mon Sep 17 00:00:00 2001 From: "Patrick McKee Jr." Date: Thu, 2 Nov 2023 13:33:07 -0400 Subject: [PATCH 5/6] updated docs --- docs/admin/prometheus.md | 21 ++++++++++++++++++++- helm/coder/values.yaml | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/admin/prometheus.md b/docs/admin/prometheus.md index 3af6a08466edb..d4cbe6b58e6f5 100644 --- a/docs/admin/prometheus.md +++ b/docs/admin/prometheus.md @@ -35,7 +35,7 @@ The Prometheus endpoint can be enabled in the [Helm chart's](https://github.com/coder/coder/tree/main/helm) `values.yml` by setting the environment variable `CODER_PROMETHEUS_ADDRESS` to `0.0.0.0:2112`. The environment variable `CODER_PROMETHEUS_ENABLE` will be enabled -automatically. +automatically. A Service Endpoint will also be exposed allowing Prometheus Service Monitors to be used. ### Prometheus configuration @@ -53,6 +53,25 @@ scrape_configs: apps: "coder" ``` +To use the Service Endpoint for prometheus to scrape the metrics, you can create a +service monitor. Below is an example: `coder-service-monitor`: + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: coder-service-monitor + namespace: coder +spec: + endpoints: + - port: prometheus-http + interval: 10s + scrapeTimeout: 10s + selector: + matchLabels: + app.kubernetes.io/name: coder +``` + ## Available metrics diff --git a/helm/coder/values.yaml b/helm/coder/values.yaml index 0c4a3924ab07f..d4fe2da0cf611 100644 --- a/helm/coder/values.yaml +++ b/helm/coder/values.yaml @@ -265,6 +265,7 @@ coder: # coder.service.httpsNodePort -- Enabled if coder.service.type is set to NodePort. # If not set, Kubernetes will allocate a port from the default range, 30000-32767. httpsNodePort: "" + # Prometheus service port is only exposed if CODER_PROMETHEUS_ENABLE=true in the env section. # coder.service.prometheusNodePort -- Enabled if coder.service.type is set to NodePort. # If not set, Kubernetes will allocate a port from the default range, 30000-32767. prometheusNodePort: "" From bdcc8dac72b20a4130aec330a4a366935ac7a42c Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Mon, 6 Nov 2023 14:40:36 +0000 Subject: [PATCH 6/6] Add helm test and fix template bug --- docs/admin/prometheus.md | 14 +- helm/coder/templates/service.yaml | 10 +- helm/coder/tests/chart_test.go | 9 +- helm/coder/tests/testdata/prometheus.golden | 204 ++++++++++++++++++++ helm/coder/tests/testdata/prometheus.yaml | 9 + helm/coder/values.yaml | 17 +- 6 files changed, 247 insertions(+), 16 deletions(-) create mode 100644 helm/coder/tests/testdata/prometheus.golden create mode 100644 helm/coder/tests/testdata/prometheus.yaml diff --git a/docs/admin/prometheus.md b/docs/admin/prometheus.md index d4cbe6b58e6f5..836b61475ce40 100644 --- a/docs/admin/prometheus.md +++ b/docs/admin/prometheus.md @@ -35,7 +35,8 @@ The Prometheus endpoint can be enabled in the [Helm chart's](https://github.com/coder/coder/tree/main/helm) `values.yml` by setting the environment variable `CODER_PROMETHEUS_ADDRESS` to `0.0.0.0:2112`. The environment variable `CODER_PROMETHEUS_ENABLE` will be enabled -automatically. A Service Endpoint will also be exposed allowing Prometheus Service Monitors to be used. +automatically. A Service Endpoint will also be exposed allowing Prometheus +Service Monitors to be used. ### Prometheus configuration @@ -53,8 +54,9 @@ scrape_configs: apps: "coder" ``` -To use the Service Endpoint for prometheus to scrape the metrics, you can create a -service monitor. Below is an example: `coder-service-monitor`: +To use the Kubernetes Prometheus operator to scrape metrics, you will need to +create a `ServiceMonitor` in your Coder deployment namespace. Below is an +example `ServiceMonitor`: ```yaml apiVersion: monitoring.coreos.com/v1 @@ -64,9 +66,9 @@ metadata: namespace: coder spec: endpoints: - - port: prometheus-http - interval: 10s - scrapeTimeout: 10s + - port: prometheus-http + interval: 10s + scrapeTimeout: 10s selector: matchLabels: app.kubernetes.io/name: coder diff --git a/helm/coder/templates/service.yaml b/helm/coder/templates/service.yaml index a65b50bb87544..4c427a699cb64 100644 --- a/helm/coder/templates/service.yaml +++ b/helm/coder/templates/service.yaml @@ -30,13 +30,19 @@ spec: {{- end }} {{- range .Values.coder.env }} {{- if eq .name "CODER_PROMETHEUS_ENABLE" }} + {{/* + This sadly has to be nested to avoid evaluating the second part + of the condition too early and potentially getting type errors if + the value is not a string (like a `valueFrom`). We do not support + `valueFrom` for this env var specifically. + */}} {{- if eq .value "true" }} - name: "prometheus-http" port: 2112 targetPort: "prometheus-http" protocol: TCP - {{ if eq .Values.coder.service.type "NodePort" }} - nodePort: {{ .Values.coder.service.prometheusNodePort }} + {{ if eq $.Values.coder.service.type "NodePort" }} + nodePort: {{ $.Values.coder.service.prometheusNodePort }} {{ end }} {{- end }} {{- end }} diff --git a/helm/coder/tests/chart_test.go b/helm/coder/tests/chart_test.go index a58ea36e675c5..8a57cdec25d69 100644 --- a/helm/coder/tests/chart_test.go +++ b/helm/coder/tests/chart_test.go @@ -80,6 +80,10 @@ var testCases = []testCase{ name: "extra_templates", expectedError: "", }, + { + name: "prometheus", + expectedError: "", + }, } type testCase struct { @@ -158,7 +162,10 @@ func TestUpdateGoldenFiles(t *testing.T) { valuesPath := tc.valuesFilePath() templateOutput, err := runHelmTemplate(t, helmPath, "..", valuesPath) - + if err != nil { + t.Logf("error running `helm template -f %q`: %v", valuesPath, err) + t.Logf("output: %s", templateOutput) + } require.NoError(t, err, "failed to run `helm template -f %q`", valuesPath) goldenFilePath := tc.goldenFilePath() diff --git a/helm/coder/tests/testdata/prometheus.golden b/helm/coder/tests/testdata/prometheus.golden new file mode 100644 index 0000000000000..d84861fa2d338 --- /dev/null +++ b/helm/coder/tests/testdata/prometheus.golden @@ -0,0 +1,204 @@ +--- +# Source: coder/templates/coder.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: coder + app.kubernetes.io/part-of: coder + app.kubernetes.io/version: 0.1.0 + helm.sh/chart: coder-0.1.0 + name: coder +--- +# Source: coder/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: coder-workspace-perms +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +--- +# Source: coder/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: "coder" +subjects: + - kind: ServiceAccount + name: "coder" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: coder-workspace-perms +--- +# Source: coder/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: coder + labels: + helm.sh/chart: coder-0.1.0 + app.kubernetes.io/name: coder + app.kubernetes.io/instance: release-name + app.kubernetes.io/part-of: coder + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: NodePort + sessionAffinity: None + ports: + - name: "http" + port: 80 + targetPort: "http" + protocol: TCP + + nodePort: + + + - name: "prometheus-http" + port: 2112 + targetPort: "prometheus-http" + protocol: TCP + + nodePort: 31112 + + selector: + app.kubernetes.io/name: coder + app.kubernetes.io/instance: release-name +--- +# Source: coder/templates/coder.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: {} + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: coder + app.kubernetes.io/part-of: coder + app.kubernetes.io/version: 0.1.0 + helm.sh/chart: coder-0.1.0 + name: coder +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: coder + template: + metadata: + annotations: {} + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: coder + app.kubernetes.io/part-of: coder + app.kubernetes.io/version: 0.1.0 + helm.sh/chart: coder-0.1.0 + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/instance + operator: In + values: + - coder + topologyKey: kubernetes.io/hostname + weight: 1 + containers: + - args: + - server + command: + - /opt/coder + env: + - name: CODER_HTTP_ADDRESS + value: 0.0.0.0:8080 + - name: CODER_PROMETHEUS_ADDRESS + value: 0.0.0.0:2112 + - name: CODER_ACCESS_URL + value: http://coder.default.svc.cluster.local + - name: KUBE_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: CODER_DERP_SERVER_RELAY_URL + value: http://$(KUBE_POD_IP):8080 + - name: CODER_PROMETHEUS_ENABLE + value: "true" + image: ghcr.io/coder/coder:latest + imagePullPolicy: IfNotPresent + lifecycle: {} + livenessProbe: + httpGet: + path: /healthz + port: http + scheme: HTTP + name: coder + ports: + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 2112 + name: prometheus-http + protocol: TCP + readinessProbe: + httpGet: + path: /healthz + port: http + scheme: HTTP + resources: {} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: null + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + volumeMounts: [] + restartPolicy: Always + serviceAccountName: coder + terminationGracePeriodSeconds: 60 + volumes: [] diff --git a/helm/coder/tests/testdata/prometheus.yaml b/helm/coder/tests/testdata/prometheus.yaml new file mode 100644 index 0000000000000..6e9fe6492c6ce --- /dev/null +++ b/helm/coder/tests/testdata/prometheus.yaml @@ -0,0 +1,9 @@ +coder: + image: + tag: latest + service: + type: NodePort + prometheusNodePort: 31112 + env: + - name: CODER_PROMETHEUS_ENABLE + value: "true" diff --git a/helm/coder/values.yaml b/helm/coder/values.yaml index d4fe2da0cf611..153d638037982 100644 --- a/helm/coder/values.yaml +++ b/helm/coder/values.yaml @@ -259,15 +259,18 @@ coder: # coder.service.annotations -- The service annotations. See: # https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer annotations: {} - # coder.service.httpNodePort -- Enabled if coder.service.type is set to NodePort. - # If not set, Kubernetes will allocate a port from the default range, 30000-32767. + # coder.service.httpNodePort -- Enabled if coder.service.type is set to + # NodePort. If not set, Kubernetes will allocate a port from the default + # range, 30000-32767. httpNodePort: "" - # coder.service.httpsNodePort -- Enabled if coder.service.type is set to NodePort. - # If not set, Kubernetes will allocate a port from the default range, 30000-32767. + # coder.service.httpsNodePort -- Enabled if coder.service.type is set to + # NodePort. If not set, Kubernetes will allocate a port from the default + # range, 30000-32767. httpsNodePort: "" - # Prometheus service port is only exposed if CODER_PROMETHEUS_ENABLE=true in the env section. - # coder.service.prometheusNodePort -- Enabled if coder.service.type is set to NodePort. - # If not set, Kubernetes will allocate a port from the default range, 30000-32767. + # coder.service.prometheusNodePort -- Enabled if coder.service.type is set + # to NodePort. If not set, Kubernetes will allocate a port from the default + # range, 30000-32767. The "prometheus-http" port on the coder service is + # only exposed if CODER_PROMETHEUS_ENABLE is set to true. prometheusNodePort: "" # coder.ingress -- The Ingress object to expose for Coder.