Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit f2ec29d

Browse files
dsessler7dsessler7
committed
Custom queries for metrics via OTel:
* Add section for metrics collection configuration to instrumentation API * Add provided custom queries file(s) to the config volume and create a new sqlquery receiver * Remove specified metrics from the default metrics; if all metrics for a given query are removed, remove the query as well * Add a collector label for discovery purposes
1 parent 39d291d commit f2ec29d

19 files changed

+773
-56
lines changed

‎config/crd/bases/postgres-operator.crunchydata.com_pgadmins.yaml

+69
Original file line numberDiff line numberDiff line change
@@ -2048,6 +2048,75 @@ spec:
20482048
- message: must be at least one hour
20492049
rule: duration("1h") <= self && self <= duration("8760h")
20502050
type: object
2051+
metrics:
2052+
description: Metrics is the place for users to configure metrics
2053+
collection.
2054+
properties:
2055+
customQueries:
2056+
description: |-
2057+
Where users can turn off built-in metrics and also provide their own
2058+
custom queries.
2059+
properties:
2060+
add:
2061+
description: User defined queries and metrics.
2062+
items:
2063+
properties:
2064+
collectionInterval:
2065+
default: 5s
2066+
description: How often the queries should be run.
2067+
format: duration
2068+
maxLength: 20
2069+
minLength: 1
2070+
pattern: ^((PT)?( *[0-9]+ *(?i:(ms|s|m)|(milli|sec|min)s?))+|0)$
2071+
type: string
2072+
x-kubernetes-validations:
2073+
- rule: duration("0") <= self && self <= duration("60m")
2074+
name:
2075+
description: |-
2076+
The name of this batch of queries, which will be used in naming the OTel
2077+
SqlQuery receiver.
2078+
maxLength: 20
2079+
pattern: ^[^\pZ\pC\pS]+$
2080+
type: string
2081+
queries:
2082+
description: A ConfigMap holding the yaml file that
2083+
contains the queries.
2084+
properties:
2085+
key:
2086+
description: Name of the data field within the
2087+
ConfigMap.
2088+
maxLength: 253
2089+
minLength: 1
2090+
pattern: ^[-._a-zA-Z0-9]+$
2091+
type: string
2092+
x-kubernetes-validations:
2093+
- message: cannot be "." or start with ".."
2094+
rule: self != "." && !self.startsWith("..")
2095+
name:
2096+
description: Name of the ConfigMap.
2097+
maxLength: 253
2098+
minLength: 1
2099+
pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?([.][a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
2100+
type: string
2101+
required:
2102+
- key
2103+
- name
2104+
type: object
2105+
x-kubernetes-map-type: atomic
2106+
required:
2107+
- name
2108+
- queries
2109+
type: object
2110+
type: array
2111+
remove:
2112+
description: |-
2113+
A list of built-in queries that should be removed. If all queries for a
2114+
given SQL statement are removed, the SQL statement will no longer be run.
2115+
items:
2116+
type: string
2117+
type: array
2118+
type: object
2119+
type: object
20512120
resources:
20522121
description: Resources holds the resource requirements for the
20532122
collector container.

‎config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml

+69
Original file line numberDiff line numberDiff line change
@@ -11695,6 +11695,75 @@ spec:
1169511695
- message: must be at least one hour
1169611696
rule: duration("1h") <= self && self <= duration("8760h")
1169711697
type: object
11698+
metrics:
11699+
description: Metrics is the place for users to configure metrics
11700+
collection.
11701+
properties:
11702+
customQueries:
11703+
description: |-
11704+
Where users can turn off built-in metrics and also provide their own
11705+
custom queries.
11706+
properties:
11707+
add:
11708+
description: User defined queries and metrics.
11709+
items:
11710+
properties:
11711+
collectionInterval:
11712+
default: 5s
11713+
description: How often the queries should be run.
11714+
format: duration
11715+
maxLength: 20
11716+
minLength: 1
11717+
pattern: ^((PT)?( *[0-9]+ *(?i:(ms|s|m)|(milli|sec|min)s?))+|0)$
11718+
type: string
11719+
x-kubernetes-validations:
11720+
- rule: duration("0") <= self && self <= duration("60m")
11721+
name:
11722+
description: |-
11723+
The name of this batch of queries, which will be used in naming the OTel
11724+
SqlQuery receiver.
11725+
maxLength: 20
11726+
pattern: ^[^\pZ\pC\pS]+$
11727+
type: string
11728+
queries:
11729+
description: A ConfigMap holding the yaml file that
11730+
contains the queries.
11731+
properties:
11732+
key:
11733+
description: Name of the data field within the
11734+
ConfigMap.
11735+
maxLength: 253
11736+
minLength: 1
11737+
pattern: ^[-._a-zA-Z0-9]+$
11738+
type: string
11739+
x-kubernetes-validations:
11740+
- message: cannot be "." or start with ".."
11741+
rule: self != "." && !self.startsWith("..")
11742+
name:
11743+
description: Name of the ConfigMap.
11744+
maxLength: 253
11745+
minLength: 1
11746+
pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?([.][a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
11747+
type: string
11748+
required:
11749+
- key
11750+
- name
11751+
type: object
11752+
x-kubernetes-map-type: atomic
11753+
required:
11754+
- name
11755+
- queries
11756+
type: object
11757+
type: array
11758+
remove:
11759+
description: |-
11760+
A list of built-in queries that should be removed. If all queries for a
11761+
given SQL statement are removed, the SQL statement will no longer be run.
11762+
items:
11763+
type: string
11764+
type: array
11765+
type: object
11766+
type: object
1169811767
resources:
1169911768
description: Resources holds the resource requirements for the
1170011769
collector container.

‎internal/collector/instance.go

+39-17
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,12 @@ func AddToPod(
4343
spec *v1beta1.InstrumentationSpec,
4444
pullPolicy corev1.PullPolicy,
4545
inInstanceConfigMap *corev1.ConfigMap,
46-
outPod *corev1.PodSpec,
46+
template *corev1.PodTemplateSpec,
4747
volumeMounts []corev1.VolumeMount,
4848
sqlQueryPassword string,
4949
logDirectories []string,
5050
includeLogrotate bool,
51+
thisPodServesMetrics bool,
5152
) {
5253
if spec == nil ||
5354
!(feature.Enabled(ctx, feature.OpenTelemetryLogs) ||
@@ -76,14 +77,13 @@ func AddToPod(
7677
}},
7778
}
7879

79-
// If the user has specified files to be mounted in the spec, add them to the projected config volume
80-
if spec != nil && spec.Config != nil && spec.Config.Files != nil {
81-
configVolume.Projected.Sources = append(configVolume.Projected.Sources, spec.Config.Files...)
80+
// If the user has specified files to be mounted in the spec, add them to
81+
// the projected config volume
82+
if spec.Config != nil && spec.Config.Files != nil {
83+
configVolume.Projected.Sources = append(configVolume.Projected.Sources,
84+
spec.Config.Files...)
8285
}
8386

84-
// Add configVolume to the pod's volumes
85-
outPod.Volumes = append(outPod.Volumes, configVolume)
86-
8787
// Create collector container
8888
container := corev1.Container{
8989
Name: naming.ContainerCollector,
@@ -113,6 +113,28 @@ func AddToPod(
113113
VolumeMounts: append(volumeMounts, configVolumeMount),
114114
}
115115

116+
// If metrics feature is enabled and this Pod serves metrics, add the
117+
// Prometheus port to this container
118+
if feature.Enabled(ctx, feature.OpenTelemetryMetrics) && thisPodServesMetrics {
119+
container.Ports = []corev1.ContainerPort{{
120+
ContainerPort: int32(PrometheusPort),
121+
Name: "otel-metrics",
122+
Protocol: corev1.ProtocolTCP,
123+
}}
124+
125+
// If the user has specified custom queries to add, put the queries
126+
// file(s) in the projected config volume
127+
if spec.Metrics != nil && spec.Metrics.CustomQueries != nil &&
128+
spec.Metrics.CustomQueries.Add != nil {
129+
for _, querySet := range spec.Metrics.CustomQueries.Add {
130+
projection := querySet.Queries.AsProjection(querySet.Name +
131+
"/" + querySet.Queries.Key)
132+
configVolume.Projected.Sources = append(configVolume.Projected.Sources,
133+
corev1.VolumeProjection{ConfigMap: &projection})
134+
}
135+
}
136+
}
137+
116138
// If this is a pod that uses logrotate for log rotation, add config volume
117139
// and mount for logrotate config
118140
if includeLogrotate {
@@ -136,18 +158,17 @@ func AddToPod(
136158
}},
137159
}
138160
container.VolumeMounts = append(container.VolumeMounts, logrotateConfigVolumeMount)
139-
outPod.Volumes = append(outPod.Volumes, logrotateConfigVolume)
161+
template.Spec.Volumes = append(template.Spec.Volumes, logrotateConfigVolume)
140162
}
141163

142-
if feature.Enabled(ctx, feature.OpenTelemetryMetrics) {
143-
container.Ports = []corev1.ContainerPort{{
144-
ContainerPort: int32(8889),
145-
Name: "otel-metrics",
146-
Protocol: corev1.ProtocolTCP,
147-
}}
148-
}
164+
// Add configVolume to the Pod's volumes and add the collector container to
165+
// the Pod's containers
166+
template.Spec.Volumes = append(template.Spec.Volumes, configVolume)
167+
template.Spec.Containers = append(template.Spec.Containers, container)
149168

150-
outPod.Containers = append(outPod.Containers, container)
169+
// add the OTel collector label to the Pod
170+
initialize.Labels(template)
171+
template.Labels[naming.LabelCollectorDiscovery] = "true"
151172
}
152173

153174
// startCommand generates the command script used by the collector container
@@ -192,7 +213,8 @@ while read -r -t 5 -u "${fd}" ||:; do
192213
done
193214
`, mkdirScript, configDirectory, logrotateCommand)
194215

195-
wrapper := `monitor() {` + startScript + `}; export directory="$1"; export -f monitor; exec -a "$0" bash -ceu monitor`
216+
wrapper := `monitor() {` + startScript +
217+
`}; export directory="$1"; export -f monitor; exec -a "$0" bash -ceu monitor`
196218

197219
return []string{"bash", "-ceu", "--", wrapper, "collector", configDirectory}
198220
}

‎internal/collector/naming.go

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const LogsBatchProcessor = "batch/logs"
1010
const OneSecondBatchProcessor = "batch/1s"
1111
const SubSecondBatchProcessor = "batch/200ms"
1212
const Prometheus = "prometheus"
13+
const PrometheusPort = 9187
1314
const PGBouncerMetrics = "metrics/pgbouncer"
1415
const PostgresMetrics = "metrics/postgres"
1516
const PatroniMetrics = "metrics/patroni"

‎internal/collector/patroni.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ package collector
77
import (
88
"context"
99
"slices"
10+
"strconv"
1011

1112
"github.com/crunchydata/postgres-operator/internal/feature"
1213
"github.com/crunchydata/postgres-operator/internal/naming"
@@ -136,7 +137,7 @@ func EnablePatroniMetrics(ctx context.Context,
136137
if feature.Enabled(ctx, feature.OpenTelemetryMetrics) {
137138
// Add Prometheus exporter
138139
outConfig.Exporters[Prometheus] = map[string]any{
139-
"endpoint": "0.0.0.0:9187",
140+
"endpoint": "0.0.0.0:" + strconv.Itoa(PrometheusPort),
140141
}
141142

142143
// Add Prometheus Receiver

‎internal/collector/pgbouncer.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"encoding/json"
1111
"fmt"
1212
"slices"
13+
"strconv"
1314

1415
"github.com/crunchydata/postgres-operator/internal/feature"
1516
"github.com/crunchydata/postgres-operator/internal/naming"
@@ -174,13 +175,14 @@ func EnablePgBouncerMetrics(ctx context.Context, config *Config, sqlQueryUsernam
174175
if feature.Enabled(ctx, feature.OpenTelemetryMetrics) {
175176
// Add Prometheus exporter
176177
config.Exporters[Prometheus] = map[string]any{
177-
"endpoint": "0.0.0.0:9187",
178+
"endpoint": "0.0.0.0:" + strconv.Itoa(PrometheusPort),
178179
}
179180

180181
// Add SqlQuery Receiver
181182
config.Receivers[SqlQuery] = map[string]any{
182183
"driver": "postgres",
183-
"datasource": fmt.Sprintf(`host=localhost dbname=pgbouncer port=5432 user=%s password=${env:PGPASSWORD}`,
184+
"datasource": fmt.Sprintf(
185+
`host=localhost dbname=pgbouncer port=5432 user=%s password=${env:PGPASSWORD}`,
184186
sqlQueryUsername),
185187
"queries": slices.Clone(pgBouncerMetricsQueries),
186188
}

0 commit comments

Comments
 (0)