From b9e1320ebe1021da8f2cba1ac0f1f109cae50bc8 Mon Sep 17 00:00:00 2001 From: John Morales Date: Sun, 17 Nov 2024 11:31:05 -0500 Subject: [PATCH 1/3] Tests progress toward issue 20655 --- cmd/metrics-v2.go | 2 +- cmd/metrics-v2_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index 3dd3ac19205a1..9197ac379812c 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -1881,7 +1881,7 @@ func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription, metrics = append(metrics, MetricV2{ Description: desc, VariableLabels: labels1, - Value: dtoMetric.Counter.GetValue(), + Value: float64(dtoMetric.Histogram.GetSampleCount()), }) } return metrics diff --git a/cmd/metrics-v2_test.go b/cmd/metrics-v2_test.go index 03f01e85f6b31..7994b39fbeb37 100644 --- a/cmd/metrics-v2_test.go +++ b/cmd/metrics-v2_test.go @@ -18,6 +18,8 @@ package cmd import ( + "slices" + "strings" "testing" "time" @@ -85,4 +87,73 @@ func TestGetHistogramMetrics(t *testing.T) { if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) { t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics)) } + + // Accumulate regular-cased API label metrics for 'PutObject' for deeper verification + capitalPutObjects := make([]MetricV2, 0, len(histBuckets)+1) + for _, metric := range metrics { + if value := metric.VariableLabels["api"]; value == "PutObject" { + capitalPutObjects = append(capitalPutObjects, metric) + } + } + if expMetricsPerApi := len(histBuckets) + 1; expMetricsPerApi != len(capitalPutObjects) { + t.Fatalf("Expected %d api=PutObject metrics but got %d", expMetricsPerApi, len(capitalPutObjects)) + } + + // Deterministic ordering + slices.SortFunc(capitalPutObjects, func(a MetricV2, b MetricV2) int { + le1 := a.VariableLabels["le"] + le2 := a.VariableLabels["le"] + return strings.Compare(le1, le2) + }) + if le := capitalPutObjects[0].VariableLabels["le"]; le != "0.050" { + t.Errorf("Expected le='0.050' api=PutObject metrics but got '%v'", le) + } + if value := capitalPutObjects[0].Value; value != 0 { + t.Errorf("Expected le='0.050' api=PutObject value to be 0 but got '%v'", value) + } + if le := capitalPutObjects[1].VariableLabels["le"]; le != "0.100" { + t.Errorf("Expected le='0.100' api=PutObject metrics but got '%v'", le) + } + if value := capitalPutObjects[1].Value; value != 1 { + t.Errorf("Expected le='0.100' api=PutObject value to be 1 but got '%v'", value) + } + if le := capitalPutObjects[2].VariableLabels["le"]; le != "0.250" { + t.Errorf("Expected le='0.250' api=PutObject metrics but got '%v'", le) + } + if value := capitalPutObjects[2].Value; value != 3 { + t.Errorf("Expected le='0.250' api=PutObject value to be 3 but got '%v'", value) + } + if le := capitalPutObjects[3].VariableLabels["le"]; le != "0.500" { + t.Errorf("Expected le='0.500' api=PutObject metrics but got '%v'", le) + } + if value := capitalPutObjects[3].Value; value != 4 { + t.Errorf("Expected le='0.500' api=PutObject value to be 4 but got '%v'", value) + } + if le := capitalPutObjects[4].VariableLabels["le"]; le != "0.750" { + t.Errorf("Expected le='0.750' api=PutObject metrics but got '%v'", le) + } + if value := capitalPutObjects[4].Value; value != 4 { + t.Errorf("Expected le='0.750' api=PutObject value to be 4 but got '%v'", value) + } + if le := capitalPutObjects[5].VariableLabels["le"]; le != "+Inf" { + t.Errorf("Expected le='+Inf' api=PutObject metrics but got '%v'", le) + } + if value := capitalPutObjects[5].Value; value != 4 { + t.Errorf("Expected le='+Inf' api=PutObject value to be 4 but got '%v'", value) + } + //expectedLabels := [] + for _, m := range capitalPutObjects { + t.Logf("Capital: %+v", m) + } + //labelNames := slices.Collect(maps.Values(actualLabelNames)) + //labelValues := slices.Collect(maps.Keys(actualLabelValues)) + //t.Logf("Label names: %s", labelNames) + //t.Logf("Label vals: %s", labelValues) + + metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true) + t.Logf("Metrics: %+v", metrics) + // additional labels for +Inf for all histogram metrics + if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) { + t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics)) + } } From 178d170e13a672f0eda144dd529989370a801397 Mon Sep 17 00:00:00 2001 From: John Morales Date: Sun, 17 Nov 2024 13:42:59 -0500 Subject: [PATCH 2/3] Issue 20655: adding test cases for labels and values --- cmd/metrics-v2.go | 6 +- cmd/metrics-v2_test.go | 136 +++++++++++++++++++++++++++++------------ 2 files changed, 103 insertions(+), 39 deletions(-) diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index 9197ac379812c..bcf9742403ff5 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -1875,7 +1875,11 @@ func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription, // add metrics with +Inf label labels1 := make(map[string]string) for _, lp := range dtoMetric.GetLabel() { - labels1[*lp.Name] = *lp.Value + if *lp.Name == "api" && toLowerAPILabels { + labels1[*lp.Name] = strings.ToLower(*lp.Value) + } else { + labels1[*lp.Name] = *lp.Value + } } labels1["le"] = fmt.Sprintf("%.3f", math.Inf(+1)) metrics = append(metrics, MetricV2{ diff --git a/cmd/metrics-v2_test.go b/cmd/metrics-v2_test.go index 7994b39fbeb37..fd10f0176c37e 100644 --- a/cmd/metrics-v2_test.go +++ b/cmd/metrics-v2_test.go @@ -26,7 +26,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -func TestGetHistogramMetrics(t *testing.T) { +func TestGetHistogramMetrics_BucketCount(t *testing.T) { histBuckets := []float64{0.05, 0.1, 0.25, 0.5, 0.75} labels := []string{"GetObject", "PutObject", "CopyObject", "CompleteMultipartUpload"} ttfbHist := prometheus.NewHistogramVec( @@ -85,10 +85,66 @@ func TestGetHistogramMetrics(t *testing.T) { metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false) // additional labels for +Inf for all histogram metrics if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) { - t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics)) + t.Fatalf("Expected %v data points when toLowerAPILabels=false but got %v", expPoints, len(metrics)) + } + + metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true) + // additional labels for +Inf for all histogram metrics + if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) { + t.Fatalf("Expected %v data points when toLowerAPILabels=true but got %v", expPoints, len(metrics)) + } +} + +func TestGetHistogramMetrics_Values(t *testing.T) { + histBuckets := []float64{0.50, 5.00} + labels := []string{"PutObject", "CopyObject"} + ttfbHist := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "s3_ttfb_seconds", + Help: "Time taken by requests served by current MinIO server instance", + Buckets: histBuckets, + }, + []string{"api"}, + ) + observations := []struct { + val float64 + label string + }{ + { + val: 0.02, + label: labels[0], + }, + { + val: 0.19, + label: labels[1], + }, + { + val: 0.31, + label: labels[1], + }, + { + val: 0.61, + label: labels[0], + }, + { + val: 6.79, + label: labels[1], + }, + } + ticker := time.NewTicker(1 * time.Millisecond) + defer ticker.Stop() + for _, obs := range observations { + // Send observations once every 1ms, to simulate delay between + // observations. This is to test the channel based + // synchronization used internally. + select { + case <-ticker.C: + ttfbHist.With(prometheus.Labels{"api": obs.label}).Observe(obs.val) + } } // Accumulate regular-cased API label metrics for 'PutObject' for deeper verification + metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false) capitalPutObjects := make([]MetricV2, 0, len(histBuckets)+1) for _, metric := range metrics { if value := metric.VariableLabels["api"]; value == "PutObject" { @@ -105,55 +161,59 @@ func TestGetHistogramMetrics(t *testing.T) { le2 := a.VariableLabels["le"] return strings.Compare(le1, le2) }) - if le := capitalPutObjects[0].VariableLabels["le"]; le != "0.050" { + if le := capitalPutObjects[0].VariableLabels["le"]; le != "0.500" { t.Errorf("Expected le='0.050' api=PutObject metrics but got '%v'", le) } - if value := capitalPutObjects[0].Value; value != 0 { - t.Errorf("Expected le='0.050' api=PutObject value to be 0 but got '%v'", value) + if value := capitalPutObjects[0].Value; value != 1 { + t.Errorf("Expected le='0.050' api=PutObject value to be 1 but got '%v'", value) } - if le := capitalPutObjects[1].VariableLabels["le"]; le != "0.100" { - t.Errorf("Expected le='0.100' api=PutObject metrics but got '%v'", le) + if le := capitalPutObjects[1].VariableLabels["le"]; le != "5.000" { + t.Errorf("Expected le='5.000' api=PutObject metrics but got '%v'", le) } - if value := capitalPutObjects[1].Value; value != 1 { - t.Errorf("Expected le='0.100' api=PutObject value to be 1 but got '%v'", value) + if value := capitalPutObjects[1].Value; value != 2 { + t.Errorf("Expected le='5.000' api=PutObject value to be 2 but got '%v'", value) } - if le := capitalPutObjects[2].VariableLabels["le"]; le != "0.250" { - t.Errorf("Expected le='0.250' api=PutObject metrics but got '%v'", le) + if le := capitalPutObjects[2].VariableLabels["le"]; le != "+Inf" { + t.Errorf("Expected le='+Inf' api=PutObject metrics but got '%v'", le) } - if value := capitalPutObjects[2].Value; value != 3 { - t.Errorf("Expected le='0.250' api=PutObject value to be 3 but got '%v'", value) + if value := capitalPutObjects[2].Value; value != 2 { + t.Errorf("Expected le='+Inf' api=PutObject value to be 2 but got '%v'", value) } - if le := capitalPutObjects[3].VariableLabels["le"]; le != "0.500" { - t.Errorf("Expected le='0.500' api=PutObject metrics but got '%v'", le) + + // Accumulate lower-cased API label metrics for 'copyobject' for deeper verification + metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true) + lowerCopyObjects := make([]MetricV2, 0, len(histBuckets)+1) + for _, metric := range metrics { + if value := metric.VariableLabels["api"]; value == "copyobject" { + lowerCopyObjects = append(lowerCopyObjects, metric) + } } - if value := capitalPutObjects[3].Value; value != 4 { - t.Errorf("Expected le='0.500' api=PutObject value to be 4 but got '%v'", value) + if expMetricsPerApi := len(histBuckets) + 1; expMetricsPerApi != len(lowerCopyObjects) { + t.Fatalf("Expected %d api=copyobject metrics but got %d", expMetricsPerApi, len(lowerCopyObjects)) } - if le := capitalPutObjects[4].VariableLabels["le"]; le != "0.750" { - t.Errorf("Expected le='0.750' api=PutObject metrics but got '%v'", le) + + // Deterministic ordering + slices.SortFunc(lowerCopyObjects, func(a MetricV2, b MetricV2) int { + le1 := a.VariableLabels["le"] + le2 := a.VariableLabels["le"] + return strings.Compare(le1, le2) + }) + if le := lowerCopyObjects[0].VariableLabels["le"]; le != "0.500" { + t.Errorf("Expected le='0.050' api=copyobject metrics but got '%v'", le) } - if value := capitalPutObjects[4].Value; value != 4 { - t.Errorf("Expected le='0.750' api=PutObject value to be 4 but got '%v'", value) + if value := lowerCopyObjects[0].Value; value != 2 { + t.Errorf("Expected le='0.050' api=copyobject value to be 2 but got '%v'", value) } - if le := capitalPutObjects[5].VariableLabels["le"]; le != "+Inf" { - t.Errorf("Expected le='+Inf' api=PutObject metrics but got '%v'", le) + if le := lowerCopyObjects[1].VariableLabels["le"]; le != "5.000" { + t.Errorf("Expected le='5.000' api=copyobject metrics but got '%v'", le) } - if value := capitalPutObjects[5].Value; value != 4 { - t.Errorf("Expected le='+Inf' api=PutObject value to be 4 but got '%v'", value) + if value := lowerCopyObjects[1].Value; value != 2 { + t.Errorf("Expected le='5.000' api=copyobject value to be 2 but got '%v'", value) } - //expectedLabels := [] - for _, m := range capitalPutObjects { - t.Logf("Capital: %+v", m) + if le := lowerCopyObjects[2].VariableLabels["le"]; le != "+Inf" { + t.Errorf("Expected le='+Inf' api=copyobject metrics but got '%v'", le) } - //labelNames := slices.Collect(maps.Values(actualLabelNames)) - //labelValues := slices.Collect(maps.Keys(actualLabelValues)) - //t.Logf("Label names: %s", labelNames) - //t.Logf("Label vals: %s", labelValues) - - metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true) - t.Logf("Metrics: %+v", metrics) - // additional labels for +Inf for all histogram metrics - if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) { - t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics)) + if value := lowerCopyObjects[2].Value; value != 3 { + t.Errorf("Expected le='+Inf' api=copyobject value to be 3 but got '%v'", value) } } From 5f06e2b423ce2951b0cd016973c3243977f5001b Mon Sep 17 00:00:00 2001 From: John Morales Date: Mon, 18 Nov 2024 11:01:28 -0500 Subject: [PATCH 3/3] lint --- cmd/metrics-v2_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/metrics-v2_test.go b/cmd/metrics-v2_test.go index fd10f0176c37e..1e5221ea3dbb7 100644 --- a/cmd/metrics-v2_test.go +++ b/cmd/metrics-v2_test.go @@ -151,8 +151,8 @@ func TestGetHistogramMetrics_Values(t *testing.T) { capitalPutObjects = append(capitalPutObjects, metric) } } - if expMetricsPerApi := len(histBuckets) + 1; expMetricsPerApi != len(capitalPutObjects) { - t.Fatalf("Expected %d api=PutObject metrics but got %d", expMetricsPerApi, len(capitalPutObjects)) + if expMetricsPerAPI := len(histBuckets) + 1; expMetricsPerAPI != len(capitalPutObjects) { + t.Fatalf("Expected %d api=PutObject metrics but got %d", expMetricsPerAPI, len(capitalPutObjects)) } // Deterministic ordering @@ -188,8 +188,8 @@ func TestGetHistogramMetrics_Values(t *testing.T) { lowerCopyObjects = append(lowerCopyObjects, metric) } } - if expMetricsPerApi := len(histBuckets) + 1; expMetricsPerApi != len(lowerCopyObjects) { - t.Fatalf("Expected %d api=copyobject metrics but got %d", expMetricsPerApi, len(lowerCopyObjects)) + if expMetricsPerAPI := len(histBuckets) + 1; expMetricsPerAPI != len(lowerCopyObjects) { + t.Fatalf("Expected %d api=copyobject metrics but got %d", expMetricsPerAPI, len(lowerCopyObjects)) } // Deterministic ordering